From 2fffc831eea0fd8013ad8ba654ceffbf4ea80654 Mon Sep 17 00:00:00 2001 From: stabldev <114811070+stabldev@users.noreply.github.com> Date: Wed, 29 Oct 2025 10:09:28 +0530 Subject: [PATCH 1/2] feat: basic sqlalchemy type_decorator and proxy class --- pyproject.toml | 1 + src/cloud_storage/base.py | 32 ++++++++-- src/cloud_storage/integrations/__init__.py | 0 src/cloud_storage/integrations/sqlalchemy.py | 34 +++++++++++ src/cloud_storage/s3.py | 36 +++++------ tests/test_s3_storage.py | 34 +++++------ uv.lock | 64 ++++++++++++++++++++ 7 files changed, 161 insertions(+), 40 deletions(-) create mode 100644 src/cloud_storage/integrations/__init__.py create mode 100644 src/cloud_storage/integrations/sqlalchemy.py diff --git a/pyproject.toml b/pyproject.toml index aa53977..eef9b01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dev = [ "pytest>=8.4.2", "pytest-aioboto3>=0.6.0", "pytest-asyncio>=1.2.0", + "sqlalchemy>=2.0.44", ] [tool.uv.build-backend] diff --git a/src/cloud_storage/base.py b/src/cloud_storage/base.py index fda7f24..2ef4d4e 100644 --- a/src/cloud_storage/base.py +++ b/src/cloud_storage/base.py @@ -3,17 +3,39 @@ class AsyncBaseStorage: - def get_secure_key(self, key: str) -> str: + def get_name(self, name: str) -> str: raise NotImplementedError() - async def get_size(self, key: str) -> int: + async def get_size(self, name: str) -> int: raise NotImplementedError() - async def get_url(self, key: str) -> str: + async def get_url(self, name: str) -> str: raise NotImplementedError() - async def upload(self, file: BinaryIO, key: str) -> str: + async def upload(self, file: BinaryIO, name: str) -> str: raise NotImplementedError() - async def delete(self, key: str) -> None: + async def delete(self, name: str) -> None: raise NotImplementedError() + + +class AsyncStorageFile: + def __init__(self, name: str, storage: AsyncBaseStorage): + self._name: str = name + self._storage: AsyncBaseStorage = storage + + @property + def name(self) -> str: + return self._name + + async def get_size(self) -> int: + return await self._storage.get_size(self._name) + + async def get_url(self) -> str: + return await self._storage.get_url(self._name) + + async def upload(self, file: BinaryIO) -> str: + return await self._storage.upload(file=file, name=self._name) + + async def delete(self) -> None: + await self._storage.delete(self._name) diff --git a/src/cloud_storage/integrations/__init__.py b/src/cloud_storage/integrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cloud_storage/integrations/sqlalchemy.py b/src/cloud_storage/integrations/sqlalchemy.py new file mode 100644 index 0000000..e29d362 --- /dev/null +++ b/src/cloud_storage/integrations/sqlalchemy.py @@ -0,0 +1,34 @@ +from typing import Any, override +from sqlalchemy.engine.interfaces import Dialect +from sqlalchemy.types import TypeDecorator, TypeEngine, Unicode + +from cloud_storage.base import AsyncBaseStorage, AsyncStorageFile + + +class AsyncFileType(TypeDecorator[Any]): + impl: TypeEngine[Any] | type[TypeEngine[Any]] = Unicode + cache_ok: bool | None = True + + def __init__(self, storage: AsyncBaseStorage, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self.storage: AsyncBaseStorage = storage + + @override + def process_bind_param(self, value: Any, dialect: Dialect) -> str: + if value is None: + return value + if isinstance(value, str): + return value + + name = getattr(value, "name", None) + if name: + return name + return str(value) + + @override + def process_result_value( + self, value: Any | None, dialect: Dialect + ) -> AsyncStorageFile | None: + if value is None: + return None + return AsyncStorageFile(name=value, storage=self.storage) diff --git a/src/cloud_storage/s3.py b/src/cloud_storage/s3.py index e9ce314..3038ce7 100644 --- a/src/cloud_storage/s3.py +++ b/src/cloud_storage/s3.py @@ -57,8 +57,8 @@ def _get_s3_client(self) -> Any: ) @override - def get_secure_key(self, key: str) -> str: - parts = Path(key).parts + def get_name(self, name: str) -> str: + parts = Path(name).parts safe_parts: list[str] = [] for part in parts: @@ -69,13 +69,13 @@ def get_secure_key(self, key: str) -> str: return str(safe_path) @override - async def get_size(self, key: str) -> int: - key = self.get_secure_key(key) + async def get_size(self, name: str) -> int: + name = self.get_name(name) async with self._get_s3_client() as s3_client: try: - response = await s3_client.head_object(Bucket=self.bucket_name, Key=key) - return int(response.get("ContentLength", 0)) + res = await s3_client.head_object(Bucket=self.bucket_name, Key=name) + return int(res.get("ContentLength", 0)) except ClientError as e: code = e.response.get("Error", {}).get("Code") status = e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") @@ -85,23 +85,23 @@ async def get_size(self, key: str) -> int: raise @override - async def get_url(self, key: str, expires_in: int = 3600) -> str: + async def get_url(self, name: str) -> str: if self.custom_domain: - return f"{self._http_scheme}://{self.custom_domain}/{key}" + return f"{self._http_scheme}://{self.custom_domain}/{name}" elif self.querystring_auth: async with self._get_s3_client() as s3_client: - params = {"Bucket": self.bucket_name, "Key": key} + params = {"Bucket": self.bucket_name, "Key": name} return await s3_client.generate_presigned_url( - "get_object", Params=params, ExpiresIn=expires_in + "get_object", Params=params ) else: - url = f"{self._http_scheme}://{self.endpoint_url}/{self.bucket_name}/{key}" + url = f"{self._http_scheme}://{self.endpoint_url}/{self.bucket_name}/{name}" return url @override - async def upload(self, file: BinaryIO, key: str) -> str: - key = self.get_secure_key(key) - content_type, _ = mimetypes.guess_type(key) + async def upload(self, file: BinaryIO, name: str) -> str: + name = self.get_name(name) + content_type, _ = mimetypes.guess_type(name) extra_args = {"ContentType": content_type or "application/octet-stream"} if self.default_acl: extra_args["ACL"] = self.default_acl @@ -109,15 +109,15 @@ async def upload(self, file: BinaryIO, key: str) -> str: async with self._get_s3_client() as s3_client: file.seek(0) await s3_client.put_object( - Bucket=self.bucket_name, Key=key, Body=file, **extra_args + Bucket=self.bucket_name, Key=name, Body=file, **extra_args ) - return key + return name @override - async def delete(self, key: str) -> None: + async def delete(self, name: str) -> None: async with self._get_s3_client() as s3_client: try: - await s3_client.delete_object(Bucket=self.bucket_name, Key=key) + await s3_client.delete_object(Bucket=self.bucket_name, Key=name) except ClientError as e: if e.response.get("Error", {}).get("Code") != "NoSuchKey": raise diff --git a/tests/test_s3_storage.py b/tests/test_s3_storage.py index 193507a..ad42669 100644 --- a/tests/test_s3_storage.py +++ b/tests/test_s3_storage.py @@ -20,25 +20,25 @@ async def test_s3_storage_methods(s3_test_env: Any): file_content = b"hello moto" file_obj = BytesIO(file_content) - key = "test/file.txt" + name = "test/file.txt" # upload test - returned_key = await storage.upload(file_obj, key) - assert returned_key == storage.get_secure_key(key) + returned_name = await storage.upload(file_obj, name) + assert returned_name == storage.get_name(name) # get url test without custom domain or querystring_auth - url = await storage.get_url(key) - assert key in url + url = await storage.get_url(name) + assert name in url # get size test - size = await storage.get_size(key) + size = await storage.get_size(name) assert size == len(file_content) # delete test (should suceed silently) - await storage.delete(key) + await storage.delete(name) # get size test after delete (should return 0) - size_after_delete = await storage.get_size(key) + size_after_delete = await storage.get_size(name) assert size_after_delete == 0 @@ -55,8 +55,8 @@ async def test_s3_storage_querystring_auth(s3_test_env: Any): querystring_auth=True, ) - key = "test/file.txt" - url = await storage.get_url(key) + name = "test/file.txt" + url = await storage.get_url(name) assert url.count("AWSAccessKeyId=") == 1 assert url.count("Signature=") == 1 @@ -76,11 +76,11 @@ async def test_s3_storage_custom_domain(s3_test_env: Any): custom_domain="cdn.example.com", ) - key = "test/file.txt" - url = await storage.get_url(key) + name = "test/file.txt" + url = await storage.get_url(name) assert url.startswith("http://cdn.example.com/") - assert key in await storage.get_url(key) + assert name in await storage.get_url(name) @pytest.mark.asyncio @@ -93,8 +93,8 @@ async def test_get_secure_key_normalization(): use_ssl=False, ) - raw_key = "../../weird ../file name.txt" - normalized_key = storage.get_secure_key(raw_key) + raw_name = "../../weird ../file name.txt" + normalized_name = storage.get_name(raw_name) - assert ".." not in normalized_key - assert ".txt" in normalized_key + assert ".." not in normalized_name + assert ".txt" in normalized_name diff --git a/uv.lock b/uv.lock index 15e2a2b..a8181ea 100644 --- a/uv.lock +++ b/uv.lock @@ -514,6 +514,7 @@ dev = [ { name = "pytest" }, { name = "pytest-aioboto3" }, { name = "pytest-asyncio" }, + { name = "sqlalchemy" }, ] [package.metadata] @@ -525,6 +526,7 @@ dev = [ { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-aioboto3", specifier = ">=0.6.0" }, { name = "pytest-asyncio", specifier = ">=1.2.0" }, + { name = "sqlalchemy", specifier = ">=2.0.44" }, ] [[package]] @@ -655,6 +657,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/4f/7297663840621022bc73c22d7d9d80dbc78b4db6297f764b545cd5dd462d/graphql_core-3.2.6-py3-none-any.whl", hash = "sha256:78b016718c161a6fb20a7d97bbf107f331cd1afe53e45566c59f776ed7f0b45f", size = 203416, upload-time = "2025-01-26T16:36:24.868Z" }, ] +[[package]] +name = "greenlet" +version = "3.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/b8/704d753a5a45507a7aab61f18db9509302ed3d0a27ac7e0359ec2905b1a6/greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d", size = 188260, upload-time = "2025-08-07T13:24:33.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, + { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, + { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, + { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, + { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, + { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, + { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, + { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0b/bc13f787394920b23073ca3b6c4a7a21396301ed75a655bcb47196b50e6e/greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc", size = 655191, upload-time = "2025-08-07T13:45:29.752Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d6/6adde57d1345a8d0f14d31e4ab9c23cfe8e2cd39c3baf7674b4b0338d266/greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a", size = 649516, upload-time = "2025-08-07T13:53:16.314Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3b/3a3328a788d4a473889a2d403199932be55b1b0060f4ddd96ee7cdfcad10/greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504", size = 652169, upload-time = "2025-08-07T13:18:32.861Z" }, + { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, + { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/a2/15/0d5e4e1a66fab130d98168fe984c509249c833c1a3c16806b90f253ce7b9/greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae", size = 1149210, upload-time = "2025-08-07T13:18:24.072Z" }, + { url = "https://files.pythonhosted.org/packages/0b/55/2321e43595e6801e105fcfdee02b34c0f996eb71e6ddffca6b10b7e1d771/greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b", size = 299685, upload-time = "2025-08-07T13:24:38.824Z" }, + { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, + { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, + { url = "https://files.pythonhosted.org/packages/c0/aa/687d6b12ffb505a4447567d1f3abea23bd20e73a5bed63871178e0831b7a/greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5", size = 699218, upload-time = "2025-08-07T13:45:30.969Z" }, + { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, + { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, + { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a5/6ddab2b4c112be95601c13428db1d8b6608a8b6039816f2ba09c346c08fc/greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01", size = 303425, upload-time = "2025-08-07T13:32:27.59Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1673,6 +1708,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.44" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/f2/840d7b9496825333f532d2e3976b8eadbf52034178aac53630d09fe6e1ef/sqlalchemy-2.0.44.tar.gz", hash = "sha256:0ae7454e1ab1d780aee69fd2aae7d6b8670a581d8847f2d1e0f7ddfbf47e5a22", size = 9819830, upload-time = "2025-10-10T14:39:12.935Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/c4/59c7c9b068e6813c898b771204aad36683c96318ed12d4233e1b18762164/sqlalchemy-2.0.44-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:72fea91746b5890f9e5e0997f16cbf3d53550580d76355ba2d998311b17b2250", size = 2139675, upload-time = "2025-10-10T16:03:31.064Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ae/eeb0920537a6f9c5a3708e4a5fc55af25900216bdb4847ec29cfddf3bf3a/sqlalchemy-2.0.44-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:585c0c852a891450edbb1eaca8648408a3cc125f18cf433941fa6babcc359e29", size = 2127726, upload-time = "2025-10-10T16:03:35.934Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d5/2ebbabe0379418eda8041c06b0b551f213576bfe4c2f09d77c06c07c8cc5/sqlalchemy-2.0.44-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b94843a102efa9ac68a7a30cd46df3ff1ed9c658100d30a725d10d9c60a2f44", size = 3327603, upload-time = "2025-10-10T15:35:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/45/e5/5aa65852dadc24b7d8ae75b7efb8d19303ed6ac93482e60c44a585930ea5/sqlalchemy-2.0.44-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:119dc41e7a7defcefc57189cfa0e61b1bf9c228211aba432b53fb71ef367fda1", size = 3337842, upload-time = "2025-10-10T15:43:45.431Z" }, + { url = "https://files.pythonhosted.org/packages/41/92/648f1afd3f20b71e880ca797a960f638d39d243e233a7082c93093c22378/sqlalchemy-2.0.44-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0765e318ee9179b3718c4fd7ba35c434f4dd20332fbc6857a5e8df17719c24d7", size = 3264558, upload-time = "2025-10-10T15:35:29.93Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/e27d7ee61a10f74b17740918e23cbc5bc62011b48282170dc4c66da8ec0f/sqlalchemy-2.0.44-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2e7b5b079055e02d06a4308d0481658e4f06bc7ef211567edc8f7d5dce52018d", size = 3301570, upload-time = "2025-10-10T15:43:48.407Z" }, + { url = "https://files.pythonhosted.org/packages/3b/3d/3116a9a7b63e780fb402799b6da227435be878b6846b192f076d2f838654/sqlalchemy-2.0.44-cp312-cp312-win32.whl", hash = "sha256:846541e58b9a81cce7dee8329f352c318de25aa2f2bbe1e31587eb1f057448b4", size = 2103447, upload-time = "2025-10-10T15:03:21.678Z" }, + { url = "https://files.pythonhosted.org/packages/25/83/24690e9dfc241e6ab062df82cc0df7f4231c79ba98b273fa496fb3dd78ed/sqlalchemy-2.0.44-cp312-cp312-win_amd64.whl", hash = "sha256:7cbcb47fd66ab294703e1644f78971f6f2f1126424d2b300678f419aa73c7b6e", size = 2130912, upload-time = "2025-10-10T15:03:24.656Z" }, + { url = "https://files.pythonhosted.org/packages/45/d3/c67077a2249fdb455246e6853166360054c331db4613cda3e31ab1cadbef/sqlalchemy-2.0.44-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ff486e183d151e51b1d694c7aa1695747599bb00b9f5f604092b54b74c64a8e1", size = 2135479, upload-time = "2025-10-10T16:03:37.671Z" }, + { url = "https://files.pythonhosted.org/packages/2b/91/eabd0688330d6fd114f5f12c4f89b0d02929f525e6bf7ff80aa17ca802af/sqlalchemy-2.0.44-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0b1af8392eb27b372ddb783b317dea0f650241cea5bd29199b22235299ca2e45", size = 2123212, upload-time = "2025-10-10T16:03:41.755Z" }, + { url = "https://files.pythonhosted.org/packages/b0/bb/43e246cfe0e81c018076a16036d9b548c4cc649de241fa27d8d9ca6f85ab/sqlalchemy-2.0.44-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b61188657e3a2b9ac4e8f04d6cf8e51046e28175f79464c67f2fd35bceb0976", size = 3255353, upload-time = "2025-10-10T15:35:31.221Z" }, + { url = "https://files.pythonhosted.org/packages/b9/96/c6105ed9a880abe346b64d3b6ddef269ddfcab04f7f3d90a0bf3c5a88e82/sqlalchemy-2.0.44-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b87e7b91a5d5973dda5f00cd61ef72ad75a1db73a386b62877d4875a8840959c", size = 3260222, upload-time = "2025-10-10T15:43:50.124Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/1857e35a47155b5ad927272fee81ae49d398959cb749edca6eaa399b582f/sqlalchemy-2.0.44-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:15f3326f7f0b2bfe406ee562e17f43f36e16167af99c4c0df61db668de20002d", size = 3189614, upload-time = "2025-10-10T15:35:32.578Z" }, + { url = "https://files.pythonhosted.org/packages/88/ee/4afb39a8ee4fc786e2d716c20ab87b5b1fb33d4ac4129a1aaa574ae8a585/sqlalchemy-2.0.44-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e77faf6ff919aa8cd63f1c4e561cac1d9a454a191bb864d5dd5e545935e5a40", size = 3226248, upload-time = "2025-10-10T15:43:51.862Z" }, + { url = "https://files.pythonhosted.org/packages/32/d5/0e66097fc64fa266f29a7963296b40a80d6a997b7ac13806183700676f86/sqlalchemy-2.0.44-cp313-cp313-win32.whl", hash = "sha256:ee51625c2d51f8baadf2829fae817ad0b66b140573939dd69284d2ba3553ae73", size = 2101275, upload-time = "2025-10-10T15:03:26.096Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/665617fe4f8c6450f42a6d8d69243f9420f5677395572c2fe9d21b493b7b/sqlalchemy-2.0.44-cp313-cp313-win_amd64.whl", hash = "sha256:c1c80faaee1a6c3428cecf40d16a2365bcf56c424c92c2b6f0f9ad204b899e9e", size = 2127901, upload-time = "2025-10-10T15:03:27.548Z" }, + { url = "https://files.pythonhosted.org/packages/9c/5e/6a29fa884d9fb7ddadf6b69490a9d45fded3b38541713010dad16b77d015/sqlalchemy-2.0.44-py3-none-any.whl", hash = "sha256:19de7ca1246fbef9f9d1bff8f1ab25641569df226364a0e40457dc5457c54b05", size = 1928718, upload-time = "2025-10-10T15:29:45.32Z" }, +] + [[package]] name = "sympy" version = "1.14.0" From 112c34b9eb1f229f9d8ecbd82d84464c86009320 Mon Sep 17 00:00:00 2001 From: stabldev <114811070+stabldev@users.noreply.github.com> Date: Wed, 29 Oct 2025 11:04:00 +0530 Subject: [PATCH 2/2] test: add test cases for sqlalchemy integration --- pyproject.toml | 1 + src/cloud_storage/__init__.py | 3 +- tests/test_integrations/conftest.py | 17 +++++ tests/test_integrations/test_sqlalchemy.py | 75 ++++++++++++++++++++++ tests/test_s3_storage.py | 18 +++--- uv.lock | 14 ++++ 6 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 tests/test_integrations/conftest.py create mode 100644 tests/test_integrations/test_sqlalchemy.py diff --git a/pyproject.toml b/pyproject.toml index eef9b01..2bd8670 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ s3 = ["aioboto3>=15.4.0"] [dependency-groups] dev = [ + "aiosqlite>=0.21.0", "pytest>=8.4.2", "pytest-aioboto3>=0.6.0", "pytest-asyncio>=1.2.0", diff --git a/src/cloud_storage/__init__.py b/src/cloud_storage/__init__.py index e5378c4..93ddba0 100644 --- a/src/cloud_storage/__init__.py +++ b/src/cloud_storage/__init__.py @@ -1,4 +1,5 @@ +from .base import AsyncStorageFile from .s3 import AsyncS3Storage __version__ = "0.1.0" -__all__ = ["AsyncS3Storage"] +__all__ = ["AsyncStorageFile", "AsyncS3Storage"] diff --git a/tests/test_integrations/conftest.py b/tests/test_integrations/conftest.py new file mode 100644 index 0000000..7b9e8f0 --- /dev/null +++ b/tests/test_integrations/conftest.py @@ -0,0 +1,17 @@ +from typing import Any +import pytest + +from cloud_storage import AsyncS3Storage + + +@pytest.fixture +async def s3_test_storage(s3_test_env: Any) -> AsyncS3Storage: + bucket_name, endpoint_without_scheme = s3_test_env + + return AsyncS3Storage( + bucket_name=bucket_name, + endpoint_url=endpoint_without_scheme, + aws_access_key_id="fake-access-key", + aws_secret_access_key="fake-secret-key", + use_ssl=False, + ) diff --git a/tests/test_integrations/test_sqlalchemy.py b/tests/test_integrations/test_sqlalchemy.py new file mode 100644 index 0000000..f9ddc83 --- /dev/null +++ b/tests/test_integrations/test_sqlalchemy.py @@ -0,0 +1,75 @@ +from io import BytesIO +from typing import Any +import pytest +from sqlalchemy import Column, Integer +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.ext.asyncio.session import async_sessionmaker +from sqlalchemy.orm import declarative_base + +from cloud_storage import AsyncStorageFile +from cloud_storage.integrations.sqlalchemy import AsyncFileType + +Base = declarative_base() + + +class Document(Base): + __tablename__: str = "documents" + id: Column[int] = Column(Integer, primary_key=True) + file: Column[str] = Column(AsyncFileType(storage=None)) # pyright: ignore[reportArgumentType] + + +@pytest.mark.asyncio +async def test_sqlalchemy_filetype_with_s3(s3_test_storage: Any): + storage = s3_test_storage + # assign s3_storage to file column + Document.__table__.columns.file.type.storage = storage + + # create async engine and session + engine = create_async_engine("sqlite+aiosqlite:///:memory:", echo=False) + async_session = async_sessionmaker( + engine, expire_on_commit=False, class_=AsyncSession + ) + + # create db tables + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + # create demo file object + file_name = "uploads/test-file.txt" + file_content = b"SQLAlchemy + S3 integration test" + file_obj = BytesIO(file_content) + + # upload to s3 storage to fetch from db and test methods + await storage.upload(file_obj, file_name) + + # insert record into db + async with async_session() as session: + doc = Document(file=file_name) + session.add(doc) + await session.commit() + doc_id = doc.id + + # fetch record back and run tests + async with async_session() as session: + doc = await session.get(Document, doc_id) + if doc is None: + return + + # check instance type + assert isinstance(doc.file, AsyncStorageFile) + assert doc.file.name == f"{file_name}" + + # methods should work + url = await doc.file.get_url() + assert file_name in url + + size = await doc.file.get_size() + assert size == len(file_content) + + # deleting should not raise + await doc.file.delete() + size_after_delete = await storage.get_size(file_name) + assert size_after_delete == 0 + + # close all connections + await engine.dispose() diff --git a/tests/test_s3_storage.py b/tests/test_s3_storage.py index ad42669..ab3122c 100644 --- a/tests/test_s3_storage.py +++ b/tests/test_s3_storage.py @@ -8,7 +8,6 @@ @pytest.mark.asyncio async def test_s3_storage_methods(s3_test_env: Any): bucket_name, endpoint_without_scheme = s3_test_env - storage = AsyncS3Storage( bucket_name=bucket_name, endpoint_url=endpoint_without_scheme, @@ -17,28 +16,27 @@ async def test_s3_storage_methods(s3_test_env: Any): use_ssl=False, ) + file_name = "test/file.txt" file_content = b"hello moto" file_obj = BytesIO(file_content) - name = "test/file.txt" - # upload test - returned_name = await storage.upload(file_obj, name) - assert returned_name == storage.get_name(name) + returned_name = await storage.upload(file_obj, file_name) + assert returned_name == storage.get_name(file_name) # get url test without custom domain or querystring_auth - url = await storage.get_url(name) - assert name in url + url = await storage.get_url(file_name) + assert file_name in url # get size test - size = await storage.get_size(name) + size = await storage.get_size(file_name) assert size == len(file_content) # delete test (should suceed silently) - await storage.delete(name) + await storage.delete(file_name) # get size test after delete (should return 0) - size_after_delete = await storage.get_size(name) + size_after_delete = await storage.get_size(file_name) assert size_after_delete == 0 diff --git a/uv.lock b/uv.lock index a8181ea..92c8d35 100644 --- a/uv.lock +++ b/uv.lock @@ -163,6 +163,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "aiosqlite" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/7d/8bca2bf9a247c2c5dfeec1d7a5f40db6518f88d314b8bca9da29670d2671/aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3", size = 13454, upload-time = "2025-02-03T07:30:16.235Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -511,6 +523,7 @@ s3 = [ [package.dev-dependencies] dev = [ + { name = "aiosqlite" }, { name = "pytest" }, { name = "pytest-aioboto3" }, { name = "pytest-asyncio" }, @@ -523,6 +536,7 @@ provides-extras = ["s3"] [package.metadata.requires-dev] dev = [ + { name = "aiosqlite", specifier = ">=0.21.0" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-aioboto3", specifier = ">=0.6.0" }, { name = "pytest-asyncio", specifier = ">=1.2.0" },