fix lints

star-whale · Oct 28, 2022 · e41e0f3 · e41e0f3
1 parent 052ecff
commit e41e0f3
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 18 deletions.
diff --git a/client/starwhale/api/_impl/dataset/loader.py b/client/starwhale/api/_impl/dataset/loader.py
@@ -66,7 +66,9 @@ def _get_store(self, row: TabularDatasetRow) -> ObjectStore:
         self._stores[_k] = _store
         return _store
 
-    def _get_key_compose(self, row: TabularDatasetRow, store: ObjectStore) -> t.Tuple[str, int, int]:
+    def _get_key_compose(
+        self, row: TabularDatasetRow, store: ObjectStore
+    ) -> t.Tuple[str, int, int]:
         if row.object_store_type == ObjectStoreType.REMOTE:
             data_uri = row.data_uri
         else:

diff --git a/client/starwhale/core/dataset/store.py b/client/starwhale/core/dataset/store.py
@@ -349,7 +349,9 @@ def __str__(self) -> str:
     __repr__ = __str__
 
     @abstractmethod
-    def _make_file(self, bucket: str, key_compose: t.Tuple[str, int, int]) -> FileLikeObj:
+    def _make_file(
+        self, bucket: str, key_compose: t.Tuple[str, int, int]
+    ) -> FileLikeObj:
         raise NotImplementedError
 
 
@@ -378,7 +380,9 @@ def __init__(
             region_name=conn.region,
         )
 
-    def _make_file(self, bucket: str, key_compose: t.Tuple[str, int, int]) -> FileLikeObj:
+    def _make_file(
+        self, bucket: str, key_compose: t.Tuple[str, int, int]
+    ) -> FileLikeObj:
         # TODO: merge connections for s3
         _key, _start, _end = key_compose
         return S3BufferedFileLike(
@@ -394,7 +398,9 @@ class LocalFSStorageBackend(StorageBackend):
     def __init__(self) -> None:
         super().__init__(kind=SWDSBackendType.LocalFS)
 
-    def _make_file(self, bucket: str, key_compose: t.Tuple[str, int, int]) -> FileLikeObj:
+    def _make_file(
+        self, bucket: str, key_compose: t.Tuple[str, int, int]
+    ) -> FileLikeObj:
         _key, _start, _end = key_compose
         bucket_path = (
             Path(bucket).expanduser() if bucket.startswith("~/") else Path(bucket)

diff --git a/client/tests/sdk/test_loader.py b/client/tests/sdk/test_loader.py
@@ -1,5 +1,6 @@
 import os
 import shutil
+from http import HTTPStatus
 from unittest.mock import patch, MagicMock
 
 from pyfakefs.fake_filesystem_unittest import TestCase
@@ -13,7 +14,7 @@
 from starwhale.core.dataset.type import Image, ArtifactType, DatasetSummary
 from starwhale.core.dataset.store import (
     DatasetStorage,
-    S3StorageBackend,
+    SignedUrlBackend,
     LocalFSStorageBackend,
 )
 from starwhale.core.dataset.tabular import TabularDatasetRow
@@ -227,8 +228,15 @@ def test_user_raw_remote_store(
     @patch("starwhale.core.dataset.store.boto3.resource")
     @patch("starwhale.core.dataset.model.CloudDataset.summary")
     @patch("starwhale.api._impl.dataset.loader.TabularDataset.scan")
+    @patch("requests.get")
+    @patch("requests.request")
     def test_swds_bin_s3(
-        self, m_scan: MagicMock, m_summary: MagicMock, m_boto3: MagicMock
+        self,
+        m_request: MagicMock,
+        m_get: MagicMock,
+        m_scan: MagicMock,
+        m_summary: MagicMock,
+        m_boto3: MagicMock,
     ) -> None:
         m_summary.return_value = DatasetSummary(
             include_user_raw=False,
@@ -266,7 +274,6 @@ def test_swds_bin_s3(
         os.environ.update(
             {
                 "SW_S3_BUCKET": "starwhale",
-                "SW_OBJECT_STORE_KEY_PREFIX": f"project/self/dataset/mnist/version/11/{version}",
                 "SW_S3_ENDPOINT": "starwhale.mock:9000",
                 "SW_S3_ACCESS_KEY": "foo",
                 "SW_S3_SECRET": "bar",
@@ -276,6 +283,14 @@ def test_swds_bin_s3(
         with open(os.path.join(self.swds_dir, fname), "rb") as f:
             swds_content = f.read(-1)
 
+        m_request.return_value = MagicMock(
+            **{"status_code": HTTPStatus.OK, "data": "a"}
+        )
+        m_get.return_value = MagicMock(
+            **{
+                "content": swds_content,
+            }
+        )
         m_boto3.return_value = MagicMock(
             **{
                 "Object.return_value": MagicMock(
@@ -301,18 +316,11 @@ def test_swds_bin_s3(
 
         assert list(loader._stores.keys()) == ["local."]
         backend = loader._stores["local."].backend
-        assert isinstance(backend, S3StorageBackend)
-        assert backend.kind == SWDSBackendType.S3
-        assert backend.s3.Object.call_args[0] == (
-            "starwhale",
-            f"project/self/dataset/mnist/version/11/{version}/{fname}",
-        )
+        assert isinstance(backend, SignedUrlBackend)
+        assert backend.kind == SWDSBackendType.SignedUrl
 
-        assert loader._stores["local."].bucket == "starwhale"
-        assert (
-            loader._stores["local."].key_prefix
-            == f"project/self/dataset/mnist/version/11/{version}"
-        )
+        assert loader._stores["local."].bucket == ""
+        assert loader._stores["local."].key_prefix == ""
 
     @patch.dict(os.environ, {})
     @patch("starwhale.core.dataset.model.StandaloneDataset.summary")