Update tests to work with controlplane changes (#2752)

Update tests to work with controlplane changes

---------

Co-authored-by: Nathan Voxland <nathan@voxland.net>
Co-authored-by: activesoull <levon@activeloop.ai>

.github/workflows/test-push.yml

@@ -142,7 +142,7 @@ jobs:
       gcp_sa_credentials_json: ${{ secrets.GCP_SA_CREDENTIALS_JSON }}
       azure_creds_json: ${{ secrets.AZURE_CREDS_JSON }}
       hub_username: ${{ secrets.ACTIVELOOP_HUB_USERNAME }}
-      hub_password: ${{ secrets.ACTIVELOOP_HUB_PASSWORD }}
+      hub_token: ${{ secrets.ACTIVELOOP_HUB_TOKEN }}
       kaggle_username: ${{ secrets.KAGGLE_USERNAME }}
       kaggle_key: ${{ secrets.KAGGLE_KEY }}
       oauth_client_id: ${{ secrets.GDRIVE_CLIENT_ID }}

deeplake/api/tests/test_dataset.py

@@ -95,16 +95,3 @@ def test_persistence_bug(local_ds_generator):
 
         ds = local_ds_generator()
         np.testing.assert_array_equal(ds[tensor_name].numpy(), np.array([[1], [2]]))
-
-
-def test_dataset_token(local_ds_generator, hub_cloud_dev_credentials):
-    username, password = hub_cloud_dev_credentials
-    CliRunner().invoke(login, f"-u {username} -p {password}")
-    ds = local_ds_generator()
-    token = ds.token
-    token_username = jwt.decode(token, options={"verify_signature": False})["id"]
-    assert token_username == username
-
-    CliRunner().invoke(logout)
-    ds = local_ds_generator()
-    assert ds.token is None

deeplake/cli/test_cli.py

@@ -5,25 +5,6 @@
 import pytest
 
 
-@pytest.mark.parametrize("method", ["creds", "token"])
-def test_cli_auth(hub_cloud_dev_credentials, hub_cloud_dev_token, method):
-    username, password = hub_cloud_dev_credentials
-
-    runner = CliRunner()
-
-    if method == "creds":
-        result = runner.invoke(login, f"-u {username} -p {password}")
-    elif method == "token":
-        result = runner.invoke(login, f"-t {hub_cloud_dev_token}")
-
-    assert result.exit_code == 0
-    assert result.output == "Successfully logged in to Activeloop.\n"
-
-    result = runner.invoke(logout)
-    assert result.exit_code == 0
-    assert result.output == "Logged out of Activeloop.\n"
-
-
 def test_bad_token():
     runner = CliRunner()
 

deeplake/client/client.py

@@ -28,7 +28,6 @@
     HUB_REST_ENDPOINT,
     HUB_REST_ENDPOINT_LOCAL,
     HUB_REST_ENDPOINT_DEV,
-    GET_TOKEN_SUFFIX,
     HUB_REST_ENDPOINT_STAGING,
     REGISTER_USER_SUFFIX,
     DEFAULT_REQUEST_TIMEOUT,
@@ -188,15 +187,10 @@ def request_auth_token(self, username: str, password: str):
             LoginException: If there is an issue retrieving the auth token.
 
         """
-        json = {"username": username, "password": password}
-        response = self.request("POST", GET_TOKEN_SUFFIX, json=json)
+        if username != "public":
+            raise LoginException("Can only request a token for the public user")
 
-        try:
-            token_dict = response.json()
-            token = token_dict["token"]
-        except Exception:
-            raise LoginException()
-        return token
+        return "PUBLIC TOKEN " + ("_" * 150)
 
     def send_register_request(self, username: str, email: str, password: str):
         """Sends a request to backend to register a new user.

deeplake/client/test_client.py

@@ -15,19 +15,6 @@
 from time import sleep
 
 
-@pytest.mark.slow
-def test_client_requests(hub_cloud_dev_credentials):
-    username, password = hub_cloud_dev_credentials
-
-    deeplake_client = DeepLakeBackendClient()
-    deeplake_client.request_auth_token(username, password)
-    with pytest.raises(Exception):
-        # request will fail as username already exists
-        deeplake_client.send_register_request(
-            "activeloop", "abc@d.com", "notactualpassword"
-        )
-
-
 def test_client_utils():
     write_token("abcdefgh")
     assert read_token() == "abcdefgh"

deeplake/constants.py

@@ -81,6 +81,9 @@
 # environment variables
 ENV_HUB_DEV_USERNAME = "ACTIVELOOP_HUB_USERNAME"
 ENV_HUB_DEV_PASSWORD = "ACTIVELOOP_HUB_PASSWORD"
+
+ENV_HUB_DEV_TOKEN = "ACTIVELOOP_HUB_TOKEN"
+
 ENV_HUB_DEV_MANAGED_CREDS_KEY = "ACTIVELOOP_HUB_MANAGED_CREDS_KEY"
 
 ENV_KAGGLE_USERNAME = "KAGGLE_USERNAME"

deeplake/core/chunk_engine.py

@@ -1156,10 +1156,14 @@ def _extend_sequence(self, samples, progressbar, link_callback, ignore_errors):
     def _prepare_samples_for_link_callback(self, samples):
         if not isinstance(samples, np.ndarray):
             samples = [
-                None
-                if is_empty_list(s)
-                or (isinstance(s, deeplake.core.tensor.Tensor) and s.is_empty_tensor)
-                else s
+                (
+                    None
+                    if is_empty_list(s)
+                    or (
+                        isinstance(s, deeplake.core.tensor.Tensor) and s.is_empty_tensor
+                    )
+                    else s
+                )
                 for s in samples
             ]
         return samples
@@ -2536,9 +2540,11 @@ def update_links_and_encoders(idx):
             """Update linked tensors and sample level encoders"""
             self.commit_diff.pop(
                 idx,
-                sample_id_tensor[idx].numpy().item()
-                if sample_id_tensor is not None
-                else None,
+                (
+                    sample_id_tensor[idx].numpy().item()
+                    if sample_id_tensor is not None
+                    else None
+                ),
             )
             if link_callback:
                 link_callback(idx)
@@ -2996,9 +3002,11 @@ def _merge_seq_shape(self, shape, sample_index):
             is_same = np.all(shape == shape[0, :], axis=0)  # type: ignore
             shape = (len(shape),) + (
                 tuple(
-                    int(shape[0, i])  # type: ignore
-                    if is_same[i]  # type: ignore
-                    else -1
+                    (
+                        int(shape[0, i])  # type: ignore
+                        if is_same[i]  # type: ignore
+                        else -1
+                    )
                     for i in range(shape.shape[1])  # type: ignore
                 )
                 or (1,)
@@ -3165,9 +3173,11 @@ def _apply_deeper_indexing(self, sample_shapes, num_samples, sample_index):
     def _sample_shapes_to_shape(self, sample_shapes, squeeze_dims, sample_ndim):
         is_same = np.all(sample_shapes == sample_shapes[0, :], axis=0)
         shape = [  # type: ignore
-            int(sample_shapes[0, i])
-            if sample_shapes[0, i] != -1 and is_same[i]
-            else None
+            (
+                int(sample_shapes[0, i])
+                if sample_shapes[0, i] != -1 and is_same[i]
+                else None
+            )
             for i in range(sample_ndim)
         ]
 
@@ -3318,9 +3328,11 @@ def _transform_callback(
                     meta = self.tensor_meta
                     vs = func(
                         samples,
-                        factor=tensor.info.downsampling_factor
-                        if func == extend_downsample
-                        else None,
+                        factor=(
+                            tensor.info.downsampling_factor
+                            if func == extend_downsample
+                            else None
+                        ),
                         compression=meta.sample_compression,
                         htype=meta.htype,
                         link_creds=self.link_creds,

deeplake/core/dataset/dataset.py

@@ -289,9 +289,9 @@ def __init__(
             else None
         )
         self._first_load_init()
-        self._initial_autoflush: List[
-            bool
-        ] = []  # This is a stack to support nested with contexts
+        self._initial_autoflush: List[bool] = (
+            []
+        )  # This is a stack to support nested with contexts
         self._indexing_history: List[int] = []
 
         if not self.read_only:
@@ -4122,16 +4122,18 @@ def _copy(
             token=token,
             overwrite=overwrite,
             public=public,
-            unlink=[
-                t
-                for t in self.tensors
-                if (
-                    self.tensors[t].base_htype != "video"
-                    or deeplake.constants._UNLINK_VIDEOS
-                )
-            ]
-            if unlink
-            else False,
+            unlink=(
+                [
+                    t
+                    for t in self.tensors
+                    if (
+                        self.tensors[t].base_htype != "video"
+                        or deeplake.constants._UNLINK_VIDEOS
+                    )
+                ]
+                if unlink
+                else False
+            ),
             verbose=verbose,
         )
 

deeplake/core/io.py

@@ -100,8 +100,7 @@ def __len__(self):
 
 class Scheduler(ABC):
     @abstractmethod
-    def schedule(self, jobs: List[IOBlock]) -> List[Schedule]:
-        ...
+    def schedule(self, jobs: List[IOBlock]) -> List[Schedule]: ...
 
 
 class SingleThreadScheduler(Scheduler):

deeplake/core/meta/dataset_meta.py

@@ -100,9 +100,11 @@ def rename_group(self, name, new_name):
         self.groups.remove(name)
         self.groups = list(
             map(
-                lambda g: posixpath.join(new_name, relpath(g, name))
-                if (g == name or g.startswith(name + "/"))
-                else g,
+                lambda g: (
+                    posixpath.join(new_name, relpath(g, name))
+                    if (g == name or g.startswith(name + "/"))
+                    else g
+                ),
                 self.groups,
             )
         )

deeplake/core/query/test/test_query.py

@@ -466,9 +466,11 @@ def test_link_materialize(local_ds, num_workers):
         ds.create_tensor("abc", htype="link[image]", sample_compression="jpg")
         ds.abc.extend(
             [
-                deeplake.link("https://picsum.photos/20/20")
-                if i % 2
-                else deeplake.link("https://picsum.photos/10/10")
+                (
+                    deeplake.link("https://picsum.photos/20/20")
+                    if i % 2
+                    else deeplake.link("https://picsum.photos/10/10")
+                )
                 for i in range(20)
             ]
         )

deeplake/core/sample.py

@@ -224,9 +224,11 @@ def _get_dicom_meta(self) -> dict:
             x.keyword: {
                 "name": x.name,
                 "tag": str(x.tag),
-                "value": x.value
-                if isinstance(x.value, (str, int, float))
-                else x.to_json_dict(None, None).get("Value", ""),  # type: ignore
+                "value": (
+                    x.value
+                    if isinstance(x.value, (str, int, float))
+                    else x.to_json_dict(None, None).get("Value", "")  # type: ignore
+                ),
                 "vr": x.VR,
             }
             for x in dcm

deeplake/core/tensor.py

@@ -1095,9 +1095,11 @@ def _extend_links(self, samples, flat: Optional[bool], progressbar: bool = False
                     func = get_link_transform(func_name)
                     vs = func(
                         samples,
-                        factor=tensor.info.downsampling_factor
-                        if func == extend_downsample
-                        else None,
+                        factor=(
+                            tensor.info.downsampling_factor
+                            if func == extend_downsample
+                            else None
+                        ),
                         compression=self.meta.sample_compression,
                         htype=self.htype,
                         link_creds=self.link_creds,
@@ -1145,9 +1147,11 @@ def _update_links(
                 val = func(
                     new_sample,
                     old_value=tensor[global_sample_index],
-                    factor=tensor.info.downsampling_factor
-                    if func == update_downsample
-                    else None,
+                    factor=(
+                        tensor.info.downsampling_factor
+                        if func == update_downsample
+                        else None
+                    ),
                     compression=self.meta.sample_compression,
                     htype=self.htype,
                     link_creds=self.link_creds,

deeplake/core/tests/test_query.py

@@ -8,38 +8,35 @@
 
 @requires_libdeeplake
 def test_single_source_query(
-    hub_cloud_dev_credentials,
+    hub_cloud_dev_token,
 ):
-    username, password = hub_cloud_dev_credentials
-    token = DeepLakeBackendClient().request_auth_token(username, password)
-    ds = deeplake.query('SELECT * FROM "hub://activeloop/mnist-train"', token=token)
+    ds = deeplake.query(
+        'SELECT * FROM "hub://activeloop/mnist-train"', token=hub_cloud_dev_token
+    )
     assert len(ds) == 60000
     assert len(ds.tensors) == 2
     assert ds.images.meta.htype == "image"
     assert ds.labels.meta.htype == "class_label"
 
     ds = deeplake.query(
-        'SELECT images FROM "hub://activeloop/mnist-train"', token=token
+        'SELECT images FROM "hub://activeloop/mnist-train"', token=hub_cloud_dev_token
     )
     assert len(ds) == 60000
     assert len(ds.tensors) == 1
     assert ds.images.meta.htype == "image"
 
 
 @requires_libdeeplake
-def test_multi_source_query(hub_cloud_dev_credentials):
-    username, password = hub_cloud_dev_credentials
-    token = DeepLakeBackendClient().request_auth_token(username, password)
-
+def test_multi_source_query(hub_cloud_dev_token):
     with pytest.raises(RuntimeError):
         ds = deeplake.query(
             'SELECT * FROM "hub://activeloop/mnist-train" UNION (SELECT * FROM "hub://activeloop/coco-train")',
-            token=token,
+            token=hub_cloud_dev_token,
         )
 
     ds = deeplake.query(
         'SELECT * FROM "hub://activeloop/mnist-train" UNION (SELECT images, categories[0] as labels FROM "hub://activeloop/coco-train")',
-        token=token,
+        token=hub_cloud_dev_token,
     )
     assert len(ds) == 178287
     assert len(ds.tensors) == 2
@@ -48,7 +45,7 @@ def test_multi_source_query(hub_cloud_dev_credentials):
 
     ds = deeplake.query(
         'SELECT * FROM (SELECT * FROM "hub://activeloop/mnist-train" UNION (SELECT images, labels FROM "hub://activeloop/cifar100-train")) WHERE labels == 0',
-        token=token,
+        token=hub_cloud_dev_token,
     )
     assert len(ds) == 6423
     assert len(ds.tensors) == 2

deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py

@@ -71,9 +71,9 @@ def __init__(
             self.bugout_reporting_path,
             "vs.initialize",
             {
-                "tensor_params": "default"
-                if tensor_params is not None
-                else tensor_params,
+                "tensor_params": (
+                    "default" if tensor_params is not None else tensor_params
+                ),
                 "embedding_function": True if embedding_function is not None else False,
                 "num_workers": num_workers,
                 "overwrite": overwrite,