add Team.get_by_name & Dataset.get_remote_dataset_registry (#763)

* update client, add Team.get_by_name(), add Dataset.get_remote_dataset_registry()

* rm unused imports

* add tests and docs

* update docs and example

* add changelog entry

* export RemoteDataset (mostly for annotations)

* incorporate PR feedback

* Update remote_dataset_registry.py

docs/mkdocs.yml

@@ -75,6 +75,7 @@ nav:
         - webknossos-py/examples/dataset_usage.md
         - webknossos-py/examples/upload_image_data.md
         - webknossos-py/examples/download_image_data.md
+        - webknossos-py/examples/remote_datasets.md
       - Annotation Examples:
         - webknossos-py/examples/apply_merger_mode.md
         - webknossos-py/examples/learned_segmenter.md

docs/src/webknossos-py/examples/remote_datasets.md

@@ -0,0 +1,9 @@
+# Remote Dataset Access
+
+This example shows how to access [remote datasets](../../api/webknossos/dataset/dataset.md#RemoteDataset). This can be done directly using [wk.Dataset.open_remote()](../../api/webknossos/dataset/dataset.md#Dataset.open_remote), or listing all available datasets via [wk.Dataset.get_remote_datasets()](../../api/webknossos/dataset/dataset.md#Dataset.get_remote_datasets).
+
+```python
+--8<--
+webknossos/examples/remote_datasets.py
+--8<--
+```

webknossos/Changelog.md

@@ -21,6 +21,8 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 - `Dataset.upload()` accepts `Layer` objects from a `RemoteDataset` in the `layers_to_link` argument list. Also, `LayerToLink` can consume those via `LayerToLink.from_remote_layer()`. [#761](https://github.com/scalableminds/webknossos-libs/pull/761)
 - `Task.create()` accepts a `RemoteDataset` for the `dataset_name` argument. [#761](https://github.com/scalableminds/webknossos-libs/pull/761)
 - Added `annotation.get_remote_base_dataset()` returning a `RemoteDataset`. [#761](https://github.com/scalableminds/webknossos-libs/pull/761)
+- Added `Team.get_by_name()`. [#763](https://github.com/scalableminds/webknossos-libs/pull/763)
+- Added `Dataset.get_remote_datasets()`. [#763](https://github.com/scalableminds/webknossos-libs/pull/763)
 
 ### Changed
 - If a token is requested from the user on the commandline, it is now stored in the current context. Before, it was discarded. [#761](https://github.com/scalableminds/webknossos-libs/pull/761)

webknossos/__generate_client.py

@@ -72,13 +72,15 @@ def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
         build_info,
         current_user_info,
         dataset_info,
+        dataset_list,
         dataset_sharing_token,
         datastore_list,
         generate_token_for_data_store,
         project_info_by_id,
         project_info_by_name,
         task_info,
         task_infos_by_project_id,
+        team_list,
         user_info_by_id,
         user_list,
         user_logged_time,
@@ -115,7 +117,6 @@ def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
         "annotationInfo",
         extract_200_response(
             annotation_info.sync_detailed(
-                typ="Explorational",
                 id=explorative_annotation_id,
                 client=client,
                 timestamp=unixtime,
@@ -134,6 +135,15 @@ def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
         ),
     )
 
+    yield (
+        "datasetList",
+        extract_200_response(
+            dataset_list.sync_detailed(
+                client=client,
+            )
+        ),
+    )
+
     yield (
         "datasetSharingToken",
         extract_200_response(
@@ -165,6 +175,15 @@ def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
         ),
     )
 
+    yield (
+        "teamList",
+        extract_200_response(
+            team_list.sync_detailed(
+                client=client,
+            ),
+        ),
+    )
+
     yield (
         "projectInfoById",
         extract_200_response(
@@ -241,7 +260,9 @@ def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
     # user and owner are optional in annotations
     "user",
     "owner",
+    "status",  # sometimes part of the dataSource dict
     "volumeInterpolationAllowed",  # added 2022-06, optional for backwards-compatibility
+    "teams",  # added 2022-07, optional for backwards-compatibility
     # isSuperUser field was added 2022-03 and only optional for backwards-compatibility with wk,
     # it can be made non-optional when needed later:
     "isSuperUser",

webknossos/examples/remote_datasets.py

@@ -0,0 +1,36 @@
+from textwrap import wrap
+
+import webknossos as wk
+
+
+def main() -> None:
+    # Remote datasets are read-only, but can be used similar to normal datasets:
+    l4_sample_dataset = wk.Dataset.open_remote(
+        "https://webknossos.org/datasets/scalable_minds/l4dense_motta_et_al_demo"
+    )
+
+    # Print information of the public l4dense_motta_et_al_demo dataset:
+    print(l4_sample_dataset.url)
+    print("\n  ".join(["Description:"] + wrap(l4_sample_dataset.description or "")))
+    print("Layers:", ", ".join(l4_sample_dataset.layers))
+    print("Tags:", ", ".join(l4_sample_dataset.tags))
+
+    # List all accessible remote datasets via get_remote_datasets():
+    own_remote_datasets = wk.Dataset.get_remote_datasets()
+
+    # Print the first 10 dataset names from your organization:
+    print()
+    print("First 10 datasets for own organization:")
+    for dataset_name in sorted(own_remote_datasets)[:10]:
+        print("*", dataset_name)
+
+    # List all accessible demo datasets of the scalable minds organization:
+    remote_demo_datasets = wk.Dataset.get_remote_datasets(
+        organization_id="scalable_minds", tags="demo"
+    )
+    print("Remote demo datasets:", list(remote_demo_datasets))
+    assert l4_sample_dataset in remote_demo_datasets.values()
+
+
+if __name__ == "__main__":
+    main()

webknossos/local_wk_setup.sh

@@ -1,7 +1,7 @@
 function export_vars {
     export WK_TOKEN=1b88db86331a38c21a0b235794b9e459856490d70408bcffb767f64ade0f83d2bdb4c4e181b9a9a30cdece7cb7c65208cc43b6c1bb5987f5ece00d348b1a905502a266f8fc64f0371cd6559393d72e031d0c2d0cabad58cccf957bb258bc86f05b5dc3d4fff3d5e3d9c0389a6027d861a21e78e3222fb6c5b7944520ef21761e
     export WK_URL=http://localhost:9000
-    export DOCKER_TAG=22.07.0
+    export DOCKER_TAG=master__18448
 }
 
 function ensure_local_test_wk {

webknossos/test.sh

@@ -6,6 +6,10 @@ source local_wk_setup.sh
 export_vars
 
 
+# Note that pytest should be executed via `python -m`, since
+# this will ensure that the current directory is added to sys.path
+# (which is standard python behavior). This is necessary so that the imports
+# refer to the checked out (and potentially modified) code.
 PYTEST="poetry run python -m pytest --suppress-no-test-exit-code"
 
 
@@ -15,10 +19,11 @@ if [ $# -gt 0 ] && [ "$1" = "--refresh-snapshots" ]; then
     rm -rf tests/cassettes
     rm -rf tests/**/cassettes
 
-    # Note that pytest should be executed via `python -m`, since
-    # this will ensure that the current directory is added to sys.path
-    # (which is standard python behavior). This is necessary so that the imports
-    # refer to the checked out (and potentially modified) code.
+    shift
+    $PYTEST --record-mode once -m "with_vcr" "$@"
+    stop_local_test_wk
+elif [ $# -gt 0 ] && [ "$1" = "--add-snapshots" ]; then
+    ensure_local_test_wk
     shift
     $PYTEST --record-mode once -m "with_vcr" "$@"
     stop_local_test_wk