From 876995cc44a58ce65e6a7160539aa7d428000b59 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 30 Aug 2023 12:02:29 +0100 Subject: [PATCH 1/4] Initial fix --- darwin/client.py | 45 +++++++++++++++++++++++++++++++------ tests/darwin/client_test.py | 42 +++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/darwin/client.py b/darwin/client.py index aad83ab5d..4fb86ecd9 100644 --- a/darwin/client.py +++ b/darwin/client.py @@ -17,7 +17,14 @@ from darwin.dataset.identifier import DatasetIdentifier from darwin.dataset.remote_dataset_v1 import RemoteDatasetV1 from darwin.dataset.remote_dataset_v2 import RemoteDatasetV2 -from darwin.datatypes import DarwinVersionNumber, Feature, ItemId, Team +from darwin.datatypes import ( + DarwinVersionNumber, + Feature, + ItemId, + JSONFreeForm, + JSONType, + Team, +) from darwin.exceptions import ( InsufficientStorage, InvalidLogin, @@ -54,6 +61,30 @@ def __init__(self, config: Config, default_team: Optional[str] = None, log: Opti else: self.log = log + @staticmethod + def _get_item_count(dataset_dict: JSONFreeForm) -> int: + """ + Returns the number of items in the dataset. + + Parameters + ---------- + dataset_dict: Dict[str, Any] + The dataset dictionary. + + Returns + ------- + int + The number of items in the dataset. + """ + num_items: Optional[int] = dataset_dict.get("num_items") + num_videos: Optional[int] = dataset_dict.get("num_videos") + num_images: Optional[int] = dataset_dict.get("num_images") + + if num_items is not None: + return num_items + + return (num_images or 0) + (num_videos or 0) + def list_local_datasets(self, team_slug: Optional[str] = None) -> Iterator[Path]: """ Returns a list of all local folders which are detected as dataset. @@ -107,7 +138,7 @@ def list_remote_datasets(self, team_slug: Optional[str] = None) -> Iterator[Remo slug=dataset["slug"], team=team_slug or self.default_team, dataset_id=dataset["id"], - item_count=dataset.get("num_items", dataset.get("num_images", 0) + dataset.get("num_videos", 0)), + item_count=self._get_item_count(dataset), progress=dataset["progress"], client=self, ) @@ -172,7 +203,7 @@ def get_remote_dataset(self, dataset_identifier: Union[str, DatasetIdentifier]) slug=dataset["slug"], team=parsed_dataset_identifier.team_slug, dataset_id=dataset["id"], - item_count=dataset.get("num_items", dataset.get("num_images", 0) + dataset.get("num_videos", 0)), + item_count=self._get_item_count(dataset), progress=0, client=self, ) @@ -182,7 +213,7 @@ def get_remote_dataset(self, dataset_identifier: Union[str, DatasetIdentifier]) slug=dataset["slug"], team=parsed_dataset_identifier.team_slug, dataset_id=dataset["id"], - item_count=dataset.get("num_items", dataset.get("num_images", 0) + dataset.get("num_videos", 0)), + item_count=self._get_item_count(dataset), progress=0, client=self, ) @@ -209,13 +240,14 @@ def create_dataset(self, name: str, team_slug: Optional[str] = None) -> RemoteDa The created dataset. """ dataset: Dict[str, Any] = cast(Dict[str, Any], self._post("/datasets", {"name": name}, team_slug=team_slug)) + if dataset.get("version", 1) == 2: return RemoteDatasetV2( name=dataset["name"], team=team_slug or self.default_team, slug=dataset["slug"], dataset_id=dataset["id"], - item_count=dataset.get("num_items", dataset.get("num_images", 0) + dataset.get("num_videos", 0)), + item_count=self._get_item_count(dataset), progress=0, client=self, ) @@ -225,7 +257,7 @@ def create_dataset(self, name: str, team_slug: Optional[str] = None) -> RemoteDa team=team_slug or self.default_team, slug=dataset["slug"], dataset_id=dataset["id"], - item_count=dataset.get("num_items", dataset.get("num_images", 0) + dataset.get("num_videos", 0)), + item_count=self._get_item_count(dataset), progress=0, client=self, ) @@ -1168,7 +1200,6 @@ def _delete( return self._decode_response(response) def _raise_if_known_error(self, response: Response, url: str) -> None: - if response.status_code == 401: raise Unauthorized() diff --git a/tests/darwin/client_test.py b/tests/darwin/client_test.py index c7c98eb7e..e24e8d544 100644 --- a/tests/darwin/client_test.py +++ b/tests/darwin/client_test.py @@ -8,7 +8,7 @@ from darwin.config import Config from darwin.dataset.remote_dataset_v1 import RemoteDatasetV1 from darwin.dataset.remote_dataset_v2 import RemoteDatasetV2 -from darwin.datatypes import Feature +from darwin.datatypes import Feature, JSONType from darwin.exceptions import NameTaken, NotFound from tests.fixtures import * @@ -293,3 +293,43 @@ def assert_dataset(dataset_1, dataset_2): assert dataset_1.dataset_id == dataset_2.dataset_id assert dataset_1.slug == dataset_2.slug assert dataset_1.item_count == dataset_2.item_count + + +# Tests for _get_item_count +def test__get_item_count_defaults_to_num_items_if_present() -> None: + dataset_return = { + "num_images": 2, # Should be ignored + "num_videos": 3, # Should be ignored + "num_items": 5, # Should get this one + } + + assert Client._get_item_count(dataset_return) == 5 + + +def test__get_item_count_returns_sum_of_others_if_num_items_not_present() -> None: + dataset_return = { + "num_images": 7, # Should be summed + "num_videos": 3, # Should be summed + } + + assert Client._get_item_count(dataset_return) == 10 + + +def test__get_item_count_should_tolerate_missing_members() -> None: + assert ( + Client._get_item_count( + { + "num_videos": 3, # Should be ignored + } + ) + == 3 + ) + + assert ( + Client._get_item_count( + { + "num_images": 2, + } + ) + == 2 + ) From 3a22f31615dd71cbe7c9183e8ece68b3c655a650 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 30 Aug 2023 13:53:52 +0100 Subject: [PATCH 2/4] Tests for the client fix --- tests/darwin/client_test.py | 95 +++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 26 deletions(-) diff --git a/tests/darwin/client_test.py b/tests/darwin/client_test.py index e24e8d544..4cf318ee2 100644 --- a/tests/darwin/client_test.py +++ b/tests/darwin/client_test.py @@ -1,16 +1,17 @@ from pathlib import Path -from typing import Any, Dict, List +from typing import List import pytest import responses from darwin.client import Client from darwin.config import Config +from darwin.dataset.remote_dataset import RemoteDataset from darwin.dataset.remote_dataset_v1 import RemoteDatasetV1 from darwin.dataset.remote_dataset_v2 import RemoteDatasetV2 -from darwin.datatypes import Feature, JSONType +from darwin.datatypes import Feature, JSONFreeForm from darwin.exceptions import NameTaken, NotFound -from tests.fixtures import * +from tests.fixtures import * # noqa: F401, F403 @pytest.fixture @@ -26,10 +27,10 @@ def darwin_client(darwin_config_path: Path, darwin_datasets_path: Path, team_slu @pytest.mark.usefixtures("file_read_write_test") class TestListRemoteDatasets: @responses.activate - def test_returns_list_of_datasets(self, darwin_client: Client): + def test_returns_list_of_datasets(self, darwin_client: Client) -> None: team_slug: str = "v7" endpoint: str = "/datasets" - json_response: List[Dict[str, Any]] = [ + json_response: List[JSONFreeForm] = [ { "name": "dataset-name-1", "slug": "dataset-slug-1", @@ -73,11 +74,53 @@ def test_returns_list_of_datasets(self, darwin_client: Client): assert_dataset(remote_datasets[0], expected_dataset_1) assert_dataset(remote_datasets[1], expected_dataset_2) + @responses.activate + def test_coalesces_null_item_counts_to_zeroes(self, darwin_client: Client) -> None: + team_slug: str = "v7" + endpoint: str = "/datasets" + json_response: List[JSONFreeForm] = [ + { + "name": "dataset-name-1", + "slug": "dataset-slug-1", + "id": 1, + "num_items": None, # As this is None, num_images and num_videos should be used + "num_images": None, # Should be coalesced to 0 + "num_videos": None, # Should be coalesced to 0 + "progress": 4, + }, + { + "name": "dataset-name-2", + "slug": "dataset-slug-2", + "id": 2, + "num_items": None, # As this is None, num_images and num_videos should be used + "num_images": 2, # Should be used + "num_videos": 3, # Should be used + "progress": 32, + }, + { + "name": "dataset-name-3", + "slug": "dataset-slug-3", + "id": 3, + "num_items": 11, # Should be used + "num_images": 2, # Should be ignored, as num_items is present + "num_videos": 3, # Should be ignored, as num_items is present + "progress": 32, + }, + ] + + responses.add(responses.GET, darwin_client.url + endpoint, json=json_response, status=200) + + remote_datasets = list(darwin_client.list_remote_datasets(team_slug)) + + expected_item_count = [0, 5, 11] + for i, ds in enumerate(remote_datasets): + assert ds.item_count == expected_item_count[i] + @pytest.mark.usefixtures("file_read_write_test") class TestGetRemoteDataset: @responses.activate - def test_raises_if_dataset_is_not_found(self, darwin_client: Client): + def test_raises_if_dataset_is_not_found(self, darwin_client: Client) -> None: endpoint: str = "/datasets" json_response = [ { @@ -96,7 +139,7 @@ def test_raises_if_dataset_is_not_found(self, darwin_client: Client): darwin_client.get_remote_dataset("v7/dataset-slug-2") @responses.activate - def test_returns_the_dataset(self, darwin_client: Client): + def test_returns_the_dataset(self, darwin_client: Client) -> None: endpoint: str = "/datasets" json_response = [ { @@ -128,9 +171,9 @@ def test_returns_the_dataset(self, darwin_client: Client): @pytest.mark.usefixtures("file_read_write_test") class TestCreateDataset: @responses.activate - def test_returns_the_created_dataset(self, darwin_client: Client): + def test_returns_the_created_dataset(self, darwin_client: Client) -> None: endpoint: str = "/datasets" - json_response: Dict[str, Any] = { + json_response: JSONFreeForm = { "name": "my-dataset", "slug": "my-dataset", "id": 1, @@ -154,9 +197,9 @@ def test_returns_the_created_dataset(self, darwin_client: Client): assert_dataset(actual_dataset, expected_dataset) @responses.activate - def test_raises_if_name_is_taken(self, darwin_client: Client): + def test_raises_if_name_is_taken(self, darwin_client: Client) -> None: endpoint: str = "/datasets" - json_response: Dict[str, Any] = {"errors": {"name": ["has already been taken"]}} + json_response: JSONFreeForm = {"errors": {"name": ["has already been taken"]}} responses.add( responses.POST, @@ -173,7 +216,7 @@ def test_raises_if_name_is_taken(self, darwin_client: Client): @pytest.mark.usefixtures("file_read_write_test") class TestFetchRemoteFiles: @responses.activate - def test_returns_remote_files(self, darwin_client: Client): + def test_returns_remote_files(self, darwin_client: Client) -> None: dataset_id = 1 endpoint: str = f"/datasets/{dataset_id}/items?page%5Bsize%5D=500&page%5Bfrom%5D=0" responses.add(responses.POST, darwin_client.url + endpoint, json={}, status=200) @@ -184,9 +227,9 @@ def test_returns_remote_files(self, darwin_client: Client): @pytest.mark.usefixtures("file_read_write_test") class TestFetchRemoteClasses: @responses.activate - def test_returns_remote_classes(self, team_slug: str, darwin_client: Client): + def test_returns_remote_classes(self, team_slug: str, darwin_client: Client) -> None: endpoint: str = f"/teams/{team_slug}/annotation_classes?include_tags=true" - response: Dict[str, Any] = { + response: JSONFreeForm = { "annotation_classes": [ { "annotation_class_image_url": None, @@ -207,8 +250,8 @@ def test_returns_remote_classes(self, team_slug: str, darwin_client: Client): responses.add(responses.GET, darwin_client.url + endpoint, json=response, status=200) - result: List[Dict[str, Any]] = darwin_client.fetch_remote_classes(team_slug) - annotation_class: Dict[str, Any] = result[0] + result: List[JSONFreeForm] = darwin_client.fetch_remote_classes(team_slug) + annotation_class: JSONFreeForm = result[0] assert annotation_class["annotation_class_image_url"] is None assert annotation_class["annotation_types"] == ["tag"] @@ -220,7 +263,7 @@ def test_returns_remote_classes(self, team_slug: str, darwin_client: Client): @pytest.mark.usefixtures("file_read_write_test") class TestGetTeamFeatures: @responses.activate - def test_returns_list_of_features(self, team_slug: str, darwin_client: Client): + def test_returns_list_of_features(self, team_slug: str, darwin_client: Client) -> None: endpoint: str = f"/teams/{team_slug}/features" json_response = [ {"enabled": False, "name": "WORKFLOW_V2"}, @@ -238,10 +281,10 @@ def test_returns_list_of_features(self, team_slug: str, darwin_client: Client): @pytest.mark.usefixtures("file_read_write_test") class TestInstantiateItem: @responses.activate - def test_raises_if_workflow_id_is_not_found(self, darwin_client: Client): + def test_raises_if_workflow_id_is_not_found(self, darwin_client: Client) -> None: item_id: int = 1234 endpoint: str = f"/dataset_items/{item_id}/workflow" - json_response: Dict[str, Any] = {} + json_response: JSONFreeForm = {} responses.add(responses.POST, darwin_client.url + endpoint, json=json_response, status=200) @@ -251,11 +294,11 @@ def test_raises_if_workflow_id_is_not_found(self, darwin_client: Client): assert str(exception.value) == f"No Workflow Id found for item_id: {item_id}" @responses.activate - def test_returns_workflow_id(self, darwin_client: Client): + def test_returns_workflow_id(self, darwin_client: Client) -> None: item_id: int = 1234 workflow_id: int = 1 endpoint: str = f"/dataset_items/{item_id}/workflow" - json_response: Dict[str, Any] = {"current_workflow_id": workflow_id} + json_response: JSONFreeForm = {"current_workflow_id": workflow_id} responses.add(responses.POST, darwin_client.url + endpoint, json=json_response, status=200) assert darwin_client.instantiate_item(item_id) == workflow_id @@ -264,10 +307,10 @@ def test_returns_workflow_id(self, darwin_client: Client): @pytest.mark.usefixtures("file_read_write_test") class TestWorkflowComment: @responses.activate - def test_raises_if_comment_id_is_not_found(self, darwin_client: Client): + def test_raises_if_comment_id_is_not_found(self, darwin_client: Client) -> None: workflow_id = 1234 endpoint: str = f"/workflows/{workflow_id}/workflow_comment_threads" - json_response: Dict[str, Any] = {} + json_response: JSONFreeForm = {} responses.add(responses.POST, darwin_client.url + endpoint, json=json_response, status=200) @@ -277,17 +320,17 @@ def test_raises_if_comment_id_is_not_found(self, darwin_client: Client): assert str(exception.value) == f"Unable to retrieve comment id for workflow: {workflow_id}." @responses.activate - def test_returns_comment_id(self, darwin_client: Client): + def test_returns_comment_id(self, darwin_client: Client) -> None: comment_id: int = 1234 workflow_id: int = 1 endpoint: str = f"/workflows/{workflow_id}/workflow_comment_threads" - json_response: Dict[str, Any] = {"id": comment_id} + json_response: JSONFreeForm = {"id": comment_id} responses.add(responses.POST, darwin_client.url + endpoint, json=json_response, status=200) assert darwin_client.post_workflow_comment(workflow_id, "My comment.") == comment_id -def assert_dataset(dataset_1, dataset_2): +def assert_dataset(dataset_1: RemoteDataset, dataset_2: RemoteDataset) -> None: assert dataset_1.name == dataset_2.name assert dataset_1.team == dataset_2.team assert dataset_1.dataset_id == dataset_2.dataset_id From 9e0242e42fd71136922d6756f1153124bbc50e84 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 30 Aug 2023 13:54:27 +0100 Subject: [PATCH 3/4] Boyscouting/type clearup --- darwin/client.py | 182 +++++++++++++++++++++++------------------------ 1 file changed, 89 insertions(+), 93 deletions(-) diff --git a/darwin/client.py b/darwin/client.py index 4fb86ecd9..ad759ea41 100644 --- a/darwin/client.py +++ b/darwin/client.py @@ -10,21 +10,15 @@ import requests from requests import Response +from requests.adapters import HTTPAdapter from darwin.backend_v2 import BackendV2 from darwin.config import Config -from darwin.dataset import RemoteDataset from darwin.dataset.identifier import DatasetIdentifier +from darwin.dataset.remote_dataset import RemoteDataset from darwin.dataset.remote_dataset_v1 import RemoteDatasetV1 from darwin.dataset.remote_dataset_v2 import RemoteDatasetV2 -from darwin.datatypes import ( - DarwinVersionNumber, - Feature, - ItemId, - JSONFreeForm, - JSONType, - Team, -) +from darwin.datatypes import DarwinVersionNumber, Feature, ItemId, JSONFreeForm, Team from darwin.exceptions import ( InsufficientStorage, InvalidLogin, @@ -53,7 +47,7 @@ def __init__(self, config: Config, default_team: Optional[str] = None, log: Opti self.features: Dict[str, List[Feature]] = {} self._newer_version: Optional[DarwinVersionNumber] = None self.session = requests.Session() - adapter = requests.adapters.HTTPAdapter(pool_maxsize=100) + adapter = HTTPAdapter(pool_maxsize=100) self.session.mount("https://", adapter) if log is None: @@ -68,7 +62,7 @@ def _get_item_count(dataset_dict: JSONFreeForm) -> int: Parameters ---------- - dataset_dict: Dict[str, Any] + dataset_dict: JSONFreeForm The dataset dictionary. Returns @@ -129,7 +123,7 @@ def list_remote_datasets(self, team_slug: Optional[str] = None) -> Iterator[Remo Iterator[RemoteDataset] List of all remote datasets """ - response: List[Dict[str, Any]] = cast(List[Dict[str, Any]], self._get("/datasets/", team_slug=team_slug)) + response: List[JSONFreeForm] = cast(List[JSONFreeForm], self._get("/datasets/", team_slug=team_slug)) for dataset in response: if dataset.get("version", 1) == 2: @@ -148,7 +142,7 @@ def list_remote_datasets(self, team_slug: Optional[str] = None) -> Iterator[Remo slug=dataset["slug"], team=team_slug or self.default_team, dataset_id=dataset["id"], - item_count=dataset.get("num_images", 0) + dataset.get("num_videos", 0), + item_count=self._get_item_count(dataset), progress=dataset["progress"], client=self, ) @@ -185,8 +179,8 @@ def get_remote_dataset(self, dataset_identifier: Union[str, DatasetIdentifier]) ] except Unauthorized: # There is a chance that we tried to access an open dataset - dataset: Dict[str, Any] = cast( - Dict[str, Any], + dataset: JSONFreeForm = cast( + JSONFreeForm, self._get(f"{parsed_dataset_identifier.team_slug}/{parsed_dataset_identifier.dataset_slug}"), ) @@ -239,7 +233,7 @@ def create_dataset(self, name: str, team_slug: Optional[str] = None) -> RemoteDa RemoteDataset The created dataset. """ - dataset: Dict[str, Any] = cast(Dict[str, Any], self._post("/datasets", {"name": name}, team_slug=team_slug)) + dataset: JSONFreeForm = cast(JSONFreeForm, self._post("/datasets", {"name": name}, team_slug=team_slug)) if dataset.get("version", 1) == 2: return RemoteDatasetV2( @@ -276,8 +270,8 @@ def archive_remote_dataset(self, dataset_id: int, team_slug: str) -> None: self._put(f"datasets/{dataset_id}/archive", payload={}, team_slug=team_slug) def fetch_remote_files( - self, dataset_id: int, cursor: Dict[str, Any], payload: Dict[str, Any], team_slug: str - ) -> Dict[str, Any]: + self, dataset_id: int, cursor: JSONFreeForm, payload: JSONFreeForm, team_slug: str + ) -> JSONFreeForm: """ Download the remote files from the given dataset. @@ -285,24 +279,24 @@ def fetch_remote_files( ---------- dataset_id: int Id of the dataset the file belong to. - cursor: Dict[str, Any] + cursor: JSONFreeForm Number of items per page and page number. Defaults to {"page[size]": 500, "page[from]": 0}. - payload: Dict[str, Any] + payload: JSONFreeForm Filter and sort parameters. team_slug: str The team slug of the dataset. Returns ------- - Dict[str, Any] + JSONFreeForm A response dictionary with the file information. """ - response: Dict[str, Any] = cast( - Dict[str, Any], self._post(f"/datasets/{dataset_id}/items?{parse.urlencode(cursor)}", payload, team_slug) + response: JSONFreeForm = cast( + JSONFreeForm, self._post(f"/datasets/{dataset_id}/items?{parse.urlencode(cursor)}", payload, team_slug) ) return response - def fetch_remote_classes(self, team_slug: Optional[str] = None) -> List[Dict[str, Any]]: + def fetch_remote_classes(self, team_slug: Optional[str] = None) -> List[JSONFreeForm]: """ Fetches all remote classes on the remote dataset. @@ -313,7 +307,7 @@ def fetch_remote_classes(self, team_slug: Optional[str] = None) -> List[Dict[str Returns ------- - Dict[str, Any] + JSONFreeForm None if no information about the team is found, a List of Annotation classes otherwise. Raises @@ -327,13 +321,13 @@ def fetch_remote_classes(self, team_slug: Optional[str] = None) -> List[Dict[str raise ValueError("No team was found.") the_team_slug: str = the_team.slug - response: Dict[str, Any] = cast( - Dict[str, Any], self._get(f"/teams/{the_team_slug}/annotation_classes?include_tags=true") + response: JSONFreeForm = cast( + JSONFreeForm, self._get(f"/teams/{the_team_slug}/annotation_classes?include_tags=true") ) return response["annotation_classes"] - def update_annotation_class(self, class_id: int, payload: Dict[str, Any]) -> Dict[str, Any]: + def update_annotation_class(self, class_id: int, payload: JSONFreeForm) -> JSONFreeForm: """ Updates the AnnotationClass with the given id. @@ -341,18 +335,18 @@ def update_annotation_class(self, class_id: int, payload: Dict[str, Any]) -> Dic ---------- class_id: int The id of the AnnotationClass to update. - payload: Dict[str, Any] + payload: JSONFreeForm A dictionary with the changes to perform. Returns ------- - Dict[str, Any] + JSONFreeForm A dictionary with the result of the operation. """ - response: Dict[str, Any] = cast(Dict[str, Any], self._put(f"/annotation_classes/{class_id}", payload)) + response: JSONFreeForm = cast(JSONFreeForm, self._put(f"/annotation_classes/{class_id}", payload)) return response - def create_annotation_class(self, dataset_id: int, type_ids: List[int], name: str) -> Dict[str, Any]: + def create_annotation_class(self, dataset_id: int, type_ids: List[int], name: str) -> JSONFreeForm: """ Creates an AnnotationClass. @@ -367,11 +361,11 @@ def create_annotation_class(self, dataset_id: int, type_ids: List[int], name: st Returns ------- - Dict[str, Any] + JSONFreeForm A dictionary with the result of the operation. """ - response: Dict[str, Any] = cast( - Dict[str, Any], + response: JSONFreeForm = cast( + JSONFreeForm, self._post( "/annotation_classes", payload={ @@ -385,7 +379,7 @@ def create_annotation_class(self, dataset_id: int, type_ids: List[int], name: st ) return response - def import_annotation(self, item_id: ItemId, payload: Dict[str, Any]) -> None: + def import_annotation(self, item_id: ItemId, payload: JSONFreeForm) -> None: """ Imports the annotation for the item with the given id. @@ -393,14 +387,14 @@ def import_annotation(self, item_id: ItemId, payload: Dict[str, Any]) -> None: ---------- item_id: ItemId Identifier of the Image or Video that we are import the annotation to. - payload: Dict[str, Any] + payload: JSONFreeForm A dictionary with the annotation to import. The default format is: `{"annotations": serialized_annotations, "overwrite": "false"}` """ self._post_raw(f"/dataset_items/{item_id}/import", payload=payload) - def fetch_remote_attributes(self, dataset_id: int) -> List[Dict[str, Any]]: + def fetch_remote_attributes(self, dataset_id: int) -> List[JSONFreeForm]: """ Fetches all attributes remotely. @@ -412,10 +406,10 @@ def fetch_remote_attributes(self, dataset_id: int) -> List[Dict[str, Any]]: Returns ------- - List[Dict[str, Any]] + List[JSONFreeForm] A List with the attributes, where each attribute is a dictionary. """ - response: List[Dict[str, Any]] = cast(List[Dict[str, Any]], self._get(f"/datasets/{dataset_id}/attributes")) + response: List[JSONFreeForm] = cast(List[JSONFreeForm], self._get(f"/datasets/{dataset_id}/attributes")) return response def load_feature_flags(self, team_slug: Optional[str] = None) -> None: @@ -450,7 +444,7 @@ def get_team_features(self, team_slug: str) -> List[Feature]: List[Feature] List of Features for the given team. """ - response: List[Dict[str, Any]] = cast(List[Dict[str, Any]], self._get(f"/teams/{team_slug}/features")) + response: List[JSONFreeForm] = cast(List[JSONFreeForm], self._get(f"/teams/{team_slug}/features")) features: List[Feature] = [] for feature in response: @@ -551,7 +545,7 @@ def confirm_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) self._put_raw(endpoint=f"/dataset_items/{dataset_item_id}/confirm_upload", payload={}, team_slug=the_team_slug) - def sign_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) -> Dict[str, Any]: + def sign_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) -> JSONFreeForm: """ Signs the upload of the given DatasetItem. @@ -564,7 +558,7 @@ def sign_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) -> Returns ------ - Dict[str, Any] + JSONFreeForm A dictionary with the signed response, or None if the Team was not found. Raises @@ -579,14 +573,12 @@ def sign_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) -> the_team_slug: str = the_team.slug - response: Dict[str, Any] = cast( - Dict[str, Any], self._get(f"/dataset_items/{dataset_item_id}/sign_upload", team_slug=the_team_slug) + response: JSONFreeForm = cast( + JSONFreeForm, self._get(f"/dataset_items/{dataset_item_id}/sign_upload", team_slug=the_team_slug) ) return response - def upload_data( - self, dataset_slug: str, payload: Dict[str, Any], team_slug: Optional[str] = None - ) -> Dict[str, Any]: + def upload_data(self, dataset_slug: str, payload: JSONFreeForm, team_slug: Optional[str] = None) -> JSONFreeForm: """ Uploads the given data to the given dataset. @@ -594,7 +586,7 @@ def upload_data( ---------- dataset_slug: str The slug of the dataset. - payload: Dict[str, Any] + payload: JSONFreeForm The data we want to upload. Usually a Dictionary with an `items` key containing a list of items to upload. team_slug: Optional[str] @@ -602,7 +594,7 @@ def upload_data( Returns ------ - Dict[str, Any] + JSONFreeForm A dictionary with the result of the operation, or None if the Team was not found. Raises @@ -617,8 +609,8 @@ def upload_data( the_team_slug: str = the_team.slug - response: Dict[str, Any] = cast( - Dict[str, Any], + response: JSONFreeForm = cast( + JSONFreeForm, self._put( endpoint=f"/teams/{the_team_slug}/datasets/{dataset_slug}/data", payload=payload, @@ -627,19 +619,19 @@ def upload_data( ) return response - def annotation_types(self) -> List[Dict[str, Any]]: + def annotation_types(self) -> List[JSONFreeForm]: """ Returns a list of annotation types. Returns ------ - List[Dict[str, Any]] + List[JSONFreeForm] A list with the annotation types as dictionaries. """ - response: List[Dict[str, Any]] = cast(List[Dict[str, Any]], self._get("/annotation_types")) + response: List[JSONFreeForm] = cast(List[JSONFreeForm], self._get("/annotation_types")) return response - def get_exports(self, dataset_id: int, team_slug: Optional[str] = None) -> List[Dict[str, Any]]: + def get_exports(self, dataset_id: int, team_slug: Optional[str] = None) -> List[JSONFreeForm]: """ Get all the exports from the given dataset. @@ -652,7 +644,7 @@ def get_exports(self, dataset_id: int, team_slug: Optional[str] = None) -> List[ Returns ------ - List[Dict[str, Any]] + List[JSONFreeForm] A list with all the exports (as dictionaries) or None if the Team was not found. Raises @@ -667,12 +659,12 @@ def get_exports(self, dataset_id: int, team_slug: Optional[str] = None) -> List[ the_team_slug: str = the_team.slug - response: List[Dict[str, Any]] = cast( - List[Dict[str, Any]], self._get(f"/datasets/{dataset_id}/exports", team_slug=the_team_slug) + response: List[JSONFreeForm] = cast( + List[JSONFreeForm], self._get(f"/datasets/{dataset_id}/exports", team_slug=the_team_slug) ) return response - def create_export(self, dataset_id: int, payload: Dict[str, Any], team_slug: str) -> None: + def create_export(self, dataset_id: int, payload: JSONFreeForm, team_slug: str) -> None: """ Create an export for the given dataset. @@ -680,7 +672,7 @@ def create_export(self, dataset_id: int, payload: Dict[str, Any], team_slug: str ---------- dataset_id: int The id of the dataset. - payload: Dict[str, Any] + payload: JSONFreeForm The export infomation as a Dictionary. team_slug: Optional[str] Team slug of the team the dataset will belong to. Defaults to None. @@ -722,7 +714,7 @@ def get_report(self, dataset_id: int, granularity: str, team_slug: Optional[str] the_team_slug, ) - def delete_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, Any]) -> None: + def delete_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: """ Gets the report for the given dataset. @@ -732,12 +724,12 @@ def delete_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, Any] The slug of the dataset. team_slug: str The slug of the team. - payload: Dict[str, Any] + payload: JSONFreeForm A filter Dictionary that defines the items to be deleted. """ self._delete(f"teams/{team_slug}/datasets/{dataset_slug}/items", payload, team_slug) - def archive_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, Any]) -> None: + def archive_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: """ Archives the item from the given dataset. @@ -747,12 +739,12 @@ def archive_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, Any The slug of the dataset. team_slug: str The slug of the team. - payload: Dict[str, Any] + payload: JSONFreeForm A filter Dictionary that defines the items to be archived. """ self._put_raw(f"teams/{team_slug}/datasets/{dataset_slug}/items/archive", payload, team_slug) - def restore_archived_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, Any]) -> None: + def restore_archived_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: """ Restores the archived item from the given dataset. @@ -762,12 +754,12 @@ def restore_archived_item(self, dataset_slug: str, team_slug: str, payload: Dict The slug of the dataset. team_slug: str The slug of the team. - payload: Dict[str, Any] + payload: JSONFreeForm A filter Dictionary that defines the items to be restored. """ self._put_raw(f"teams/{team_slug}/datasets/{dataset_slug}/items/restore", payload, team_slug) - def move_item_to_new(self, dataset_slug: str, team_slug: str, payload: Dict[str, Any]) -> None: + def move_item_to_new(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: """ Moves the given item's status to new. @@ -777,12 +769,12 @@ def move_item_to_new(self, dataset_slug: str, team_slug: str, payload: Dict[str, The slug of the dataset. team_slug: str The slug of the team. - payload: Dict[str, Any] + payload: JSONFreeForm A filter Dictionary that defines the items to have the 'new' status. """ self._put_raw(f"teams/{team_slug}/datasets/{dataset_slug}/items/move_to_new", payload, team_slug) - def reset_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, Any]) -> None: + def reset_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: """ Resets the given item. @@ -792,12 +784,12 @@ def reset_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, Any]) The slug of the dataset. team_slug: str The slug of the team. - payload: Dict[str, Any] + payload: JSONFreeForm A filter Dictionary that defines the items to be reset. """ self._put_raw(f"teams/{team_slug}/datasets/{dataset_slug}/items/reset", payload, team_slug) - def move_to_stage(self, dataset_slug: str, team_slug: str, filters: Dict[str, Any], stage_id: int) -> None: + def move_to_stage(self, dataset_slug: str, team_slug: str, filters: JSONFreeForm, stage_id: int) -> None: """ Moves the given items to the specified stage @@ -807,12 +799,12 @@ def move_to_stage(self, dataset_slug: str, team_slug: str, filters: Dict[str, An The slug of the dataset. team_slug: str The slug of the team. - filters: Dict[str, Any] + filters: JSONFreeForm A filter Dictionary that defines the items to have the new, selected stage. stage_id: int ID of the stage to set. """ - payload: Dict[str, Any] = { + payload: JSONFreeForm = { "filter": filters, "workflow_stage_template_id": stage_id, } @@ -844,8 +836,8 @@ def post_workflow_comment( int The id of the created comment. """ - response: Dict[str, Any] = cast( - Dict[str, Any], + response: JSONFreeForm = cast( + JSONFreeForm, self._post( f"workflows/{workflow_id}/workflow_comment_threads", {"bounding_box": {"x": x, "y": y, "w": w, "h": h}, "workflow_comments": [{"body": text}]}, @@ -880,7 +872,7 @@ def instantiate_item(self, item_id: int, include_metadata: bool = False) -> Unio ValueError If due to an error, no workflow was instantiated for this item an therefore no workflow id can be returned. """ - response: Dict[str, Any] = cast(Dict[str, Any], self._post(f"dataset_items/{item_id}/workflow")) + response: JSONFreeForm = cast(JSONFreeForm, self._post(f"dataset_items/{item_id}/workflow")) id: Optional[int] = response.get("current_workflow_id") if id is None: @@ -1000,7 +992,7 @@ def from_api_key(cls, api_key: str, datasets_dir: Optional[Path] = None) -> "Cli if not response.ok: raise InvalidLogin() - data: Dict[str, Any] = response.json() + data: JSONFreeForm = response.json() team: str = data["selected_team"]["slug"] config: Config = Config(path=None) @@ -1048,7 +1040,7 @@ def _get_headers(self, team_slug: Optional[str] = None, compressed: bool = False if compressed: headers["X-Darwin-Payload-Compression-Version"] = "1" - from darwin import __version__ + from darwin.version import __version__ headers["User-Agent"] = f"darwin-py/{__version__}" return headers @@ -1082,12 +1074,12 @@ def _get_raw( def _get( self, endpoint: str, team_slug: Optional[str] = None, retry: bool = False - ) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + ) -> Union[JSONFreeForm, List[JSONFreeForm]]: response = self._get_raw(endpoint, team_slug, retry) return self._decode_response(response) def _put_raw( - self, endpoint: str, payload: Dict[str, Any], team_slug: Optional[str] = None, retry: bool = False + self, endpoint: str, payload: JSONFreeForm, team_slug: Optional[str] = None, retry: bool = False ) -> Response: response: requests.Response = self.session.put( urljoin(self.url, endpoint), json=payload, headers=self._get_headers(team_slug) @@ -1111,15 +1103,15 @@ def _put_raw( return response def _put( - self, endpoint: str, payload: Dict[str, Any], team_slug: Optional[str] = None, retry: bool = False - ) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + self, endpoint: str, payload: JSONFreeForm, team_slug: Optional[str] = None, retry: bool = False + ) -> Union[JSONFreeForm, List[JSONFreeForm]]: response: Response = self._put_raw(endpoint, payload, team_slug, retry) return self._decode_response(response) def _post_raw( self, endpoint: str, - payload: Optional[Dict[Any, Any]] = None, + payload: Optional[JSONFreeForm] = None, team_slug: Optional[str] = None, retry: bool = False, ) -> Response: @@ -1161,20 +1153,20 @@ def _post_raw( def _post( self, endpoint: str, - payload: Optional[Dict[Any, Any]] = None, + payload: Optional[JSONFreeForm] = None, team_slug: Optional[str] = None, retry: bool = False, - ) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + ) -> Union[JSONFreeForm, List[JSONFreeForm]]: response: Response = self._post_raw(endpoint, payload, team_slug, retry) return self._decode_response(response) def _delete( self, endpoint: str, - payload: Optional[Dict[Any, Any]] = None, + payload: Optional[JSONFreeForm] = None, team_slug: Optional[str] = None, retry: bool = False, - ) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + ) -> Union[JSONFreeForm, List[JSONFreeForm]]: if payload is None: payload = {} @@ -1236,7 +1228,7 @@ def _raise_if_known_error(self, response: Response, url: str) -> None: if error_code == "INSUFFICIENT_REMAINING_STORAGE": raise InsufficientStorage() - def _decode_response(self, response: requests.Response) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + def _decode_response(self, response: requests.Response) -> Union[JSONFreeForm, List[JSONFreeForm]]: """Decode the response as JSON entry or return a dictionary with the error Parameters @@ -1269,14 +1261,14 @@ def parse_version(version: str) -> DarwinVersionNumber: (major, minor, patch) = version.split(".") return (int(major), int(minor), int(patch)) - from darwin import __version__ + from darwin.version import __version__ current_version = parse_version(__version__) latest_version = parse_version(server_latest_version) if current_version >= latest_version: return self._newer_version = latest_version - except: + except Exception: pass @property @@ -1287,5 +1279,9 @@ def __str__(self) -> str: return f"Client(default_team={self.default_team})" @property - def api_v2(self): - return BackendV2(self, self.config.get_default_team().slug) + def api_v2(self) -> BackendV2: + team = self.config.get_default_team() + if not team: + raise ValueError("No team was found.") + + return BackendV2(self, team.slug) From 91653af41c2eadddbf2711fbd56b56e6874ac48d Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 30 Aug 2023 14:04:08 +0100 Subject: [PATCH 4/4] Changed JSONFreeForm fixes to UnknownType as it is more appropriate. --- darwin/client.py | 171 ++++++++++++++++++++++++++--------------------- 1 file changed, 93 insertions(+), 78 deletions(-) diff --git a/darwin/client.py b/darwin/client.py index ad759ea41..6b9b0d676 100644 --- a/darwin/client.py +++ b/darwin/client.py @@ -5,7 +5,7 @@ import zlib from logging import Logger from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast +from typing import Dict, Iterator, List, Optional, Tuple, Union, cast from urllib import parse import requests @@ -18,7 +18,7 @@ from darwin.dataset.remote_dataset import RemoteDataset from darwin.dataset.remote_dataset_v1 import RemoteDatasetV1 from darwin.dataset.remote_dataset_v2 import RemoteDatasetV2 -from darwin.datatypes import DarwinVersionNumber, Feature, ItemId, JSONFreeForm, Team +from darwin.datatypes import DarwinVersionNumber, Feature, ItemId, Team, UnknownType from darwin.exceptions import ( InsufficientStorage, InvalidLogin, @@ -56,13 +56,13 @@ def __init__(self, config: Config, default_team: Optional[str] = None, log: Opti self.log = log @staticmethod - def _get_item_count(dataset_dict: JSONFreeForm) -> int: + def _get_item_count(dataset_dict: Dict[str, UnknownType]) -> int: """ Returns the number of items in the dataset. Parameters ---------- - dataset_dict: JSONFreeForm + dataset_dict: Dict[str, UnknownType] The dataset dictionary. Returns @@ -123,7 +123,9 @@ def list_remote_datasets(self, team_slug: Optional[str] = None) -> Iterator[Remo Iterator[RemoteDataset] List of all remote datasets """ - response: List[JSONFreeForm] = cast(List[JSONFreeForm], self._get("/datasets/", team_slug=team_slug)) + response: List[Dict[str, UnknownType]] = cast( + List[Dict[str, UnknownType]], self._get("/datasets/", team_slug=team_slug) + ) for dataset in response: if dataset.get("version", 1) == 2: @@ -179,8 +181,8 @@ def get_remote_dataset(self, dataset_identifier: Union[str, DatasetIdentifier]) ] except Unauthorized: # There is a chance that we tried to access an open dataset - dataset: JSONFreeForm = cast( - JSONFreeForm, + dataset: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], self._get(f"{parsed_dataset_identifier.team_slug}/{parsed_dataset_identifier.dataset_slug}"), ) @@ -233,7 +235,9 @@ def create_dataset(self, name: str, team_slug: Optional[str] = None) -> RemoteDa RemoteDataset The created dataset. """ - dataset: JSONFreeForm = cast(JSONFreeForm, self._post("/datasets", {"name": name}, team_slug=team_slug)) + dataset: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], self._post("/datasets", {"name": name}, team_slug=team_slug) + ) if dataset.get("version", 1) == 2: return RemoteDatasetV2( @@ -270,8 +274,8 @@ def archive_remote_dataset(self, dataset_id: int, team_slug: str) -> None: self._put(f"datasets/{dataset_id}/archive", payload={}, team_slug=team_slug) def fetch_remote_files( - self, dataset_id: int, cursor: JSONFreeForm, payload: JSONFreeForm, team_slug: str - ) -> JSONFreeForm: + self, dataset_id: int, cursor: Dict[str, UnknownType], payload: Dict[str, UnknownType], team_slug: str + ) -> Dict[str, UnknownType]: """ Download the remote files from the given dataset. @@ -279,24 +283,25 @@ def fetch_remote_files( ---------- dataset_id: int Id of the dataset the file belong to. - cursor: JSONFreeForm + cursor: Dict[str, UnknownType] Number of items per page and page number. Defaults to {"page[size]": 500, "page[from]": 0}. - payload: JSONFreeForm + payload: Dict[str, UnknownType] Filter and sort parameters. team_slug: str The team slug of the dataset. Returns ------- - JSONFreeForm + Dict[str, UnknownType] A response dictionary with the file information. """ - response: JSONFreeForm = cast( - JSONFreeForm, self._post(f"/datasets/{dataset_id}/items?{parse.urlencode(cursor)}", payload, team_slug) + response: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], + self._post(f"/datasets/{dataset_id}/items?{parse.urlencode(cursor)}", payload, team_slug), ) return response - def fetch_remote_classes(self, team_slug: Optional[str] = None) -> List[JSONFreeForm]: + def fetch_remote_classes(self, team_slug: Optional[str] = None) -> List[Dict[str, UnknownType]]: """ Fetches all remote classes on the remote dataset. @@ -307,7 +312,7 @@ def fetch_remote_classes(self, team_slug: Optional[str] = None) -> List[JSONFree Returns ------- - JSONFreeForm + Dict[str, UnknownType] None if no information about the team is found, a List of Annotation classes otherwise. Raises @@ -321,13 +326,13 @@ def fetch_remote_classes(self, team_slug: Optional[str] = None) -> List[JSONFree raise ValueError("No team was found.") the_team_slug: str = the_team.slug - response: JSONFreeForm = cast( - JSONFreeForm, self._get(f"/teams/{the_team_slug}/annotation_classes?include_tags=true") + response: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], self._get(f"/teams/{the_team_slug}/annotation_classes?include_tags=true") ) return response["annotation_classes"] - def update_annotation_class(self, class_id: int, payload: JSONFreeForm) -> JSONFreeForm: + def update_annotation_class(self, class_id: int, payload: Dict[str, UnknownType]) -> Dict[str, UnknownType]: """ Updates the AnnotationClass with the given id. @@ -335,18 +340,20 @@ def update_annotation_class(self, class_id: int, payload: JSONFreeForm) -> JSONF ---------- class_id: int The id of the AnnotationClass to update. - payload: JSONFreeForm + payload: Dict[str, UnknownType] A dictionary with the changes to perform. Returns ------- - JSONFreeForm + Dict[str, UnknownType] A dictionary with the result of the operation. """ - response: JSONFreeForm = cast(JSONFreeForm, self._put(f"/annotation_classes/{class_id}", payload)) + response: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], self._put(f"/annotation_classes/{class_id}", payload) + ) return response - def create_annotation_class(self, dataset_id: int, type_ids: List[int], name: str) -> JSONFreeForm: + def create_annotation_class(self, dataset_id: int, type_ids: List[int], name: str) -> Dict[str, UnknownType]: """ Creates an AnnotationClass. @@ -361,11 +368,11 @@ def create_annotation_class(self, dataset_id: int, type_ids: List[int], name: st Returns ------- - JSONFreeForm + Dict[str, UnknownType] A dictionary with the result of the operation. """ - response: JSONFreeForm = cast( - JSONFreeForm, + response: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], self._post( "/annotation_classes", payload={ @@ -379,7 +386,7 @@ def create_annotation_class(self, dataset_id: int, type_ids: List[int], name: st ) return response - def import_annotation(self, item_id: ItemId, payload: JSONFreeForm) -> None: + def import_annotation(self, item_id: ItemId, payload: Dict[str, UnknownType]) -> None: """ Imports the annotation for the item with the given id. @@ -387,14 +394,14 @@ def import_annotation(self, item_id: ItemId, payload: JSONFreeForm) -> None: ---------- item_id: ItemId Identifier of the Image or Video that we are import the annotation to. - payload: JSONFreeForm + payload: Dict[str, UnknownType] A dictionary with the annotation to import. The default format is: `{"annotations": serialized_annotations, "overwrite": "false"}` """ self._post_raw(f"/dataset_items/{item_id}/import", payload=payload) - def fetch_remote_attributes(self, dataset_id: int) -> List[JSONFreeForm]: + def fetch_remote_attributes(self, dataset_id: int) -> List[Dict[str, UnknownType]]: """ Fetches all attributes remotely. @@ -406,10 +413,12 @@ def fetch_remote_attributes(self, dataset_id: int) -> List[JSONFreeForm]: Returns ------- - List[JSONFreeForm] + List[Dict[str, UnknownType]] A List with the attributes, where each attribute is a dictionary. """ - response: List[JSONFreeForm] = cast(List[JSONFreeForm], self._get(f"/datasets/{dataset_id}/attributes")) + response: List[Dict[str, UnknownType]] = cast( + List[Dict[str, UnknownType]], self._get(f"/datasets/{dataset_id}/attributes") + ) return response def load_feature_flags(self, team_slug: Optional[str] = None) -> None: @@ -444,7 +453,9 @@ def get_team_features(self, team_slug: str) -> List[Feature]: List[Feature] List of Features for the given team. """ - response: List[JSONFreeForm] = cast(List[JSONFreeForm], self._get(f"/teams/{team_slug}/features")) + response: List[Dict[str, UnknownType]] = cast( + List[Dict[str, UnknownType]], self._get(f"/teams/{team_slug}/features") + ) features: List[Feature] = [] for feature in response: @@ -545,7 +556,7 @@ def confirm_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) self._put_raw(endpoint=f"/dataset_items/{dataset_item_id}/confirm_upload", payload={}, team_slug=the_team_slug) - def sign_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) -> JSONFreeForm: + def sign_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) -> Dict[str, UnknownType]: """ Signs the upload of the given DatasetItem. @@ -558,7 +569,7 @@ def sign_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) -> Returns ------ - JSONFreeForm + Dict[str, UnknownType] A dictionary with the signed response, or None if the Team was not found. Raises @@ -573,12 +584,14 @@ def sign_upload(self, dataset_item_id: int, team_slug: Optional[str] = None) -> the_team_slug: str = the_team.slug - response: JSONFreeForm = cast( - JSONFreeForm, self._get(f"/dataset_items/{dataset_item_id}/sign_upload", team_slug=the_team_slug) + response: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], self._get(f"/dataset_items/{dataset_item_id}/sign_upload", team_slug=the_team_slug) ) return response - def upload_data(self, dataset_slug: str, payload: JSONFreeForm, team_slug: Optional[str] = None) -> JSONFreeForm: + def upload_data( + self, dataset_slug: str, payload: Dict[str, UnknownType], team_slug: Optional[str] = None + ) -> Dict[str, UnknownType]: """ Uploads the given data to the given dataset. @@ -586,7 +599,7 @@ def upload_data(self, dataset_slug: str, payload: JSONFreeForm, team_slug: Optio ---------- dataset_slug: str The slug of the dataset. - payload: JSONFreeForm + payload: Dict[str, UnknownType] The data we want to upload. Usually a Dictionary with an `items` key containing a list of items to upload. team_slug: Optional[str] @@ -594,7 +607,7 @@ def upload_data(self, dataset_slug: str, payload: JSONFreeForm, team_slug: Optio Returns ------ - JSONFreeForm + Dict[str, UnknownType] A dictionary with the result of the operation, or None if the Team was not found. Raises @@ -609,8 +622,8 @@ def upload_data(self, dataset_slug: str, payload: JSONFreeForm, team_slug: Optio the_team_slug: str = the_team.slug - response: JSONFreeForm = cast( - JSONFreeForm, + response: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], self._put( endpoint=f"/teams/{the_team_slug}/datasets/{dataset_slug}/data", payload=payload, @@ -619,19 +632,19 @@ def upload_data(self, dataset_slug: str, payload: JSONFreeForm, team_slug: Optio ) return response - def annotation_types(self) -> List[JSONFreeForm]: + def annotation_types(self) -> List[Dict[str, UnknownType]]: """ Returns a list of annotation types. Returns ------ - List[JSONFreeForm] + List[Dict[str, UnknownType]] A list with the annotation types as dictionaries. """ - response: List[JSONFreeForm] = cast(List[JSONFreeForm], self._get("/annotation_types")) + response: List[Dict[str, UnknownType]] = cast(List[Dict[str, UnknownType]], self._get("/annotation_types")) return response - def get_exports(self, dataset_id: int, team_slug: Optional[str] = None) -> List[JSONFreeForm]: + def get_exports(self, dataset_id: int, team_slug: Optional[str] = None) -> List[Dict[str, UnknownType]]: """ Get all the exports from the given dataset. @@ -644,7 +657,7 @@ def get_exports(self, dataset_id: int, team_slug: Optional[str] = None) -> List[ Returns ------ - List[JSONFreeForm] + List[Dict[str, UnknownType]] A list with all the exports (as dictionaries) or None if the Team was not found. Raises @@ -659,12 +672,12 @@ def get_exports(self, dataset_id: int, team_slug: Optional[str] = None) -> List[ the_team_slug: str = the_team.slug - response: List[JSONFreeForm] = cast( - List[JSONFreeForm], self._get(f"/datasets/{dataset_id}/exports", team_slug=the_team_slug) + response: List[Dict[str, UnknownType]] = cast( + List[Dict[str, UnknownType]], self._get(f"/datasets/{dataset_id}/exports", team_slug=the_team_slug) ) return response - def create_export(self, dataset_id: int, payload: JSONFreeForm, team_slug: str) -> None: + def create_export(self, dataset_id: int, payload: Dict[str, UnknownType], team_slug: str) -> None: """ Create an export for the given dataset. @@ -672,7 +685,7 @@ def create_export(self, dataset_id: int, payload: JSONFreeForm, team_slug: str) ---------- dataset_id: int The id of the dataset. - payload: JSONFreeForm + payload: Dict[str, UnknownType] The export infomation as a Dictionary. team_slug: Optional[str] Team slug of the team the dataset will belong to. Defaults to None. @@ -714,7 +727,7 @@ def get_report(self, dataset_id: int, granularity: str, team_slug: Optional[str] the_team_slug, ) - def delete_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: + def delete_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, UnknownType]) -> None: """ Gets the report for the given dataset. @@ -724,12 +737,12 @@ def delete_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) The slug of the dataset. team_slug: str The slug of the team. - payload: JSONFreeForm + payload: Dict[str, UnknownType] A filter Dictionary that defines the items to be deleted. """ self._delete(f"teams/{team_slug}/datasets/{dataset_slug}/items", payload, team_slug) - def archive_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: + def archive_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, UnknownType]) -> None: """ Archives the item from the given dataset. @@ -739,12 +752,12 @@ def archive_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) The slug of the dataset. team_slug: str The slug of the team. - payload: JSONFreeForm + payload: Dict[str, UnknownType] A filter Dictionary that defines the items to be archived. """ self._put_raw(f"teams/{team_slug}/datasets/{dataset_slug}/items/archive", payload, team_slug) - def restore_archived_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: + def restore_archived_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, UnknownType]) -> None: """ Restores the archived item from the given dataset. @@ -754,12 +767,12 @@ def restore_archived_item(self, dataset_slug: str, team_slug: str, payload: JSON The slug of the dataset. team_slug: str The slug of the team. - payload: JSONFreeForm + payload: Dict[str, UnknownType] A filter Dictionary that defines the items to be restored. """ self._put_raw(f"teams/{team_slug}/datasets/{dataset_slug}/items/restore", payload, team_slug) - def move_item_to_new(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: + def move_item_to_new(self, dataset_slug: str, team_slug: str, payload: Dict[str, UnknownType]) -> None: """ Moves the given item's status to new. @@ -769,12 +782,12 @@ def move_item_to_new(self, dataset_slug: str, team_slug: str, payload: JSONFreeF The slug of the dataset. team_slug: str The slug of the team. - payload: JSONFreeForm + payload: Dict[str, UnknownType] A filter Dictionary that defines the items to have the 'new' status. """ self._put_raw(f"teams/{team_slug}/datasets/{dataset_slug}/items/move_to_new", payload, team_slug) - def reset_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) -> None: + def reset_item(self, dataset_slug: str, team_slug: str, payload: Dict[str, UnknownType]) -> None: """ Resets the given item. @@ -784,12 +797,12 @@ def reset_item(self, dataset_slug: str, team_slug: str, payload: JSONFreeForm) - The slug of the dataset. team_slug: str The slug of the team. - payload: JSONFreeForm + payload: Dict[str, UnknownType] A filter Dictionary that defines the items to be reset. """ self._put_raw(f"teams/{team_slug}/datasets/{dataset_slug}/items/reset", payload, team_slug) - def move_to_stage(self, dataset_slug: str, team_slug: str, filters: JSONFreeForm, stage_id: int) -> None: + def move_to_stage(self, dataset_slug: str, team_slug: str, filters: Dict[str, UnknownType], stage_id: int) -> None: """ Moves the given items to the specified stage @@ -799,12 +812,12 @@ def move_to_stage(self, dataset_slug: str, team_slug: str, filters: JSONFreeForm The slug of the dataset. team_slug: str The slug of the team. - filters: JSONFreeForm + filters: Dict[str, UnknownType] A filter Dictionary that defines the items to have the new, selected stage. stage_id: int ID of the stage to set. """ - payload: JSONFreeForm = { + payload: Dict[str, UnknownType] = { "filter": filters, "workflow_stage_template_id": stage_id, } @@ -836,8 +849,8 @@ def post_workflow_comment( int The id of the created comment. """ - response: JSONFreeForm = cast( - JSONFreeForm, + response: Dict[str, UnknownType] = cast( + Dict[str, UnknownType], self._post( f"workflows/{workflow_id}/workflow_comment_threads", {"bounding_box": {"x": x, "y": y, "w": w, "h": h}, "workflow_comments": [{"body": text}]}, @@ -872,7 +885,7 @@ def instantiate_item(self, item_id: int, include_metadata: bool = False) -> Unio ValueError If due to an error, no workflow was instantiated for this item an therefore no workflow id can be returned. """ - response: JSONFreeForm = cast(JSONFreeForm, self._post(f"dataset_items/{item_id}/workflow")) + response: Dict[str, UnknownType] = cast(Dict[str, UnknownType], self._post(f"dataset_items/{item_id}/workflow")) id: Optional[int] = response.get("current_workflow_id") if id is None: @@ -992,7 +1005,7 @@ def from_api_key(cls, api_key: str, datasets_dir: Optional[Path] = None) -> "Cli if not response.ok: raise InvalidLogin() - data: JSONFreeForm = response.json() + data: Dict[str, UnknownType] = response.json() team: str = data["selected_team"]["slug"] config: Config = Config(path=None) @@ -1074,12 +1087,12 @@ def _get_raw( def _get( self, endpoint: str, team_slug: Optional[str] = None, retry: bool = False - ) -> Union[JSONFreeForm, List[JSONFreeForm]]: + ) -> Union[Dict[str, UnknownType], List[Dict[str, UnknownType]]]: response = self._get_raw(endpoint, team_slug, retry) return self._decode_response(response) def _put_raw( - self, endpoint: str, payload: JSONFreeForm, team_slug: Optional[str] = None, retry: bool = False + self, endpoint: str, payload: Dict[str, UnknownType], team_slug: Optional[str] = None, retry: bool = False ) -> Response: response: requests.Response = self.session.put( urljoin(self.url, endpoint), json=payload, headers=self._get_headers(team_slug) @@ -1103,15 +1116,15 @@ def _put_raw( return response def _put( - self, endpoint: str, payload: JSONFreeForm, team_slug: Optional[str] = None, retry: bool = False - ) -> Union[JSONFreeForm, List[JSONFreeForm]]: + self, endpoint: str, payload: Dict[str, UnknownType], team_slug: Optional[str] = None, retry: bool = False + ) -> Union[Dict[str, UnknownType], List[Dict[str, UnknownType]]]: response: Response = self._put_raw(endpoint, payload, team_slug, retry) return self._decode_response(response) def _post_raw( self, endpoint: str, - payload: Optional[JSONFreeForm] = None, + payload: Optional[Dict[str, UnknownType]] = None, team_slug: Optional[str] = None, retry: bool = False, ) -> Response: @@ -1153,20 +1166,20 @@ def _post_raw( def _post( self, endpoint: str, - payload: Optional[JSONFreeForm] = None, + payload: Optional[Dict[str, UnknownType]] = None, team_slug: Optional[str] = None, retry: bool = False, - ) -> Union[JSONFreeForm, List[JSONFreeForm]]: + ) -> Union[Dict[str, UnknownType], List[Dict[str, UnknownType]]]: response: Response = self._post_raw(endpoint, payload, team_slug, retry) return self._decode_response(response) def _delete( self, endpoint: str, - payload: Optional[JSONFreeForm] = None, + payload: Optional[Dict[str, UnknownType]] = None, team_slug: Optional[str] = None, retry: bool = False, - ) -> Union[JSONFreeForm, List[JSONFreeForm]]: + ) -> Union[Dict[str, UnknownType], List[Dict[str, UnknownType]]]: if payload is None: payload = {} @@ -1228,7 +1241,9 @@ def _raise_if_known_error(self, response: Response, url: str) -> None: if error_code == "INSUFFICIENT_REMAINING_STORAGE": raise InsufficientStorage() - def _decode_response(self, response: requests.Response) -> Union[JSONFreeForm, List[JSONFreeForm]]: + def _decode_response( + self, response: requests.Response + ) -> Union[Dict[str, UnknownType], List[Dict[str, UnknownType]]]: """Decode the response as JSON entry or return a dictionary with the error Parameters