From 404db056342adea5e902daf4f7fa83807bb8def1 Mon Sep 17 00:00:00 2001 From: "takahiro.tamenishi" Date: Wed, 6 Dec 2023 02:03:11 +0900 Subject: [PATCH 1/3] add dataset training --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ fastlabel/__init__.py | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bdd1fd3..84d058a 100644 --- a/README.md +++ b/README.md @@ -3051,6 +3051,48 @@ Example of two training jobs. ] ``` +### Execute training job + +Get training jobs. + +```python +training_job = client.execute_training_job( + dataset_name="dataset_name", + base_model_name="fastlabel_object_detection_light", // "fastlabel_object_detection_light" or "fastlabel_object_detection_high_accuracy" + epoch=300, + use_dataset_train_val=True +) + +``` + +#### Response + +Example of two training jobs. + +```python +{ + "trainingJobId": "f40c5838-4c3a-482f-96b7-f77e16c96fed", + "status": "in_progress", + "baseModelName": "FastLabel Object Detection High Accuracy - 汎用", + "instanceType": "ml.p3.2xlarge", + "epoch": 300, + "projects": [ + "image-bbox" + ], + "statuses": [], + "tags": [], + "contentCount": 23, + "userName": "Admin", + "createdAt": "2023-10-31T07:10:28.306Z", + "completedAt": null, + "customModel": { + "modelId": "", + "modelName": "", + "modelURL": "", + "classes": [] + } +} +``` ### Execute endpoint diff --git a/fastlabel/__init__.py b/fastlabel/__init__.py index daf08b4..50830ed 100644 --- a/fastlabel/__init__.py +++ b/fastlabel/__init__.py @@ -3918,7 +3918,7 @@ def find_dataset_object(self, dataset_id: str, object_name: str) -> dict: """ Find a dataset object. """ - endpoint = "datasets-v2/" + dataset_id + "/objects/" + object_name + endpoint = "datasets-v2/" + dataset_id + "/objects/" + object_name return self.api.get_request(endpoint) def get_dataset_objects( @@ -4049,7 +4049,7 @@ def delete_dataset_object(self, dataset_id: str, object_name: str) -> None: """ Delete a dataset object. """ - endpoint = "datasets-v2/" + dataset_id + "/objects/" + object_name + endpoint = "datasets-v2/" + dataset_id + "/objects/" + object_name self.api.delete_request(endpoint) def update_aws_s3_storage( @@ -4172,6 +4172,41 @@ def get_training_jobs( return self.api.get_request(endpoint, params=params) + def execute_training_job( + self, + dataset_name: str, + base_model_name: str, + epoch: int, + dataset_revision_id: str = None, + use_dataset_train_val: bool = False, + instance_type: str = "ml.p3.2xlarge", + batch_size: int = None, + learning_rate: float = None, + ) -> list: + """ + Returns a list of training jobs. + Returns up to 1000 at a time, to get more, set offset as the starting position + to fetch. + offset is the starting position number to fetch (Optional). + limit is the max number to fetch (Optional). + """ + endpoint = "trainings" + payload = { + "datasetName": dataset_name, + "baseModelName": base_model_name, + "epoch": epoch, + "useDatasetTrainVal": use_dataset_train_val, + "datasetRevisionId": dataset_revision_id, + "instanceType": instance_type, + "batchSize": batch_size, + "learningRate": learning_rate, + } + + return self.api.post_request( + endpoint, + payload={key: value for key, value in payload.items() if value is not None}, + ) + def execute_endpoint( self, endpoint_name: str, From 44d1a49bc4b4a0d6098a736226f0d718a74e7dc3 Mon Sep 17 00:00:00 2001 From: "takahiro.tamenishi" Date: Thu, 21 Dec 2023 18:24:35 +0900 Subject: [PATCH 2/3] add dataset evaluation --- README.md | 113 ++++++++++++++++++++++++++++++++++-------- fastlabel/__init__.py | 83 +++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 84d058a..9f4b0ff 100644 --- a/README.md +++ b/README.md @@ -2992,6 +2992,14 @@ def get_training_jobs() -> list[dict]: ``` +#### Find Training job + +Find a single training job. + +```python +task = client.find_training_job(id="YOUR_TRAINING_ID") +``` + #### Response Example of two training jobs. @@ -3065,33 +3073,94 @@ training_job = client.execute_training_job( ``` +### Get evaluation jobs + +Get evaluation jobs. + +```python +def get_evaluation_jobs() -> list[dict]: + all_evaluation_jobs = [] + offset = None + while True: + time.sleep(1) + + evaluation_jobs = client.get_evaluation_jobs(offset=offset) + all_evaluation_jobs.extend(evaluation_jobs) + + if len(evaluation_jobs) > 0: + offset = len(all_evaluation_jobs) + else: + break + return all_evaluation_jobs + +``` + +#### Find Evaluation job + +Find a single evaluation job. + +```python +evaluation_job = client.find_evaluation_job(id="YOUR_EVALUATION_ID") +``` + #### Response -Example of two training jobs. +Example of two evaluation jobs. ```python + { - "trainingJobId": "f40c5838-4c3a-482f-96b7-f77e16c96fed", - "status": "in_progress", - "baseModelName": "FastLabel Object Detection High Accuracy - 汎用", - "instanceType": "ml.p3.2xlarge", - "epoch": 300, - "projects": [ - "image-bbox" - ], - "statuses": [], - "tags": [], - "contentCount": 23, - "userName": "Admin", - "createdAt": "2023-10-31T07:10:28.306Z", - "completedAt": null, - "customModel": { - "modelId": "", - "modelName": "", - "modelURL": "", - "classes": [] - } -} + id: "50873ea1-e008-48db-a368-241ca88d6f67", + version: 59, + status: "in_progress", + modelType: "builtin", + modelName: "FastLabel Object Detection Light - 汎用", + customModelId: None, + iouThreshold: 0.8, + confidenceThreshold: 0.4, + contentCount: 0, + gtCount: 0, + predCount: 0, + mAP: 0, + recall: 0, + precision: 0, + f1: 0, + confusionMatrix: None, + duration: 0, + evaluationSource: "dataset", + projects: [], + statuses: [], + tags: [], + datasetId: "deacbe6d-406f-4086-bd87-80ffb1c1a393", + dataset: { + id: "deacbe6d-406f-4086-bd87-80ffb1c1a393", + workspaceId: "df201d3c-af00-423a-aa7f-827376fd96de", + name: "sample-dataset", + createdAt: "2023-12-20T10:44:12.198Z", + updatedAt: "2023-12-20T10:44:12.198Z", + }, + datasetRevisionId: "2d26ab64-dfc0-482d-9211-ce8feb3d480b", + useDatasetTest: True, + userName: "", + completedAt: None, + createdAt: "2023-12-21T09:08:16.111Z", + updatedAt: "2023-12-21T09:08:18.414Z", +}; + +``` + +### Execute evaluation job + +Execute evaluation jobs. + +```python +training_job = client.execute_evaluation_job( + dataset_name="dataset_name", + base_model_name="fastlabel_object_detection_light", // "fastlabel_object_detection_light" or "fastlabel_object_detection_high_accuracy" + epoch=300, + use_dataset_train_val=True +) + ``` ### Execute endpoint diff --git a/fastlabel/__init__.py b/fastlabel/__init__.py index 50830ed..b6dd0fb 100644 --- a/fastlabel/__init__.py +++ b/fastlabel/__init__.py @@ -4207,6 +4207,89 @@ def execute_training_job( payload={key: value for key, value in payload.items() if value is not None}, ) + def find_training_job(self, id: str) -> list: + """ + Returns training job. + id is id of training job (Required). + """ + + endpoint = f"trainings/{id}" + + return self.api.get_request( + endpoint, + ) + + def get_evaluation_jobs( + self, + offset: int = None, + limit: int = 1000, + ) -> list: + """ + Returns a list of training jobs. + Returns up to 1000 at a time, to get more, set offset as the starting position + to fetch. + offset is the starting position number to fetch (Optional). + limit is the max number to fetch (Optional). + """ + if limit > 1000: + raise FastLabelInvalidException( + "Limit must be less than or equal to 1000.", 422 + ) + endpoint = "evaluations" + params = {} + if offset: + params["offset"] = offset + if limit: + params["limit"] = limit + + return self.api.get_request(endpoint, params=params) + + def find_evaluation_job(self, id: str) -> list: + """ + Returns evaluation job. + id is id of evaluation job (Required). + """ + + endpoint = f"evaluations/{id}" + + return self.api.get_request( + endpoint, + ) + + def execute_evaluation_job( + self, + dataset_name: str, + model_name: str, + iou_threshold: float = 0.5, + confidence_threshold: float = 0.4, + dataset_revision_id: str = None, + use_dataset_test: bool = False, + instance_type: str = "ml.p3.2xlarge", + batch_size: int = None, + learning_rate: float = None, + ) -> list: + """ + Returns a list of training jobs. + Returns up to 1000 at a time, to get more, set offset as the starting position + to fetch. + offset is the starting position number to fetch (Optional). + limit is the max number to fetch (Optional). + """ + endpoint = "evaluations" + payload = { + "modelName": model_name, + "datasetName": dataset_name, + "iouThreshold": iou_threshold, + "confidenceThreshold": confidence_threshold, + "datasetRevisionId": dataset_revision_id, + "useDatasetTest": use_dataset_test, + } + + return self.api.post_request( + endpoint, + payload={key: value for key, value in payload.items() if value is not None}, + ) + def execute_endpoint( self, endpoint_name: str, From d8efed193083e3cee897fe73aed57baed8df6d68 Mon Sep 17 00:00:00 2001 From: "takahiro.tamenishi" Date: Thu, 21 Dec 2023 18:28:16 +0900 Subject: [PATCH 3/3] fix --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9f4b0ff..706e9cc 100644 --- a/README.md +++ b/README.md @@ -3155,10 +3155,11 @@ Execute evaluation jobs. ```python training_job = client.execute_evaluation_job( - dataset_name="dataset_name", - base_model_name="fastlabel_object_detection_light", // "fastlabel_object_detection_light" or "fastlabel_object_detection_high_accuracy" - epoch=300, - use_dataset_train_val=True + dataset_name="DATASET_NAME", + model_name="fastlabel_object_detection_light", + // If you want to use the built-in model, select the following. "fastlabel_object_detection_light" or "fastlabel_object_detection_high_accuracy" + // If you want to use the custom model, please fill　out model name. + use_dataset_test=True, ) ```