Skip to content

Commit bf09ee5

Browse files
ritheek-pitti-doritheek-pitti-do
authored
Model Evals - Added New APIs (#1179)
1 parent 3512e76 commit bf09ee5

8 files changed

Lines changed: 191 additions & 0 deletions

‎specification/DigitalOcean-public.v2.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2644,6 +2644,8 @@ paths:
26442644

26452645

26462646
/v2/gen-ai/evaluation_datasets:
2647+
get:
2648+
$ref: 'resources/gen-ai/genai_list_evaluation_datasets.yml'
26472649

26482650
post:
26492651
$ref: 'resources/gen-ai/genai_create_evaluation_dataset.yml'
@@ -2798,6 +2800,9 @@ paths:
27982800
get:
27992801
$ref: 'resources/gen-ai/genai_get_model_evaluation_preset.yml'
28002802

2803+
delete:
2804+
$ref: 'resources/gen-ai/genai_delete_model_evaluation_preset.yml'
2805+
28012806

28022807
/v2/gen-ai/model_evaluation_runs:
28032808
get:
@@ -2815,6 +2820,11 @@ paths:
28152820
$ref: 'resources/gen-ai/genai_delete_model_evaluation_run.yml'
28162821

28172822

2823+
/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}/cancel:
2824+
put:
2825+
$ref: 'resources/gen-ai/genai_cancel_model_evaluation_run.yml'
2826+
2827+
28182828
/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}/results/download_url:
28192829
get:
28202830
$ref: 'resources/gen-ai/genai_get_model_evaluation_run_results_download_url.yml'

‎specification/resources/gen-ai/definitions.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1662,6 +1662,22 @@ apiCancelKnowledgeBaseIndexingJobOutput:
16621662
job:
16631663
$ref: '#/apiIndexingJob'
16641664
type: object
1665+
apiCancelModelEvaluationRunInputPublic:
1666+
properties:
1667+
eval_run_uuid:
1668+
description: |-
1669+
UUID of the model evaluation run to cancel. Returned by `CreateModelEvaluationRun`
1670+
and listed via `ListModelEvaluationRuns`. The run must be in a non-terminal status
1671+
(queued, running_dataset, or evaluating_results); already-terminal runs return an
1672+
error.
1673+
example: '"12345678-1234-1234-1234-123456789012"'
1674+
type: string
1675+
type: object
1676+
apiCancelModelEvaluationRunOutput:
1677+
properties:
1678+
run:
1679+
$ref: '#/apiModelEvaluationRunSummary'
1680+
type: object
16651681
apiCandidateInferenceConfig:
16661682
description: Inference configuration for the candidate model during evaluation.
16671683
properties:
@@ -2900,6 +2916,8 @@ apiDeleteModelAPIKeyOutput:
29002916
api_key_info:
29012917
$ref: '#/apiModelAPIKeyInfo'
29022918
type: object
2919+
apiDeleteModelEvaluationPresetOutput:
2920+
type: object
29032921
apiDeleteModelEvaluationRunInputPublic:
29042922
properties:
29052923
eval_run_uuid:

‎specification/resources/gen-ai/examples/curl/genai_cancel_model_evaluation_run.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
lang: cURL
2+
source: |-
3+
curl -X PUT \
4+
-H "Content-Type: application/json" \
5+
-H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
6+
"https://api.digitalocean.com/v2/gen-ai/model_evaluation_runs/12345678-1234-1234-1234-123456789012/cancel" \
7+
-d '{
8+
"eval_run_uuid": "12345678-1234-1234-1234-123456789012"
9+
}'

‎specification/resources/gen-ai/examples/curl/genai_delete_model_evaluation_preset.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
lang: cURL
2+
source: |-
3+
curl -X DELETE \
4+
-H "Content-Type: application/json" \
5+
-H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
6+
"https://api.digitalocean.com/v2/gen-ai/model_evaluation_presets/12345678-1234-1234-1234-123456789012"

‎specification/resources/gen-ai/examples/curl/genai_list_evaluation_datasets.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
lang: cURL
2+
source: |-
3+
curl -X GET \
4+
-H "Content-Type: application/json" \
5+
-H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
6+
"https://api.digitalocean.com/v2/gen-ai/evaluation_datasets"

‎specification/resources/gen-ai/genai_cancel_model_evaluation_run.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
description: To cancel an in-progress model evaluation run, send a PUT request to
2+
`/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}/cancel`.
3+
operationId: genai_cancel_model_evaluation_run
4+
parameters:
5+
- description: |-
6+
UUID of the model evaluation run to cancel. Returned by `CreateModelEvaluationRun`
7+
and listed via `ListModelEvaluationRuns`. The run must be in a non-terminal status
8+
(queued, running_dataset, or evaluating_results); already-terminal runs return an
9+
error.
10+
example: '"123e4567-e89b-12d3-a456-426614174000"'
11+
in: path
12+
name: eval_run_uuid
13+
required: true
14+
schema:
15+
type: string
16+
requestBody:
17+
content:
18+
application/json:
19+
schema:
20+
$ref: ./definitions.yml#/apiCancelModelEvaluationRunInputPublic
21+
responses:
22+
"200":
23+
content:
24+
application/json:
25+
schema:
26+
$ref: ./definitions.yml#/apiCancelModelEvaluationRunOutput
27+
description: A successful response.
28+
headers:
29+
ratelimit-limit:
30+
$ref: ../../shared/headers.yml#/ratelimit-limit
31+
ratelimit-remaining:
32+
$ref: ../../shared/headers.yml#/ratelimit-remaining
33+
ratelimit-reset:
34+
$ref: ../../shared/headers.yml#/ratelimit-reset
35+
"401":
36+
$ref: ../../shared/responses/unauthorized.yml
37+
"404":
38+
$ref: ../../shared/responses/not_found.yml
39+
"429":
40+
$ref: ../../shared/responses/too_many_requests.yml
41+
"500":
42+
$ref: ../../shared/responses/server_error.yml
43+
default:
44+
$ref: ../../shared/responses/unexpected_error.yml
45+
security:
46+
- bearer_auth:
47+
- genai:update
48+
summary: Cancel Model Evaluation Run
49+
tags:
50+
- GradientAI Platform
51+
x-codeSamples:
52+
- $ref: examples/curl/genai_cancel_model_evaluation_run.yml

‎specification/resources/gen-ai/genai_delete_model_evaluation_preset.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
description: To delete a saved model evaluation preset, send a DELETE request to
2+
`/v2/gen-ai/model_evaluation_presets/{eval_preset_uuid}`.
3+
operationId: genai_delete_model_evaluation_preset
4+
parameters:
5+
- description: UUID of the evaluation preset to delete.
6+
example: '"123e4567-e89b-12d3-a456-426614174000"'
7+
in: path
8+
name: eval_preset_uuid
9+
required: true
10+
schema:
11+
type: string
12+
responses:
13+
"200":
14+
content:
15+
application/json:
16+
schema:
17+
$ref: ./definitions.yml#/apiDeleteModelEvaluationPresetOutput
18+
description: A successful response.
19+
headers:
20+
ratelimit-limit:
21+
$ref: ../../shared/headers.yml#/ratelimit-limit
22+
ratelimit-remaining:
23+
$ref: ../../shared/headers.yml#/ratelimit-remaining
24+
ratelimit-reset:
25+
$ref: ../../shared/headers.yml#/ratelimit-reset
26+
"401":
27+
$ref: ../../shared/responses/unauthorized.yml
28+
"404":
29+
$ref: ../../shared/responses/not_found.yml
30+
"429":
31+
$ref: ../../shared/responses/too_many_requests.yml
32+
"500":
33+
$ref: ../../shared/responses/server_error.yml
34+
default:
35+
$ref: ../../shared/responses/unexpected_error.yml
36+
security:
37+
- bearer_auth:
38+
- genai:delete
39+
summary: Delete Model Evaluation Preset
40+
tags:
41+
- GradientAI Platform
42+
x-codeSamples:
43+
- $ref: examples/curl/genai_delete_model_evaluation_preset.yml

‎specification/resources/gen-ai/genai_list_evaluation_datasets.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
description: To list evaluation datasets, send a GET request to `/v2/gen-ai/evaluation_datasets`.
2+
operationId: genai_list_evaluation_datasets
3+
parameters:
4+
- description: Filter by evaluation dataset type.
5+
example: EVALUATION_DATASET_TYPE_UNKNOWN
6+
in: query
7+
name: dataset_type
8+
schema:
9+
default: EVALUATION_DATASET_TYPE_UNKNOWN
10+
enum:
11+
- EVALUATION_DATASET_TYPE_UNKNOWN
12+
- EVALUATION_DATASET_TYPE_ADK
13+
- EVALUATION_DATASET_TYPE_NON_ADK
14+
- EVALUATION_DATASET_TYPE_MODEL
15+
type: string
16+
responses:
17+
"200":
18+
content:
19+
application/json:
20+
schema:
21+
$ref: ./definitions.yml#/apiListEvaluationDatasetsOutput
22+
description: A successful response.
23+
headers:
24+
ratelimit-limit:
25+
$ref: ../../shared/headers.yml#/ratelimit-limit
26+
ratelimit-remaining:
27+
$ref: ../../shared/headers.yml#/ratelimit-remaining
28+
ratelimit-reset:
29+
$ref: ../../shared/headers.yml#/ratelimit-reset
30+
"401":
31+
$ref: ../../shared/responses/unauthorized.yml
32+
"404":
33+
$ref: ../../shared/responses/not_found.yml
34+
"429":
35+
$ref: ../../shared/responses/too_many_requests.yml
36+
"500":
37+
$ref: ../../shared/responses/server_error.yml
38+
default:
39+
$ref: ../../shared/responses/unexpected_error.yml
40+
security:
41+
- bearer_auth:
42+
- genai:read
43+
summary: List Evaluation Datasets
44+
tags:
45+
- GradientAI Platform
46+
x-codeSamples:
47+
- $ref: examples/curl/genai_list_evaluation_datasets.yml

0 commit comments

Comments
 (0)