From ce5ac67bc3db1c25ea199b99286dd2e7fc75b4c8 Mon Sep 17 00:00:00 2001 From: lcawl Date: Wed, 17 Nov 2021 09:41:02 -0800 Subject: [PATCH 1/3] Edits ML get model snapshot API --- specification/ml/_types/Model.ts | 117 ++++++++++++++++-- .../MlGetModelSnapshotsRequest.ts | 25 +++- 2 files changed, 126 insertions(+), 16 deletions(-) diff --git a/specification/ml/_types/Model.ts b/specification/ml/_types/Model.ts index 746f0663ee..ebe442501b 100644 --- a/specification/ml/_types/Model.ts +++ b/specification/ml/_types/Model.ts @@ -34,7 +34,10 @@ export class ModelSnapshot { min_version: VersionString /** Summary information describing the model. */ model_size_stats: ModelSizeStats - /** If true, this snapshot will not be deleted during automatic cleanup of snapshots older than model_snapshot_retention_days. However, this snapshot will be deleted when the job is deleted. The default value is false. */ + /** If true, this snapshot will not be deleted during automatic cleanup of snapshots older than + * `model_snapshot_retention_days`. However, this snapshot will be deleted when the job is deleted. + * @server_default false. + */ retain: boolean /** For internal use only. */ snapshot_doc_count: long @@ -45,36 +48,130 @@ export class ModelSnapshot { } export class ModelSizeStats { + /** + * Indicates where to find the memory requirement that is used to decide where the job runs. The possible values are: + * + * * `model_memory_limit`: The job’s memory requirement is calculated on the basis that its model memory will grow to + * the `model_memory_limit` specified in the `analysis_limits` of its config. + * * `current_model_bytes`: The job’s memory requirement is calculated on the basis that its current model memory + * size is a good reflection of what it will be in the future. + * * `peak_model_bytes`: The job’s memory requirement is calculated on the basis that its peak model memory size + * is a good reflection of what the model size will be in the future. + */ + assignment_memory_basis?: string + /** + * The number of buckets for which entities were not processed due to memory limit constraints. + */ bucket_allocation_failures_count: long + /** + * The number of documents that have had a field categorized. + */ + categorized_doc_count: integer + /** + * The status of categorization for this job. + */ + categorization_status: CategorizationStatus + /** + * The number of categories created by categorization that will never be assigned again because another category’s + * definition makes it a superset of the dead category. Dead categories are a side effect of the way categorization + * has no prior training. + */ + dead_category_count: integer + /** + * The number of times that categorization wanted to create a new category but couldn’t because the job had hit its + * `model_memory_limit`. This count does not track which specific categories failed to be created. Therefore, you + * cannot use this value to determine the number of unique categories that were missed. + */ + failed_category_count: integer + /** + * The number of categories that match more than 1% of categorized documents. + */ + frequent_category_count: integer + /** + * Identifier for the anomaly detection job. + */ job_id: Id + /** + * The timestamp that the model size stats were recorded, according to server-time. + */ log_time: Time + /** + * The status of the memory in relation to its `model_memory_limit`. + */ memory_status: MemoryStatus + /** + * An approximation of the memory resources required for this analysis. + */ model_bytes: ByteSize + /** + * The number of bytes over the high limit for memory usage at the last allocation failure. + */ model_bytes_exceeded?: ByteSize + /** + * The upper limit for memory usage, checked on increasing values. + */ model_bytes_memory_limit?: ByteSize + /** + * The highest recorded value for the model memory usage. + */ peak_model_bytes?: ByteSize - assignment_memory_basis?: string + /** + * The number of categories that match just one categorized document. + */ + rare_category_count: integer + /** + * Internal. This value is always `model_size_stats`. + */ result_type: string + /** + * The timestamp that the model size stats were recorded, according to the bucket timestamp of the data. + */ + timestamp?: long + /** + * The number of by field values analyzed. Note that these are counted separately for each detector and partition. + */ total_by_field_count: long + /** + * The number of categories created by categorization. + */ + total_category_count: integer + /** + * The number of over field values analyzed. Note that these are counted separately for each detector and partition. + */ total_over_field_count: long + /** + * The number of partition field values analyzed. + */ total_partition_field_count: long - categorization_status: CategorizationStatus - categorized_doc_count: integer - dead_category_count: integer - failed_category_count: integer - frequent_category_count: integer - rare_category_count: integer - total_category_count: integer - timestamp?: long } export enum CategorizationStatus { + /** + * This status indicates that categorization is performing acceptably well (or not being used at all). + */ ok = 0, + /** + * This status indicates that categorization is detecting a distribution of categories that suggests the input data + * is inappropriate for categorization. Problems could be that there is only one category, more than 90% of + * categories are rare, the number of categories is greater than 50% of the number of categorized documents, there + * are no frequently matched categories, or more than 50% of categories are dead. + */ warn = 1 } export enum MemoryStatus { + /** + * This status indicates that the internal models stayed below the configured value. + */ ok = 0, + /** + * This status indicates that the internal models require more than 60% of the configured memory limit and more + * aggressive pruning will be performed in order to try to reclaim space. + */ soft_limit = 1, + /** + * This status indicates that the internal models require more space than the configured memory limit. Some incoming + * data could not be processed. + */ hard_limit = 2 } diff --git a/specification/ml/get_model_snapshots/MlGetModelSnapshotsRequest.ts b/specification/ml/get_model_snapshots/MlGetModelSnapshotsRequest.ts index 675dd7d677..744e482f34 100644 --- a/specification/ml/get_model_snapshots/MlGetModelSnapshotsRequest.ts +++ b/specification/ml/get_model_snapshots/MlGetModelSnapshotsRequest.ts @@ -36,13 +36,16 @@ export interface Request extends RequestBase { */ job_id: Id /** - * A numerical character string that uniquely identifies the model snapshot. + * A numerical character string that uniquely identifies the model snapshot. You can get information for multiple + * snapshots by using a comma-separated list or a wildcard expression. You can get all snapshots by using `_all`, + * by specifying `*` as the snapshot ID, or by omitting the snapshot ID. */ snapshot_id?: Id } query_parameters: { /** * If true, the results are sorted in descending order. + * @server_default false */ desc?: boolean /** @@ -51,10 +54,12 @@ export interface Request extends RequestBase { end?: Time /** * Skips the specified number of snapshots. + * @server_default 0 */ from?: integer /** * Specifies the maximum number of snapshots to obtain. + * @server_default 100 */ size?: integer /** @@ -69,14 +74,22 @@ export interface Request extends RequestBase { } body: { /** - * Returns snapshots with timestamps after this time. Defaults to unset, - * which means results are not limited to specific timestamps. + * Refer to the description for the `desc` query parameter. + * @server_default false */ - start?: Time + desc?: boolean /** - * Returns snapshots with timestamps earlier than this time. Defaults to - * unset, which means results are not limited to specific timestamps. + * Refer to the description for the `end` query parameter. */ end?: Time + page?: Page + /** + * Refer to the description for the `sort` query parameter. + */ + sort?: Field + /** + * Refer to the description for the `start` query parameter. + */ + start?: Time } } From 7783b282e07cbeb6a27d5a6d87d84aa1a455ddb4 Mon Sep 17 00:00:00 2001 From: lcawl Date: Wed, 17 Nov 2021 09:43:45 -0800 Subject: [PATCH 2/3] Removes default value --- specification/ml/_types/Model.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/specification/ml/_types/Model.ts b/specification/ml/_types/Model.ts index ebe442501b..ed80008767 100644 --- a/specification/ml/_types/Model.ts +++ b/specification/ml/_types/Model.ts @@ -36,7 +36,6 @@ export class ModelSnapshot { model_size_stats: ModelSizeStats /** If true, this snapshot will not be deleted during automatic cleanup of snapshots older than * `model_snapshot_retention_days`. However, this snapshot will be deleted when the job is deleted. - * @server_default false. */ retain: boolean /** For internal use only. */ From bc1296400451fb5d91b87c8a2fa6089093969bd0 Mon Sep 17 00:00:00 2001 From: lcawl Date: Wed, 17 Nov 2021 10:00:38 -0800 Subject: [PATCH 3/3] Makes contrib --- output/schema/schema.json | 49 +++++++- output/typescript/types.ts | 5 +- specification/ml/_types/Model.ts | 116 ++---------------- .../MlGetModelSnapshotsRequest.ts | 3 +- 4 files changed, 60 insertions(+), 113 deletions(-) diff --git a/output/schema/schema.json b/output/schema/schema.json index ff0a06ee9c..356866301a 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -119526,8 +119526,21 @@ "kind": "properties", "properties": [ { - "description": "Returns snapshots with timestamps after this time. Defaults to unset,\nwhich means results are not limited to specific timestamps.", - "name": "start", + "description": "Refer to the description for the `desc` query parameter.", + "name": "desc", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "internal" + } + } + }, + { + "description": "Refer to the description for the `end` query parameter.", + "name": "end", "required": false, "type": { "kind": "instance_of", @@ -119538,8 +119551,31 @@ } }, { - "description": "Returns snapshots with timestamps earlier than this time. Defaults to\nunset, which means results are not limited to specific timestamps.", - "name": "end", + "name": "page", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "Page", + "namespace": "ml._types" + } + } + }, + { + "description": "Refer to the description for the `sort` query parameter.", + "name": "sort", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + }, + { + "description": "Refer to the description for the `start` query parameter.", + "name": "start", "required": false, "type": { "kind": "instance_of", @@ -119577,7 +119613,7 @@ } }, { - "description": "A numerical character string that uniquely identifies the model snapshot.", + "description": "A numerical character string that uniquely identifies the model snapshot. You can get information for multiple\nsnapshots by using a comma-separated list or a wildcard expression. You can get all snapshots by using `_all`,\nby specifying `*` as the snapshot ID, or by omitting the snapshot ID.", "name": "snapshot_id", "required": false, "type": { @@ -119594,6 +119630,7 @@ "description": "If true, the results are sorted in descending order.", "name": "desc", "required": false, + "serverDefault": false, "type": { "kind": "instance_of", "type": { @@ -119618,6 +119655,7 @@ "description": "Skips the specified number of snapshots.", "name": "from", "required": false, + "serverDefault": 0, "type": { "kind": "instance_of", "type": { @@ -119630,6 +119668,7 @@ "description": "Specifies the maximum number of snapshots to obtain.", "name": "size", "required": false, + "serverDefault": 100, "type": { "kind": "instance_of", "type": { diff --git a/output/typescript/types.ts b/output/typescript/types.ts index 7902337cf2..62a2b2cc75 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -12136,8 +12136,11 @@ export interface MlGetModelSnapshotsRequest extends RequestBase { sort?: Field start?: Time body?: { - start?: Time + desc?: boolean end?: Time + page?: MlPage + sort?: Field + start?: Time } } diff --git a/specification/ml/_types/Model.ts b/specification/ml/_types/Model.ts index ed80008767..746f0663ee 100644 --- a/specification/ml/_types/Model.ts +++ b/specification/ml/_types/Model.ts @@ -34,9 +34,7 @@ export class ModelSnapshot { min_version: VersionString /** Summary information describing the model. */ model_size_stats: ModelSizeStats - /** If true, this snapshot will not be deleted during automatic cleanup of snapshots older than - * `model_snapshot_retention_days`. However, this snapshot will be deleted when the job is deleted. - */ + /** If true, this snapshot will not be deleted during automatic cleanup of snapshots older than model_snapshot_retention_days. However, this snapshot will be deleted when the job is deleted. The default value is false. */ retain: boolean /** For internal use only. */ snapshot_doc_count: long @@ -47,130 +45,36 @@ export class ModelSnapshot { } export class ModelSizeStats { - /** - * Indicates where to find the memory requirement that is used to decide where the job runs. The possible values are: - * - * * `model_memory_limit`: The job’s memory requirement is calculated on the basis that its model memory will grow to - * the `model_memory_limit` specified in the `analysis_limits` of its config. - * * `current_model_bytes`: The job’s memory requirement is calculated on the basis that its current model memory - * size is a good reflection of what it will be in the future. - * * `peak_model_bytes`: The job’s memory requirement is calculated on the basis that its peak model memory size - * is a good reflection of what the model size will be in the future. - */ - assignment_memory_basis?: string - /** - * The number of buckets for which entities were not processed due to memory limit constraints. - */ bucket_allocation_failures_count: long - /** - * The number of documents that have had a field categorized. - */ - categorized_doc_count: integer - /** - * The status of categorization for this job. - */ - categorization_status: CategorizationStatus - /** - * The number of categories created by categorization that will never be assigned again because another category’s - * definition makes it a superset of the dead category. Dead categories are a side effect of the way categorization - * has no prior training. - */ - dead_category_count: integer - /** - * The number of times that categorization wanted to create a new category but couldn’t because the job had hit its - * `model_memory_limit`. This count does not track which specific categories failed to be created. Therefore, you - * cannot use this value to determine the number of unique categories that were missed. - */ - failed_category_count: integer - /** - * The number of categories that match more than 1% of categorized documents. - */ - frequent_category_count: integer - /** - * Identifier for the anomaly detection job. - */ job_id: Id - /** - * The timestamp that the model size stats were recorded, according to server-time. - */ log_time: Time - /** - * The status of the memory in relation to its `model_memory_limit`. - */ memory_status: MemoryStatus - /** - * An approximation of the memory resources required for this analysis. - */ model_bytes: ByteSize - /** - * The number of bytes over the high limit for memory usage at the last allocation failure. - */ model_bytes_exceeded?: ByteSize - /** - * The upper limit for memory usage, checked on increasing values. - */ model_bytes_memory_limit?: ByteSize - /** - * The highest recorded value for the model memory usage. - */ peak_model_bytes?: ByteSize - /** - * The number of categories that match just one categorized document. - */ - rare_category_count: integer - /** - * Internal. This value is always `model_size_stats`. - */ + assignment_memory_basis?: string result_type: string - /** - * The timestamp that the model size stats were recorded, according to the bucket timestamp of the data. - */ - timestamp?: long - /** - * The number of by field values analyzed. Note that these are counted separately for each detector and partition. - */ total_by_field_count: long - /** - * The number of categories created by categorization. - */ - total_category_count: integer - /** - * The number of over field values analyzed. Note that these are counted separately for each detector and partition. - */ total_over_field_count: long - /** - * The number of partition field values analyzed. - */ total_partition_field_count: long + categorization_status: CategorizationStatus + categorized_doc_count: integer + dead_category_count: integer + failed_category_count: integer + frequent_category_count: integer + rare_category_count: integer + total_category_count: integer + timestamp?: long } export enum CategorizationStatus { - /** - * This status indicates that categorization is performing acceptably well (or not being used at all). - */ ok = 0, - /** - * This status indicates that categorization is detecting a distribution of categories that suggests the input data - * is inappropriate for categorization. Problems could be that there is only one category, more than 90% of - * categories are rare, the number of categories is greater than 50% of the number of categorized documents, there - * are no frequently matched categories, or more than 50% of categories are dead. - */ warn = 1 } export enum MemoryStatus { - /** - * This status indicates that the internal models stayed below the configured value. - */ ok = 0, - /** - * This status indicates that the internal models require more than 60% of the configured memory limit and more - * aggressive pruning will be performed in order to try to reclaim space. - */ soft_limit = 1, - /** - * This status indicates that the internal models require more space than the configured memory limit. Some incoming - * data could not be processed. - */ hard_limit = 2 } diff --git a/specification/ml/get_model_snapshots/MlGetModelSnapshotsRequest.ts b/specification/ml/get_model_snapshots/MlGetModelSnapshotsRequest.ts index 744e482f34..43261ff57e 100644 --- a/specification/ml/get_model_snapshots/MlGetModelSnapshotsRequest.ts +++ b/specification/ml/get_model_snapshots/MlGetModelSnapshotsRequest.ts @@ -17,6 +17,7 @@ * under the License. */ +import { Page } from '@ml/_types/Page' import { RequestBase } from '@_types/Base' import { Field, Id } from '@_types/common' import { integer } from '@_types/Numeric' @@ -77,7 +78,7 @@ export interface Request extends RequestBase { * Refer to the description for the `desc` query parameter. * @server_default false */ - desc?: boolean + desc?: boolean /** * Refer to the description for the `end` query parameter. */