Skip to content

Commit

Permalink
feat(client-glue): Add an option to run non-urgent or non-time sensit…
Browse files Browse the repository at this point in the history 
…ive Glue Jobs on spare capacity
  • Loading branch information
awstools committed Aug 8, 2022
1 parent 962583c commit 3040530
Show file tree
Hide file tree
Showing 7 changed files with 192 additions and 44 deletions.
8 changes: 6 additions & 2 deletions clients/client-glue/src/commands/DeleteClassifierCommand.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,12 @@ import {
} from "@aws-sdk/types";

import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
import { DeleteClassifierRequest, DeleteClassifierRequestFilterSensitiveLog } from "../models/models_0";
import { DeleteClassifierResponse, DeleteClassifierResponseFilterSensitiveLog } from "../models/models_1";
import {
DeleteClassifierRequest,
DeleteClassifierRequestFilterSensitiveLog,
DeleteClassifierResponse,
DeleteClassifierResponseFilterSensitiveLog,
} from "../models/models_1";
import {
deserializeAws_json1_1DeleteClassifierCommand,
serializeAws_json1_1DeleteClassifierCommand,
Expand Down
5 changes: 2 additions & 3 deletions clients/client-glue/src/commands/QuerySchemaVersionMetadataCommand.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,11 @@ import {
} from "@aws-sdk/types";

import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
import { QuerySchemaVersionMetadataInput, QuerySchemaVersionMetadataInputFilterSensitiveLog } from "../models/models_1";
import {
QuerySchemaVersionMetadataInput,
QuerySchemaVersionMetadataInputFilterSensitiveLog,
QuerySchemaVersionMetadataResponse,
QuerySchemaVersionMetadataResponseFilterSensitiveLog,
} from "../models/models_1";
} from "../models/models_2";
import {
deserializeAws_json1_1QuerySchemaVersionMetadataCommand,
serializeAws_json1_1QuerySchemaVersionMetadataCommand,
Expand Down
41 changes: 27 additions & 14 deletions clients/client-glue/src/models/models_0.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3890,6 +3890,11 @@ export interface ConnectionsList {
Connections?: string[];
}

export enum ExecutionClass {
FLEX = "FLEX",
STANDARD = "STANDARD",
}

/**
* <p>An execution property of a job.</p>
*/
Expand Down Expand Up @@ -4040,6 +4045,7 @@ export interface EventBatchingCondition {
export enum CrawlState {
CANCELLED = "CANCELLED",
CANCELLING = "CANCELLING",
ERROR = "ERROR",
FAILED = "FAILED",
RUNNING = "RUNNING",
SUCCEEDED = "SUCCEEDED",
Expand All @@ -4050,13 +4056,15 @@ export enum LogicalOperator {
}

export enum JobRunState {
ERROR = "ERROR",
FAILED = "FAILED",
RUNNING = "RUNNING",
STARTING = "STARTING",
STOPPED = "STOPPED",
STOPPING = "STOPPING",
SUCCEEDED = "SUCCEEDED",
TIMEOUT = "TIMEOUT",
WAITING = "WAITING",
}

/**
Expand Down Expand Up @@ -4479,6 +4487,15 @@ export interface JobRun {
* <p>This field populates only for Auto Scaling job runs, and represents the total time each executor ran during the lifecycle of a job run in seconds, multiplied by a DPU factor (1 for <code>G.1X</code>, 2 for <code>G.2X</code>, or 0.25 for <code>G.025X</code> workers). This value may be different than the <code>executionEngineRuntime</code> * <code>MaxCapacity</code> as in the case of Auto Scaling jobs, as the number of executors running at a given time may be less than the <code>MaxCapacity</code>. Therefore, it is possible that the value of <code>DPUSeconds</code> is less than <code>executionEngineRuntime</code> * <code>MaxCapacity</code>.</p>
*/
DPUSeconds?: number;

/**
* <p>Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.</p>
*
* <p>The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary. </p>
*
* <p>Only jobs with Glue version 3.0 and above and command type <code>glueetl</code> will be allowed to set <code>ExecutionClass</code> to <code>FLEX</code>. The flexible execution class is available for Spark jobs.</p>
*/
ExecutionClass?: ExecutionClass | string;
}

/**
Expand Down Expand Up @@ -4608,6 +4625,16 @@ export interface WorkflowRunStatistics {
* <p>Total number Actions in running state.</p>
*/
RunningActions?: number;

/**
* <p>Indicates the count of job runs in the ERROR state in the workflow run.</p>
*/
ErroredActions?: number;

/**
* <p>Indicates the count of job runs in WAITING state in the workflow run.</p>
*/
WaitingActions?: number;
}

export enum WorkflowRunStatus {
Expand Down Expand Up @@ -7131,13 +7158,6 @@ export interface DeleteBlueprintResponse {
Name?: string;
}

export interface DeleteClassifierRequest {
/**
* <p>Name of the classifier to remove.</p>
*/
Name: string | undefined;
}

/**
* @internal
*/
Expand Down Expand Up @@ -8779,10 +8799,3 @@ export const DeleteBlueprintRequestFilterSensitiveLog = (obj: DeleteBlueprintReq
export const DeleteBlueprintResponseFilterSensitiveLog = (obj: DeleteBlueprintResponse): any => ({
...obj,
});

/**
* @internal
*/
export const DeleteClassifierRequestFilterSensitiveLog = (obj: DeleteClassifierRequest): any => ({
...obj,
});
36 changes: 13 additions & 23 deletions clients/client-glue/src/models/models_1.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ import {
WorkflowRun,
} from "./models_0";

export interface DeleteClassifierRequest {
/**
* <p>Name of the classifier to remove.</p>
*/
Name: string | undefined;
}

export interface DeleteClassifierResponse {}

export interface DeleteColumnStatisticsForPartitionRequest {
Expand Down Expand Up @@ -5738,22 +5745,12 @@ export interface MetadataInfo {
OtherMetadataValueList?: OtherMetadataValueListItem[];
}

export interface QuerySchemaVersionMetadataResponse {
/**
* <p>A map of a metadata key and associated values.</p>
*/
MetadataInfoMap?: Record<string, MetadataInfo>;

/**
* <p>The unique version ID of the schema version.</p>
*/
SchemaVersionId?: string;

/**
* <p>A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.</p>
*/
NextToken?: string;
}
/**
* @internal
*/
export const DeleteClassifierRequestFilterSensitiveLog = (obj: DeleteClassifierRequest): any => ({
...obj,
});

/**
* @internal
Expand Down Expand Up @@ -7706,10 +7703,3 @@ export const OtherMetadataValueListItemFilterSensitiveLog = (obj: OtherMetadataV
export const MetadataInfoFilterSensitiveLog = (obj: MetadataInfo): any => ({
...obj,
});

/**
* @internal
*/
export const QuerySchemaVersionMetadataResponseFilterSensitiveLog = (obj: QuerySchemaVersionMetadataResponse): any => ({
...obj,
});
62 changes: 62 additions & 0 deletions clients/client-glue/src/models/models_2.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
DynamoDBCatalogSource,
ErrorDetail,
EventBatchingCondition,
ExecutionClass,
ExecutionProperty,
FillMissingValues,
Filter,
Expand Down Expand Up @@ -82,12 +83,30 @@ import {
import {
ColumnStatistics,
JobBookmarkEntry,
MetadataInfo,
MetadataKeyValuePair,
ResourceShareType,
SchemaVersionNumber,
Table,
} from "./models_1";

export interface QuerySchemaVersionMetadataResponse {
/**
* <p>A map of a metadata key and associated values.</p>
*/
MetadataInfoMap?: Record<string, MetadataInfo>;

/**
* <p>The unique version ID of the schema version.</p>
*/
SchemaVersionId?: string;

/**
* <p>A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.</p>
*/
NextToken?: string;
}

export interface RegisterSchemaVersionInput {
/**
* <p>This is a wrapper structure to contain schema identity fields. The structure contains:</p>
Expand Down Expand Up @@ -679,6 +698,15 @@ export interface StartJobRunRequest {
* <p>The number of workers of a defined <code>workerType</code> that are allocated when a job runs.</p>
*/
NumberOfWorkers?: number;

/**
* <p>Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.</p>
*
* <p>The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary. </p>
*
* <p>Only jobs with Glue version 3.0 and above and command type <code>glueetl</code> will be allowed to set <code>ExecutionClass</code> to <code>FLEX</code>. The flexible execution class is available for Spark jobs.</p>
*/
ExecutionClass?: ExecutionClass | string;
}

export interface StartJobRunResponse {
Expand Down Expand Up @@ -2285,6 +2313,15 @@ export interface CreateJobRequest {
* <p>The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.</p>
*/
CodeGenConfigurationNodes?: Record<string, CodeGenConfigurationNode>;

/**
* <p>Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.</p>
*
* <p>The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary. </p>
*
* <p>Only jobs with Glue version 3.0 and above and command type <code>glueetl</code> will be allowed to set <code>ExecutionClass</code> to <code>FLEX</code>. The flexible execution class is available for Spark jobs.</p>
*/
ExecutionClass?: ExecutionClass | string;
}

/**
Expand Down Expand Up @@ -2451,6 +2488,15 @@ export interface Job {
* <p>The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.</p>
*/
CodeGenConfigurationNodes?: Record<string, CodeGenConfigurationNode>;

/**
* <p>Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.</p>
*
* <p>The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary. </p>
*
* <p>Only jobs with Glue version 3.0 and above and command type <code>glueetl</code> will be allowed to set <code>ExecutionClass</code> to <code>FLEX</code>. The flexible execution class is available for Spark jobs.</p>
*/
ExecutionClass?: ExecutionClass | string;
}

/**
Expand Down Expand Up @@ -2599,6 +2645,15 @@ export interface JobUpdate {
* <p>The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.</p>
*/
CodeGenConfigurationNodes?: Record<string, CodeGenConfigurationNode>;

/**
* <p>Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.</p>
*
* <p>The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary. </p>
*
* <p>Only jobs with Glue version 3.0 and above and command type <code>glueetl</code> will be allowed to set <code>ExecutionClass</code> to <code>FLEX</code>. The flexible execution class is available for Spark jobs.</p>
*/
ExecutionClass?: ExecutionClass | string;
}

export interface GetJobResponse {
Expand Down Expand Up @@ -2644,6 +2699,13 @@ export interface GetJobsResponse {
NextToken?: string;
}

/**
* @internal
*/
export const QuerySchemaVersionMetadataResponseFilterSensitiveLog = (obj: QuerySchemaVersionMetadataResponse): any => ({
...obj,
});

/**
* @internal
*/
Expand Down
11 changes: 9 additions & 2 deletions clients/client-glue/src/protocols/Aws_json1_1.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,6 @@ import {
Datatype,
DeleteBlueprintRequest,
DeleteBlueprintResponse,
DeleteClassifierRequest,
DeltaTarget,
DevEndpoint,
DirectKafkaSource,
Expand Down Expand Up @@ -653,6 +652,7 @@ import {
DateColumnStatisticsData,
DecimalColumnStatisticsData,
DecimalNumber,
DeleteClassifierRequest,
DeleteClassifierResponse,
DeleteColumnStatisticsForPartitionRequest,
DeleteColumnStatisticsForPartitionResponse,
Expand Down Expand Up @@ -882,7 +882,6 @@ import {
PutWorkflowRunPropertiesRequest,
PutWorkflowRunPropertiesResponse,
QuerySchemaVersionMetadataInput,
QuerySchemaVersionMetadataResponse,
RegistryListItem,
SchedulerTransitioningException,
SchemaColumn,
Expand Down Expand Up @@ -928,6 +927,7 @@ import {
MLTransformNotReadyException,
NoScheduleException,
PropertyPredicate,
QuerySchemaVersionMetadataResponse,
RegisterSchemaVersionInput,
RegisterSchemaVersionResponse,
RemoveSchemaVersionMetadataInput,
Expand Down Expand Up @@ -14334,6 +14334,7 @@ const serializeAws_json1_1CreateJobRequest = (input: CreateJobRequest, context:
DefaultArguments: serializeAws_json1_1GenericMap(input.DefaultArguments, context),
}),
...(input.Description != null && { Description: input.Description }),
...(input.ExecutionClass != null && { ExecutionClass: input.ExecutionClass }),
...(input.ExecutionProperty != null && {
ExecutionProperty: serializeAws_json1_1ExecutionProperty(input.ExecutionProperty, context),
}),
Expand Down Expand Up @@ -15954,6 +15955,7 @@ const serializeAws_json1_1JobUpdate = (input: JobUpdate, context: __SerdeContext
DefaultArguments: serializeAws_json1_1GenericMap(input.DefaultArguments, context),
}),
...(input.Description != null && { Description: input.Description }),
...(input.ExecutionClass != null && { ExecutionClass: input.ExecutionClass }),
...(input.ExecutionProperty != null && {
ExecutionProperty: serializeAws_json1_1ExecutionProperty(input.ExecutionProperty, context),
}),
Expand Down Expand Up @@ -17316,6 +17318,7 @@ const serializeAws_json1_1StartJobRunRequest = (input: StartJobRunRequest, conte
return {
...(input.AllocatedCapacity != null && { AllocatedCapacity: input.AllocatedCapacity }),
...(input.Arguments != null && { Arguments: serializeAws_json1_1GenericMap(input.Arguments, context) }),
...(input.ExecutionClass != null && { ExecutionClass: input.ExecutionClass }),
...(input.JobName != null && { JobName: input.JobName }),
...(input.JobRunId != null && { JobRunId: input.JobRunId }),
...(input.MaxCapacity != null && { MaxCapacity: __serializeFloat(input.MaxCapacity) }),
Expand Down Expand Up @@ -21377,6 +21380,7 @@ const deserializeAws_json1_1Job = (output: any, context: __SerdeContext): Job =>
DefaultArguments:
output.DefaultArguments != null ? deserializeAws_json1_1GenericMap(output.DefaultArguments, context) : undefined,
Description: __expectString(output.Description),
ExecutionClass: __expectString(output.ExecutionClass),
ExecutionProperty:
output.ExecutionProperty != null
? deserializeAws_json1_1ExecutionProperty(output.ExecutionProperty, context)
Expand Down Expand Up @@ -21474,6 +21478,7 @@ const deserializeAws_json1_1JobRun = (output: any, context: __SerdeContext): Job
: undefined,
DPUSeconds: __limitedParseDouble(output.DPUSeconds),
ErrorMessage: __expectString(output.ErrorMessage),
ExecutionClass: __expectString(output.ExecutionClass),
ExecutionTime: __expectInt32(output.ExecutionTime),
GlueVersion: __expectString(output.GlueVersion),
Id: __expectString(output.Id),
Expand Down Expand Up @@ -24174,12 +24179,14 @@ const deserializeAws_json1_1WorkflowRuns = (output: any, context: __SerdeContext

const deserializeAws_json1_1WorkflowRunStatistics = (output: any, context: __SerdeContext): WorkflowRunStatistics => {
return {
ErroredActions: __expectInt32(output.ErroredActions),
FailedActions: __expectInt32(output.FailedActions),
RunningActions: __expectInt32(output.RunningActions),
StoppedActions: __expectInt32(output.StoppedActions),
SucceededActions: __expectInt32(output.SucceededActions),
TimeoutActions: __expectInt32(output.TimeoutActions),
TotalActions: __expectInt32(output.TotalActions),
WaitingActions: __expectInt32(output.WaitingActions),
} as any;
};

Expand Down

0 comments on commit 3040530

Please sign in to comment.