From 8de50d624c8703a12713dcffbc764688868f22b0 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Fri, 10 Feb 2023 03:36:39 +0900 Subject: [PATCH] feat(glue): support Ray jobs (#23822) Glue launched a new job type: `AWS Glue for Ray`. https://aws.amazon.com/about-aws/whats-new/2022/11/aws-glue-ray-preview/ This PR is to support the Ray job type in AWS CDK. ---- ### All Submissions: * [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) ### Adding new Construct Runtime Dependencies: * [ ] This PR adds new construct runtime dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-construct-runtime-dependencies) ### New Features * [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)? * [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-glue/README.md | 17 ++ .../@aws-cdk/aws-glue/lib/job-executable.ts | 44 +++- packages/@aws-cdk/aws-glue/lib/job.ts | 12 ++ .../aws-glue-job.assets.json | 4 +- .../aws-glue-job.template.json | 139 +++++++++++- .../test/integ.job.js.snapshot/manifest.json | 20 +- .../test/integ.job.js.snapshot/tree.json | 201 +++++++++++++++++- packages/@aws-cdk/aws-glue/test/integ.job.ts | 20 ++ .../aws-glue/test/job-executable.test.ts | 25 ++- packages/@aws-cdk/aws-glue/test/job.test.ts | 31 +++ 10 files changed, 497 insertions(+), 16 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index d5260e12d771d..83cb4a1c82434 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -90,6 +90,23 @@ new glue.Job(this, 'PythonShellJob', { }); ``` +### Ray Jobs + +These jobs run in a Ray environment managed by AWS Glue. + +```ts +new glue.Job(this, 'RayJob', { + executable: glue.JobExecutable.pythonRay({ + glueVersion: glue.GlueVersion.V4_0, + pythonVersion: glue.PythonVersion.THREE_NINE, + script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')), + }), + workerType: glue.WorkerType.Z_2X, + workerCount: 2, + description: 'an example Ray job' +}); +``` + See [documentation](https://docs.aws.amazon.com/glue/latest/dg/add-job.html) for more information on adding jobs in Glue. ## Connection diff --git a/packages/@aws-cdk/aws-glue/lib/job-executable.ts b/packages/@aws-cdk/aws-glue/lib/job-executable.ts index 092cdca11552d..8ad29c1108b70 100644 --- a/packages/@aws-cdk/aws-glue/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue/lib/job-executable.ts @@ -95,12 +95,12 @@ export enum PythonVersion { */ export class JobType { /** - * Command for running a Glue ETL job. + * Command for running a Glue Spark job. */ public static readonly ETL = new JobType('glueetl'); /** - * Command for running a Glue streaming job. + * Command for running a Glue Spark streaming job. */ public static readonly STREAMING = new JobType('gluestreaming'); @@ -109,6 +109,11 @@ export class JobType { */ public static readonly PYTHON_SHELL = new JobType('pythonshell'); + /** + * Command for running a Glue Ray job. + */ + public static readonly RAY = new JobType('glueray'); + /** * Custom type name * @param name type name @@ -211,6 +216,11 @@ export interface PythonSparkJobExecutableProps extends SharedSparkJobExecutableP */ export interface PythonShellExecutableProps extends SharedJobExecutableProps, PythonExecutableProps {} +/** + * Props for creating a Python Ray job executable + */ +export interface PythonRayExecutableProps extends SharedJobExecutableProps, PythonExecutableProps {} + /** * The executable properties related to the Glue job's GlueVersion, JobType and code */ @@ -281,6 +291,19 @@ export class JobExecutable { }); } + /** + * Create Python executable props for Ray jobs. + * + * @param props Ray Job props. + */ + public static pythonRay(props: PythonRayExecutableProps): JobExecutable { + return new JobExecutable({ + ...props, + type: JobType.RAY, + language: JobLanguage.PYTHON, + }); + } + /** * Create a custom JobExecutable. * @@ -297,10 +320,18 @@ export class JobExecutable { if (config.language !== JobLanguage.PYTHON) { throw new Error('Python shell requires the language to be set to Python'); } - if ([GlueVersion.V0_9, GlueVersion.V2_0, GlueVersion.V3_0, GlueVersion.V4_0].includes(config.glueVersion)) { + if ([GlueVersion.V0_9, GlueVersion.V3_0, GlueVersion.V4_0].includes(config.glueVersion)) { throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Python Shell`); } } + if (JobType.RAY === config.type) { + if (config.language !== JobLanguage.PYTHON) { + throw new Error('Ray requires the language to be set to Python'); + } + if ([GlueVersion.V0_9, GlueVersion.V1_0, GlueVersion.V2_0, GlueVersion.V3_0].includes(config.glueVersion)) { + throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Ray`); + } + } if (config.extraJarsFirst && [GlueVersion.V0_9, GlueVersion.V1_0].includes(config.glueVersion)) { throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support extraJarsFirst`); } @@ -310,8 +341,11 @@ export class JobExecutable { if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) { throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON'); } - if (config.pythonVersion === PythonVersion.THREE_NINE && config.type !== JobType.PYTHON_SHELL) { - throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell'); + if (config.pythonVersion === PythonVersion.THREE_NINE && config.type !== JobType.PYTHON_SHELL && config.type !== JobType.RAY) { + throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray'); + } + if (config.pythonVersion === PythonVersion.THREE && config.type === JobType.RAY) { + throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray'); } this.config = config; } diff --git a/packages/@aws-cdk/aws-glue/lib/job.ts b/packages/@aws-cdk/aws-glue/lib/job.ts index eebb8b1acbdb0..95c208bdc930f 100644 --- a/packages/@aws-cdk/aws-glue/lib/job.ts +++ b/packages/@aws-cdk/aws-glue/lib/job.ts @@ -32,6 +32,16 @@ export class WorkerType { */ public static readonly G_2X = new WorkerType('G.2X'); + /** + * Each worker maps to 0.25 DPU (2 vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per worker. Suitable for low volume streaming jobs. + */ + public static readonly G_025X = new WorkerType('G.025X'); + + /** + * Each worker maps to 2 high-memory DPU [M-DPU] (8 vCPU, 64 GB of memory, 128 GB disk). Supported in Ray jobs. + */ + public static readonly Z_2X = new WorkerType('Z.2X'); + /** * Custom worker type * @param workerType custom worker type @@ -726,6 +736,8 @@ export class Job extends JobBase { private setupSparkUI(executable: JobExecutableConfig, role: iam.IRole, props: SparkUIProps) { if (JobType.PYTHON_SHELL === executable.type) { throw new Error('Spark UI is not available for JobType.PYTHON_SHELL jobs'); + } else if (JobType.RAY === executable.type) { + throw new Error('Spark UI is not available for JobType.RAY jobs'); } const bucket = props.bucket ?? new s3.Bucket(this, 'SparkUIBucket'); diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.assets.json index 8e739ca15edca..5519b93f322d2 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -14,7 +14,7 @@ } } }, - "977a2f07e22679bb04b03ce83cc1fac3e6cc269a794e38248ec67106ee39f0a2": { + "b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "977a2f07e22679bb04b03ce83cc1fac3e6cc269a794e38248ec67106ee39f0a2.json", + "objectKey": "b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.template.json index 47f34d95c01f7..06115404e2a3c 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -350,9 +350,11 @@ }, "GlueVersion": "2.0", "Name": "StreamingJob2.0", + "NumberOfWorkers": 10, "Tags": { "key": "value" - } + }, + "WorkerType": "G.025X" } }, "EtlJob30ServiceRole8E675579": { @@ -705,9 +707,11 @@ }, "GlueVersion": "3.0", "Name": "StreamingJob3.0", + "NumberOfWorkers": 10, "Tags": { "key": "value" - } + }, + "WorkerType": "G.025X" } }, "EtlJob40ServiceRoleBDD9998A": { @@ -1060,9 +1064,11 @@ }, "GlueVersion": "4.0", "Name": "StreamingJob4.0", + "NumberOfWorkers": 10, "Tags": { "key": "value" - } + }, + "WorkerType": "G.025X" } }, "ShellJobServiceRoleCF97BC4B": { @@ -1314,6 +1320,133 @@ "key": "value" } } + }, + "RayJobServiceRole51433C3D": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "RayJobServiceRoleDefaultPolicyA615640D": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "RayJobServiceRoleDefaultPolicyA615640D", + "Roles": [ + { + "Ref": "RayJobServiceRole51433C3D" + } + ] + } + }, + "RayJob2F7864D9": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "glueray", + "PythonVersion": "3.9", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, + "DefaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "GlueVersion": "4.0", + "Name": "RayJob", + "NumberOfWorkers": 2, + "Tags": { + "key": "value" + }, + "WorkerType": "Z.2X" + } } }, "Parameters": { diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/manifest.json index 9a6172107f0bc..e12d21e1befbd 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/977a2f07e22679bb04b03ce83cc1fac3e6cc269a794e38248ec67106ee39f0a2.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -213,6 +213,24 @@ "data": "ShellJob390C141361" } ], + "/aws-glue-job/RayJob/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "RayJobServiceRole51433C3D" + } + ], + "/aws-glue-job/RayJob/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "RayJobServiceRoleDefaultPolicyA615640D" + } + ], + "/aws-glue-job/RayJob/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "RayJob2F7864D9" + } + ], "/aws-glue-job/BootstrapVersion": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/tree.json index 712c057e96df5..e641a09da50d7 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/tree.json @@ -532,9 +532,11 @@ }, "glueVersion": "2.0", "name": "StreamingJob2.0", + "numberOfWorkers": 10, "tags": { "key": "value" - } + }, + "workerType": "G.025X" } }, "constructInfo": { @@ -1046,9 +1048,11 @@ }, "glueVersion": "3.0", "name": "StreamingJob3.0", + "numberOfWorkers": 10, "tags": { "key": "value" - } + }, + "workerType": "G.025X" } }, "constructInfo": { @@ -1560,9 +1564,11 @@ }, "glueVersion": "4.0", "name": "StreamingJob4.0", + "numberOfWorkers": 10, "tags": { "key": "value" - } + }, + "workerType": "G.025X" } }, "constructInfo": { @@ -1950,6 +1956,195 @@ "version": "0.0.0" } }, + "RayJob": { + "id": "RayJob", + "path": "aws-glue-job/RayJob", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job/RayJob/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/RayJob/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/RayJob/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job/RayJob/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/RayJob/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "RayJobServiceRoleDefaultPolicyA615640D", + "roles": [ + { + "Ref": "RayJobServiceRole51433C3D" + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Role", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/RayJob/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "glueray", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3.9" + }, + "role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, + "defaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "glueVersion": "4.0", + "name": "RayJob", + "numberOfWorkers": 2, + "tags": { + "key": "value" + }, + "workerType": "Z.2X" + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Job", + "version": "0.0.0" + } + }, "BootstrapVersion": { "id": "BootstrapVersion", "path": "aws-glue-job/BootstrapVersion", diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.ts b/packages/@aws-cdk/aws-glue/test/integ.job.ts index 791fd734fb0ca..28450cfaf6a3a 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.job.ts @@ -63,6 +63,8 @@ const script = glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world. glueVersion, script, }), + workerType: glue.WorkerType.G_025X, + workerCount: 10, defaultArguments: { arg1: 'value1', arg2: 'value2', @@ -105,4 +107,22 @@ new glue.Job(stack, 'ShellJob39', { }, }); +new glue.Job(stack, 'RayJob', { + jobName: 'RayJob', + executable: glue.JobExecutable.pythonRay({ + glueVersion: glue.GlueVersion.V4_0, + pythonVersion: glue.PythonVersion.THREE_NINE, + script, + }), + workerType: glue.WorkerType.Z_2X, + workerCount: 2, + defaultArguments: { + arg1: 'value1', + arg2: 'value2', + }, + tags: { + key: 'value', + }, +}); + app.synth(); diff --git a/packages/@aws-cdk/aws-glue/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue/test/job-executable.test.ts index 6de1ad9859509..ca5beb95d59bb 100644 --- a/packages/@aws-cdk/aws-glue/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue/test/job-executable.test.ts @@ -31,6 +31,8 @@ describe('JobType', () => { test('.PYTHON_SHELL should set the name correctly', () => expect(glue.JobType.PYTHON_SHELL.name).toEqual('pythonshell')); + test('.RAY should set the name correctly', () => expect(glue.JobType.RAY.name).toEqual('glueray')); + test('of(customName) should set the name correctly', () => expect(glue.JobType.of('CustomName').name).toEqual('CustomName')); }); @@ -65,6 +67,15 @@ describe('JobExecutable', () => { })).toThrow(/Python shell requires the language to be set to Python/); }); + test('with JobType.RAY and a language other than JobLanguage.PYTHON should throw', () => { + expect(() => glue.JobExecutable.of({ + glueVersion: glue.GlueVersion.V4_0, + type: glue.JobType.RAY, + language: glue.JobLanguage.SCALA, + script, + })).toThrow(/Ray requires the language to be set to Python/); + }); + test('with a non JobLanguage.PYTHON and extraPythonFiles set should throw', () => { expect(() => glue.JobExecutable.of({ glueVersion: glue.GlueVersion.V3_0, @@ -76,7 +87,7 @@ describe('JobExecutable', () => { })).toThrow(/extraPythonFiles is not supported for languages other than JobLanguage.PYTHON/); }); - [glue.GlueVersion.V0_9, glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { + [glue.GlueVersion.V0_9, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { test(`with JobType.PYTHON_SHELL and GlueVersion ${glueVersion} should throw`, () => { expect(() => glue.JobExecutable.of({ type: glue.JobType.PYTHON_SHELL, @@ -113,7 +124,7 @@ describe('JobExecutable', () => { }); }); - test('with PythonVersion set to PythonVersion.THREE_NINE and JobType not pythonshell should throw', () => { + test('with PythonVersion set to PythonVersion.THREE_NINE and JobType etl should throw', () => { expect(() => glue.JobExecutable.of({ type: glue.JobType.ETL, language: glue.JobLanguage.PYTHON, @@ -132,5 +143,15 @@ describe('JobExecutable', () => { script, })).toBeDefined(); }); + + test('with PythonVersion PythonVersion.THREE_NINE and JobType ray should succeed', () => { + expect(glue.JobExecutable.of({ + type: glue.JobType.RAY, + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + script, + })).toBeDefined(); + }); }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/job.test.ts b/packages/@aws-cdk/aws-glue/test/job.test.ts index f03d41d243494..31b6a2421e9a5 100644 --- a/packages/@aws-cdk/aws-glue/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue/test/job.test.ts @@ -14,6 +14,10 @@ describe('WorkerType', () => { test('.G_2X should set the name correctly', () => expect(glue.WorkerType.G_2X.name).toEqual('G.2X')); + test('.G_025X should set the name correctly', () => expect(glue.WorkerType.G_025X.name).toEqual('G.025X')); + + test('.Z_2X should set the name correctly', () => expect(glue.WorkerType.Z_2X.name).toEqual('Z.2X')); + test('of(customType) should set name correctly', () => expect(glue.WorkerType.of('CustomType').name).toEqual('CustomType')); }); @@ -604,6 +608,33 @@ describe('Job', () => { }); }); + describe('ray job', () => { + test('with unsupported glue version should throw', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonRay({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE_NINE, + script, + }), + workerType: glue.WorkerType.Z_2X, + workerCount: 2, + })).toThrow('Specified GlueVersion 3.0 does not support Ray'); + }); + + test('with unsupported Spark UI prop should throw', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonRay({ + glueVersion: glue.GlueVersion.V4_0, + pythonVersion: glue.PythonVersion.THREE_NINE, + script, + }), + workerType: glue.WorkerType.Z_2X, + workerCount: 2, + sparkUI: { enabled: true }, + })).toThrow('Spark UI is not available for JobType.RAY'); + }); + }); + test('etl job with all props should synthesize correctly', () => { new glue.Job(stack, 'Job', {