-
Notifications
You must be signed in to change notification settings - Fork 0
/
glue_convert_job.tf
40 lines (32 loc) · 1.42 KB
/
glue_convert_job.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
resource "aws_cloudwatch_log_group" "convert_to_parquet" {
name = "convert-to-parquet-job"
}
resource "aws_glue_job" "convert_to_parquet" {
name = "convert_json_to_parquet"
role_arn = aws_iam_role.glue.arn
worker_type = "Standard"
number_of_workers = 2
glue_version = "3.0"
command {
script_location = "s3://${aws_s3_bucket.glue_assets.bucket}/convert_to_parquet.py"
# name = "pythonshell"
python_version = 3
}
security_configuration = aws_glue_security_configuration.raw_bucket.id
connections = [ format("%s-glue-connection", var.project_name) ]
default_arguments = {
"--continuous-log-logGroup" = aws_cloudwatch_log_group.convert_to_parquet.name
"--enable-continuous-cloudwatch-log" = true
"--enable-continuous-log-filter" = true
"--enable-spark-ui" = true
"--enable-metrics" = true
"--LOG_LEVEL" = "DEBUG"
"--job-language" = "python"
"--source_bucket" = aws_s3_bucket.raw_bucket.id
"--lake_bucket" = aws_s3_bucket.lake_bucket.id
"--number_of_partitions" = 3
}
depends_on = [
aws_glue_connection.glue_connection
]
}