-
Notifications
You must be signed in to change notification settings - Fork 0
/
glue_workflow_complex.tf
74 lines (65 loc) · 1.64 KB
/
glue_workflow_complex.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
module "glue_workflow_complex" {
source = "../modules/glue-workflow"
workflow_name = "etl-workflow-complex"
security_name = "glueSecurityConfigComplex"
}
###################### Glue Triggers and DAG ########################################
locals {
jobs_0_1 = [
module.dummy_job["dummy_job_0"].job_name,
module.dummy_job["dummy_job_1"].job_name
]
job_2 = module.dummy_job["dummy_job_2"].job_name
jobs_3_5 = [
module.dummy_job["dummy_job_3"].job_name,
module.dummy_job["dummy_job_4"].job_name,
module.dummy_job["dummy_job_5"].job_name
]
}
# starts 2 jobs
resource "aws_glue_trigger" "start_complex" {
name = "start_complex"
type = "ON_DEMAND"
workflow_name = module.glue_workflow_complex.workflow_name
dynamic "actions" {
for_each = local.jobs_0_1
content {
job_name = actions.value
}
}
}
# waits for first 2 jobs to finish
resource "aws_glue_trigger" "complex_stage_2" {
name = "complex_stage_2"
type = "CONDITIONAL"
workflow_name = module.glue_workflow_complex.workflow_name
actions {
job_name = local.job_2
}
predicate {
dynamic "conditions" {
for_each = local.jobs_0_1
content {
job_name = conditions.value
state = "SUCCEEDED"
}
}
}
}
resource "aws_glue_trigger" "wait_for_second" {
name = "complex_stage_3"
type = "CONDITIONAL"
workflow_name = module.glue_workflow_complex.workflow_name
dynamic "actions" {
for_each = local.jobs_3_5
content {
job_name = actions.value
}
}
predicate {
conditions {
job_name = local.job_2
state = "SUCCEEDED"
}
}
}