-
Notifications
You must be signed in to change notification settings - Fork 7
/
main.tf
153 lines (136 loc) · 4.49 KB
/
main.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
locals {
tags = { "tecton-accessible:${var.deployment_name}" : "true" }
hive_config = [
{
Classification : "hive-site",
Properties : {
"hive.metastore.client.factory.class" : "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory",
"hive.metastore.glue.catalogid" : var.glue_account_id
}
},
{
Classification : "spark-defaults",
Properties : {
"hive.metastore.client.factory.class" : "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory",
"hive.metastore.glue.catalogid" : var.glue_account_id
}
},
{
Classification : "spark-hive-site",
Properties : {
"hive.metastore.client.factory.class" : "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory",
"hive.metastore.glue.catalogid" : var.glue_account_id
}
}
]
base_config = [
{
Classification : "livy-env",
Properties : {}
Configurations : [
{
Classification : "export",
Properties : {
"CLUSTER_REGION" : var.region,
"TECTON_CLUSTER_NAME" : var.deployment_name
}
}
]
},
{
Classification : "yarn-env",
Properties : {},
Configurations : [
{
Classification : "export",
Properties : {
"CLUSTER_REGION" : var.region,
"TECTON_CLUSTER_NAME" : var.deployment_name
}
}
]
}
]
// bootstrap_regions
// ---
// EMR bootstrapping only supports bootstrap scripts from s3 buckets. The current way the s3
// client within EMR is retrieving the bootstrap scripts causes a failure to retrieve the file in
// certain regions. Currently Tecton supports serving bootstrap scripts from the following
// regions. (including us-west-2 by default) Reach out to customer support for further information.
bootstrap_regions = {
"eu-central-1" : "-eu-central-1",
"eu-west-1" : "-eu-west-1",
"us-east-2" : "-us-east-2",
}
bootstrap_action = [
{
name = "tecton_emr_setup"
path = format(
"s3://tecton.ai.public%s/install_scripts/setup_emr_notebook_cluster_v2.sh",
lookup(local.bootstrap_regions, var.region, ""),
)
}
]
}
resource "aws_emr_cluster" "cluster" {
name = "tecton-${var.deployment_name}-notebook-cluster"
release_label = "emr-6.4.0"
applications = ["Spark", "Livy", "Hive", "JupyterEnterpriseGateway"]
ec2_attributes {
subnet_id = var.subnet_id
emr_managed_master_security_group = var.emr_security_group_id
emr_managed_slave_security_group = var.emr_security_group_id
instance_profile = var.instance_profile_arn
service_access_security_group = var.emr_service_security_group_id
}
master_instance_group {
instance_type = var.instance_type
}
core_instance_group {
instance_type = var.instance_type
instance_count = var.instance_count
ebs_config {
size = var.ebs_size
type = var.ebs_type
volumes_per_instance = var.ebs_count
}
}
dynamic "bootstrap_action" {
iterator = bootstrap_action
for_each = concat(local.bootstrap_action, var.extra_bootstrap_actions)
content {
name = lookup(bootstrap_action.value, "name", null)
path = lookup(bootstrap_action.value, "path", null)
args = lookup(bootstrap_action.value, "args", null)
}
}
service_role = var.emr_service_role_id
configurations_json = (var.has_glue ?
jsonencode(concat(local.hive_config, local.base_config, var.extra_cluster_config)) :
jsonencode(concat(local.base_config, var.extra_cluster_config))
)
step {
action_on_failure = "TERMINATE_CLUSTER"
name = "Setup Hadoop Debugging"
hadoop_jar_step {
jar = "command-runner.jar"
args = ["state-pusher-script"]
}
}
tags = {
notebook = "true",
"tecton-accessible:${var.deployment_name}" = "true",
tecton-owned = "true"
}
}
### AWS Secrets
resource "aws_secretsmanager_secret" "api_service" {
name = "tecton-${var.deployment_name}/API_SERVICE"
}
resource "aws_secretsmanager_secret_version" "api_service" {
secret_id = aws_secretsmanager_secret.api_service.id
secret_string = "https://${var.deployment_name}.tecton.ai/api"
}
resource "aws_secretsmanager_secret" "tecton_api_key" {
name = "tecton-${var.deployment_name}/TECTON_API_KEY"
}