Skip to content

Commit

Permalink
feat(emr): set different primary, core, task node sizes (#14751)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongbo-miao
hongbo-miao committed Feb 22, 2024
1 parent d13b5b4 commit c627cbc
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 61 deletions.
100 changes: 60 additions & 40 deletions cloud-infrastructure/terraform/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ terraform {
version = "0.70.0"
}
}
required_version = ">= 1.6"
# terraform version
required_version = ">= 1.7"
}

provider "aws" {
Expand Down Expand Up @@ -48,16 +49,19 @@ module "hm_trino_s3_set_up_script" {
amazon_s3_key = "amazon-emr/hm-amazon-emr-cluster-trino/bootstrap-actions/set_up.sh"
local_file_path = "./data/amazon-emr/hm-amazon-emr-cluster-trino/bootstrap-actions/set_up.sh"
}
module "hm_trino" {
source = "./modules/hm_amazon_emr_cluster"
amazon_emr_cluster_name = "hm-trino"
amazon_emr_version = "emr-7.0.0"
applications = ["Trino"]
primary_instance_type = "r7gd.xlarge"
core_instance_type = "r7gd.2xlarge"
core_target_on_demand_capacity = 1
bootstrap_set_up_script_s3_uri = module.hm_trino_s3_set_up_script.uri
configurations_json_string = <<EOF
module "hm_trino_emr" {
source = "./modules/hm_amazon_emr_cluster"
amazon_emr_cluster_name = "hm-trino"
amazon_emr_version = "emr-7.0.0"
applications = ["Trino"]
primary_instance_target_on_demand_capacity = 1
primary_instance_weighted_capacity = 1
primary_instance_type = "r7g.xlarge"
core_instance_target_on_demand_capacity = 1
core_instance_weighted_capacity = 1
core_instance_type = "r7g.xlarge"
bootstrap_set_up_script_s3_uri = module.hm_trino_s3_set_up_script.uri
configurations_json_string = <<EOF
[
{
"Classification": "delta-defaults",
Expand All @@ -81,20 +85,21 @@ module "hm_trino" {
}
]
EOF
iam_role_arn = "arn:aws:iam::272394222652:role/service-role/AmazonEMR-ServiceRole-hm"
environment = var.environment
team = var.team
iam_role_arn = "arn:aws:iam::272394222652:role/service-role/AmazonEMR-ServiceRole-hm"
environment = var.environment
team = var.team
}
module "hm_trino_task_instance_fleet" {
source = "./modules/hm_amazon_emr_cluster_task_instance_fleet"
amazon_emr_cluster_id = module.hm_trino.id
task_instance_type = "r7gd.2xlarge"
task_target_spot_capacity = 7
source = "./modules/hm_amazon_emr_cluster_task_instance_fleet"
amazon_emr_cluster_id = module.hm_trino_emr.id
task_instance_target_spot_capacity = 1
task_instance_weighted_capacity = 1
task_instance_type = "r7g.xlarge"
}
data "aws_instance" "hm_trino_primary_node_ec2_instance" {
filter {
name = "private-dns-name"
values = [module.hm_trino.master_public_dns]
values = [module.hm_trino_emr.master_public_dns]
}
}
module "hm_route_53_record" {
Expand Down Expand Up @@ -123,15 +128,18 @@ module "hm_sedona_s3_set_up_jupyterlab_script" {
local_file_path = "./data/amazon-emr/hm-amazon-emr-cluster-sedona/steps/set_up_jupyterlab.sh"
}
module "hm_sedona_emr" {
source = "./modules/hm_amazon_emr_cluster"
amazon_emr_cluster_name = "hm-sedona"
amazon_emr_version = "emr-7.0.0"
applications = ["Hadoop", "Hive", "JupyterEnterpriseGateway", "Spark"]
primary_instance_type = "r7gd.2xlarge"
core_instance_type = "r7gd.2xlarge"
core_target_on_demand_capacity = 1
bootstrap_set_up_script_s3_uri = module.hm_sedona_s3_set_up_script.uri
configurations_json_string = <<EOF
source = "./modules/hm_amazon_emr_cluster"
amazon_emr_cluster_name = "hm-sedona"
amazon_emr_version = "emr-7.0.0"
applications = ["Hadoop", "Hive", "JupyterEnterpriseGateway", "Livy", "Spark"]
primary_instance_target_on_demand_capacity = 1
primary_instance_weighted_capacity = 1
primary_instance_type = "r7g.xlarge"
core_instance_target_on_demand_capacity = 1
core_instance_weighted_capacity = 1
core_instance_type = "r7g.xlarge"
bootstrap_set_up_script_s3_uri = module.hm_sedona_s3_set_up_script.uri
configurations_json_string = <<EOF
[
{
"Classification" : "delta-defaults",
Expand All @@ -156,26 +164,38 @@ module "hm_sedona_emr" {
}
]
EOF
iam_role_arn = "arn:aws:iam::272394222652:role/service-role/AmazonEMR-ServiceRole-hm"
environment = var.environment
team = var.team
iam_role_arn = "arn:aws:iam::272394222652:role/service-role/AmazonEMR-ServiceRole-hm"
environment = var.environment
team = var.team
}
module "hm_sedona_emr_task_instance_fleet" {
source = "./modules/hm_amazon_emr_cluster_task_instance_fleet"
amazon_emr_cluster_id = module.hm_sedona_emr.id
task_instance_type = "r7gd.2xlarge"
task_target_spot_capacity = 1
source = "./modules/hm_amazon_emr_cluster_task_instance_fleet"
amazon_emr_cluster_id = module.hm_sedona_emr.id
task_instance_target_spot_capacity = 2
task_instance_weighted_capacity = 2
task_instance_type = "r7g.2xlarge"
}
module "hm_sedona_emr_managed_scaling_policy" {
source = "./modules/hm_amazon_emr_managed_scaling_policy"
amazon_emr_cluster_id = module.hm_sedona_emr.id
max_capacity_units = 10
max_capacity_units = 60
}
data "aws_instance" "hm_sedona_emr_primary_node" {
filter {
name = "private-dns-name"
values = [module.hm_sedona_emr.master_public_dns]
}
}
module "hm_sedona_route_53" {
source = "./modules/hm_amazon_route_53"
amazon_route_53_record_name = "hm-sedona"
amazon_route_53_record_values = [data.aws_instance.hm_sedona_emr_primary_node.private_ip]
}

module "hm_sedona_emr_studio_iam" {
source = "./modules/hm_amazon_emr_studio_iam"
amazon_emr_studio_name = "hm-sedona-emr-studio"
s3_bucket = "hongbomiao-bucke"
s3_bucket = "hongbomiao-bucket"
environment = var.environment
team = var.team
}
Expand All @@ -193,7 +213,7 @@ module "hm_sedona_emr_studio" {
module "hm_glue_databrew_recipe_job_write_adsb_2x_flight_trace_csv_to_parquet_iam" {
source = "./modules/hm_aws_glue_databrew_iam"
aws_glue_databrew_job_nickname = "write-adsb-csv-to-parquet"
input_s3_bucket = "track-data-adsbexchange-transfer-archer"
input_s3_bucket = "hongbomiao-bucket"
output_s3_bucket = "hongbomiao-bucket"
environment = var.environment
team = var.team
Expand All @@ -202,8 +222,8 @@ module "hm_glue_databrew_recipe_job_write_adsb_2x_flight_trace_csv_to_parquet_da
source = "./modules/hm_aws_glue_databrew_dataset_adsb_raw_data"
count = length(var.adsb_2x_flight_trace_raw_data_dates)
aws_glue_databrew_dataset_name = "adsb-2x-flight-trace-dataset-raw-data-${replace(var.adsb_2x_flight_trace_raw_data_dates[count.index], "/", "-")}"
input_s3_bucket = "track-data-adsbexchange-transfer-archer"
input_s3_dir = "hires-traces-json-gz/hires-traces/${var.adsb_2x_flight_trace_raw_data_dates[count.index]}/"
input_s3_bucket = "hongbomiao-bucket"
input_s3_dir = "data/raw/adsb_2x_flight_trace_data/${var.adsb_2x_flight_trace_raw_data_dates[count.index]}/"
environment = var.environment
team = var.team
}
Expand Down
15 changes: 7 additions & 8 deletions cloud-infrastructure/terraform/modules/hm_amazon_emr_cluster/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ resource "aws_emr_cluster" "hm_amazon_emr_cluster" {
name = var.amazon_emr_cluster_name
release_label = var.amazon_emr_version
applications = var.applications
termination_protection = true
termination_protection = false
keep_job_flow_alive_when_no_steps = true
log_uri = "s3://hongbomiao-bucket/amazon-emr/logs/"
ec2_attributes {
Expand All @@ -16,46 +16,45 @@ resource "aws_emr_cluster" "hm_amazon_emr_cluster" {
}
master_instance_fleet {
name = "Primary"
target_on_demand_capacity = 1
target_on_demand_capacity = var.primary_instance_target_on_demand_capacity
launch_specifications {
on_demand_specification {
allocation_strategy = "lowest-price"
}
}
instance_type_configs {
weighted_capacity = var.primary_instance_weighted_capacity
instance_type = var.primary_instance_type
weighted_capacity = 1
}
}
core_instance_fleet {
name = "Core"
target_on_demand_capacity = var.core_target_on_demand_capacity
target_on_demand_capacity = var.core_instance_target_on_demand_capacity
launch_specifications {
on_demand_specification {
allocation_strategy = "lowest-price"
}
}
instance_type_configs {
weighted_capacity = var.core_instance_weighted_capacity
instance_type = var.core_instance_type
weighted_capacity = 1
}
}
bootstrap_action {
name = "set_up"
path = var.bootstrap_set_up_script_s3_uri
}
configurations_json = var.configurations_json_string
step = var.steps
configurations_json = var.configurations_json_string
service_role = var.iam_role_arn
tags = {
for-use-with-amazon-emr-managed-policies = true
Environment = var.environment
Team = var.team
Name = var.amazon_emr_cluster_name
}
# Avoid forcing replacement
# https://github.com/hashicorp/terraform-provider-aws/issues/12683#issuecomment-752899019
lifecycle {
# https://github.com/hashicorp/terraform-provider-aws/issues/12683#issuecomment-752899019
ignore_changes = [
configurations_json,
step
Expand Down
19 changes: 14 additions & 5 deletions cloud-infrastructure/terraform/modules/hm_amazon_emr_cluster/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,34 @@ variable "amazon_emr_version" {
variable "applications" {
type = list(string)
}
variable "primary_instance_target_on_demand_capacity" {
type = number
}
variable "primary_instance_weighted_capacity" {
type = number
}
variable "primary_instance_type" {
type = string
}
variable "core_instance_type" {
type = string
variable "core_instance_target_on_demand_capacity" {
type = number
}
variable "core_target_on_demand_capacity" {
variable "core_instance_weighted_capacity" {
type = number
}
variable "bootstrap_set_up_script_s3_uri" {
variable "core_instance_type" {
type = string
}
variable "configurations_json_string" {
variable "bootstrap_set_up_script_s3_uri" {
type = string
}
variable "steps" {
type = list(any)
default = []
}
variable "configurations_json_string" {
type = string
}
variable "iam_role_arn" {
type = string
}
Expand Down
8 changes: 3 additions & 5 deletions cloud-infrastructure/terraform/modules/hm_amazon_emr_cluster_task_instance_fleet/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
resource "aws_emr_instance_fleet" "hm_amazon_emr_cluster_task_instance_fleet" {
cluster_id = var.amazon_emr_cluster_id
name = "Task"
target_spot_capacity = var.task_target_spot_capacity
target_spot_capacity = var.task_instance_target_spot_capacity
instance_type_configs {
weighted_capacity = var.task_instance_weighted_capacity
instance_type = var.task_instance_type
weighted_capacity = 1
bid_price_as_percentage_of_on_demand_price = 100
}
launch_specifications {
Expand All @@ -17,9 +17,7 @@ resource "aws_emr_instance_fleet" "hm_amazon_emr_cluster_task_instance_fleet" {
}
lifecycle {
ignore_changes = [
# https://github.com/hashicorp/terraform-provider-aws/issues/34151
launch_specifications[0].spot_specification["allocation_strategy"],
# https://stackoverflow.com/questions/77442744/why-does-emr-cluster-has-more-on-demand-instances-over-than-the-max-limit
# https://stackoverflow.com/questions/77442744/how-can-i-make-emr-cluster-auto-scaling-to-utilize-on-demand-instances-while-sta
target_on_demand_capacity,
target_spot_capacity
]
Expand Down
9 changes: 6 additions & 3 deletions ...d-infrastructure/terraform/modules/hm_amazon_emr_cluster_task_instance_fleet/variables.tf
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
variable "amazon_emr_cluster_id" {
type = string
}
variable "task_instance_type" {
type = string
variable "task_instance_target_spot_capacity" {
type = number
}
variable "task_target_spot_capacity" {
variable "task_instance_weighted_capacity" {
type = number
}
variable "task_instance_type" {
type = string
}

0 comments on commit c627cbc

Please sign in to comment.