Skip to content

Commit

Permalink
Viz RDS & DB-Pipeline
Browse files Browse the repository at this point in the history 
This initial version of the db-pipeline branch includes the Viz RDS components that Corey setup (including RDS Bastion) as well as the DB-Pipeline Lambda functions (including small change to SNS).

SmartSheet Card: https://app.smartsheetgov.com/sheets/FwqPVjGh6Qwv9GWh8hG35rj2Rr3g7RFMF3jcj4h1?rowId=7952738123310980
  • Loading branch information
@TylerSchrag-NOAA
TylerSchrag-NOAA committed Jan 21, 2022
1 parent 95b7bc5 commit 3ea4b09
Show file tree
Hide file tree
Showing 15 changed files with 6,903 additions and 3,014 deletions.
156 changes: 107 additions & 49 deletions Core/EC2/RDSBastion/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,19 @@
###############

variable "ami_owner_account_id" {
type = string
type = string
}

variable "ec2_instance_subnet" {
type = string
type = string
}

variable "ec2_instance_availability_zone" {
type = string
type = string
}

variable "ec2_instance_sgs" {
type = list(string)
type = list(string)
}

variable "ec2_instance_profile_name" {
Expand All @@ -32,26 +32,39 @@ variable "kms_key_arn" {
type = string
}

variable "db_ingest_secret_string" {
variable "ingest_db_secret_string" {
type = string
}

variable "db_ingest_address" {
variable "ingest_db_address" {
type = string
}

variable "db_ingest_port" {
variable "ingest_db_port" {
type = string
}

variable "viz_db_secret_string" {
type = string
}

variable "viz_db_address" {
type = string
}

variable "viz_db_port" {
type = string
}

variable "data_deployment_bucket" {
type = string
}

variable "mq_ingest_secret_string" {
variable "ingest_mq_secret_string" {
type = string
}

variable "mq_ingest_endpoint" {
variable "ingest_mq_endpoint" {
type = string
}

Expand All @@ -75,22 +88,30 @@ variable "location_ro_user_secret_string" {
type = string
}

variable "viz_proc_admin_rw_secret_string" {
type = string
}

locals {
rfc_db_users = "rfc_fcst, rfc_fcst_ro"
location_db_users = "rfc_fcst_ro, location_ro_user_grp"
forecast_db = "rfcfcst"
location_db = "wrds_location3"
nwm_viz_ro_password = jsondecode(var.nwm_viz_ro_secret_string)["password"]
rfc_fcst_password = jsondecode(var.rfc_fcst_secret_string)["password"]
rfc_fcst_ro_user_password = jsondecode(var.rfc_fcst_ro_user_secret_string)["password"]
rfc_fcst_user_password = jsondecode(var.rfc_fcst_user_secret_string)["password"]
location_ro_user_password = jsondecode(var.location_ro_user_secret_string)["password"]
ingest_db_users = "rfc_fcst, rfc_fcst_ro"
location_db_users = "rfc_fcst_ro, location_ro_user_grp"
viz_db_users = "viz_proc_admin_rw_user"
forecast_db = "rfcfcst"
location_db = "wrds_location3"
viz_db = "vizprocessing"
nwm_viz_ro_password = jsondecode(var.nwm_viz_ro_secret_string)["password"]
rfc_fcst_password = jsondecode(var.rfc_fcst_secret_string)["password"]
rfc_fcst_ro_user_password = jsondecode(var.rfc_fcst_ro_user_secret_string)["password"]
rfc_fcst_user_password = jsondecode(var.rfc_fcst_user_secret_string)["password"]
location_ro_user_password = jsondecode(var.location_ro_user_secret_string)["password"]
viz_proc_admin_rw_password = jsondecode(var.viz_proc_admin_rw_secret_string)["password"]
home_dir = "/home/ec2-user"

mq_vhost = {
"dev" : "development",
"development" : "development",
"ti" : "testing_integration",
"uat" : "user_acceptance_testing",
"uat" : "user_acceptance-testing",
"prod" : "production",
"production" : "production",
}
Expand Down Expand Up @@ -144,25 +165,23 @@ data "aws_ami" "linux" {
owners = [var.ami_owner_account_id]
}

data "template_file" "postgresql_setup" {
template = file("${path.module}/templates/postgres/postgresql_setup.sh")
data "template_file" "ingest_postgresql_setup" {
template = file("${path.module}/scripts/ingest/postgresql_setup.sh")
vars = {
FORECASTDB = local.forecast_db
LOCATIONDB = local.location_db
RFCDBUSERS = local.rfc_db_users
INGESTDBUSERS = local.ingest_db_users
LOCATIONDBUSERS = local.location_db_users
PGHOST = var.db_ingest_address
PGPORT = var.db_ingest_port
PGUSERNAME = jsondecode(var.db_ingest_secret_string)["username"]
PGPASSWORD = jsondecode(var.db_ingest_secret_string)["password"]
DBHOST = var.ingest_db_address
DBPORT = var.ingest_db_port
DBUSERNAME = jsondecode(var.ingest_db_secret_string)["username"]
DBPASSWORD = jsondecode(var.ingest_db_secret_string)["password"]
DEPLOYMENT_BUCKET = var.data_deployment_bucket

INITIALIZATION_SCRIPT = "${file("${path.module}/templates/postgres/postgresql_initialization.sh")}"
}
}

data "template_file" "db_users" {
template = file("${path.module}/templates/postgres/db_users.sql")
data "template_file" "ingest_users" {
template = file("${path.module}/scripts/ingest/ingest_users.sql")
vars = {
NWM_VIZ_RO = local.nwm_viz_ro_password
RFC_FCST = local.rfc_fcst_password
Expand All @@ -173,27 +192,50 @@ data "template_file" "db_users" {
}

data "template_file" "rabbitmq_setup" {
template = file("${path.module}/templates/rabbitmq/rabbitmq_setup.sh")
template = file("${path.module}/scripts/rabbitmq/rabbitmq_setup.sh")
vars = {
MQINGESTENDPOINT = var.mq_ingest_endpoint
MQUSERNAME = jsondecode(var.mq_ingest_secret_string)["username"]
MQPASSWORD = jsondecode(var.mq_ingest_secret_string)["password"]
MQINGESTENDPOINT = var.ingest_mq_endpoint
MQUSERNAME = jsondecode(var.ingest_mq_secret_string)["username"]
MQPASSWORD = jsondecode(var.ingest_mq_secret_string)["password"]
RFC_FCST_USER = jsondecode(var.rfc_fcst_user_secret_string)["username"]
RFC_FCST_USER_PASSWORD = jsondecode(var.rfc_fcst_user_secret_string)["password"]
MQVHOST = local.mq_vhost[var.environment]
}
}

INITIALIZATION_SCRIPT = "${file("${path.module}/templates/rabbitmq/rabbitmq_initialization.sh")}"
data "template_file" "viz_postgresql_setup" {
template = file("${path.module}/scripts/viz/postgresql_setup.sh")
vars = {
DBNAME = local.viz_db
DBHOST = var.viz_db_address
DBPORT = var.viz_db_port
DBUSERNAME = jsondecode(var.viz_db_secret_string)["username"]
DBPASSWORD = jsondecode(var.viz_db_secret_string)["password"]
DEPLOYMENT_BUCKET = var.data_deployment_bucket
DBUSERS = local.viz_db_users
HOME = local.home_dir
}
}

data "template_file" "viz_setup" {
template = file("${path.module}/scripts/viz/viz_setup.sql")
vars = {
VIZ_PROC_ADMIN_RW_PASS = local.viz_proc_admin_rw_password
RECURR_FLOW_CONUS = "rf_2_0_17c"
RECURR_FLOW_HI = "rf_2_0"
RECURR_FLOW_PRVI = "rf_2_0"
HOME = local.home_dir
}
}

data "cloudinit_config" "startup" {
gzip = false
base64_encode = false
gzip = true
base64_encode = true

part {
content_type = "text/x-shellscript"
filename = "postgres_setup.sh"
content = data.template_file.postgresql_setup.rendered
filename = "ingest_postgresql_setup.sh"
content = data.template_file.ingest_postgresql_setup.rendered
}

part {
Expand All @@ -202,23 +244,35 @@ data "cloudinit_config" "startup" {
content = data.template_file.rabbitmq_setup.rendered
}

part {
content_type = "text/x-shellscript"
filename = "viz_postgresql_setup.sh"
content = data.template_file.viz_postgresql_setup.rendered
}

part {
content_type = "text/cloud-config"
filename = "cloud-config.yaml"
content = <<-END
#cloud-config
${jsonencode({
write_files = [
{
path = "/deploy_files/db_users.sql"
permissions = "0400"
owner = "ec2-user:ec2-user"
content = data.template_file.db_users.rendered
}
]
})}
write_files = [
{
path = "/deploy_files/ingest_users.sql"
permissions = "0400"
owner = "ec2-user:ec2-user"
content = data.template_file.ingest_users.rendered
},
{
path = "/deploy_files/viz_setup.sql"
permissions = "0400"
owner = "ec2-user:ec2-user"
content = data.template_file.viz_setup.rendered
}
]
})}
END
}
}
}

output "forecast_db" {
Expand All @@ -228,3 +282,7 @@ output "forecast_db" {
output "location_db" {
value = local.location_db
}

output "viz_db" {
value = local.viz_db
}
47 changes: 47 additions & 0 deletions Core/EC2/RDSBastion/scripts/ingest/ingest_users.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
-- ROLES

CREATE ROLE nwm_viz_ro;
ALTER ROLE nwm_viz_ro WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN NOBYPASSRLS CONNECTION LIMIT 45 ENCRYPTED PASSWORD '${NWM_VIZ_RO}';
COMMENT ON ROLE rfc_fcst IS 'Read-write group role for the Viz databases';

CREATE ROLE rfc_fcst;
ALTER ROLE rfc_fcst WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS ENCRYPTED PASSWORD '${RFC_FCST}';
COMMENT ON ROLE rfc_fcst IS 'Read-write group role for the RFC Forecast databases';

CREATE ROLE rfc_fcst_ro_user;
ALTER ROLE rfc_fcst_ro_user WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN NOBYPASSRLS CONNECTION LIMIT 45 ENCRYPTED PASSWORD '${RFC_FCST_RO_USER}';
COMMENT ON ROLE rfc_fcst_ro_user IS 'Read-only user role for the RFC Forecast databases';

CREATE ROLE rfc_fcst_user;
ALTER ROLE rfc_fcst_user WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN NOBYPASSRLS ENCRYPTED PASSWORD '${RFC_FCST_USER}';
COMMENT ON ROLE rfc_fcst_user IS 'Read-write user role for the RFC Forecast databases';

CREATE ROLE location_ro_user;
ALTER ROLE location_ro_user WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN NOBYPASSRLS CONNECTION LIMIT 45 ENCRYPTED PASSWORD '${LOCATION_RO_USER}';
COMMENT ON ROLE location_ro_user IS 'Read-only user role for the Location databases';

CREATE ROLE wrds_svc_rw_user;
ALTER ROLE wrds_svc_rw_user WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS;
COMMENT ON ROLE wrds_svc_rw_user IS 'Read-write user role for the Location databases';

-- GROUPS

CREATE ROLE rfc_fcst_ro;
ALTER ROLE rfc_fcst_ro WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS;
COMMENT ON ROLE rfc_fcst_ro IS 'Read-only group role for the RFC Forecast databases';

CREATE ROLE location_ro_user_grp;
ALTER ROLE location_ro_user_grp WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS;
COMMENT ON ROLE location_ro_user_grp IS 'Read-only group role for the Location databases';

CREATE ROLE nwm_ro_user_grp;
ALTER ROLE nwm_ro_user_grp WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS;
COMMENT ON ROLE nwm_ro_user_grp IS 'Read-only group role for the Location databases';

-- GROUP ASSIGNMENT

GRANT rfc_fcst_ro TO rfc_fcst_ro_user ;
GRANT rfc_fcst TO rfc_fcst_user ;
GRANT rfc_fcst_ro TO nwm_viz_ro ;
GRANT location_ro_user_grp TO location_ro_user ;

Loading

0 comments on commit 3ea4b09

Please sign in to comment.