Viz RDS & DB-Pipeline

Core/EC2/RDSBastion/main.tf

@@ -3,19 +3,19 @@
 ###############
 
 variable "ami_owner_account_id" {
-  type        = string
+  type = string
 }
 
 variable "ec2_instance_subnet" {
-  type    = string
+  type = string
 }
 
 variable "ec2_instance_availability_zone" {
-  type    = string
+  type = string
 }
 
 variable "ec2_instance_sgs" {
-  type    = list(string)
+  type = list(string)
 }
 
 variable "ec2_instance_profile_name" {
@@ -32,26 +32,39 @@ variable "kms_key_arn" {
   type = string
 }
 
-variable "db_ingest_secret_string" {
+variable "ingest_db_secret_string" {
   type = string
 }
 
-variable "db_ingest_address" {
+variable "ingest_db_address" {
   type = string
 }
 
-variable "db_ingest_port" {
+variable "ingest_db_port" {
   type = string
 }
+
+variable "viz_db_secret_string" {
+  type = string
+}
+
+variable "viz_db_address" {
+  type = string
+}
+
+variable "viz_db_port" {
+  type = string
+}
+
 variable "data_deployment_bucket" {
   type = string
 }
 
-variable "mq_ingest_secret_string" {
+variable "ingest_mq_secret_string" {
   type = string
 }
 
-variable "mq_ingest_endpoint" {
+variable "ingest_mq_endpoint" {
   type = string
 }
 
@@ -75,22 +88,30 @@ variable "location_ro_user_secret_string" {
   type = string
 }
 
+variable "viz_proc_admin_rw_secret_string" {
+  type = string
+}
+
 locals {
-  rfc_db_users              = "rfc_fcst, rfc_fcst_ro"
-  location_db_users         = "rfc_fcst_ro, location_ro_user_grp"
-  forecast_db               = "rfcfcst"
-  location_db               = "wrds_location3"
-  nwm_viz_ro_password       = jsondecode(var.nwm_viz_ro_secret_string)["password"]
-  rfc_fcst_password         = jsondecode(var.rfc_fcst_secret_string)["password"]
-  rfc_fcst_ro_user_password = jsondecode(var.rfc_fcst_ro_user_secret_string)["password"]
-  rfc_fcst_user_password    = jsondecode(var.rfc_fcst_user_secret_string)["password"]
-  location_ro_user_password = jsondecode(var.location_ro_user_secret_string)["password"]
+  ingest_db_users            = "rfc_fcst, rfc_fcst_ro"
+  location_db_users          = "rfc_fcst_ro, location_ro_user_grp"
+  viz_db_users               = "viz_proc_admin_rw_user"
+  forecast_db                = "rfcfcst"
+  location_db                = "wrds_location3"
+  viz_db                     = "vizprocessing"
+  nwm_viz_ro_password        = jsondecode(var.nwm_viz_ro_secret_string)["password"]
+  rfc_fcst_password          = jsondecode(var.rfc_fcst_secret_string)["password"]
+  rfc_fcst_ro_user_password  = jsondecode(var.rfc_fcst_ro_user_secret_string)["password"]
+  rfc_fcst_user_password     = jsondecode(var.rfc_fcst_user_secret_string)["password"]
+  location_ro_user_password  = jsondecode(var.location_ro_user_secret_string)["password"]
+  viz_proc_admin_rw_password = jsondecode(var.viz_proc_admin_rw_secret_string)["password"]
+  home_dir                   = "/home/ec2-user"
 
   mq_vhost = {
     "dev" : "development",
     "development" : "development",
     "ti" : "testing_integration",
-    "uat" : "user_acceptance_testing",
+    "uat" : "user_acceptance-testing",
     "prod" : "production",
     "production" : "production",
   }
@@ -144,25 +165,23 @@ data "aws_ami" "linux" {
   owners = [var.ami_owner_account_id]
 }
 
-data "template_file" "postgresql_setup" {
-  template = file("${path.module}/templates/postgres/postgresql_setup.sh")
+data "template_file" "ingest_postgresql_setup" {
+  template = file("${path.module}/scripts/ingest/postgresql_setup.sh")
   vars = {
     FORECASTDB        = local.forecast_db
     LOCATIONDB        = local.location_db
-    RFCDBUSERS        = local.rfc_db_users
+    INGESTDBUSERS     = local.ingest_db_users
     LOCATIONDBUSERS   = local.location_db_users
-    PGHOST            = var.db_ingest_address
-    PGPORT            = var.db_ingest_port
-    PGUSERNAME        = jsondecode(var.db_ingest_secret_string)["username"]
-    PGPASSWORD        = jsondecode(var.db_ingest_secret_string)["password"]
+    DBHOST            = var.ingest_db_address
+    DBPORT            = var.ingest_db_port
+    DBUSERNAME        = jsondecode(var.ingest_db_secret_string)["username"]
+    DBPASSWORD        = jsondecode(var.ingest_db_secret_string)["password"]
     DEPLOYMENT_BUCKET = var.data_deployment_bucket
-
-    INITIALIZATION_SCRIPT = "${file("${path.module}/templates/postgres/postgresql_initialization.sh")}"
   }
 }
 
-data "template_file" "db_users" {
-  template = file("${path.module}/templates/postgres/db_users.sql")
+data "template_file" "ingest_users" {
+  template = file("${path.module}/scripts/ingest/ingest_users.sql")
   vars = {
     NWM_VIZ_RO       = local.nwm_viz_ro_password
     RFC_FCST         = local.rfc_fcst_password
@@ -173,27 +192,50 @@ data "template_file" "db_users" {
 }
 
 data "template_file" "rabbitmq_setup" {
-  template = file("${path.module}/templates/rabbitmq/rabbitmq_setup.sh")
+  template = file("${path.module}/scripts/rabbitmq/rabbitmq_setup.sh")
   vars = {
-    MQINGESTENDPOINT       = var.mq_ingest_endpoint
-    MQUSERNAME             = jsondecode(var.mq_ingest_secret_string)["username"]
-    MQPASSWORD             = jsondecode(var.mq_ingest_secret_string)["password"]
+    MQINGESTENDPOINT       = var.ingest_mq_endpoint
+    MQUSERNAME             = jsondecode(var.ingest_mq_secret_string)["username"]
+    MQPASSWORD             = jsondecode(var.ingest_mq_secret_string)["password"]
     RFC_FCST_USER          = jsondecode(var.rfc_fcst_user_secret_string)["username"]
     RFC_FCST_USER_PASSWORD = jsondecode(var.rfc_fcst_user_secret_string)["password"]
     MQVHOST                = local.mq_vhost[var.environment]
+  }
+}
 
-    INITIALIZATION_SCRIPT = "${file("${path.module}/templates/rabbitmq/rabbitmq_initialization.sh")}"
+data "template_file" "viz_postgresql_setup" {
+  template = file("${path.module}/scripts/viz/postgresql_setup.sh")
+  vars = {
+    DBNAME            = local.viz_db
+    DBHOST            = var.viz_db_address
+    DBPORT            = var.viz_db_port
+    DBUSERNAME        = jsondecode(var.viz_db_secret_string)["username"]
+    DBPASSWORD        = jsondecode(var.viz_db_secret_string)["password"]
+    DEPLOYMENT_BUCKET = var.data_deployment_bucket
+    DBUSERS           = local.viz_db_users
+    HOME              = local.home_dir
+  }
+}
+
+data "template_file" "viz_setup" {
+  template = file("${path.module}/scripts/viz/viz_setup.sql")
+  vars = {
+    VIZ_PROC_ADMIN_RW_PASS = local.viz_proc_admin_rw_password
+    RECURR_FLOW_CONUS      = "rf_2_0_17c"
+    RECURR_FLOW_HI         = "rf_2_0"
+    RECURR_FLOW_PRVI       = "rf_2_0"
+    HOME                   = local.home_dir
   }
 }
 
 data "cloudinit_config" "startup" {
-  gzip          = false
-  base64_encode = false
+  gzip          = true
+  base64_encode = true
 
   part {
     content_type = "text/x-shellscript"
-    filename     = "postgres_setup.sh"
-    content      = data.template_file.postgresql_setup.rendered
+    filename     = "ingest_postgresql_setup.sh"
+    content      = data.template_file.ingest_postgresql_setup.rendered
   }
 
   part {
@@ -202,23 +244,35 @@ data "cloudinit_config" "startup" {
     content      = data.template_file.rabbitmq_setup.rendered
   }
 
+  part {
+    content_type = "text/x-shellscript"
+    filename     = "viz_postgresql_setup.sh"
+    content      = data.template_file.viz_postgresql_setup.rendered
+  }
+
   part {
     content_type = "text/cloud-config"
     filename     = "cloud-config.yaml"
     content = <<-END
       #cloud-config
       ${jsonencode({
-      write_files = [
-        {
-          path        = "/deploy_files/db_users.sql"
-          permissions = "0400"
-          owner       = "ec2-user:ec2-user"
-          content     = data.template_file.db_users.rendered
-        }
-      ]
-      })}
+    write_files = [
+      {
+        path        = "/deploy_files/ingest_users.sql"
+        permissions = "0400"
+        owner       = "ec2-user:ec2-user"
+        content     = data.template_file.ingest_users.rendered
+      },
+      {
+        path        = "/deploy_files/viz_setup.sql"
+        permissions = "0400"
+        owner       = "ec2-user:ec2-user"
+        content     = data.template_file.viz_setup.rendered
+      }
+    ]
+})}
     END
-  }
+}
 }
 
 output "forecast_db" {
@@ -228,3 +282,7 @@ output "forecast_db" {
 output "location_db" {
   value = local.location_db
 }
+
+output "viz_db" {
+  value = local.viz_db
+}

Core/EC2/RDSBastion/scripts/ingest/ingest_users.sql

@@ -0,0 +1,47 @@
+-- ROLES
+
+CREATE ROLE nwm_viz_ro;
+ALTER ROLE nwm_viz_ro WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN NOBYPASSRLS CONNECTION LIMIT 45 ENCRYPTED PASSWORD '${NWM_VIZ_RO}';
+COMMENT ON ROLE rfc_fcst IS 'Read-write group role for the Viz databases';
+
+CREATE ROLE rfc_fcst;
+ALTER ROLE rfc_fcst WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS ENCRYPTED PASSWORD '${RFC_FCST}';
+COMMENT ON ROLE rfc_fcst IS 'Read-write group role for the RFC Forecast databases';
+
+CREATE ROLE rfc_fcst_ro_user;
+ALTER ROLE rfc_fcst_ro_user WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN NOBYPASSRLS CONNECTION LIMIT 45 ENCRYPTED PASSWORD '${RFC_FCST_RO_USER}';
+COMMENT ON ROLE rfc_fcst_ro_user IS 'Read-only user role for the RFC Forecast databases';
+
+CREATE ROLE rfc_fcst_user;
+ALTER ROLE rfc_fcst_user WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN NOBYPASSRLS ENCRYPTED PASSWORD '${RFC_FCST_USER}';
+COMMENT ON ROLE rfc_fcst_user IS 'Read-write user role for the RFC Forecast databases';
+
+CREATE ROLE location_ro_user;
+ALTER ROLE location_ro_user WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN NOBYPASSRLS CONNECTION LIMIT 45 ENCRYPTED PASSWORD '${LOCATION_RO_USER}';
+COMMENT ON ROLE location_ro_user IS 'Read-only user role for the Location databases';
+
+CREATE ROLE wrds_svc_rw_user;
+ALTER ROLE wrds_svc_rw_user WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS;
+COMMENT ON ROLE wrds_svc_rw_user IS 'Read-write user role for the Location databases';
+
+-- GROUPS
+
+CREATE ROLE rfc_fcst_ro;
+ALTER ROLE rfc_fcst_ro WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS;
+COMMENT ON ROLE rfc_fcst_ro IS 'Read-only group role for the RFC Forecast databases';
+
+CREATE ROLE location_ro_user_grp;
+ALTER ROLE location_ro_user_grp WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS;
+COMMENT ON ROLE location_ro_user_grp IS 'Read-only group role for the Location databases';
+
+CREATE ROLE nwm_ro_user_grp;
+ALTER ROLE nwm_ro_user_grp WITH INHERIT NOCREATEROLE NOCREATEDB NOLOGIN NOBYPASSRLS;
+COMMENT ON ROLE nwm_ro_user_grp IS 'Read-only group role for the Location databases';
+
+-- GROUP ASSIGNMENT
+
+GRANT rfc_fcst_ro TO rfc_fcst_ro_user ;
+GRANT rfc_fcst TO rfc_fcst_user ;
+GRANT rfc_fcst_ro TO nwm_viz_ro ;
+GRANT location_ro_user_grp TO location_ro_user ;
+