Add shared-vpc support on data-playground blueprint

blueprints/data-solutions/data-playground/README.md

@@ -12,12 +12,15 @@ This sample creates several distinct groups of resources:
 
 - project
 - networking
-    - VPC network with a default subnet and CloudNat
-    - Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC
 - Vertex AI Workbench notebook configured with a private IP and using a dedicated Service Account
 - One GCS bucket
 - One BigQuery dataset
 
+## Virtual Private Cloud (VPC) design
+As is often the case in real-world configurations, this blueprint accepts as input an existing Shared-VPC via the network_config variable. Make sure that 'container.googleapis.com', 'notebooks.googleapis.com' and 'servicenetworking.googleapis.com' are enabled in the VPC host project.
+
+If the network_config variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
+
 ## Deploy your enviroment
 We assume the identiy running the following steps has the following role:
 
@@ -47,12 +50,12 @@ You can now connect to the Vertex AI notbook to perform your data analysy.
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [prefix](variables.tf#L22) | Prefix used for resource names. | <code>string</code> | ✓ |  |
-| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ |  |
+| [prefix](variables.tf#L32) | Prefix used for resource names. | <code>string</code> | ✓ |  |
+| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ |  |
 | [location](variables.tf#L16) | The location where resources will be deployed. | <code>string</code> |  | <code>&#34;EU&#34;</code> |
-| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent format:  folders/folder_id or organizations/org_id. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; string&#10;  parent             &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [region](variables.tf#L45) | The region where resources will be deployed. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
-| [vpc_config](variables.tf#L61) | Parameters to create a VPC. | <code title="object&#40;&#123;&#10;  ip_cidr_range &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  ip_cidr_range &#61; &#34;10.0.0.0&#47;20&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
+| [network_config](variables.tf#L22) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10;  host_project      &#61; string&#10;  network_self_link &#61; string&#10;  subnet_self_link  &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent format:  folders/folder_id or organizations/org_id. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; string&#10;  parent             &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [region](variables.tf#L55) | The region where resources will be deployed. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
 
 ## Outputs
 

blueprints/data-solutions/data-playground/main.tf

@@ -17,6 +17,39 @@
 ###############################################################################
 locals {
   service_encryption_keys = var.service_encryption_keys
+  shared_vpc_project      = try(var.network_config.host_project, null)
+
+  subnet = (
+    local.use_shared_vpc
+    ? var.network_config.subnet_self_link
+    : values(module.vpc.0.subnet_self_links)[0]
+  )
+  vpc = (
+    local.use_shared_vpc
+    ? var.network_config.network_self_link
+    : module.vpc.0.self_link
+  )
+  use_shared_vpc = var.network_config != null
+
+  shared_vpc_bindings = {
+    "roles/compute.networkUser" = [
+      "robot-df", "notebooks"
+    ]
+  }
+
+  shared_vpc_role_members = {
+    robot-df  = "serviceAccount:${module.project.service_accounts.robots.dataflow}"
+    notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
+  }
+
+  # reassemble in a format suitable for for_each
+  shared_vpc_bindings_map = {
+    for binding in flatten([
+      for role, members in local.shared_vpc_bindings : [
+        for member in members : { role = role, member = member }
+      ]
+    ]) : "${binding.role}-${binding.member}" => binding
+  }
 }
 
 module "project" {
@@ -27,6 +60,7 @@ module "project" {
   project_create  = var.project_create != null
   prefix          = var.project_create == null ? null : var.prefix
   services = [
+    "aiplatform.googleapis.com",
     "bigquery.googleapis.com",
     "bigquerystorage.googleapis.com",
     "bigqueryreservation.googleapis.com",
@@ -42,17 +76,26 @@ module "project" {
     "storage.googleapis.com",
     "storage-component.googleapis.com"
   ]
+
+  shared_vpc_service_config = local.shared_vpc_project == null ? null : {
+    attach       = true
+    host_project = local.shared_vpc_project
+  }
+
   org_policies = {
     # "constraints/compute.requireOsLogin" = {
     #   enforce = false
     # }
-    # Example of applying a project wide policy, mainly useful for Composer
+    # Example of applying a project wide policy, mainly useful for Composer 1
   }
   service_encryption_key_ids = {
     compute = [try(local.service_encryption_keys.compute, null)]
     bq      = [try(local.service_encryption_keys.bq, null)]
     storage = [try(local.service_encryption_keys.storage, null)]
   }
+  service_config = {
+    disable_on_destroy = false, disable_dependent_services = false
+  }
 }
 
 ###############################################################################
@@ -61,11 +104,12 @@ module "project" {
 
 module "vpc" {
   source     = "../../../modules/net-vpc"
+  count      = local.use_shared_vpc ? 0 : 1
   project_id = module.project.project_id
   name       = "${var.prefix}-vpc"
   subnets = [
     {
-      ip_cidr_range = var.vpc_config.ip_cidr_range
+      ip_cidr_range = "10.0.0.0/20"
       name          = "${var.prefix}-subnet"
       region        = var.region
     }
@@ -74,10 +118,11 @@ module "vpc" {
 
 module "vpc-firewall" {
   source     = "../../../modules/net-vpc-firewall"
+  count      = local.use_shared_vpc ? 0 : 1
   project_id = module.project.project_id
-  network    = module.vpc.name
+  network    = module.vpc.0.name
   default_rules_config = {
-    admin_ranges = [var.vpc_config.ip_cidr_range]
+    admin_ranges = ["10.0.0.0/20"]
   }
   ingress_rules = {
     #TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed
@@ -92,12 +137,21 @@ module "vpc-firewall" {
 
 module "cloudnat" {
   source         = "../../../modules/net-cloudnat"
+  count          = local.use_shared_vpc ? 0 : 1
   project_id     = module.project.project_id
   name           = "${var.prefix}-default"
   region         = var.region
-  router_network = module.vpc.name
+  router_network = module.vpc.0.name
 }
 
+resource "google_project_iam_member" "shared_vpc" {
+  count   = local.use_shared_vpc ? 1 : 0
+  project = var.network_config.host_project
+  role    = "roles/compute.networkUser"
+  member  = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
+}
+
+
 ###############################################################################
 #                              Storage                                        #
 ###############################################################################
@@ -121,8 +175,6 @@ module "dataset" {
 ###############################################################################
 #                         Vertex AI Notebook                                   #
 ###############################################################################
-# TODO: Add encryption_key to Vertex AI notebooks as well
-# TODO: Add shared VPC support
 
 module "service-account-notebook" {
   source     = "../../../modules/iam-service-account"
@@ -160,11 +212,14 @@ resource "google_notebooks_instance" "playground" {
   no_public_ip    = true
   no_proxy_access = false
 
-  network = module.vpc.network.id
-  subnet  = module.vpc.subnets[format("%s/%s", var.region, "${var.prefix}-subnet")].id
+  network = local.vpc
+  subnet  = local.subnet
 
   service_account = module.service-account-notebook.email
 
   #TODO Uncomment once terraform-provider-google/issues/9273 is fixed
   # tags = ["ssh"]
+  depends_on = [
+    google_project_iam_member.shared_vpc,
+  ]
 }

blueprints/data-solutions/data-playground/outputs.tf

@@ -37,5 +37,5 @@ output "project" {
 
 output "vpc" {
   description = "VPC Network."
-  value       = module.vpc.name
+  value       = local.vpc
 }

blueprints/data-solutions/data-playground/variables.tf

@@ -19,6 +19,16 @@ variable "location" {
   default     = "EU"
 }
 
+variable "network_config" {
+  description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
+  type = object({
+    host_project      = string
+    network_self_link = string
+    subnet_self_link  = string
+  })
+  default = null
+}
+
 variable "prefix" {
   description = "Prefix used for resource names."
   type        = string
@@ -57,13 +67,3 @@ variable "service_encryption_keys" { # service encription key
   })
   default = null
 }
-
-variable "vpc_config" {
-  description = "Parameters to create a VPC."
-  type = object({
-    ip_cidr_range = string
-  })
-  default = {
-    ip_cidr_range = "10.0.0.0/20"
-  }
-}

modules/project/service-accounts.tf

@@ -45,6 +45,7 @@ locals {
     # TODO: jit?
     gke-mcs                  = "service-%s@gcp-sa-mcsd"
     monitoring-notifications = "service-%s@gcp-sa-monitoring-notification"
+    notebooks                = "service-%s@gcp-sa-notebooks"
     pubsub                   = "service-%s@gcp-sa-pubsub"
     secretmanager            = "service-%s@gcp-sa-secretmanager"
     sql                      = "service-%s@gcp-sa-cloud-sql"

tests/blueprints/data_solutions/data_playground/test_plan.py

@@ -22,4 +22,4 @@ def test_resources(e2e_plan_runner):
   "Test that plan works and the numbers of resources is as expected."
   modules, resources = e2e_plan_runner(FIXTURES_DIR)
   assert len(modules) == 7
-  assert len(resources) == 37
+  assert len(resources) == 38