Dataproc module bug fix (#1848)

* bug fix * bug fix * fix dinamic for_each --------- Co-authored-by: Francesco Spinelli <francesco.spinelli@nttdata.com>
GoogleCloudPlatform · Nov 9, 2023 · 82c74e4 · 82c74e4
1 parent 03937f2
commit 82c74e4
Show file tree

Hide file tree

Showing 7 changed files with 76 additions and 136 deletions.
diff --git a/blueprints/factories/project-factory/README.md b/blueprints/factories/project-factory/README.md
@@ -29,10 +29,6 @@ In addition to the yaml files describing projects, the project factory accepts t
 
 Some examples on where to use each of the three sets are provided below.
 
-## Overriding project id
-
-By default, the project id is derived from the file path relative to the data root. Path slashes are replaced with dashes to allow for multilevel data folders (e.g. `app01/fe.yaml`, `app02/fe.yaml`, etc.). The project id can be overridden by specifying a custom `name` attribute in the yaml file. Refer to the example below for details.
-
 ## Example
 
 ```hcl
@@ -61,7 +57,7 @@ module "project-factory" {
   # location where the yaml files are read from
   factory_data_path = "data"
 }
-# tftest modules=8 resources=32 files=app01,app02-be,app03-be,app03-fe inventory=example.yaml
+# tftest modules=7 resources=26 files=prj-app-1,prj-app-2,prj-app-3 inventory=example.yaml
 ```
 
 ```yaml
@@ -84,12 +80,12 @@ service_accounts:
   app-1-fe:
     display_name: "Test app 1 frontend."
 
-# tftest-file id=app01 path=data/app01.yaml
+# tftest-file id=prj-app-1 path=data/prj-app-1.yaml
 ```
 
 ```yaml
 labels:
- app: app02
+ app: app-2
  team: foo
 parent: folders/12345678
 service_accounts:
@@ -101,29 +97,17 @@ services:
 shared_vpc_service_config:
   host_project: foo-host
 
-# tftest-file id=app02-be path=data/app02/be.yaml
+# tftest-file id=prj-app-2 path=data/prj-app-2.yaml
 ```
 
 ```yaml
-name: app03-be-0
 parent: folders/12345678
 services:
 - run.googleapis.com
 - storage.googleapis.com
 
-# tftest-file id=app03-be path=data/app03/be.yaml
+# tftest-file id=prj-app-3 path=data/prj-app-3.yaml
 ```
-
-```yaml
-name: app03-fe-0
-parent: folders/12345678
-services:
-- run.googleapis.com
-- storage.googleapis.com
-
-# tftest-file id=app03-fe path=data/app03/fe.yaml
-```
-
 <!-- BEGIN TFDOC -->
 ## Variables
 

diff --git a/blueprints/factories/project-factory/factory.tf b/blueprints/factories/project-factory/factory.tf
@@ -44,7 +44,6 @@ locals {
         try(v.metric_scopes, null),
         var.data_defaults.metric_scopes
       )
-      name         = try(v.name, replace(k, "/", "-"))
       org_policies = try(v.org_policies, {})
       parent = coalesce(
         var.data_overrides.parent,

diff --git a/blueprints/factories/project-factory/main.tf b/blueprints/factories/project-factory/main.tf
@@ -18,7 +18,7 @@ module "projects" {
   source              = "../../../modules/project"
   for_each            = local.projects
   billing_account     = each.value.billing_account
-  name                = each.value.name
+  name                = each.key
   parent              = try(each.value.parent, null)
   prefix              = each.value.prefix
   auto_create_network = try(each.value.auto_create_network, false)

diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md
diff --git a/modules/dataproc/main.tf b/modules/dataproc/main.tf
@@ -108,8 +108,8 @@ resource "google_dataproc_cluster" "cluster" {
           dynamic "accelerators" {
             for_each = var.dataproc_config.cluster_config.worker_config.accelerators == null ? [] : [""]
             content {
-              accelerator_type  = var.dataproc_config.cluster_config.accelerators.accelerator_type
-              accelerator_count = var.dataproc_config.cluster_config.accelerators.accelerator_count
+              accelerator_type  = var.dataproc_config.cluster_config.worker_config.accelerators.accelerator_type
+              accelerator_count = var.dataproc_config.cluster_config.worker_config.accelerators.accelerator_count
             }
           }
         }
@@ -185,10 +185,10 @@ resource "google_dataproc_cluster" "cluster" {
         for_each = var.dataproc_config.cluster_config.dataproc_metric_config == null ? [] : [""]
         content {
           dynamic "metrics" {
-            for_each = var.dataproc_config.cluster_config.dataproc_metric_config.metrics == null ? [] : [""]
+            for_each = coalesce(var.dataproc_config.cluster_config.dataproc_metric_config.metrics, [])
             content {
-              metric_source    = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_source
-              metric_overrides = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_overrides
+              metric_source    = metrics.value.metric_source
+              metric_overrides = metrics.value.metric_overrides
             }
           }
         }

diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf
@@ -128,7 +128,7 @@ variable "dataproc_config" {
       dataproc_metric_config = optional(object({
         metrics = list(object({
           metric_source    = string
-          metric_overrides = optional(string)
+          metric_overrides = optional(list(string))
         }))
       }))
       metastore_config = optional(object({