Skip to content

Commit

Permalink
bug fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@Francesco-cloud24
Francesco Spinelli authored and Francesco-cloud24 committed Nov 9, 2023
1 parent 03937f2 commit ddbcc3f
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 7 deletions.
2 changes: 1 addition & 1 deletion modules/dataproc/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ module "processing-dp-cluster" {
| [name](variables.tf#L235) | Cluster name. | <code>string</code> | ✓ | |
| [project_id](variables.tf#L250) | Project ID. | <code>string</code> | ✓ | |
| [region](variables.tf#L255) | Dataproc region. | <code>string</code> | ✓ | |
| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | <code title="object&#40;&#123;&#10; graceful_decommission_timeout &#61; optional&#40;string&#41;&#10; cluster_config &#61; optional&#40;object&#40;&#123;&#10; staging_bucket &#61; optional&#40;string&#41;&#10; temp_bucket &#61; optional&#40;string&#41;&#10; gce_cluster_config &#61; optional&#40;object&#40;&#123;&#10; zone &#61; optional&#40;string&#41;&#10; network &#61; optional&#40;string&#41;&#10; subnetwork &#61; optional&#40;string&#41;&#10; service_account &#61; optional&#40;string&#41;&#10; service_account_scopes &#61; optional&#40;list&#40;string&#41;&#41;&#10; tags &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; internal_ip_only &#61; optional&#40;bool&#41;&#10; metadata &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; reservation_affinity &#61; optional&#40;object&#40;&#123;&#10; consume_reservation_type &#61; string&#10; key &#61; string&#10; values &#61; string&#10; &#125;&#41;&#41;&#10; node_group_affinity &#61; optional&#40;object&#40;&#123;&#10; node_group_uri &#61; string&#10; &#125;&#41;&#41;&#10;&#10;&#10; shielded_instance_config &#61; optional&#40;object&#40;&#123;&#10; enable_secure_boot &#61; bool&#10; enable_vtpm &#61; bool&#10; enable_integrity_monitoring &#61; bool&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; master_config &#61; optional&#40;object&#40;&#123;&#10; num_instances &#61; number&#10; machine_type &#61; string&#10; min_cpu_platform &#61; string&#10; image_uri &#61; string&#10; disk_config &#61; optional&#40;object&#40;&#123;&#10; boot_disk_type &#61; string&#10; boot_disk_size_gb &#61; number&#10; num_local_ssds &#61; number&#10; &#125;&#41;&#41;&#10; accelerators &#61; optional&#40;object&#40;&#123;&#10; accelerator_type &#61; string&#10; accelerator_count &#61; number&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; worker_config &#61; optional&#40;object&#40;&#123;&#10; num_instances &#61; number&#10; machine_type &#61; string&#10; min_cpu_platform &#61; string&#10; disk_config &#61; optional&#40;object&#40;&#123;&#10; boot_disk_type &#61; string&#10; boot_disk_size_gb &#61; number&#10; num_local_ssds &#61; number&#10; &#125;&#41;&#41;&#10; image_uri &#61; string&#10; accelerators &#61; optional&#40;object&#40;&#123;&#10; accelerator_type &#61; string&#10; accelerator_count &#61; number&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; preemptible_worker_config &#61; optional&#40;object&#40;&#123;&#10; num_instances &#61; number&#10; preemptibility &#61; string&#10; disk_config &#61; optional&#40;object&#40;&#123;&#10; boot_disk_type &#61; string&#10; boot_disk_size_gb &#61; number&#10; num_local_ssds &#61; number&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; software_config &#61; optional&#40;object&#40;&#123;&#10; image_version &#61; optional&#40;string&#41;&#10; override_properties &#61; map&#40;string&#41;&#10; optional_components &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;&#10; security_config &#61; optional&#40;object&#40;&#123;&#10; kerberos_config &#61; object&#40;&#123;&#10; cross_realm_trust_admin_server &#61; optional&#40;string&#41;&#10; cross_realm_trust_kdc &#61; optional&#40;string&#41;&#10; cross_realm_trust_realm &#61; optional&#40;string&#41;&#10; cross_realm_trust_shared_password_uri &#61; optional&#40;string&#41;&#10; enable_kerberos &#61; optional&#40;string&#41;&#10; kdc_db_key_uri &#61; optional&#40;string&#41;&#10; key_password_uri &#61; optional&#40;string&#41;&#10; keystore_uri &#61; optional&#40;string&#41;&#10; keystore_password_uri &#61; optional&#40;string&#41;&#10; kms_key_uri &#61; string&#10; realm &#61; optional&#40;string&#41;&#10; root_principal_password_uri &#61; string&#10; tgt_lifetime_hours &#61; optional&#40;string&#41;&#10; truststore_password_uri &#61; optional&#40;string&#41;&#10; truststore_uri &#61; optional&#40;string&#41;&#10; &#125;&#41;&#10; &#125;&#41;&#41;&#10; autoscaling_config &#61; optional&#40;object&#40;&#123;&#10; policy_uri &#61; string&#10; &#125;&#41;&#41;&#10; initialization_action &#61; optional&#40;object&#40;&#123;&#10; script &#61; string&#10; timeout_sec &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10; encryption_config &#61; optional&#40;object&#40;&#123;&#10; kms_key_name &#61; string&#10; &#125;&#41;&#41;&#10; lifecycle_config &#61; optional&#40;object&#40;&#123;&#10; idle_delete_ttl &#61; optional&#40;string&#41;&#10; auto_delete_time &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10; endpoint_config &#61; optional&#40;object&#40;&#123;&#10; enable_http_port_access &#61; string&#10; &#125;&#41;&#41;&#10; dataproc_metric_config &#61; optional&#40;object&#40;&#123;&#10; metrics &#61; list&#40;object&#40;&#123;&#10; metric_source &#61; string&#10; metric_overrides &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; metastore_config &#61; optional&#40;object&#40;&#123;&#10; dataproc_metastore_service &#61; string&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10;&#10;&#10; virtual_cluster_config &#61; optional&#40;object&#40;&#123;&#10; staging_bucket &#61; optional&#40;string&#41;&#10; auxiliary_services_config &#61; optional&#40;object&#40;&#123;&#10; metastore_config &#61; optional&#40;object&#40;&#123;&#10; dataproc_metastore_service &#61; string&#10; &#125;&#41;&#41;&#10; spark_history_server_config &#61; optional&#40;object&#40;&#123;&#10; dataproc_cluster &#61; string&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; kubernetes_cluster_config &#61; object&#40;&#123;&#10; kubernetes_namespace &#61; optional&#40;string&#41;&#10; kubernetes_software_config &#61; object&#40;&#123;&#10; component_version &#61; list&#40;map&#40;string&#41;&#41;&#10; properties &#61; optional&#40;list&#40;map&#40;string&#41;&#41;&#41;&#10; &#125;&#41;&#10;&#10;&#10; gke_cluster_config &#61; object&#40;&#123;&#10; gke_cluster_target &#61; optional&#40;string&#41;&#10; node_pool_target &#61; optional&#40;object&#40;&#123;&#10; node_pool &#61; string&#10; roles &#61; list&#40;string&#41;&#10; node_pool_config &#61; optional&#40;object&#40;&#123;&#10; autoscaling &#61; optional&#40;object&#40;&#123;&#10; min_node_count &#61; optional&#40;number&#41;&#10; max_node_count &#61; optional&#40;number&#41;&#10; &#125;&#41;&#41;&#10;&#10;&#10; config &#61; object&#40;&#123;&#10; machine_type &#61; optional&#40;string&#41;&#10; preemptible &#61; optional&#40;bool&#41;&#10; local_ssd_count &#61; optional&#40;number&#41;&#10; min_cpu_platform &#61; optional&#40;string&#41;&#10; spot &#61; optional&#40;bool&#41;&#10; &#125;&#41;&#10;&#10;&#10; locations &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#10; &#125;&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | <code title="object&#40;&#123;&#10; graceful_decommission_timeout &#61; optional&#40;string&#41;&#10; cluster_config &#61; optional&#40;object&#40;&#123;&#10; staging_bucket &#61; optional&#40;string&#41;&#10; temp_bucket &#61; optional&#40;string&#41;&#10; gce_cluster_config &#61; optional&#40;object&#40;&#123;&#10; zone &#61; optional&#40;string&#41;&#10; network &#61; optional&#40;string&#41;&#10; subnetwork &#61; optional&#40;string&#41;&#10; service_account &#61; optional&#40;string&#41;&#10; service_account_scopes &#61; optional&#40;list&#40;string&#41;&#41;&#10; tags &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; internal_ip_only &#61; optional&#40;bool&#41;&#10; metadata &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; reservation_affinity &#61; optional&#40;object&#40;&#123;&#10; consume_reservation_type &#61; string&#10; key &#61; string&#10; values &#61; string&#10; &#125;&#41;&#41;&#10; node_group_affinity &#61; optional&#40;object&#40;&#123;&#10; node_group_uri &#61; string&#10; &#125;&#41;&#41;&#10;&#10;&#10; shielded_instance_config &#61; optional&#40;object&#40;&#123;&#10; enable_secure_boot &#61; bool&#10; enable_vtpm &#61; bool&#10; enable_integrity_monitoring &#61; bool&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; master_config &#61; optional&#40;object&#40;&#123;&#10; num_instances &#61; number&#10; machine_type &#61; string&#10; min_cpu_platform &#61; string&#10; image_uri &#61; string&#10; disk_config &#61; optional&#40;object&#40;&#123;&#10; boot_disk_type &#61; string&#10; boot_disk_size_gb &#61; number&#10; num_local_ssds &#61; number&#10; &#125;&#41;&#41;&#10; accelerators &#61; optional&#40;object&#40;&#123;&#10; accelerator_type &#61; string&#10; accelerator_count &#61; number&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; worker_config &#61; optional&#40;object&#40;&#123;&#10; num_instances &#61; number&#10; machine_type &#61; string&#10; min_cpu_platform &#61; string&#10; disk_config &#61; optional&#40;object&#40;&#123;&#10; boot_disk_type &#61; string&#10; boot_disk_size_gb &#61; number&#10; num_local_ssds &#61; number&#10; &#125;&#41;&#41;&#10; image_uri &#61; string&#10; accelerators &#61; optional&#40;object&#40;&#123;&#10; accelerator_type &#61; string&#10; accelerator_count &#61; number&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; preemptible_worker_config &#61; optional&#40;object&#40;&#123;&#10; num_instances &#61; number&#10; preemptibility &#61; string&#10; disk_config &#61; optional&#40;object&#40;&#123;&#10; boot_disk_type &#61; string&#10; boot_disk_size_gb &#61; number&#10; num_local_ssds &#61; number&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; software_config &#61; optional&#40;object&#40;&#123;&#10; image_version &#61; optional&#40;string&#41;&#10; override_properties &#61; map&#40;string&#41;&#10; optional_components &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;&#10; security_config &#61; optional&#40;object&#40;&#123;&#10; kerberos_config &#61; object&#40;&#123;&#10; cross_realm_trust_admin_server &#61; optional&#40;string&#41;&#10; cross_realm_trust_kdc &#61; optional&#40;string&#41;&#10; cross_realm_trust_realm &#61; optional&#40;string&#41;&#10; cross_realm_trust_shared_password_uri &#61; optional&#40;string&#41;&#10; enable_kerberos &#61; optional&#40;string&#41;&#10; kdc_db_key_uri &#61; optional&#40;string&#41;&#10; key_password_uri &#61; optional&#40;string&#41;&#10; keystore_uri &#61; optional&#40;string&#41;&#10; keystore_password_uri &#61; optional&#40;string&#41;&#10; kms_key_uri &#61; string&#10; realm &#61; optional&#40;string&#41;&#10; root_principal_password_uri &#61; string&#10; tgt_lifetime_hours &#61; optional&#40;string&#41;&#10; truststore_password_uri &#61; optional&#40;string&#41;&#10; truststore_uri &#61; optional&#40;string&#41;&#10; &#125;&#41;&#10; &#125;&#41;&#41;&#10; autoscaling_config &#61; optional&#40;object&#40;&#123;&#10; policy_uri &#61; string&#10; &#125;&#41;&#41;&#10; initialization_action &#61; optional&#40;object&#40;&#123;&#10; script &#61; string&#10; timeout_sec &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10; encryption_config &#61; optional&#40;object&#40;&#123;&#10; kms_key_name &#61; string&#10; &#125;&#41;&#41;&#10; lifecycle_config &#61; optional&#40;object&#40;&#123;&#10; idle_delete_ttl &#61; optional&#40;string&#41;&#10; auto_delete_time &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10; endpoint_config &#61; optional&#40;object&#40;&#123;&#10; enable_http_port_access &#61; string&#10; &#125;&#41;&#41;&#10; dataproc_metric_config &#61; optional&#40;object&#40;&#123;&#10; metrics &#61; list&#40;object&#40;&#123;&#10; metric_source &#61; string&#10; metric_overrides &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; metastore_config &#61; optional&#40;object&#40;&#123;&#10; dataproc_metastore_service &#61; string&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10;&#10;&#10; virtual_cluster_config &#61; optional&#40;object&#40;&#123;&#10; staging_bucket &#61; optional&#40;string&#41;&#10; auxiliary_services_config &#61; optional&#40;object&#40;&#123;&#10; metastore_config &#61; optional&#40;object&#40;&#123;&#10; dataproc_metastore_service &#61; string&#10; &#125;&#41;&#41;&#10; spark_history_server_config &#61; optional&#40;object&#40;&#123;&#10; dataproc_cluster &#61; string&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; kubernetes_cluster_config &#61; object&#40;&#123;&#10; kubernetes_namespace &#61; optional&#40;string&#41;&#10; kubernetes_software_config &#61; object&#40;&#123;&#10; component_version &#61; list&#40;map&#40;string&#41;&#41;&#10; properties &#61; optional&#40;list&#40;map&#40;string&#41;&#41;&#41;&#10; &#125;&#41;&#10;&#10;&#10; gke_cluster_config &#61; object&#40;&#123;&#10; gke_cluster_target &#61; optional&#40;string&#41;&#10; node_pool_target &#61; optional&#40;object&#40;&#123;&#10; node_pool &#61; string&#10; roles &#61; list&#40;string&#41;&#10; node_pool_config &#61; optional&#40;object&#40;&#123;&#10; autoscaling &#61; optional&#40;object&#40;&#123;&#10; min_node_count &#61; optional&#40;number&#41;&#10; max_node_count &#61; optional&#40;number&#41;&#10; &#125;&#41;&#41;&#10;&#10;&#10; config &#61; object&#40;&#123;&#10; machine_type &#61; optional&#40;string&#41;&#10; preemptible &#61; optional&#40;bool&#41;&#10; local_ssd_count &#61; optional&#40;number&#41;&#10; min_cpu_platform &#61; optional&#40;string&#41;&#10; spot &#61; optional&#40;bool&#41;&#10; &#125;&#41;&#10;&#10;&#10; locations &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#10; &#125;&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
| [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam_bindings](variables.tf#L199) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10; members &#61; list&#40;string&#41;&#10; role &#61; string&#10; condition &#61; optional&#40;object&#40;&#123;&#10; expression &#61; string&#10; title &#61; string&#10; description &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
Expand Down
10 changes: 5 additions & 5 deletions modules/dataproc/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ resource "google_dataproc_cluster" "cluster" {
dynamic "accelerators" {
for_each = var.dataproc_config.cluster_config.worker_config.accelerators == null ? [] : [""]
content {
accelerator_type = var.dataproc_config.cluster_config.accelerators.accelerator_type
accelerator_count = var.dataproc_config.cluster_config.accelerators.accelerator_count
accelerator_type = var.dataproc_config.cluster_config.worker_config.accelerators.accelerator_type
accelerator_count = var.dataproc_config.cluster_config.worker_config.accelerators.accelerator_count
}
}
}
Expand Down Expand Up @@ -185,10 +185,10 @@ resource "google_dataproc_cluster" "cluster" {
for_each = var.dataproc_config.cluster_config.dataproc_metric_config == null ? [] : [""]
content {
dynamic "metrics" {
for_each = var.dataproc_config.cluster_config.dataproc_metric_config.metrics == null ? [] : [""]
for_each = var.dataproc_config.cluster_config.dataproc_metric_config.metrics == null ? [] : var.dataproc_config.cluster_config.dataproc_metric_config.metrics
content {
metric_source = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_source
metric_overrides = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_overrides
metric_source = metrics.value.metric_source
metric_overrides = metrics.value.metric_overrides
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion modules/dataproc/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ variable "dataproc_config" {
dataproc_metric_config = optional(object({
metrics = list(object({
metric_source = string
metric_overrides = optional(string)
metric_overrides = optional(list(string))
}))
}))
metastore_config = optional(object({
Expand Down

0 comments on commit ddbcc3f

Please sign in to comment.