Merge pull request #690 from GoogleCloudPlatform/lcaggio/fix-readme-2…

…0220620 Fix stages tfvars links
GoogleCloudPlatform · Jun 21, 2022 · e0b1231 · e0b1231
2 parents 9691577 + d9ad654
commit e0b1231
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 32 deletions.
diff --git a/fast/stages/02-networking-nva/README.md b/fast/stages/02-networking-nva/README.md
@@ -225,8 +225,8 @@ If you have set a valid value for `outputs_location` in the bootstrap and in the
 
 ```bash
 # `outputs_location` is set to `~/fast-config`
-ln -s ../../configs/example/02-networking/terraform-bootstrap.auto.tfvars.json
-ln -s ../../configs/example/02-networking/terraform-resman.auto.tfvars.json
+ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
 # also copy the tfvars file used for the bootstrap stage
 cp ../00-bootstrap/terraform.tfvars .
 ```

diff --git a/fast/stages/02-networking-peering/README.md b/fast/stages/02-networking-peering/README.md
@@ -155,8 +155,8 @@ If you have set a valid value for `outputs_location` in the bootstrap and in the
 
 ```bash
 # `outputs_location` is set to `~/fast-config`
-ln -s ../../configs/example/02-networking/terraform-bootstrap.auto.tfvars.json
-ln -s ../../configs/example/02-networking/terraform-resman.auto.tfvars.json
+ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
 # also copy the tfvars file used for the bootstrap stage
 cp ../00-bootstrap/terraform.tfvars .
 ```

diff --git a/fast/stages/03-data-platform/dev/README.md b/fast/stages/03-data-platform/dev/README.md
@@ -108,6 +108,14 @@ If you're running this on top of Fast, you should run the following commands to
 ln -s ~/fast-config/providers/03-data-platform-dev-providers.tf .
 ```
 
+If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs:
+
+```bash
+cd ../../01-resman
+terraform output -json providers | jq -r '.["03-data-platform-dev"]' \
+  > ../03-data-platform/dev/providers.tf
+```
+
 ### Variable configuration
 
 There are two broad sets of variables that can be configured:
@@ -124,6 +132,8 @@ If you configured a valid path for `outputs_location` in the bootstrap security
 ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
 ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json . 
 ln -s ~/fast-config/tfvars/02-networking.auto.tfvars.json .
+# also copy the tfvars file used for the bootstrap stage
+cp ../../00-bootstrap/terraform.tfvars .
 ```
 
 If you're not using FAST or its output files, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning.
@@ -146,43 +156,44 @@ You can find examples in the `[demo](../../../../examples/data-solutions/data-pl
 
 ## Files
 
-| name | description | modules |
-|---|---|---|
-| [main.tf](./main.tf) | Data Platformy. | <code>data-platform-foundations</code> |
-| [outputs.tf](./outputs.tf) | Output variables. |  |
-| [variables.tf](./variables.tf) | Terraform Variables. |  |
+| name | description | modules | resources |
+|---|---|---|---|
+| [main.tf](./main.tf) | Data Platformy. | <code>data-platform-foundations</code> |  |
+| [outputs.tf](./outputs.tf) | Output variables. |  | <code>google_storage_bucket_object</code> · <code>local_file</code> |
+| [variables.tf](./variables.tf) | Terraform Variables. |  |  |
 
 ## Variables
 
 | name | description | type | required | default | producer |
 |---|---|:---:|:---:|:---:|:---:|
-| [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object&#40;&#123;&#10;  id              &#61; string&#10;  organization_id &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-globals</code> |
-| [folder_ids](variables.tf#L56) | Folder to be used for the networking resources in folders/nnnn format. | <code title="object&#40;&#123;&#10;  data-platform &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>01-resman</code> |
-| [host_project_ids](variables.tf#L74) | Shared VPC project ids. | <code title="object&#40;&#123;&#10;  dev-spoke-0 &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>02-networking</code> |
-| [organization](variables.tf#L100) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-globals</code> |
-| [prefix](variables.tf#L116) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | <code>string</code> | ✓ |  | <code>00-globals</code> |
-| [composer_config](variables.tf#L26) |  | <code title="object&#40;&#123;&#10;  node_count      &#61; number&#10;  airflow_version &#61; string&#10;  env_variables   &#61; map&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  node_count      &#61; 3&#10;  airflow_version &#61; &#34;composer-1.17.5-airflow-2.1.4&#34;&#10;  env_variables   &#61; &#123;&#125;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
-| [data_catalog_tags](variables.tf#L39) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map&#40;map&#40;list&#40;string&#41;&#41;&#41;</code> |  | <code title="&#123;&#10;  &#34;3_Confidential&#34; &#61; null&#10;  &#34;2_Private&#34;      &#61; null&#10;  &#34;1_Sensitive&#34;    &#61; null&#10;&#125;">&#123;&#8230;&#125;</code> |  |
-| [data_force_destroy](variables.tf#L50) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | <code>bool</code> |  | <code>false</code> |  |
-| [groups](variables.tf#L64) | Groups. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  data-analysts  &#61; &#34;gcp-data-analysts&#34;&#10;  data-engineers &#61; &#34;gcp-data-engineers&#34;&#10;  data-security  &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
-| [network_config_composer](variables.tf#L82) | Network configurations to use for Composer. | <code title="object&#40;&#123;&#10;  cloudsql_range    &#61; string&#10;  gke_master_range  &#61; string&#10;  gke_pods_name     &#61; string&#10;  gke_services_name &#61; string&#10;  web_server_range  &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  cloudsql_range    &#61; &#34;192.168.254.0&#47;24&#34;&#10;  gke_master_range  &#61; &#34;192.168.255.0&#47;28&#34;&#10;  gke_pods_name     &#61; &#34;pods&#34;&#10;  gke_services_name &#61; &#34;services&#34;&#10;  web_server_range  &#61; &#34;192.168.255.16&#47;28&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
-| [outputs_location](variables.tf#L110) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | <code>string</code> |  | <code>null</code> |  |
-| [project_services](variables.tf#L122) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> |  | <code title="&#91;&#10;  &#34;cloudresourcemanager.googleapis.com&#34;,&#10;  &#34;iam.googleapis.com&#34;,&#10;  &#34;serviceusage.googleapis.com&#34;,&#10;  &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |  |
-| [region](variables.tf#L133) | Region used for regional resources. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |  |
-| [service_encryption_keys](variables.tf#L139) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10;  bq       &#61; string&#10;  composer &#61; string&#10;  dataflow &#61; string&#10;  storage  &#61; string&#10;  pubsub   &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
-| [subnet_self_links](variables.tf#L151) | Shared VPC subnet self links. | <code title="object&#40;&#123;&#10;  dev-spoke-0 &#61; map&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> | <code>02-networking</code> |
-| [vpc_self_links](variables.tf#L160) | Shared VPC self links. | <code title="object&#40;&#123;&#10;  dev-spoke-0 &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> | <code>02-networking</code> |
+| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-bootstrap</code> |
+| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object&#40;&#123;&#10;  id              &#61; string&#10;  organization_id &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-globals</code> |
+| [folder_ids](variables.tf#L64) | Folder to be used for the networking resources in folders/nnnn format. | <code title="object&#40;&#123;&#10;  data-platform &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>01-resman</code> |
+| [host_project_ids](variables.tf#L82) | Shared VPC project ids. | <code title="object&#40;&#123;&#10;  dev-spoke-0 &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>02-networking</code> |
+| [organization](variables.tf#L108) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-globals</code> |
+| [prefix](variables.tf#L124) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | <code>string</code> | ✓ |  | <code>00-globals</code> |
+| [composer_config](variables.tf#L34) |  | <code title="object&#40;&#123;&#10;  node_count      &#61; number&#10;  airflow_version &#61; string&#10;  env_variables   &#61; map&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  node_count      &#61; 3&#10;  airflow_version &#61; &#34;composer-1.17.5-airflow-2.1.4&#34;&#10;  env_variables   &#61; &#123;&#125;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
+| [data_catalog_tags](variables.tf#L47) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map&#40;map&#40;list&#40;string&#41;&#41;&#41;</code> |  | <code title="&#123;&#10;  &#34;3_Confidential&#34; &#61; null&#10;  &#34;2_Private&#34;      &#61; null&#10;  &#34;1_Sensitive&#34;    &#61; null&#10;&#125;">&#123;&#8230;&#125;</code> |  |
+| [data_force_destroy](variables.tf#L58) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | <code>bool</code> |  | <code>false</code> |  |
+| [groups](variables.tf#L72) | Groups. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  data-analysts  &#61; &#34;gcp-data-analysts&#34;&#10;  data-engineers &#61; &#34;gcp-data-engineers&#34;&#10;  data-security  &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
+| [network_config_composer](variables.tf#L90) | Network configurations to use for Composer. | <code title="object&#40;&#123;&#10;  cloudsql_range    &#61; string&#10;  gke_master_range  &#61; string&#10;  gke_pods_name     &#61; string&#10;  gke_services_name &#61; string&#10;  web_server_range  &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  cloudsql_range    &#61; &#34;192.168.254.0&#47;24&#34;&#10;  gke_master_range  &#61; &#34;192.168.255.0&#47;28&#34;&#10;  gke_pods_name     &#61; &#34;pods&#34;&#10;  gke_services_name &#61; &#34;services&#34;&#10;  web_server_range  &#61; &#34;192.168.255.16&#47;28&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
+| [outputs_location](variables.tf#L118) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | <code>string</code> |  | <code>null</code> |  |
+| [project_services](variables.tf#L130) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> |  | <code title="&#91;&#10;  &#34;cloudresourcemanager.googleapis.com&#34;,&#10;  &#34;iam.googleapis.com&#34;,&#10;  &#34;serviceusage.googleapis.com&#34;,&#10;  &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |  |
+| [region](variables.tf#L141) | Region used for regional resources. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |  |
+| [service_encryption_keys](variables.tf#L147) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10;  bq       &#61; string&#10;  composer &#61; string&#10;  dataflow &#61; string&#10;  storage  &#61; string&#10;  pubsub   &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
+| [subnet_self_links](variables.tf#L159) | Shared VPC subnet self links. | <code title="object&#40;&#123;&#10;  dev-spoke-0 &#61; map&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> | <code>02-networking</code> |
+| [vpc_self_links](variables.tf#L168) | Shared VPC self links. | <code title="object&#40;&#123;&#10;  dev-spoke-0 &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> | <code>02-networking</code> |
 
 ## Outputs
 
 | name | description | sensitive | consumers |
 |---|---|:---:|---|
-| [bigquery_datasets](outputs.tf#L17) | BigQuery datasets. |  |  |
-| [demo_commands](outputs.tf#L47) | Demo commands. |  |  |
-| [gcs_buckets](outputs.tf#L22) | GCS buckets. |  |  |
-| [kms_keys](outputs.tf#L27) | Cloud MKS keys. |  |  |
-| [projects](outputs.tf#L32) | GCP Projects informations. |  |  |
-| [vpc_network](outputs.tf#L37) | VPC network. |  |  |
-| [vpc_subnet](outputs.tf#L42) | VPC subnetworks. |  |  |
+| [bigquery_datasets](outputs.tf#L42) | BigQuery datasets. |  |  |
+| [demo_commands](outputs.tf#L72) | Demo commands. |  |  |
+| [gcs_buckets](outputs.tf#L47) | GCS buckets. |  |  |
+| [kms_keys](outputs.tf#L52) | Cloud MKS keys. |  |  |
+| [projects](outputs.tf#L57) | GCP Projects informations. |  |  |
+| [vpc_network](outputs.tf#L62) | VPC network. |  |  |
+| [vpc_subnet](outputs.tf#L67) | VPC subnetworks. |  |  |
 
 <!-- END TFDOC -->
diff --git a/fast/stages/03-data-platform/dev/outputs.tf b/fast/stages/03-data-platform/dev/outputs.tf
@@ -14,6 +14,31 @@
 
 # tfdoc:file:description Output variables.
 
+locals {
+  tfvars = {
+    bigquery_dataset = module.data-platform.bigquery-datasets
+    gcs_buckets      = module.data-platform.gcs-buckets
+    projects         = module.data-platform.projects
+  }
+}
+
+# generate tfvars file for subsequent stages
+
+resource "local_file" "tfvars" {
+  for_each        = var.outputs_location == null ? {} : { 1 = 1 }
+  file_permission = "0644"
+  filename        = "${pathexpand(var.outputs_location)}/tfvars/03-data-platform-dev.auto.tfvars.json"
+  content         = jsonencode(local.tfvars)
+}
+
+resource "google_storage_bucket_object" "tfvars" {
+  bucket  = var.automation.outputs_bucket
+  name    = "tfvars/03-data-platform-dev.auto.tfvars.json"
+  content = jsonencode(local.tfvars)
+}
+
+# outputs
+
 output "bigquery_datasets" {
   description = "BigQuery datasets."
   value       = module.data-platform.bigquery-datasets

diff --git a/fast/stages/03-data-platform/dev/variables.tf b/fast/stages/03-data-platform/dev/variables.tf
@@ -14,6 +14,14 @@
 
 # tfdoc:file:description Terraform Variables.
 
+variable "automation" {
+  # tfdoc:variable:source 00-bootstrap
+  description = "Automation resources created by the bootstrap stage."
+  type = object({
+    outputs_bucket = string
+  })
+}
+
 variable "billing_account" {
   # tfdoc:variable:source 00-globals
   description = "Billing account id and organization id ('nnnnnnnn' or null)."

diff --git a/tests/fast/stages/s03_data_platform/fixture/main.tf b/tests/fast/stages/s03_data_platform/fixture/main.tf
@@ -18,6 +18,9 @@
 
 module "stage" {
   source = "../../../../../fast/stages/03-data-platform/dev/"
+  automation = {
+    outputs_bucket = "test"
+  }
   billing_account = {
     id              = "012345-67890A-BCDEF0",
     organization_id = 123456