FAST: bootstrap and extra stage CI/CD improvements and fixes (#956)

* add clone commands output * always create secret key for repos, fix module source * optional modules ref * tfdoc * create secrets in the right repositories * add publick key to modules repository * bump Terraform version in CI templates * add template to populated files * tfdoc * do not error out writing ci/cd workflows when output files are disabled * update README * fix apply file outputs when outputs_location is changed to null
GoogleCloudPlatform · Nov 8, 2022 · dff7b69 · dff7b69
1 parent 02368df
commit dff7b69
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 28 deletions.
diff --git a/fast/assets/templates/workflow-github.yaml b/fast/assets/templates/workflow-github.yaml
@@ -30,7 +30,7 @@ env:
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
   TF_PROVIDERS_FILE: ${tf_providers_file}
   TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n    ", tf_var_files)}
-  TF_VERSION: 1.1.7
+  TF_VERSION: 1.3.2
 
 jobs:
   fast-pr:

diff --git a/fast/assets/templates/workflow-sourcerepo.yaml b/fast/assets/templates/workflow-sourcerepo.yaml
@@ -95,4 +95,4 @@ substitutions:
   _FAST_OUTPUTS_BUCKET: ${outputs_bucket}
   _TF_PROVIDERS_FILE: ${tf_providers_file}
   _TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n    ", tf_var_files)}
-  _TF_VERSION: 1.1.7
+  _TF_VERSION: 1.3.2
diff --git a/fast/extras/00-cicd-github/README.md b/fast/extras/00-cicd-github/README.md
@@ -8,11 +8,16 @@ This stage is designed for quick repository creation in a GitHub organization, a
 
 Initial file population of repositories is controlled via the `populate_from` attribute, and needs a bit of care:
 
-- never run this stage gain with the same variables used for population once the repository starts being used,  as **Terraform will manage file state and revert any changes at each apply**, which is probably not what you want.
-- be mindful when enabling initial population of the modules repository, as the number of resulting files to manage is very close to the GitHub hourly limit for their API
+- never run this stage with the same variables used for population once the repository starts being used, as **Terraform will manage file state and revert any changes at each apply**, which is probably not what you want.
+- initial population of the modules repository is discouraged, as the number of resulting files Terraform needs to manage is very close to the GitHub hourly limit for their API, it's much easier to populate modules via regular git commands
 
 The scenario for which this stage has been designed is one-shot creation and/or population of stage repositories, running it multiple times with different variables and Terraform states if incremental creation is needed for subsequent FAST stages (e.g. GKE, data platform, etc.).
 
+Once initial population is done, you need to manually push to the repository
+
+- the `.tfvars` file with custom variable values for your stages
+- the workflow configuration file generated by FAST stages
+
 ## GitHub provider credentials
 
 A [GitHub token](https://github.com/settings/tokens) is needed to authenticate against their API. The token needs organization-level permissions, like shown in this screenshot:
@@ -77,16 +82,24 @@ When initial population is configured for a repository, this stage also adds a s
 | name | description | resources |
 |---|---|---|
 | [cicd-versions.tf](./cicd-versions.tf) | Provider version. |  |
-| [main.tf](./main.tf) | Module-level locals and resources. | <code>github_actions_secret</code> · <code>github_repository</code> · <code>github_repository_file</code> · <code>tls_private_key</code> |
+| [main.tf](./main.tf) | Module-level locals and resources. | <code>github_actions_secret</code> · <code>github_repository</code> · <code>github_repository_deploy_key</code> · <code>github_repository_file</code> · <code>tls_private_key</code> |
+| [outputs.tf](./outputs.tf) | Module outputs. |  |
 | [providers.tf](./providers.tf) | Provider configuration. |  |
 | [variables.tf](./variables.tf) | Module variables. |  |
 
 ## Variables
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [organization](variables.tf#L28) | GitHub organization. | <code>string</code> | ✓ |  |
+| [organization](variables.tf#L34) | GitHub organization. | <code>string</code> | ✓ |  |
 | [commmit_config](variables.tf#L17) | Configure commit metadata. | <code title="object&#40;&#123;&#10;  author  &#61; optional&#40;string, &#34;FAST loader&#34;&#41;&#10;  email   &#61; optional&#40;string, &#34;fast-loader&#64;fast.gcp.tf&#34;&#41;&#10;  message &#61; optional&#40;string, &#34;FAST initial loading&#34;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [repositories](variables.tf#L33) | Repositories to create. | <code title="map&#40;object&#40;&#123;&#10;  create_options &#61; optional&#40;object&#40;&#123;&#10;    allow &#61; optional&#40;object&#40;&#123;&#10;      auto_merge   &#61; optional&#40;bool&#41;&#10;      merge_commit &#61; optional&#40;bool&#41;&#10;      rebase_merge &#61; optional&#40;bool&#41;&#10;      squash_merge &#61; optional&#40;bool&#41;&#10;    &#125;&#41;&#41;&#10;    auto_init   &#61; optional&#40;bool&#41;&#10;    description &#61; optional&#40;string&#41;&#10;    features &#61; optional&#40;object&#40;&#123;&#10;      issues   &#61; optional&#40;bool&#41;&#10;      projects &#61; optional&#40;bool&#41;&#10;      wiki     &#61; optional&#40;bool&#41;&#10;    &#125;&#41;&#41;&#10;    templates &#61; optional&#40;object&#40;&#123;&#10;      gitignore &#61; optional&#40;string, &#34;Terraform&#34;&#41;&#10;      license   &#61; optional&#40;string&#41;&#10;      repository &#61; optional&#40;object&#40;&#123;&#10;        name  &#61; string&#10;        owner &#61; string&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;, &#123;&#125;&#41;&#10;    visibility &#61; optional&#40;string, &#34;private&#34;&#41;&#10;  &#125;&#41;&#41;&#10;  has_modules   &#61; optional&#40;bool, false&#41;&#10;  populate_from &#61; optional&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [modules_ref](variables.tf#L28) | Optional git ref used in module sources. | <code>string</code> |  | <code>null</code> |
+| [repositories](variables.tf#L39) | Repositories to create. | <code title="map&#40;object&#40;&#123;&#10;  create_options &#61; optional&#40;object&#40;&#123;&#10;    allow &#61; optional&#40;object&#40;&#123;&#10;      auto_merge   &#61; optional&#40;bool&#41;&#10;      merge_commit &#61; optional&#40;bool&#41;&#10;      rebase_merge &#61; optional&#40;bool&#41;&#10;      squash_merge &#61; optional&#40;bool&#41;&#10;    &#125;&#41;&#41;&#10;    auto_init   &#61; optional&#40;bool&#41;&#10;    description &#61; optional&#40;string&#41;&#10;    features &#61; optional&#40;object&#40;&#123;&#10;      issues   &#61; optional&#40;bool&#41;&#10;      projects &#61; optional&#40;bool&#41;&#10;      wiki     &#61; optional&#40;bool&#41;&#10;    &#125;&#41;&#41;&#10;    templates &#61; optional&#40;object&#40;&#123;&#10;      gitignore &#61; optional&#40;string, &#34;Terraform&#34;&#41;&#10;      license   &#61; optional&#40;string&#41;&#10;      repository &#61; optional&#40;object&#40;&#123;&#10;        name  &#61; string&#10;        owner &#61; string&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;, &#123;&#125;&#41;&#10;    visibility &#61; optional&#40;string, &#34;private&#34;&#41;&#10;  &#125;&#41;&#41;&#10;  has_modules   &#61; optional&#40;bool, false&#41;&#10;  populate_from &#61; optional&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| [clone](outputs.tf#L17) | Clone repository commands. |  |
 
 <!-- END TFDOC -->
diff --git a/fast/extras/00-cicd-github/main.tf b/fast/extras/00-cicd-github/main.tf
@@ -30,6 +30,7 @@ locals {
       }
     ] if v.populate_from != null
   ])
+  modules_ref = var.modules_ref == null ? "" : "?ref=${var.modules_ref}"
   modules_repository = (
     length(local._modules_repository) > 0
     ? local._modules_repository.0
@@ -39,13 +40,24 @@ locals {
     for k, v in var.repositories :
     k => v.create_options == null ? k : github_repository.default[k].name
   }
-  repository_files = {
-    for k in local._repository_files :
-    "${k.repository}/${k.name}" => k
-    if !endswith(k.name, ".tf") || (
-      !startswith(k.name, "0") && k.name != "globals.tf"
-    )
-  }
+  repository_files = merge(
+    {
+      for k in local._repository_files :
+      "${k.repository}/${k.name}" => k
+      if !endswith(k.name, ".tf") || (
+        !startswith(k.name, "0") && k.name != "globals.tf"
+      )
+    },
+    {
+      for k, v in var.repositories :
+      "${k}/templates/providers.tf.tpl" => {
+        repository = k
+        file       = "../../assets/templates/providers.tf.tpl"
+        name       = "templates/providers.tf.tpl"
+      }
+      if v.populate_from != null
+    }
+  )
 }
 
 resource "github_repository" "default" {
@@ -88,15 +100,20 @@ resource "tls_private_key" "default" {
   algorithm = "ED25519"
 }
 
+resource "github_repository_deploy_key" "exdefaultample_repository_deploy_key" {
+  count      = local.modules_repository == null ? 0 : 1
+  title      = "Modules repository access"
+  repository = local.modules_repository
+  key        = tls_private_key.default.0.public_key_openssh
+  read_only  = true
+}
+
 resource "github_actions_secret" "default" {
   for_each = local.modules_repository == null ? {} : {
     for k, v in local.repositories :
-    k => v if(
-      k != local.modules_repository &&
-      var.repositories[k].populate_from != null
-    )
+    k => v if k != local.modules_repository
   }
-  repository      = local.repositories[local.modules_repository]
+  repository      = each.key
   secret_name     = "CICD_MODULES_KEY"
   plaintext_value = tls_private_key.default.0.private_key_openssh
 }
@@ -112,8 +129,8 @@ resource "github_repository_file" "default" {
     endswith(each.value.name, ".tf") && local.modules_repository != null
     ? replace(
       file(each.value.file),
-      "/source\\s*=\\s*\"../../../",
-      "source = \"git@github.com:${var.organization}/${local.modules_repository}.git/"
+      "/source\\s*=\\s*\"../../../modules/([^/\"]+)\"/",
+      "source = \"git@github.com:${var.organization}/${local.modules_repository}.git//$1${local.modules_ref}\"" # "
     )
     : file(each.value.file)
   )

diff --git a/fast/extras/00-cicd-github/outputs.tf b/fast/extras/00-cicd-github/outputs.tf
@@ -0,0 +1,23 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "clone" {
+  description = "Clone repository commands."
+  value = {
+    for k, v in var.repositories :
+    k => "git clone git@github.com:${var.organization}/${k}.git"
+  }
+}
diff --git a/fast/extras/00-cicd-github/variables.tf b/fast/extras/00-cicd-github/variables.tf
@@ -25,6 +25,12 @@ variable "commmit_config" {
   nullable = false
 }
 
+variable "modules_ref" {
+  description = "Optional git ref used in module sources."
+  type        = string
+  default     = null
+}
+
 variable "organization" {
   description = "GitHub organization."
   type        = string

diff --git a/fast/stages/00-bootstrap/outputs-files.tf b/fast/stages/00-bootstrap/outputs-files.tf
@@ -19,27 +19,27 @@
 resource "local_file" "providers" {
   for_each        = var.outputs_location == null ? {} : local.providers
   file_permission = "0644"
-  filename        = "${pathexpand(var.outputs_location)}/providers/${each.key}-providers.tf"
-  content         = each.value
+  filename        = "${try(pathexpand(var.outputs_location), "")}/providers/${each.key}-providers.tf"
+  content         = try(each.value, null)
 }
 
 resource "local_file" "tfvars" {
   for_each        = var.outputs_location == null ? {} : { 1 = 1 }
   file_permission = "0644"
-  filename        = "${pathexpand(var.outputs_location)}/tfvars/00-bootstrap.auto.tfvars.json"
+  filename        = "${try(pathexpand(var.outputs_location), "")}/tfvars/00-bootstrap.auto.tfvars.json"
   content         = jsonencode(local.tfvars)
 }
 
 resource "local_file" "tfvars_globals" {
   for_each        = var.outputs_location == null ? {} : { 1 = 1 }
   file_permission = "0644"
-  filename        = "${pathexpand(var.outputs_location)}/tfvars/globals.auto.tfvars.json"
+  filename        = "${try(pathexpand(var.outputs_location), "")}/tfvars/globals.auto.tfvars.json"
   content         = jsonencode(local.tfvars_globals)
 }
 
 resource "local_file" "workflows" {
-  for_each        = local.cicd_workflows
+  for_each        = var.outputs_location == null ? {} : local.cicd_workflows
   file_permission = "0644"
-  filename        = "${pathexpand(var.outputs_location)}/workflows/${each.key}-workflow.yaml"
-  content         = each.value
+  filename        = "${try(pathexpand(var.outputs_location), "")}/workflows/${each.key}-workflow.yaml"
+  content         = try(each.value, null)
 }