Terraform GH action automation & bootstraping

.github/workflows/auto-plan.yml

@@ -0,0 +1,122 @@
+name: Terraform Plan
+
+# created action based on the following guides:
+# - https://learn.hashicorp.com/tutorials/terraform/github-actions
+# - https://github.com/marketplace/actions/hashicorp-setup-terraform#usage
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+  workflow_dispatch:
+
+
+defaults:
+  run:
+    # temporary till I write a script to find all terraform files and apply/plan them all
+    #   based off of this: git diff <branch_name> --name-only | grep '.tf$' | xargs -I {} dirname '{}' | sort -u
+    working-directory: terraform/dns/
+env:
+  AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+  AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" 
+  CLOUDFLARE_API_TOKEN: "${{ secrets.CLOUDFLARE_API_TOKEN }}" 
+
+jobs:
+  terraform_lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: hashicorp/setup-terraform@v2
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -check
+        # TODO: should be removed after files are formatted
+        continue-on-error: true
+
+  terraform_plan:
+    needs: terraform_lint
+    runs-on: ubuntu-latest
+    environment: production
+    steps:
+      - uses: actions/checkout@v2
+      - uses: hashicorp/setup-terraform@v2
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+
+      - name: Terraform Plan
+        id: plan
+        run: terraform plan -no-color -out=terraform.plan
+        continue-on-error: true
+
+      - uses: actions/github-script@v6
+        if: github.event_name == 'pull_request'
+        env:
+          PLAN: "terraform\n${{ steps.plan.outputs.stdout }}"
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // 1. Retrieve existing bot comments for the PR
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            })
+            const botComment = comments.find(comment => {
+              return comment.user.type === 'Bot' && comment.body.includes('Terraform Format and Style')
+            })
+
+            // 2. Prepare format of the comment
+            const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+            #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+            #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
+            <details><summary>Validation Output</summary>
+
+            \`\`\`\n
+            ${{ steps.validate.outputs.stdout }}
+            \`\`\`
+
+            </details>
+
+            #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+
+            <details><summary>Show Plan</summary>
+
+            \`\`\`\n
+            ${process.env.PLAN}
+            \`\`\`
+
+            </details>
+
+            *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Working Directory: \`${{ env.tf_actions_working_dir }}\`, Workflow: \`${{ github.workflow }}\`*`;
+
+            // 3. If we have a comment, update it, otherwise create a new one
+            if (botComment) {
+              github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: output
+              })
+            } else {
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: output
+              })
+            }
+
+      - name: Terraform Plan Status
+        if: steps.plan.outcome == 'failure'
+        run: exit 1
+
+      - name: Terraform Apply
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        run: terraform apply -auto-approve -input=false terraform.plan

.gitignore

@@ -2,12 +2,16 @@
 galaxy_roles/
 .vscode
 
-# Created by https://www.toptal.com/developers/gitignore/api/linux,macos,visualstudiocode,ansible,terraform
-# Edit at https://www.toptal.com/developers/gitignore?templates=linux,macos,visualstudiocode,ansible,terraform
+# Created by https://www.toptal.com/developers/gitignore/api/linux,macos,ansible,terraform,visualstudiocode,direnv
+# Edit at https://www.toptal.com/developers/gitignore?templates=linux,macos,ansible,terraform,visualstudiocode,direnv
 
 ### Ansible ###
 *.retry
 
+### direnv ###
+.direnv
+.envrc
+
 ### Linux ###
 *~
 
@@ -51,6 +55,10 @@ Network Trash Folder
 Temporary Items
 .apdisk
 
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
 ### Terraform ###
 # Local .terraform directories
 **/.terraform/*
@@ -61,12 +69,14 @@ Temporary Items
 
 # Crash log files
 crash.log
+crash.*.log
 
-# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
-# .tfvars files are managed as part of configuration and so should be included in
-# version control.
-#
-# example.tfvars
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
 
 # Ignore override files as they are usually used to override resources locally and so
 # are not checked in
@@ -81,17 +91,33 @@ override.tf.json
 # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
 # example: *tfplan*
 
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+
 ### VisualStudioCode ###
 .vscode/*
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
-*.code-workspace
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
 
 ### VisualStudioCode Patch ###
 # Ignore all local history of files
 .history
+.ionide
+
+# Support for Project snippet scope
+.vscode/*.code-snippets
+
+# Ignore code-workspaces
+*.code-workspace
 
-# End of https://www.toptal.com/developers/gitignore/api/linux,macos,visualstudiocode,ansible,terraform
-.terraform.lock.hcl
+# End of https://www.toptal.com/developers/gitignore/api/linux,macos,ansible,terraform,visualstudiocode,direnv

terraform/.envrc.template

@@ -0,0 +1,2 @@
+export AWS_ACCESS_KEY_ID='<Linode_s3_access_key>' \
+    AWS_SECRET_ACCESS_KEY='<Linode_s3_secret_key>'

terraform/README.md

@@ -0,0 +1,38 @@
+# Overview
+
+## In this repo
+
+There are accompanying README.md files in the different terraform directories.
+
+They in general cover:
+
+- what terraform is
+- how to setup the local development environment so it's similar to the CI
+- the current required CI environment variables needing to be setup in the GH repo's settings
+
+## What is terraform?
+
+[![what is terraform](http://img.youtube.com/vi/HmxkYNv1ksg/0.jpg)](https://youtu.be/HmxkYNv1ksg "What is terraform")
+
+## Setup
+
+### Local Development
+
+- Install terraform: <https://learn.hashicorp.com/tutorials/terraform/install-cli>
+- Install direnv: <https://direnv.net/#basic-installation>
+  - :arrow_up: is used to help [export necessary environment variables](https://direnv.net/#how-it-works).
+- Copy all the .envrc.template files to .envrc
+- Fill out information you can (initial bootstrap will just be [Linode PAT](bootstrap/README.md#linode-pat))
+- When following a guide, make sure you cd into the guide's directory
+- do a `direnv allow .` (after you've looked at the file to check for malicious code), after you've filled out the .envrc file with the needed information
+
+### CI
+
+1. Run the [bootstrap steps](bootstrap/README.md) locally, so you can bootstrap terraform's remote state
+2. Define necessary [environment variables](https://blog.elreydetoda.site/github-action-security/)
+3. API tokens needed so far are:
+    - AWS_ACCESS_KEY_ID = your linode s3 access key (follow the [bootstrap notes](bootstrap/README.md) to get this)
+    - AWS_SECRET_ACCESS_KEY = your linode s3 secret key (follow the [bootstrap notes](bootstrap/README.md) to get this)
+    - CLOUDFLARE_API_KEY = your cloudflare API key (guide [here](dns/README.md#cloudflare-api-key) on how to create)
+
+([here](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions) are some security guidelines from GitHub on hardening GitHub Actions (also environment variables).)

terraform/bootstrap/.envrc.template

@@ -0,0 +1 @@
+export LINODE_TOKEN=''

terraform/bootstrap/README.md

@@ -0,0 +1,24 @@
+# Overview
+
+This will create the initial Linode's S3 object storage, which terraform will use ([S3 docs](https://www.terraform.io/docs/backends/types/s3.html)) for hosting the remote [terraform state](https://www.terraform.io/language/settings/backends/configuration).
+
+## Setup
+
+### Local Development
+
+Follow steps from [here](../README.md#local-development).
+
+### Linode PAT
+
+Generate a [Linode PAT (Personal Access Token)](https://www.linode.com/docs/products/tools/linode-api/guides/get-access-token/)
+
+## Bootstrap
+
+**NOTE:** Only run this locally! It's only meant for provisioning the initial bootstrap for terraform's remote state.
+
+1. After doing the [local setup](#local-development), you can then run `terraform plan` (if you want to see what it'll create)
+2. `terraform apply` and type `yes` if everything looks good
+3. To grab the access & secret key you can run the following command: `terraform output -raw bucket_access_key && echo && terraform output -raw bucket_secret_key`
+4. Then this guide tells you how you need to format your backend.tf: <https://adriano.fyi/post/2020-05-29-how-to-use-linode-object-storage-as-a-terraform-backend/>
+    - an example is [here](../dns/backend.tf)
+5. Fill in the .envrc template at the root of the terraform directory with these keys

terraform/bootstrap/main.tf

@@ -0,0 +1,26 @@
+# generated a PAT (Personal Access Token) from here: https://www.linode.com/docs/products/tools/linode-api/guides/get-access-token/
+#   give access to whatever terraform will need to read and write to
+
+provider "linode" {
+#   token = var.linode_token
+}
+
+data "linode_object_storage_cluster" "primary" {
+  id = "us-east-1"
+}
+
+resource "linode_object_storage_key" "terraform_access" {
+  label = "terraform_backend_access"
+}
+
+resource "linode_object_storage_bucket" "terraform_backend" {
+  cluster = data.linode_object_storage_cluster.primary.id
+  label = "terraform-backend"
+  access_key = linode_object_storage_key.terraform_access.access_key
+  secret_key = linode_object_storage_key.terraform_access.secret_key
+  # default enabled in web-ui
+  acl = "private"
+  cors_enabled = true
+  # best practice for terraform state buckets (allows you to rollback if needed)
+  versioning = true
+}

terraform/bootstrap/outputs.tf

@@ -0,0 +1,21 @@
+output "bucket_endpoint" {
+  description = "S3 bucket endpoint URL"
+  value = linode_object_storage_bucket.terraform_backend.hostname
+}
+
+output "bucket_region" {
+  description = "Region the bucket is in"
+  value = data.linode_object_storage_cluster.primary.region
+}
+
+output "bucket_access_key" {
+  description = "Access Key (Secret)"
+  value = linode_object_storage_key.terraform_access.access_key
+  sensitive = true
+}
+
+output "bucket_secret_key" {
+  description = "Secret Key (Secret)"
+  value = linode_object_storage_key.terraform_access.secret_key
+  sensitive = true
+}

terraform/bootstrap/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    linode = {
+        source = "linode/linode"
+        version = ">=1.29.2"
+    }
+  }
+}

terraform/dns/.envrc.template

@@ -0,0 +1,2 @@
+source ../.envrc 
+export CLOUDFLARE_API_TOKEN='<global_api_key_or_api_token>'