diff --git a/.envrc b/.envrc index 23ea8d5..509d351 100644 --- a/.envrc +++ b/.envrc @@ -1,6 +1,12 @@ +#!/bin/env sh + if [ -z "${NIX_ENV_LOADED}" ]; then - echo "entering dev environment..." - export NIX_ENV_LOADED=$(pwd) + printf "entering environment..." + NIX_ENV_LOADED="$(pwd)" + export NIX_ENV_LOADED + + nix flake update --extra-experimental-features nix-command --extra-experimental-features flakes; + echo "nix store is using $(du -hs /nix/store)" nix develop \ --ignore-environment \ @@ -15,30 +21,31 @@ if [ -z "${NIX_ENV_LOADED}" ]; then --keep AWS_ACCESS_KEY_ID \ --keep AWS_SECRET_ACCESS_KEY \ --keep AWS_SESSION_TOKEN \ - --keep UPDATECLI_GPGTOKEN \ - --keep UPDATECLI_GITHUB_TOKEN \ - --keep UPDATECLI_GITHUB_ACTOR \ - --keep GPG_SIGNING_KEY \ - --keep NIX_ENV_LOADED \ --keep TERM \ - $(pwd) + --keep XDG_DATA_DIRS \ + --keep NIX_ENV_LOADED \ + "$(pwd)" || unset NIX_ENV_LOADED; + else - echo "setting up dev environment..." + printf "setting up dev environment...\n" + unset NIX_ENV_LOADED - source .aliases - source .functions - source .variables - source .rcs -fi -if [ -z "$SSH_AUTH_SOCK" ]; then - echo "Unable to find SSH_AUTH_SOCK, is your agent running?"; -fi -if [ -z "$(ssh-add -l | grep -v 'The agent has no identities.')" ]; then - echo "Your agent doesn't appear to have any identities loaded, please load a key or forward your agent."; -fi -if [ -z "$(env | grep 'AWS')" ]; then - echo "Unable to find AWS authentication information in the environment, please make sure you authenticate with AWS."; -fi -if [ -z "$(env | grep 'GITHUB_TOKEN')" ]; then - echo "Unable to find GITHUB authentication information in the environment, please make sure you authenticate with GITHUB."; + . .functions + . .variables + . .rcs + . .aliases + + + if [ -z "$SSH_AUTH_SOCK" ]; then eval "$(ssh-agent -s)"; ssh-add; fi + if [ "" = "$(env | grep 'AWS')" ]; then + printf "Unable to find AWS authentication information in the environment, \ + please make sure you authenticate with AWS. \ + Try using the 'aws' cli included in the environment.\n"; + fi + if env | grep -q 'GITHUB_TOKEN'; then + printf "Unable to find GITHUB authentication information in the environment, \ + please make sure you authenticate with GITHUB. \ + Try using the 'gh' cli included in the environment.\n"; + fi fi +unset NIX_ENV_LOADED \ No newline at end of file diff --git a/.functions b/.functions index 5a12cc3..4e2c629 100644 --- a/.functions +++ b/.functions @@ -1,46 +1,193 @@ +#!/bin/env bash # get current branch in git repo -function parse_git_branch() { - BRANCH=`git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/\1/'` - if [ ! "${BRANCH}" == "" ] - then - STAT=`parse_git_dirty` - echo "[${BRANCH}${STAT}]" - else - echo "" +git_status() { + BRANCH="$(git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/\1/')" + if [ ! "${BRANCH}" = "" ]; then + STAT="$(parse_git_dirty)" + if printf "%s" "$STAT" | grep -q -e '!' -e '?' -e '+' -e '>' -e 'x' -e '*'; then + printf "%s[%s %s]%s" "$(red)" "$BRANCH" "$STAT" "$(ce)" + else + printf "%s[%s%s]%s" "$(green)" "$BRANCH" "$STAT" "$(ce)" + fi fi } +get_repo_owner() { + REPO="$(basename "$(git rev-parse --show-toplevel)")" + OWNER="$(basename "$(git rev-parse --show-toplevel | sed s/"$REPO"//g)")" + printf "%s" "$OWNER" +} + # get current status of git repo -function parse_git_dirty { - status=`git status 2>&1 | tee` - dirty=`echo -n "${status}" 2> /dev/null | grep "modified:" &> /dev/null; echo "$?"` - untracked=`echo -n "${status}" 2> /dev/null | grep "Untracked files" &> /dev/null; echo "$?"` - ahead=`echo -n "${status}" 2> /dev/null | grep "Your branch is ahead of" &> /dev/null; echo "$?"` - newfile=`echo -n "${status}" 2> /dev/null | grep "new file:" &> /dev/null; echo "$?"` - renamed=`echo -n "${status}" 2> /dev/null | grep "renamed:" &> /dev/null; echo "$?"` - deleted=`echo -n "${status}" 2> /dev/null | grep "deleted:" &> /dev/null; echo "$?"` - bits='' - if [ "${renamed}" == "0" ]; then - bits=">${bits}" - fi - if [ "${ahead}" == "0" ]; then - bits="*${bits}" - fi - if [ "${newfile}" == "0" ]; then - bits="+${bits}" +parse_git_dirty() { + status="$(git status 2>&1 | tee)" + if [ "0" = "$(printf "%s" "${status}" 2> /dev/null | grep "Your branch is up to date with 'origin/main'" >/dev/null 2>&1; printf "%s" $?)" ]; then printf "%s" ""; fi # clean + if [ "0" = "$(printf "%s" "${status}" 2> /dev/null | grep "modified:" >/dev/null 2>&1; printf "%s" $?)" ]; then printf "%s" "!"; fi # dirty + if [ "0" = "$(printf "%s" "${status}" 2> /dev/null | grep "Untracked files" >/dev/null 2>&1; printf "%s" $?)" ]; then printf "%s" "?"; fi # untracked + if [ "0" = "$(printf "%s" "${status}" 2> /dev/null | grep "new file:" >/dev/null 2>&1; printf "%s" $?)" ]; then printf "%s" "+"; fi # new files + if [ "0" = "$(printf "%s" "${status}" 2> /dev/null | grep "renamed:" >/dev/null 2>&1; printf "%s" $?)" ]; then printf "%s" ">"; fi # renamed files + if [ "0" = "$(printf "%s" "${status}" 2> /dev/null | grep "deleted:" >/dev/null 2>&1; printf "%s" $?)" ]; then printf "%s" "x"; fi # deleted files + if [ "0" = "$(printf "%s" "${status}" 2> /dev/null | grep "Your branch is ahead of" >/dev/null 2>&1; printf "%s" $?)" ]; then printf "%s" "*"; fi # ahead of +} + +encrypt_secrets() { + workspace="$(git rev-parse --show-toplevel)"; + dir="$(pwd)"; + cd "$workspace" || return; + while read -r file; do + if [ -f "$file" ]; then + if [ -f "$file.backup" ]; then + rm -rf "$file.backup" + cp "$file" "$file.backup" + fi + rm -rf "$file.age" + age -e -R "$workspace"/age_recipients.txt -o "$file.age" "$file" fi - if [ "${untracked}" == "0" ]; then - bits="?${bits}" + done terraform.tfstate + rm -f terraform.tfstate.age + age -e -R "$workspace/age_recipients.txt" -o terraform.tfstate.age terraform.tfstate + done + cd "$dir" || return; +} + +set_terminal_size(){ + row="$1" + col="$2" + if [ "$row" == "" ]; then row=70; fi + if [ "$col" == "" ]; then col=300; fi + stty rows "$row" + stty cols "$col" +} + +set_repo_name() { + new_name="$1" + if [ -z "$new_name" ]; then echo "set new name as $1"; exit 1; fi + for file in $(git grep \"matttrach-demo\" | awk -F':' '{print $1}'| uniq | tr '\n' ' '); do + sed -i 's/matttrach-demo/generic-demo/g' "$file"; + done +} diff --git a/.github/workflows/cleanup.yaml b/.github/workflows/cleanup.yaml index 2f6f519..f97bee4 100644 --- a/.github/workflows/cleanup.yaml +++ b/.github/workflows/cleanup.yaml @@ -20,32 +20,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - uses: nicknovitski/nix-develop@v1.1.0 with: - arguments: | - --ignore-environment \ - --extra-experimental-features nix-command \ - --extra-experimental-features flakes \ - --keep HOME \ - --keep SSH_AUTH_SOCK \ - --keep GITHUB_TOKEN \ - --keep AWS_ROLE \ - --keep AWS_REGION \ - --keep AWS_DEFAULT_REGION \ - --keep AWS_ACCESS_KEY_ID \ - --keep AWS_SECRET_ACCESS_KEY \ - --keep AWS_SESSION_TOKEN \ - --keep UPDATECLI_GPGTOKEN \ - --keep UPDATECLI_GITHUB_TOKEN \ - --keep UPDATECLI_GITHUB_ACTOR \ - --keep GPG_SIGNING_KEY \ - --keep NIX_ENV_LOADED \ - --keep TERM \ - ${{ github.workspace }} + fetch-depth: 0 + - uses: matttrach/nix-installer-action@main - name: Get Ids id: get_ids + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} # 86400 = 24 hours in seconds (24 * 60 * 60) # you might increase this number if you need to look back further for leftovers run: | @@ -66,6 +46,7 @@ jobs: # WARNING! if '--filter=""' then you will find everything in a region # WARNING! if '-d' is missing you will delete everything that is found - name: find-leftovers + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} run: | check_leftovers() { local id="$1" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 941c73c..94ccd6f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -32,57 +32,27 @@ jobs: if: steps.release-please.outputs.pr with: token: ${{secrets.GITHUB_TOKEN}} - - uses: DeterminateSystems/nix-installer-action@main - if: steps.release-please.outputs.pr - - uses: nicknovitski/nix-develop@v1.1.0 - if: steps.release-please.outputs.pr - with: - arguments: | - --ignore-environment \ - --extra-experimental-features nix-command \ - --extra-experimental-features flakes \ - --keep HOME \ - --keep SSH_AUTH_SOCK \ - --keep GITHUB_TOKEN \ - --keep AWS_ROLE \ - --keep AWS_REGION \ - --keep AWS_DEFAULT_REGION \ - --keep AWS_ACCESS_KEY_ID \ - --keep AWS_SECRET_ACCESS_KEY \ - --keep AWS_SESSION_TOKEN \ - --keep UPDATECLI_GPGTOKEN \ - --keep UPDATECLI_GITHUB_TOKEN \ - --keep UPDATECLI_GITHUB_ACTOR \ - --keep GPG_SIGNING_KEY \ - --keep NIX_ENV_LOADED \ - --keep TERM \ - ${{ github.workspace }} + fetch-depth: 0 - uses: aws-actions/configure-aws-credentials@v4 if: steps.release-please.outputs.pr with: role-to-assume: ${{env.AWS_ROLE}} role-session-name: ${{github.job}}-${{github.run_id}}-${{github.run_number}}-${{github.run_attempt}} aws-region: ${{env.AWS_REGION}} - - uses: actions/cache/restore@v4 - id: cache-terraform-restore + - uses: matttrach/nix-installer-action@main if: steps.release-please.outputs.pr - with: - path: ${{ github.workspace }}/.terraform - key: terraform-${{hashFiles('**/versions.tf','**/main.tf')}} - - run: terraform init -upgrade - if: steps.release-please.outputs.pr - - uses: actions/cache/save@v4 - id: cache-terraform-save - if: steps.release-please.outputs.pr - with: - path: ${{ github.workspace }}/.terraform - key: ${{ steps.cache-terraform-restore.outputs.cache-primary-key }} - - run: go version && cd ${{github.workspace}}/tests && go test -v -timeout=40m -parallel=10 && cd ${{github.workspace}} + - name: Run Tests if: steps.release-please.outputs.pr + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} GITHUB_OWNER: rancher IDENTIFIER: ${{github.job}}-${{github.run_id}}-${{github.run_number}}-${{github.run_attempt}} + run: | + go version + cd ${{github.workspace}}/tests + go test -v -timeout=40m -parallel=10 + cd ${{github.workspace}} - uses: peter-evans/create-or-update-comment@v4 name: 'Report Success' if: steps.release-please.outputs.pr diff --git a/.github/workflows/update.yaml b/.github/workflows/update.yaml index 1fb0611..6f910af 100644 --- a/.github/workflows/update.yaml +++ b/.github/workflows/update.yaml @@ -15,30 +15,11 @@ jobs: if: github.ref == 'refs/heads/main' steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: nicknovitski/nix-develop@v1.1.0 with: - arguments: | - --ignore-environment \ - --extra-experimental-features nix-command \ - --extra-experimental-features flakes \ - --keep HOME \ - --keep SSH_AUTH_SOCK \ - --keep GITHUB_TOKEN \ - --keep AWS_ROLE \ - --keep AWS_REGION \ - --keep AWS_DEFAULT_REGION \ - --keep AWS_ACCESS_KEY_ID \ - --keep AWS_SECRET_ACCESS_KEY \ - --keep AWS_SESSION_TOKEN \ - --keep UPDATECLI_GPGTOKEN \ - --keep UPDATECLI_GITHUB_TOKEN \ - --keep UPDATECLI_GITHUB_ACTOR \ - --keep GPG_SIGNING_KEY \ - --keep NIX_ENV_LOADED \ - --keep TERM \ - ${{ github.workspace }} + fetch-depth: 0 + - uses: matttrach/nix-installer-action@main - name: Updatecli + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} # Never use '--debug' option, because it might leak the access tokens. run: | gpgconf --kill all diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 7f620ba..17d3331 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -2,135 +2,142 @@ name: validate on: pull_request: - branches: - - main + branches: main + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_OWNER: ${{ github.repository_owner }} jobs: terraform: name: 'Terraform' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: nicknovitski/nix-develop@v1.1.0 - with: - arguments: | - --ignore-environment \ - --extra-experimental-features nix-command \ - --extra-experimental-features flakes \ - --keep HOME \ - --keep SSH_AUTH_SOCK \ - --keep GITHUB_TOKEN \ - --keep AWS_ROLE \ - --keep AWS_REGION \ - --keep AWS_DEFAULT_REGION \ - --keep AWS_ACCESS_KEY_ID \ - --keep AWS_SECRET_ACCESS_KEY \ - --keep AWS_SESSION_TOKEN \ - --keep UPDATECLI_GPGTOKEN \ - --keep UPDATECLI_GITHUB_TOKEN \ - --keep UPDATECLI_GITHUB_ACTOR \ - --keep GPG_SIGNING_KEY \ - --keep NIX_ENV_LOADED \ - --keep TERM \ - ${{ github.workspace }} - - uses: actions/cache/restore@v4 - id: cache-terraform-restore - with: - path: ${{ github.workspace }}/.terraform - key: terraform - - run: terraform init -upgrade - - uses: actions/cache/save@v4 - id: cache-terraform-save - with: - path: ${{ github.workspace }}/.terraform - key: ${{ steps.cache-terraform-restore.outputs.cache-primary-key }} - - run: cd ${{ github.workspace }}/examples/basic && terraform version && terraform init -upgrade && terraform validate && cd ${{ github.workspace }} - - run: terraform fmt -check -recursive + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: matttrach/nix-installer-action@main + - name: lint terraform + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} + run: | + terraform fmt -check -recursive + tflint --recursive actionlint: name: 'Lint Workflows' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: nicknovitski/nix-develop@v1.1.0 - with: - arguments: | - --ignore-environment \ - --extra-experimental-features nix-command \ - --extra-experimental-features flakes \ - --keep HOME \ - --keep SSH_AUTH_SOCK \ - --keep GITHUB_TOKEN \ - --keep AWS_ROLE \ - --keep AWS_REGION \ - --keep AWS_DEFAULT_REGION \ - --keep AWS_ACCESS_KEY_ID \ - --keep AWS_SECRET_ACCESS_KEY \ - --keep AWS_SESSION_TOKEN \ - --keep UPDATECLI_GPGTOKEN \ - --keep UPDATECLI_GITHUB_TOKEN \ - --keep UPDATECLI_GITHUB_ACTOR \ - --keep GPG_SIGNING_KEY \ - --keep NIX_ENV_LOADED \ - --keep TERM \ - ${{ github.workspace }} - - run: actionlint - - tflint: - name: 'TFLint' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: terraform-linters/setup-tflint@v4 - with: - tflint_version: latest - - run: tflint --version - - run: tflint --init - - run: tflint -f compact + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: matttrach/nix-installer-action@main + - name: action lint + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} + run: actionlint shellcheck: - name: Shellcheck runs-on: ubuntu-latest steps: - - uses: ludeeus/action-shellcheck@master + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: matttrach/nix-installer-action@main + - name: shell check + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} + run: | + while read -r file; do + echo "checking $file..." + shellcheck -x "$file" + done <<<"$(grep -Rl -e '^#!' | grep -v '.terraform'| grep -v '.git')" validate-commit-message: - name: Validate Commit Message runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch all history so that we can validate the commit messages + - uses: matttrach/nix-installer-action@main + - name: Check commit message + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} + run: | + set -e + # Check commit messages + # This steps enforces https://www.conventionalcommits.org/en/v1.0.0/ + # This format enables automatic generation of changelogs and versioning + filter() { + COMMIT="$1" + ouput="$(echo "$COMMIT" | grep -e '^fix: ' -e '^feature: ' -e '^feat: ' -e 'refactor!: ' -e 'feature!: ' -e 'feat!: ' -e '^chore(main): ')" + echo "$output" + } + prefix_check() { + message="$1" + if [ "" != "$(filter "$message")" ]; then + echo "...Commit message does not start with the required prefix. + Please use one of the following prefixes: fix:, feature:, feat:, refactor!:, feature!:, feat:!. + 'chore(main): ' is also allowed for release PRs. + This enables release-please to automatically determine the type of release (major, minor, patch) based on the commit message. + $message" + exit 1 + else + echo "...Commit message starts with the required prefix." + fi + } + empty_check() { + message="$1" + if [ "" == "$message" ]; then + echo "...Empty commit message." + exit 1 + else + echo "...Commit message isnt empty." + fi + } + length_check() { + message="$1" + if [ "$(wc -m <<<"$message")" -gt 50 ]; then + echo "...Commit message subject line should be less than 50 characters, found $(wc -m "$message")." + exit 1 + else + echo "...Commit message subject line is less than 50 characters." + fi + } + spell_check() { + message="$1" + WORDS="$(aspell list <<<"$message")" + if [ "" != "$WORDS" ]; then + echo "...Commit message contains spelling errors on: ^$WORDS\$" + echo "...Also try updating the PR title." + exit 1 + else + echo "...Commit message doesnt contain spelling errors." + fi + } + + # Fetch the commit messages + + COMMIT_MESSAGES="$(gh pr view ${{github.event.number}} --json commits | jq -r '.commits[].messageHeadline')" + echo "Commit messages found: " + echo "$COMMIT_MESSAGES" + + while read -r message; do + echo "checking message ^$message\$" + prefix_check "$message" + empty_check "$message" + length_check "$message" + spell_check "$message" + echo "message ^$message\$ passed all checks" + done <<<"$COMMIT_MESSAGES" gitleaks: name: 'Scan for Secrets' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: nicknovitski/nix-develop@v1.1.0 with: - arguments: | - --ignore-environment \ - --extra-experimental-features nix-command \ - --extra-experimental-features flakes \ - --keep HOME \ - --keep SSH_AUTH_SOCK \ - --keep GITHUB_TOKEN \ - --keep AWS_ROLE \ - --keep AWS_REGION \ - --keep AWS_DEFAULT_REGION \ - --keep AWS_ACCESS_KEY_ID \ - --keep AWS_SECRET_ACCESS_KEY \ - --keep AWS_SESSION_TOKEN \ - --keep UPDATECLI_GPGTOKEN \ - --keep UPDATECLI_GITHUB_TOKEN \ - --keep UPDATECLI_GITHUB_ACTOR \ - --keep GPG_SIGNING_KEY \ - --keep NIX_ENV_LOADED \ - --keep TERM \ - ${{ github.workspace }} - - run: gitleaks detect --no-banner -v --no-git - - run: gitleaks detect --no-banner -v + fetch-depth: 0 + - uses: matttrach/nix-installer-action@main + - name: Check for secrets + shell: nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep GITHUB_TOKEN --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_ENV_LOADED --keep TERM --command bash -e {0} + run: | + gitleaks detect --no-banner -v --no-git + gitleaks detect --no-banner -v diff --git a/.rcs b/.rcs index e327e42..b629f56 100644 --- a/.rcs +++ b/.rcs @@ -1,3 +1,10 @@ -source ~/.config/aws/default/rc # add personal aws auth vars -source ~/.config/alias/default/rc # add personal aliases -source ~/.config/github/default/rc # add personal github auth vars \ No newline at end of file +#!/bin/env sh +# shellcheck disable=SC1090 # disable shellcheck for external sourced files using dynamic directory +# shellcheck disable=SC1091 # disable shellcheck for external sourced files using dynamic directory +. "$HOME/.config/aws/default" # add personal aws auth vars +. "$HOME/.config/github/default" # add personal github auth vars +. "$HOME/.config/functions/default" # add personal functions +. "$HOME/.config/alias/default" # add personal aliases +. "$HOME/.config/age/default" # add age secrets +. "$HOME/.config/updatecli/default" # add updatecli secrets +. "$HOME/.config/docker/default" # add docker specific stuff \ No newline at end of file diff --git a/.variables b/.variables index 4ad4e93..266f935 100644 --- a/.variables +++ b/.variables @@ -1,5 +1,10 @@ +#!/bin/env sh export TF_IN_AUTOMATION=1 +TF_VAR_ip="$(curl -s 'https://api.ipify.org')" +export TF_VAR_ip export ACME_SERVER_URL="https://acme-v02.api.letsencrypt.org/directory" - -# expects parse_git_branch function to be defined -export PS1="\[\e[31m\]Dev:\[\e[m\] \[\e[32m\]\u\[\e[m\]\[\e[32m\]@\[\e[m\]\[\e[32m\]\h\[\e[m\] \[\e[34m\]\W\[\e[m\] \`parse_git_branch\` " \ No newline at end of file +tty_row="70" +tty_col="300" +set_terminal_size "$tty_row" "$tty_col" +# shellcheck disable=SC2140 +export PS1=""\$\(blue\)""\$\(get_repo_owner\)"/"\$\(ce\)""\$\(orange\)"\W "\$\(ce\)""\$\(git_status\)" " diff --git a/README.md b/README.md index c158514..be2917e 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,16 @@ ## AWS Access -The first step to using the AWS modules is having an AWS account, [here](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-creating.html) is a document describing this process. -You will need an API access key id and API secret key, you can get the API keys [following this tutorial](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey). +The first step to using the AWS modules is having an AWS account, + [here](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-creating.html) is a document describing this process. +You will need an API access key id and API secret key, + you can get the API keys [following this tutorial](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey). The Terraform AWS provider uses the AWS Go SDK, which allows the use of either environment variables or config files for authentication. https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-settings You do not need the AWS cli to generate the files, just place them in the proper place and Terraform will find and read them. -We use environment variables to configure the AWS provider and load them by sourcing an RC file. +In development, we use environment variables to configure the AWS provider and load them by sourcing an RC file. +In CI we use OIDC connection to AWS to authenticate. ``` export AWS_ACCESS_KEY_ID='ABC123' @@ -26,7 +29,7 @@ export AWS_REGION='us-west-1' ``` These help the tests set you as the owner on the testing infra and generate the proper key. -The `.envrc` file sources `.rcs` file which assumes a local file at path `~/.config/aws/default/rc` exists with the above information. +The `.envrc` file sources `.rcs` file which assumes a local file at path `~/.config/aws/default` exists with the above information. ## Examples diff --git a/examples/override/main.tf b/examples/override/main.tf index 3fd49d9..c3b2896 100644 --- a/examples/override/main.tf +++ b/examples/override/main.tf @@ -7,7 +7,6 @@ provider "aws" { } locals { identifier = var.identifier - name = "tf-override-${local.identifier}" security_group_name = var.security_group_name key_name = var.key_name } diff --git a/examples/skipvpc/main.tf b/examples/skipvpc/main.tf index d63e60b..3f035bd 100644 --- a/examples/skipvpc/main.tf +++ b/examples/skipvpc/main.tf @@ -7,7 +7,6 @@ provider "aws" { } locals { identifier = var.identifier - name = "tf-skipsubnet-${local.identifier}" key = var.key key_name = var.key_name } diff --git a/flake.lock b/flake.lock index 770554c..0940f1a 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1706683685, - "narHash": "sha256-FtPPshEpxH/ewBOsdKBNhlsL2MLEFv1hEnQ19f/bFsQ=", + "lastModified": 1711106783, + "narHash": "sha256-PDwAcHahc6hEimyrgGmFdft75gmLrJOZ0txX7lFqq+I=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5ad9903c16126a7d949101687af0aa589b1d7d3d", + "rev": "a3ed7406349a9335cb4c2a71369b697cecd9d351", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 12055a3..2afafaf 100644 --- a/flake.nix +++ b/flake.nix @@ -89,24 +89,34 @@ leftovers-wrapper = pkgs.writeShellScriptBin "leftovers" '' exec ${leftovers} "$@" ''; + aspellWithDicts = pkgs.aspellWithDicts (d: [d.en d.en-computers]); in { devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ + act # run workflows locally with Docker actionlint + age + aspellWithDicts bashInteractive curl + docker + gh git gitleaks - go # need go for terratest - gnupg # need gpg for signing commits + gnupg + go jq + kubectl less - openssh # need openssh for running remote provisioners + ncurses + vim # for easily editing files that are not in this directory structure + openssh_hpn shellcheck tflint tfswitch - vim + tree # for easily finding files in this directory structure and helping generate docs + xterm ]; shellHook = '' homebin=$HOME/bin; diff --git a/modules/security_group/versions.tf b/modules/security_group/versions.tf new file mode 100644 index 0000000..7c63b5d --- /dev/null +++ b/modules/security_group/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.5.0, < 1.6" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.4" + } + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + http = { + source = "hashicorp/http" + version = ">= 3.4" + } + } +} \ No newline at end of file diff --git a/modules/ssh_key/versions.tf b/modules/ssh_key/versions.tf new file mode 100644 index 0000000..7c63b5d --- /dev/null +++ b/modules/ssh_key/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.5.0, < 1.6" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.4" + } + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + http = { + source = "hashicorp/http" + version = ">= 3.4" + } + } +} \ No newline at end of file diff --git a/modules/subnet/versions.tf b/modules/subnet/versions.tf new file mode 100644 index 0000000..7c63b5d --- /dev/null +++ b/modules/subnet/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.5.0, < 1.6" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.4" + } + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + http = { + source = "hashicorp/http" + version = ">= 3.4" + } + } +} \ No newline at end of file diff --git a/modules/vpc/versions.tf b/modules/vpc/versions.tf new file mode 100644 index 0000000..7c63b5d --- /dev/null +++ b/modules/vpc/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.5.0, < 1.6" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.4" + } + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + http = { + source = "hashicorp/http" + version = ">= 3.4" + } + } +} \ No newline at end of file diff --git a/terraform.md b/terraform.md index 35008a5..5aec501 100644 --- a/terraform.md +++ b/terraform.md @@ -49,7 +49,7 @@ The word "Module" is used in three contexts: - an example of this would be a security group and its rules - while rules can be added separately from the group and are their own resources they do not make a lot of sense to have in their own external module - it may be useful to separate out the rules from the group in logical form to keep top level (implementation) modules clean -3. As an implementation of resources (Implementation Module/IMod) +3. As an implementation of resources (Implementation Module/Root module/IMod) - modules are generally considered a way to pull code into a terraform file, but eventually a "root" must be created - the "root module" or "impementation module" orchestrates a group of modules with the intent of actually provisioning resources (rather than just as a template or library) - using the git ops paradigm the implementation module should be considered the source of truth for the infrastructure @@ -99,6 +99,9 @@ For instance, you should not need to provision a new VPC for every implementatio This technique allows you to only generate a VPC once (or never, if you create it manually) by querying the provider before generating the resource. This also prevents users from having to know or pass the unique ids of resources into modules. Modules need selectors to accomplish this, usually in the form of some kind of name or default. +There is a trade off to this approach where dependency chains become coupled in a way the forces recreation of resources due to unknown values. +We diminish this trade off by explicitly ignoring this cases which Terraform calls "changes". +This in turn may cause infrastructure to stay around when you might expect it to go away, which is how we err on the side of caution. ## Parenthesis Around Ternaries @@ -115,6 +118,7 @@ Many times, variables need to be processed after an initial implementation is in variables can not be processed in the variables section, and processing the variable in multiple places throughout the config is prone to error, this standard will prevent unnecessary changes to the variables and the config as a whole. Basically, place everything in locals so you don't have to worry about moving them there later. +This also makes the interface able to be more stable than the underlying code. ## Embedded Scripts Should Use Heredoc @@ -168,16 +172,19 @@ There shouldn't be more than 3 levels of nested independent modules: (Core, Prim These independent modules represent provider resources, they should not have any nested independant modules. Core Modules should only call resources. +Most times these are local modules within a Primary module. ### Primary Modules These independent modules represent groups of core modules, they should not call resources. Primary Modules should only call Core Modules. +There is an exception for null_resource or terraform_data resources which many times are necessary to configure services. ### Secondary Modules These modules represent large systems, they should only call Primary Modules. Secondary Modules should only call Primary Modules. +There is an exception for null_resource or terraform_data resources which many times are necessary to configure services. ## Test Size @@ -196,3 +203,4 @@ Integration tests show that any two local modules work together. In this code base an "End to End" or "E2E" test refers to testing all of the units together. A module might have several E2E tests validating different configurations. +The vast majority of tests will be e2e, since that represents the highest value to our users.