Merge pull request #2 from masterpointio/feature/shared-task-modules

README.md

@@ -1,13 +1,27 @@
 # Taskit
 
-Taskit (said Task-kit or TK for short) is the Masterpoint [Taskfile](https://taskfile.dev/) Kit. We utilize this set of Taskfiles across clients as a means by which to share repeatable tasks like repo setup, tool automation, and similar shared scripting.
+Taskit (said task-kit or TK for short) is the Masterpoint [Taskfile](https://taskfile.dev/) Kit. We utilize this set of Taskfiles across clients as a means by which to share repeatable tasks like repo setup, tool automation, and similar shared scripting.
 
 ## Using Taskit
 
-Taskit is built around the idea of being consumed by downstream repos. It is pulled through our standard installation `Taskfile.yaml`
+Taskit is built around the idea of being consumed by downstream repos. It is pulled through a standard Taskfile that you can find in [`exports/Taskfile.dist.yaml`](./exports/Taskfile.dist.yaml).
+
+The procedure to add it to a project is to do the following:
+
+1. Copy `exports/Taskfile.dist.yaml` to your project by running the following command:
+
+   ```bash
+   curl -sL https://raw.githubusercontent.com/masterpointio/taskit/main/exports/Taskfile.dist.yaml -o Taskfile.dist.yaml
+   ```
+
+1. Run `task init` to initialize taskit by downloading this repo into your remote repo.
+   1. Note, `git` is a requirement.
+1. Run `task --list` to list all newly available tasks from taskit.
+1. (Optional) Add a `.env.taskit` file which can include overrides for any variables in taskit.
+1. Now that the setup process for taskit is complete, you should commit and push the new configuration files to your repo. Well done 👏
 
 ## TODO
 
+- [x] Upstream various tasks from our distributed usage
 - [ ] Create a test harness + tests around tasks
-- [ ] Upstream various tasks from our distributed usage
 - [ ] Publish versions

Taskfile.yaml

@@ -2,7 +2,8 @@ version: "3"
 
 includes:
   aqua: lib/aqua/Taskfile.yaml
+  aws: lib/aws/Taskfile.yaml
+  snaplet: lib/snaplet/Taskfile.yaml
+  toolbox: lib/toolbox/Taskfile.yaml
 
-tasks:
-  default:
-    cmds: []
+tasks: {}

exports/Taskfile.dist.yaml

@@ -0,0 +1,71 @@
+# This is the Taskfile responsible for managing taskit in any consuming repository
+# More information is available at https://github.com/masterpointio/taskit
+# This file is typically cloned into the repo as-is and any required edits should be made both here and in the taskit repo (if applicable)
+# If any project specific changes are required, they should be made in a separate taskfiles/<module name>/Taskfile.yaml and then imported here.
+
+version: "3"
+
+includes:
+  taskit:
+    taskfile: .taskit/Taskfile.yaml
+    optional: true # Optional only until we have init'd taskit
+    aliases: [t]
+  local:
+    taskfile: taskfiles/Taskfile.yaml
+    optional: true # Optional as we may not have any local taskfiles
+    aliases: [l]
+
+# Load our project specific variables from .env.taskit
+dotenv: [.env.taskit]
+
+vars:
+  TASKIT_BRANCH: main
+  TASKIT_LOCAL_PATH: ""
+
+tasks:
+  default:
+    desc: Initializes taskit
+    summary: |
+      This sets up taskit for this project by cloning the taskit project to .taskit/.
+      It also adds the taskit gitignore entries to the project's .gitignore file.
+      You can find more information at https://github.com/masterpointio/taskit
+
+      To set the branch that is cloned, set the `TASKIT_BRANCH`.
+
+      To work on taskit modules locally, set the `TASKIT_LOCAL_PATH` to the path of the local taskit repo.
+      This sets taskit to copy the files from the local repo instead of cloning the remote repo, whichs allows you to work on taskit locally.
+    deps: [clean]
+    silent: true
+    aliases: [init]
+    cmds:
+      - printf "🚀 Initializing taskit...\n\n"
+      - |
+        if [[ -f .gitignore ]]; then
+          echo "Adding taskit gitignore entries to existing .gitignore..."
+        else
+          echo "Creating .gitignore..."
+          touch .gitignore
+        fi
+
+        if ! grep -q ".taskit" .gitignore; then
+          echo -e "\n## Taskit files\n.taskit/\n.task/\n.snaplet/snapshots/" >> .gitignore
+        fi
+
+      - |
+        echo -e "\n"
+        if [[ -z "{{.TASKIT_LOCAL_PATH}}" ]]; then
+          echo "Copying taskit from local path..."
+          rsync -rq --exclude=.git --exclude= {{.TASKIT_LOCAL_PATH}} .taskit
+        else
+          echo "Downloading taskit from remote repo..."
+          git clone -b {{.TASKIT_BRANCH}} git@github.com:masterpointio/taskit.git .taskit
+        fi
+
+        echo -e "\n\n⚡ Taskit successfully initialized! ⚡\n"
+
+  clean:
+    desc: Cleans up the project's Task setup
+    silent: true
+    cmds:
+      - rm -rf .taskit/
+      - mkdir -p .taskit

lib/aqua/Taskfile.yaml

@@ -3,13 +3,24 @@ version: "3"
 tasks:
   install:
     desc: Installs the Aqua CLI using the universal installer script, sets in PATH, and upgrades.
+    summary: |
+      Installs the Aqua CLI using the universal installer script, sets in PATH, and upgrades.
+
+      NOTE: The instructions for downloading the aqua-installer have a standard checksum verification step.
+      This requires knowing the checksum for all versions, which we do not so we have decided to accept the risk and skip that step.
+
+      Provide a version to install with the AQUA_INSTALLER_VERSION variable.
+      example:
+      > task aqua:install AQUA_INSTALLER_VERSION=v2.1.1
+
     silent: true
+    vars:
+      AQUA_INSTALLER_VERSION: v2.1.2
     cmds:
       - defer: rm ./aqua-installer
       - |
         touch ./aqua-installer
-        curl -sSfL -O https://raw.githubusercontent.com/aquaproj/aqua-installer/v2.1.1/aqua-installer
-        echo "c2af02bdd15da6794f9c98db40332c804224930212f553a805425441f8331665  aqua-installer" | sha256sum -c
+        curl -sSfL -O https://raw.githubusercontent.com/aquaproj/aqua-installer/{{.AQUA_INSTALLER_VERSION}}/aqua-installer
         chmod +x aqua-installer
         ./aqua-installer
 

lib/aws/Taskfile.yaml

@@ -0,0 +1,24 @@
+version: "3"
+
+tasks:
+  validate-access:
+    desc: Validate access to AWS for the given command
+    silent: true
+    internal: true
+    set: [e]
+    vars:
+      AWS_REGION: us-east-1
+      AWS_COMMAND: '{{.AWS_COMMAND | default "sts get-caller-identity"}}'
+    requires:
+      vars:
+        - MESSAGE
+    cmds:
+      - |
+        # Check the exit status of the command
+        if aws {{.AWS_COMMAND}} >/dev/null 2>&1; then
+            # This means the command succeeded, so do nothing, all is good.
+            exit 0;
+        else
+            printf "\n\n🚨 AWS access validation failed. Please check your AWS credentials and try again.\n\nMessage:\n{{.MESSAGE}}\n"
+            exit 1;
+        fi

lib/snaplet/Taskfile.yaml

@@ -0,0 +1,110 @@
+version: "3"
+
+includes:
+  aws:
+    taskfile: ../aws/Taskfile.yaml
+    internal: true
+
+vars:
+  SNAPSHOT_NAME:
+    sh: date +%Y-%m-%d # YYYY-MM-DD
+  SNAPSHOT_PATH: ".snaplet/snapshots/{{.SNAPSHOT_NAME}}"
+
+tasks:
+  validate-snaplet-bucket-access:
+    desc: Validate access to the given S3 Bucket
+    silent: true
+    internal: true
+    requires:
+      vars:
+        - SNAPLET_BUCKET
+    cmds:
+      - task: aws:validate-access
+        vars:
+          AWS_COMMAND: s3 ls {{.SNAPLET_BUCKET}}
+          MESSAGE: |
+            AWS CLI access is required to invoke this task.
+            Please ensure you're using the correct AWS credentials that have access to the {{.SNAPLET_BUCKET}} bucket.
+
+  list:
+    desc: List all available snapshots in the given S3 Bucket
+    silent: true
+    deps:
+      - validate-snaplet-bucket-access
+    requires:
+      vars:
+        - SNAPLET_BUCKET
+    cmds:
+      - printf "\n\n📸 Snapshots available:\n"
+      - aws s3 ls {{.SNAPLET_BUCKET}} | sed 's/PRE//g'
+
+  clean:
+    desc: Clean up snapshot at SNAPSHOT_PATH
+    silent: true
+    requires:
+      vars:
+        - SNAPSHOT_PATH
+    cmds:
+      - rm -rf {{.SNAPSHOT_PATH}}
+      - printf "\n\n🗑️ Cleaned up snapshot at {{.SNAPSHOT_PATH}}"
+
+  capture:
+    desc: Helper for running snaplet snapshot capture.
+    silent: true
+    deps:
+      - clean
+    requires:
+      vars:
+        - SNAPLET_SOURCE_DATABASE_URL
+    cmds:
+      - SNAPLET_SOURCE_DATABASE_URL={{.SNAPLET_SOURCE_DATABASE_URL}} snaplet snapshot capture {{.SNAPSHOT_PATH}}
+      - zip -r {{.SNAPSHOT_PATH}}.zip {{.SNAPSHOT_PATH}}
+      - printf "\n\n🗑️ Captured snapshot at {{.SNAPSHOT_PATH}}"
+
+  upload:
+    desc: Helper for uploading a snaplet snapshot to a given S3 Bucket
+    deps:
+      - validate-snaplet-bucket-access
+    vars:
+      UPLOAD_AS_LATEST: true
+    requires:
+      vars:
+        - SNAPLET_BUCKET
+    cmds:
+      - aws s3 cp --sse=AES256 --recursive {{.SNAPSHOT_PATH}} {{.SNAPLET_BUCKET}}/{{.SNAPSHOT_NAME}}
+      - |
+        if [ "{{.UPLOAD_AS_LATEST}}" -eq "true" ]; then
+          # We first remove everything so there are no artifacts leftover from a previous upload that we're not overwriting.
+          aws s3 rm --recursive {{.SNAPLET_BUCKET}}/latest
+          aws s3 cp --sse=AES256 --recursive {{.SNAPSHOT_PATH}} {{.SNAPLET_BUCKET}}/latest
+        fi
+
+  download:
+    desc: Downloads a snaplet snapshot from a given S3 Bucket. Pass in SNAPSHOT_NAME to download a specific snapshot from the bucket.
+    silent: true
+    deps:
+      - clean
+      - validate-snaplet-bucket-access
+    vars:
+      SNAPSHOT_NAME: latest
+      SNAPSHOT_PATH: ".snaplet/snapshots/{{.SNAPSHOT_NAME}}"
+    requires:
+      vars:
+        - SNAPLET_BUCKET
+    cmds:
+      - aws s3 cp --quiet --sse=AES256 --recursive {{.SNAPLET_BUCKET}}/{{.SNAPSHOT_NAME}} {{.SNAPSHOT_PATH}}
+      - printf "\n\n⬇ Snapshot downloaded to {{.SNAPSHOT_PATH}}"
+
+  restore:
+    desc: Restores the given SNAPSHOT_NAME to the SNAPLET_TARGET_DATABASE_URL database.
+    prompt: This will wipe all of the data in the given SNAPLET_TARGET_DATABASE_URL. Are you sure?
+    silent: true
+    vars:
+      SNAPSHOT_NAME: latest
+      SNAPSHOT_PATH: ".snaplet/snapshots/{{.SNAPSHOT_NAME}}"
+    requires:
+      vars:
+        - SNAPLET_TARGET_DATABASE_URL
+    cmds:
+      - snaplet snapshot restore {{.SNAPSHOT_PATH}}
+      - printf "\n\n💯 Restored snapshot from {{.SNAPSHOT_PATH}}\n"

lib/toolbox/Taskfile.yaml

@@ -0,0 +1,59 @@
+# See more information in docs/toolbox.md
+version: "3"
+vars:
+  IMAGE_NAME: toolbox
+  LATEST_IMAGE_TAG: "{{.IMAGE_NAME}}:latest"
+  VERSION_IMAGE_TAG: "{{.IMAGE_NAME}}:{{.VERSION}}"
+  VERSION:
+    sh: git rev-parse --short HEAD
+  REGISTRY_URL_FULL: "{{.REGISTRY_URL}}/{{.LATEST_IMAGE_TAG}}"
+  CONTAINERFILE_PATH: ./Dockerfile
+  AWS_REGION: '{{.AWS_REGION | default "us-east-1"}}'
+
+tasks:
+  all:
+    desc: Build, install, and run the toolbox image
+    cmds:
+      - task: build
+      - task: install
+      - task: run
+
+  build:
+    desc: Build and tag our toolbox image
+    requires:
+      vars:
+        - IMAGE_NAME
+        - CONTAINERFILE_PATH
+    cmds:
+      - docker build --tag {{.LASTEST_IMAGE_TAG}} --tag {{.VERSION_IMAGE_TAG}} {{.CONTAINERFILE_PATH}}
+      - echo "Done building and tagging toolbox 💯 Tagged as {{.VERSION_IMAGE_TAG}} + {{.LATEST_IMAGE_TAG}}"
+
+  install:
+    desc: Install wrapper script from geodesic container
+    requires:
+      vars:
+        - IMAGE_NAME
+    cmds:
+      - docker run --rm {{.LATEST_IMAGE_TAG}} init | bash -s {{.IMAGE_NAME}} || (echo 'Try "sudo make install"'; exit 1)
+
+  pull:
+    desc: Pull our toolbox image from registry
+    requires:
+      vars:
+        - REGISTRY_URL_BASE
+        - REGISTRY_URL_FULL
+    cmds:
+      - |
+        aws ecr get-login-password --region $(AWS_REGION) | docker login --username AWS --password-stdin {{.REGISTRY_URL_BASE}}
+        docker pull {{.REGISTRY_URL_FULL}}
+
+  run:
+    desc: Run our toolbox image while also mounting your $HOME folder to `/localhost` in the container
+    cmds:
+      - /usr/local/bin/{{.IMAGE_NAME}}
+# TODO
+# ## Publish the toolbox image to our ECR repo
+# publish: build
+# 	@export VERSION=$(shell git rev-parse --short HEAD); \
+# 		make docker/image/push TARGET_VERSION=$$VERSION
+# 		make docker/image/push TARGET_VERSION=latest;