Feat: GitOps YAML Migration Tool

.github/workflows/build-gitops-migrate.yml

@@ -0,0 +1,86 @@
+name: Build GitOps Migrate
+
+on:
+  push:
+    branches: [main] # TODO: Disabled for testing.
+    paths: ['cmd/gitops-migrate/**/*.go']
+
+concurrency:
+  # Only allow a single occurrence of this job to run at any given time.
+  group: ${{ github.workflow }} # Group: 'Build GitOps Migrate'
+  # Newly queued runs terminate existing in-progress runs.
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: (${{ matrix.GOOS }})(${{ matrix.GOARCH }})
+    runs-on: ubuntu-latest
+    env:
+      # Ex: gitops-migrate-windows-amd64.exe
+      BIN_NAME: gitops-migrate-${{ matrix.GOOS }}-${{ matrix.GOARCH }}${{ matrix.GOOS == 'windows' && '.exe' }}
+      # Ex: gitops-migrate-windows-amd64.exe.sha256
+      BIN_HASH_NAME: gitops-migrate-${{ matrix.GOOS }}-${{ matrix.GOARCH }}${{ matrix.GOOS == 'windows' && '.exe' }}.sha256
+      # This serves as the root path we `aws s3 cp` all built binaries to.
+      S3_URI: s3://download/tools
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          # Define the OS and architecture permutations we want to build for.
+          - GOOS: windows
+            GOARCH: amd64
+          - GOOS: windows
+            GOARCH: arm64
+          - GOOS: linux
+            GOARCH: amd64
+          - GOOS: linux
+            GOARCH: arm64
+          - GOOS: darwin
+            GOARCH: arm64
+          - GOOS: darwin
+            GOARCH: amd64
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
+        with:
+          egress-policy: audit
+
+      - name: Checkout Repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          sparse-checkout: cmd/gitops-migrate
+
+      - name: Setup Go
+        uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
+        with:
+          go-version-file: go.mod
+
+      - name: Build GitOps Migrate
+        shell: bash
+        env:
+          GOOS: ${{ matrix.GOOS }}
+          GOARCH: ${{ matrix.GOARCH }}
+        run: go build -o ${{ env.BIN_NAME }} ./cmd/gitops-migrate
+
+      - name: Produce SHA-256 Hash of Built Binary
+        shell: bash
+        run: sha256sum ${{ env.BIN_NAME }} > ${{ env.BIN_HASH_NAME }}
+
+      - name: Upload Binary Artifact
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        with:
+          name: gitops-migrate-${{ matrix.GOOS }}-${{ matrix.GOARCH }}
+          path: |-
+            ${{ env.BIN_NAME }}
+            ${{ env.BIN_HASH_NAME }}
+
+      - name: Upload Binary & SHA-256 Hash to Cloudflare R2 Bucket
+        shell: bash
+        env:
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_DOWNLOAD_ACCESS_KEY_SECRET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.R2_DOWNLOAD_ACCESS_KEY_ID }}
+          AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT }}
+          AWS_DEFAULT_REGION: auto
+        run: |-
+          aws s3 cp '${{ env.BIN_NAME }}' '${{ env.S3_URI }}'
+          aws s3 cp '${{ env.BIN_HASH_NAME }}' '${{ env.S3_URI }}'

cmd/gitops-migrate/README.md

@@ -0,0 +1,183 @@
+# Overview
+
+This directory contains the `gitops-migrate` tool, designed leading up to the `4.74` Fleet release to automate necessary GitOps YAML transformations.
+
+# 4.74 YAML Changes
+
+The `4.74` release moves GitOps YAML keys: `self_service`, `categories`, `labels_exclude_any`, `labels_include_any` and `setup_experience` from the software files ([example](https://github.com/fleetdm/fleet/blob/c9a02741950f6510f9f1be48a2c19bc524417f70/cmd/fleetctl/gitops-migrate/testdata/mozilla-firefox.yml#L2-L9)) to the team files ([example](https://github.com/fleetdm/fleet/blob/c9a02741950f6510f9f1be48a2c19bc524417f70/it-and-security/teams/workstations.yml#L47-L70)).
+
+# Installation
+
+## Method 1: Download the Binary (Recommended)
+
+Download the appropriate binary for your operating system and architecture:
+
+| Operating System | Architecture | Download Link                                                                                                                                                       |
+| ---------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `macos`          | `arm64`      | [Download](https://download.fleetdm.com/tools/gitops-migrate-darwin-arm64)([Hash](https://download.fleetdm.com/tools/gitops-migrate-darwin-arm64.sha256))           |
+| `macos`          | `amd64`      | [Download](https://download.fleetdm.com/tools/gitops-migrate-darwin-amd64)([Hash](https://download.fleetdm.com/tools/gitops-migrate-darwin-amd64.sha256))           |
+| `windows`        | `amd64`      | [Download](https://download.fleetdm.com/tools/gitops-migrate-windows-amd64.exe)([Hash](https://download.fleetdm.com/tools/gitops-migrate-windows-amd64.exe.sha256)) |
+| `windows`        | `arm64`      | [Download](https://download.fleetdm.com/tools/gitops-migrate-windows-arm64.exe)([Hash](https://download.fleetdm.com/tools/gitops-migrate-windows-arm64.exe.sha256)) |
+| `linux`          | `amd64`      | [Download](https://download.fleetdm.com/tools/gitops-migrate-linux-amd64)([Hash](https://download.fleetdm.com/tools/gitops-migrate-linux-amd64.sha256))             |
+| `linux`          | `arm64`      | [Download](https://download.fleetdm.com/tools/gitops-migrate-linux-arm64)([Hash](https://download.fleetdm.com/tools/gitops-migrate-linux-arm64.sha256))             |
+
+## Method 2: Go Install
+
+[Install Go](https://go.dev/doc/install) and install `gitops-migrate` by running:
+
+```shell
+$ go install github.com/fleetdm/fleet/v4/cmd/gitops-migrate@latest
+```
+
+You can verify the installation was successful by running `gitops-migrate usage` which should display the help text.
+
+> [!NOTE]
+> If the `go install` is successful but you're not able to run `gitops-migrate`, you may need to add your `GOBIN` directory to `PATH` in the way appropriate for your operating system ([Windows](https://www.architectryan.com/2018/03/17/add-to-the-path-on-windows-10/), [Mac](https://medium.com/@B-Treftz/macos-adding-a-directory-to-your-path-fe7f19edd2f7), [Linux](https://pimylifeup.com/ubuntu-add-to-path/)). The path to add can be found by running: `go env GOBIN`.
+
+# Running the Migration
+
+The migration will unfold in two primary steps: `format` and `migrate`.
+
+> [!IMPORTANT]
+> If your GitOps files are version-controlled (stored in GitHub or similar) it is recommended to perform these steps in order, opening a pull request, moving through your standard review process and merging that pull request **before** moving to the next step.
+
+## Step 1: Format
+
+### Overview
+
+When manipulating YAML files with this tool, the output will always alphabetize the keys. This means the following YAML file:
+```yaml
+a: []
+c: []
+b: []
+```
+Will become:
+```yaml
+a: []
+b: []
+c: []
+```
+
+This _can_ mean, if your GitOps files are version-controlled (stored in GitHub or similar), you could see a very large number of changed lines which might make it more difficult to spot the **actual** transformations.
+
+Considering the above, we recommend running the `gitops-migrate` `format` command **before** performing the migration.
+
+### Steps
+
+Run the `gitops-migrate` tool, specifying the `format` command followed by the path to your GitOps YAML files.
+
+**Linux/Mac:**
+```bash
+# If 'gitops-migrate' is in the current working directory.
+$ ./gitops-migrate format ./fleet_gitops
+# If 'gitops-migrate' is in PATH.
+$ gitops-migrate format ./fleet_gitops
+```
+
+**Windows:**
+```powershell
+# If 'gitops-migrate' is in the current working directory.
+PS> .\gitops-migrate format .\fleet_gitops
+# If 'gitops-migrate' is in PATH.
+PS> gitops-migrate format .\fleet_gitops
+```
+
+Your YAML files should now all be alphabetized!
+
+> [!TIP]
+> It's recommended to pause here, commit changes, then open a pull request for the formatting changes _only_. Then move onto the next section once that pull request has been reviewed and merged.
+
+## Step 2: Migrate
+
+### Overview
+
+Now we'll run the `gitops-migrate` `migrate` command which will:
+- Perform a backup of your GitOps files, outputting an archive to your operating system's `TEMP` directory (**the path to this backup will be shown at the start of the output of the command, be sure to take note of it**).
+- Migrate all YAML files in the provided directory (changes outlined [above](#474-yaml-changes)).
+
+### Steps
+
+Run the `gitops-migrate` tool, specifying the `migrate` command and the path to your GitOps YAML files.
+
+**Linux/Mac**:
+```bash
+# If 'gitops-migrate' is in the current working directory.
+$ ./gitops-migrate migrate ./gitops_files
+# If 'gitops-migrate' is in PATH.
+$ gitops-migrate migrate ./gitops_files
+```
+
+**Windows:**
+```powershell
+# If 'gitops-migrate' is in the current working directory.
+PS> .\gitops-migrate migrate .\gitops_files
+# If 'gitops-migrate' is in PATH.
+PS> gitops-migrate migrate .\gitops_files
+```
+
+### Did it work?
+
+**In the command output,** you should see messages like the following:
+```shell
+> Successfully applied transforms to team file.
+┣━ [Team File]=>[it-and-security/teams/workstations.yml]
+┗━ [Count]=>[39]
+```
+
+In cases where the _team_ file previously contained software packages which referenced software files containing the fields [described above](#474-yaml-changes), you can spot-check the results by confirming these fields are now present in the software packages array items, right alongside the `path` key(s).
+
+**When looking at a `git diff`** you should see changes similar to the following:
+
+**Software file (`./slack.yml`):**
+```diff
+url: https://downloads.slack-edge.com/desktop-releases/linux/x64/4.41.105/slack-desktop-4.41.105-amd64.deb
+- self_service: true
+- categories:
+-  - Productivity
+-  - Communication
+- labels_include_any:
+-  - "Debian-based Linux hosts"
+```
+
+**Team File (`./my_team.yml`):**
+```diff
+software:
+  packages:
+    - path: slack.yml
++     self_service: true
++     categories:
++       - Productivity
++       - Communication
++     labels_include_any:
++       - "Debian-based Linux hosts"
+```
+
+### Help, something has gone wrong!
+
+In the event you've attempted the migration and encounter any issues, you can quickly revert your GitOps file states by simply restoring the backup taken automatically during the `migrate` process.
+
+To do this, locate the backup archive path in the log output:
+
+```bash
+> Performing Fleet GitOps file backup.
+┣━ [Source]=>[fleet_gitops]
+┗━ [Destination]=>[/tmp/fleet-gitops-1916163188/fleet-gitops-backup-8-31-2025_4-47-29.tar.gz] # <-- Here
+```
+
+Then simply run the `gitops-migrate` `restore` command to restore this backup, specifying the **archive** path as the first arg and the path to restore the archive **to** as the second arg:
+
+**Linux/Mac**:
+```bash
+# If 'gitops-migrate' is in the current working directory.
+$ ./gitops-migrate restore /tmp/fleet-gitops-1916163188/fleet-gitops-backup-8-31-2025_4-47-29.tar.gz ./fleet_gitops
+# If 'gitops-migrate' is in PATH.
+$ gitops-migrate restore /tmp/fleet-gitops-1916163188/fleet-gitops-backup-8-31-2025_4-47-29.tar.gz ./fleet_gitops
+```
+
+**Windows:**
+```powershell
+# If 'gitops-migrate' is in the current working directory.
+PS> .\gitops-migrate restore 'C:\Users\am\AppData\Local\Temp\fleet-gitops-1916163188/fleet-gitops-backup-8-31-2025_4-47-29.tar.gz' .\fleet_gitops
+# If 'gitops-migrate' is in PATH.
+PS> gitops-migrate restore 'C:\Users\am\AppData\Local\Temp\fleet-gitops-1916163188/fleet-gitops-backup-8-31-2025_4-47-29.tar.gz' .\fleet_gitops
+```

cmd/gitops-migrate/ansi/colorizer.go

@@ -0,0 +1,10 @@
+package ansi
+
+// Colorizer is a simple closure generator. You stick a color in and get back
+// a closure which accepts a string and wraps that string in the ANSI color
+// specified.
+func Colorizer(color string) func(string) string {
+	return func(s string) string {
+		return color + s + Reset
+	}
+}

cmd/gitops-migrate/ansi/colors.go

@@ -0,0 +1,26 @@
+package ansi
+
+const (
+	// Regular colors.
+	Black   = "\x1b[0;30m"
+	Green   = "\x1b[0;32m"
+	Yellow  = "\x1b[0;33m"
+	Blue    = "\x1b[0;34m"
+	Magenta = "\x1b[0;35m"
+	Cyan    = "\x1b[0;36m"
+	Red     = "\x1b[0;31m"
+	White   = "\x1b[0;37m"
+
+	// Bold colors.
+	BoldBlack   = "\x1b[1;30m"
+	BoldRed     = "\x1b[1;31m"
+	BoldGreen   = "\x1b[1;32m"
+	BoldYellow  = "\x1b[1;33m"
+	BoldBlue    = "\x1b[1;34m"
+	BoldMagenta = "\x1b[1;35m"
+	BoldCyan    = "\x1b[1;36m"
+	BoldWhite   = "\x1b[1;37m"
+
+	// Standard reset.
+	Reset = "\x1b[0m"
+)

cmd/gitops-migrate/args.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/fleetdm/fleet/v4/cmd/gitops-migrate/log"
+)
+
+type Args struct {
+	Debug    bool
+	Help     bool
+	Commands []string
+}
+
+func parseArgs() Args {
+	var args Args
+
+	// Override the default flag package's usage text.
+	flag.Usage = func() {
+		err := usageText(os.Stderr)
+		if err != nil {
+			log.Fatal("Failed to write usage text to stderr :|.")
+		}
+	}
+
+	// --debug
+	flag.BoolVar(&args.Debug, "debug", false, "")
+
+	// --help
+	flag.BoolVar(&args.Help, "help", false, "")
+	flag.BoolVar(&args.Help, "h", false, "")
+
+	// Parse command-line inputs.
+	flag.Parse()
+
+	// Capture positional args.
+	args.Commands = flag.Args()
+
+	return args
+}