refactor: deprecates Earthly cloud support and adds remote satellite support

.github/workflows/ci.yml

@@ -22,10 +22,6 @@ on:
         required: false
         type: boolean
         default: false
-    secrets:
-      earthly_token:
-        description: Optional Earthly token used to login to Earthly cloud during local builds of Forge CLI
-        required: false
 
 env:
   FORGE_REGEX_CHECK: ^check(-.*)?$
@@ -52,10 +48,9 @@ jobs:
         with:
           version: ${{ inputs.forge_version }}
       - name: Install Local Forge
+        id: install-local
         uses: input-output-hk/catalyst-forge/actions/install-local@master
         if: ${{ inputs.forge_version == 'local' }}
-        with:
-          earthly_token: ${{ secrets.earthly_token }}
       - name: Check forge version
         id: local
         run: |
@@ -71,7 +66,8 @@ jobs:
         with:
           skip_docker: 'true'
           skip_github: 'true'
-          skip_earthly: ${{ steps.local.outputs.skip }}
+          skip_earthly_install: ${{ steps.install-local.outputs.cache-hit == false }}
+          skip_earthly_satellite: ${{ steps.install-local.outputs.cache-hit == false }}
       - name: Discovery
         id: discovery
         uses: input-output-hk/catalyst-forge/actions/discovery@master

.github/workflows/dogfood.yml

@@ -17,6 +17,4 @@ jobs:
     with:
       forge_version: local
       verbosity: debug
-      nightly: true
-    secrets:
-      earthly_token: ${{ secrets.EARTHLY_TOKEN }}
+      nightly: true

.github/workflows/run.yml

@@ -47,6 +47,7 @@ jobs:
         with:
           version: ${{ inputs.forge_version }}
       - name: Install Local Forge
+        id: install-local
         uses: input-output-hk/catalyst-forge/actions/install-local@master
         if: ${{ inputs.forge_version == 'local' }}
         with:
@@ -64,7 +65,8 @@ jobs:
       - name: Setup CI
         uses: input-output-hk/catalyst-forge/actions/setup@master
         with:
-          skip_earthly: ${{ steps.local.outputs.skip }}
+          skip_earthly_install: ${{ steps.install-local.outputs.cache-hit == false }}
+          skip_earthly_satellite: ${{ steps.install-local.outputs.cache-hit == false }}
       - name: Run
         uses: input-output-hk/catalyst-forge/actions/run@master
         with:

actions/install-local/action.yml

@@ -1,44 +1,87 @@
 name: Install local forge CLI
 description: Installs a local version of the forge CLI
-inputs:
-  earthly_token:
-        description: Earthly token used to login to Earthly cloud for local build
-        required: true
 
 runs:
   using: composite
   steps:
+    - name: Cache binary
+      id: cache-binary
+      uses: actions/cache@v4
+      with:
+        path: /usr/local/bin/forge
+        key: ${{ runner.os }}-${{ github.sha }}
     - shell: bash
+      if: steps.cache-binary.outputs.cache-hit == false
       run: echo "Building Forge CLI locally..."
     - name: Cache Earthly binary
       id: cache-earthly
+      if: steps.cache-binary.outputs.cache-hit == false
       uses: actions/cache@v4
       with:
         path: /usr/local/bin/earthly
         key: ${{ runner.os }}
     - name: Install Earthly
-      if: steps.cache-earthly.outputs.cache-hit == false
+      if: steps.cache-earthly.outputs.cache-hit == false && steps.cache-binary.outputs.cache-hit == false
       shell: bash
       run: |
         wget -q https://github.com/earthly/earthly/releases/latest/download/earthly-linux-amd64 -O /usr/local/bin/earthly
         chmod +x /usr/local/bin/earthly
         /usr/local/bin/earthly bootstrap
-    - name: Login to Earthly Cloud
+    - name: Install CUE
+      uses: cue-lang/setup-cue@v1.0.0
+      if: steps.cache-binary.outputs.cache-hit == false
+      with:
+        version: latest
+    - name: Get AWS configuration
+      id: aws
       if: steps.cache-binary.outputs.cache-hit == false
       shell: bash
       run: |
-        earthly account login --token ${{ inputs.earthly_token }}
-        earthly org select Catalyst
-    - name: Cache binary
-      id: cache-binary
-      uses: actions/cache@v4
+        AWS=$(cue export -e global.ci.providers.aws ./blueprint.cue)
+        REGION=$(echo "$AWS" | jq -r .region)
+        ROLE=$(echo "$AWS" | jq -r .role)
+
+        echo "region=$REGION" >> $GITHUB_OUTPUT
+        echo "role=$ROLE" >> $GITHUB_OUTPUT
+    - name: Login to AWS
+      uses: aws-actions/configure-aws-credentials@v4
+      if: steps.cache-binary.outputs.cache-hit == false
       with:
-        path: /usr/local/bin/forge
-        key: ${{ runner.os }}-${{ github.sha }}
+        aws-region: ${{ steps.aws.outputs.region }}
+        role-to-assume: ${{ steps.aws.outputs.role }}
+    - name: Configure Earthly satellite credentials
+      if: steps.cache-binary.outputs.cache-hit == false
+      shell: bash
+      run: |
+        mkdir -p "$HOME/.earthly"
+
+        EARTHLY=$(cue export -e global.ci.providers.earthly.satellite.credentials ./blueprint.cue)
+        SECRET_ID=$(echo "$EARTHLY" | jq -r .path)
+
+        SECRET=$(aws secretsmanager get-secret-value --secret-id "$SECRET_ID")
+        echo "$SECRET" | jq -r .SecretString | jq -r .ca_certificate | base64 -d > ~/.earthly/ca.pem
+        echo "$SECRET" | jq -r .SecretString | jq -r .certificate | base64 -d > ~/.earthly/cert.pem
+        echo "$SECRET" | jq -r .SecretString | jq -r .private_key | base64 -d > ~/.earthly/key.pem
+
+        cat <<EOF > "$HOME/.earthly/config.yml"
+        global:
+          buildkit_host: $(echo "$SECRET" | jq -r .SecretString | jq -r .host)
+          tlsca: ca.pem
+          tlscert: cert.pem
+          tlskey: key.pem
+        EOF
+
+        echo "::add-mask::$(echo "$SECRET" | jq -r .SecretString | jq -r .host)"
     - name: Build Forge CLI
       if: steps.cache-binary.outputs.cache-hit == false
       shell: bash
       run: |
-        echo "::group::Forge CLI Earthly Build"
-        earthly --sat ci --artifact ./cli+build/forge /usr/local/bin/forge
-        echo "::endgroup::"
+        earthly --artifact ./cli+build/forge /usr/local/bin/forge
+    - name: Set output cache-hit
+      shell: bash
+      run: echo "cache-hit=${{ steps.cache-binary.outputs.cache-hit }}" >> $GITHUB_OUTPUT
+
+outputs:
+  cache-hit:
+    description: "Whether the forge binary was restored from cache"
+    value: ${{ steps.cache-binary.outputs.cache-hit }}

actions/setup/action.yml

@@ -17,8 +17,12 @@ inputs:
     description: If true, skip authenticating to DockerHub
     required: false
     default: "false"
-  skip_earthly:
-    description: If true, skip installing Earthly and authenticating to Earthly Cloud
+  skip_earthly_install:
+    description: If true, skip installing Earthly
+    required: false
+    default: "false"
+  skip_earthly_satellite:
+    description: If true, skip adding authentication for the remote Earthly satellite
     required: false
     default: "false"
   skip_github:
@@ -133,7 +137,7 @@ runs:
 
     # Earthly Provider
     - name: Get Earthly provider configuration
-      if: inputs.skip_earthly == 'false'
+      if: inputs.skip_earthly_install == 'false' && inputs.skip_earthly_satellite == 'false'
       id: earthly
       shell: bash
       run: |
@@ -144,40 +148,30 @@ runs:
 
         EARTHLY=$(echo "$BP" | jq -r .global.ci.providers.earthly)
         if [[ "$EARTHLY" != "null" ]]; then
-          ORG=$(echo "$BP" | jq -r .global.ci.providers.earthly.org)
           VERSION=$(echo "$BP" | jq -r .global.ci.providers.earthly.version)
 
-          echo "org=$ORG" >> $GITHUB_OUTPUT
-
-          EARTHLY_CREDS=$(echo "$BP" | jq -r .global.ci.providers.earthly.credentials)
-          if [[ "$EARTHLY_CREDS" != "null" ]]; then
-            SECRET=$(forge secret get --project . global.ci.providers.earthly.credentials)
-            TOKEN=$(echo "$SECRET" | jq -r .token)
-
-            if [[ "$TOKEN" == "null" ]]; then
-              echo "Error: the earthly provider secret must map the secret value to 'token'"
-              exit 1
-            fi
-
-            echo "::add-mask::$TOKEN"
-            echo "token=$TOKEN" >> $GITHUB_OUTPUT
+          SATELLITE_CREDS=$(echo "$BP" | jq -r .global.ci.providers.earthly.satellite.credentials)
+          if [[ "$SATELLITE_CREDS" != "null" ]]; then
+            CONFIG_SAT="true"
           else
-            echo "No configuration found for Earthly Cloud provider"
+            CONFIG_SAT="false"
+            echo "No configuration found for remote Earthly satellite"
           fi
         else
-          echo "No configuration found for Earthly Cloud provider"
+          echo "No configuration found for Earthly provider"
         fi
 
+        echo "sat=$CONFIG_SAT" >> GITHUB_OUTPUT
         echo "version=$VERSION" >> $GITHUB_OUTPUT
     - name: Cache Earthly binary
       id: cache-binary
       uses: actions/cache@v4
-      if: inputs.skip_earthly == 'false'
+      if: inputs.skip_earthly_install == 'false'
       with:
         path: /usr/local/bin/earthly
         key: ${{ runner.os }}-${{ steps.earthly.outputs.version }}
     - name: Install Earthly
-      if: inputs.skip_earthly == 'false' && steps.cache-binary.outputs.cache-hit == false
+      if: inputs.skip_earthly_install == 'false' && steps.cache-binary.outputs.cache-hit == false
       shell: bash
       run: |
         if [[ "${{ steps.earthly.outputs.version }}" == "latest" ]]; then
@@ -188,16 +182,12 @@ runs:
 
         chmod +x /usr/local/bin/earthly
         /usr/local/bin/earthly bootstrap
-    # - name: Login to Earthly Cloud
-    #   if: steps.earthly.outputs.token != '' && steps.earthly.conclusion == 'success'
-    #   shell: bash
-    #   run: |
-    #     earthly account login --token "${{ steps.earthly.outputs.token }}"
-    # - name: Set Earthly organization
-    #   if: steps.earthly.outputs.token != '' && steps.earthly.conclusion == 'success'
-    #   shell: bash
-    #   run: |
-    #     earthly org select "${{ steps.earthly.outputs.org }}"
+    - name: Configure Earthly satellite credentials
+      if: inputs.skip_earthly_satellite == 'false' && steps.earthly.conclusion == 'success'
+      shell: bash
+      run: |
+        rm -rf "$HOME/.earthly"
+        forge configure-satellite -vvv --ci
 
     # Timoni Provider
     - name: Get Timoni provider configuration

blueprint.cue

@@ -33,13 +33,11 @@ global: {
 			}
 
 			earthly: {
-				credentials: {
+				satellite: credentials: {
 					provider: "aws"
-					path:     "global/ci/earthly"
+					path:     "global/ci/ci-tls"
 				}
-				org:       "Catalyst"
-				satellite: "ci"
-				version:   "0.8.15"
+				version: "0.8.15"
 			}
 
 			git: credentials: {

cli/cmd/cmds/configure_satellite.go

@@ -0,0 +1,62 @@
+package cmds
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/input-output-hk/catalyst-forge/cli/pkg/earthly/satellite"
+	"github.com/input-output-hk/catalyst-forge/cli/pkg/run"
+	"github.com/input-output-hk/catalyst-forge/lib/tools/fs/billy"
+	"github.com/input-output-hk/catalyst-forge/lib/tools/git"
+	"github.com/input-output-hk/catalyst-forge/lib/tools/walker"
+)
+
+type ConfigureSatelliteCmd struct {
+	Path string `short:"p" help:"Path to place the Earthly config and certificates."`
+}
+
+func (c *ConfigureSatelliteCmd) Run(ctx run.RunContext) error {
+	fs := billy.NewBaseOsFS()
+	cwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("failed to get current working directory: %w", err)
+	}
+
+	ctx.Logger.Debug("Finding git root", "path", cwd)
+	w := walker.NewCustomReverseFSWalker(fs, ctx.Logger)
+	gitRoot, err := git.FindGitRoot(cwd, &w)
+	if err != nil {
+		return fmt.Errorf("failed to find git root: %w", err)
+	}
+	ctx.Logger.Debug("Git root found", "path", gitRoot)
+
+	ctx.Logger.Debug("Loading project", "path", gitRoot)
+	project, err := ctx.ProjectLoader.Load(gitRoot)
+	if err != nil {
+		return err
+	}
+
+	if c.Path == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return fmt.Errorf("failed to get user's home directory: %w", err)
+		}
+
+		c.Path = filepath.Join(home, ".earthly")
+	}
+
+	ctx.Logger.Info("Configuring satellite", "path", c.Path)
+	satellite := satellite.NewEarthlySatellite(
+		&project,
+		c.Path,
+		ctx.Logger,
+		satellite.WithSecretStore(ctx.SecretStore),
+		satellite.WithCI(ctx.CI),
+	)
+	if err := satellite.Configure(); err != nil {
+		return fmt.Errorf("failed to configure satellite: %w", err)
+	}
+
+	return nil
+}

cli/cmd/main.go

@@ -32,15 +32,16 @@ type GlobalArgs struct {
 var cli struct {
 	GlobalArgs
 
-	Dump     cmds.DumpCmd     `cmd:"" help:"Dumps a project's blueprint to JSON."`
-	CI       cmds.CICmd       `cmd:"" help:"Simulate a CI run."`
-	Mod      module.ModuleCmd `kong:"cmd" help:"Commands for working with deployment modules."`
-	Release  cmds.ReleaseCmd  `cmd:"" help:"Release a project."`
-	Run      cmds.RunCmd      `cmd:"" help:"Run an Earthly target."`
-	Scan     cmds.ScanCmd     `cmd:"" help:"Scan for Earthfiles."`
-	Secret   cmds.SecretCmd   `cmd:"" help:"Manage secrets."`
-	Validate cmds.ValidateCmd `cmd:"" help:"Validates a project."`
-	Version  VersionCmd       `cmd:"" help:"Print the version."`
+	Dump               cmds.DumpCmd               `cmd:"" help:"Dumps a project's blueprint to JSON."`
+	CI                 cmds.CICmd                 `cmd:"" help:"Simulate a CI run."`
+	ConfigureSatellite cmds.ConfigureSatelliteCmd `cmd:"" help:"Configure the local system to use a remote Earthly Satellite."`
+	Mod                module.ModuleCmd           `kong:"cmd" help:"Commands for working with deployment modules."`
+	Release            cmds.ReleaseCmd            `cmd:"" help:"Release a project."`
+	Run                cmds.RunCmd                `cmd:"" help:"Run an Earthly target."`
+	Scan               cmds.ScanCmd               `cmd:"" help:"Scan for Earthfiles."`
+	Secret             cmds.SecretCmd             `cmd:"" help:"Manage secrets."`
+	Validate           cmds.ValidateCmd           `cmd:"" help:"Validates a project."`
+	Version            VersionCmd                 `cmd:"" help:"Print the version."`
 
 	InstallCompletions kongplete.InstallCompletions `cmd:"" help:"install shell completions"`
 }

cli/pkg/earthly/earthly_test.go

@@ -164,15 +164,6 @@ func TestEarthlyExecutor_buildArguments(t *testing.T) {
 			platform: GetBuildPlatform(),
 			expect:   []string{"--platform", GetBuildPlatform(), "--allow-privileged", "/test/dir+foo"},
 		},
-		{
-			name: "with satellite",
-			e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{},
-				testutils.NewNoopLogger(),
-				WithSatellite("satellite"),
-			),
-			platform: GetBuildPlatform(),
-			expect:   []string{"--platform", GetBuildPlatform(), "--sat", "satellite", "/test/dir+foo"},
-		},
 	}
 
 	for _, tt := range tests {