Replace Bash fix-permissions script with Go

* Easier to test
* Can test more things
* Prevents symlink shenanigans

.buildkite/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '3'
+
+services:
+  fixperms-tests:
+    image: golang:latest
+    working_dir: /code
+    volumes:
+      - ..:/code:ro
+    command: go test -v ./...
+
+  fixperms-build:
+    image: golang:latest
+    working_dir: /code
+    volumes:
+      - ..:/code
+      - /var/lib/buildkite-agent/git-mirrors:/var/lib/buildkite-agent/git-mirrors
+    command: .buildkite/steps/build-fixperms.sh

.buildkite/pipeline.yaml

@@ -7,14 +7,28 @@ steps:
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
 
-  - id: "bats-tests"
-    name: ":bash: Unit tests"
+  - id: "fixperms-tests"
+    name: ":go: fixperms tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     plugins:
-      docker-compose#v2.1.0:
-        run: unit-tests
-        config: docker-compose.unit-tests.yml
+      - docker-compose#v2.1.0:
+          run: fixperms-tests
+          config: .buildkite/docker-compose.yml
+
+  - id: "fixperms-build"
+    name: ":go: fixperms build"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    depends_on:
+      - "fixperms-tests"
+    artifact_paths: "build/fix-perms-*"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: fixperms-build
+          config: .buildkite/docker-compose.yml
+      - artifacts#v1.9.0:
+          upload: "builds/fix-perms-*"
 
   - id: "deploy-service-role-stack"
     name: ":aws-iam: :cloudformation:"
@@ -23,7 +37,8 @@ steps:
     command: .buildkite/steps/deploy-service-role-stack.sh
     depends_on:
       - "lint"
-      - "bats-tests"
+      - "fixperms-tests"
+      - "fixperms-build"
 
   - id: "packer-windows-amd64"
     name: ":packer: :windows:"
@@ -34,7 +49,8 @@ steps:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     depends_on:
       - "lint"
-      - "bats-tests"
+      - "fixperms-tests"
+      - "fixperms-build"
 
   - id: "launch-windows-amd64"
     name: ":cloudformation: :windows: AMD64 Launch"
@@ -77,7 +93,8 @@ steps:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     depends_on:
       - "lint"
-      - "bats-tests"
+      - "fixperms-tests"
+      - "fixperms-build"
 
   - id: "launch-linux-amd64"
     name: ":cloudformation: :linux: AMD64 Launch"
@@ -119,7 +136,8 @@ steps:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     depends_on:
       - "lint"
-      - "bats-tests"
+      - "fixperms-tests"
+      - "fixperms-build"
 
   - id: "launch-linux-arm64"
     name: ":cloudformation: :linux: ARM64 Launch"

.buildkite/steps/build-fixperms.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+for arch in amd64 arm64; do
+  GOOS=linux GOARCH="${arch}" go build -v -o "build/fix-perms-linux-${arch}" ./internal/fixperms
+done

.buildkite/steps/packer.sh

@@ -16,6 +16,11 @@ fi
 
 mkdir -p "build/"
 
+if [[ "$os" == "linux" ]] ; then
+  buildkite-agent artifact download "build/fix-perms-linux-${arch}" ./build
+  mv "build/fix-perms-linux-${arch}" packer/linux/conf/buildkite-agent/scripts/fix-buildkite-agent-builds-permissions
+fi
+
 # Build a hash of packer files and the agent versions
 packer_files_sha=$(find Makefile "packer/${os}" plugins/ -type f -print0 | xargs -0 sha1sum | awk '{print $1}' | sort | sha1sum | awk '{print $1}')
 stable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/stable/latest/${agent_binary}.sha256")

go.mod

@@ -0,0 +1,8 @@
+module github.com/buildkite/elastic-ci-stack-for-aws/v6
+
+go 1.20
+
+require (
+	github.com/google/go-cmp v0.5.9
+	golang.org/x/sys v0.12.0
+)

go.sum

@@ -0,0 +1,4 @@
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/fixperms/fdfs/fdfs.go

@@ -0,0 +1,64 @@
+//go:build linux
+
+// Package fdfs is like os.DirFS, but with a file descriptor and openat(2),
+// fchownat(2), etc, to ensure symlinks do not escape.
+package fdfs
+
+import (
+	"io/fs"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+const resolveFlags = unix.RESOLVE_BENEATH | unix.RESOLVE_NO_SYMLINKS | unix.RESOLVE_NO_MAGICLINKS | unix.RESOLVE_NO_XDEV
+
+// FS uses a file descriptor for a directory as the base of a fs.FS.
+type FS uintptr
+
+// DirFS opens the directory dir, and returns an FS rooted at that directory.
+// It uses open(2) with O_PATH+O_DIRECTORY+O_CLOEXEC.
+func DirFS(dir string) (FS, error) {
+	bd, err := os.OpenFile(dir, unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
+	if err != nil {
+		return 0, err
+	}
+	return FS(bd.Fd()), nil
+}
+
+// Close closes the file descriptor.
+func (s FS) Close() error {
+	return unix.Close(int(s))
+}
+
+// Open wraps openat2(2) with O_RDONLY+O_NOFOLLOW+O_CLOEXEC.
+func (s FS) Open(path string) (fs.File, error) {
+	fd, err := unix.Openat2(int(s), path, &unix.OpenHow{
+		Flags:   unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC,
+		Mode:    0,
+		Resolve: resolveFlags,
+	})
+	if err != nil {
+		return nil, err
+	}
+	f := os.NewFile(uintptr(fd), path)
+	return f, nil
+}
+
+// Lchown wraps fchownat(2) (with AT_SYMLINK_NOFOLLOW).
+func (s FS) Lchown(path string, uid, gid int) error {
+	return unix.Fchownat(int(s), path, uid, gid, unix.AT_SYMLINK_NOFOLLOW)
+}
+
+// Sub wraps openat2(2) (with O_PATH+O_DIRECTORY+O_NOFOLLOW+O_CLOEXEC), and returns an FS.
+func (s FS) Sub(dir string) (FS, error) {
+	subFD, err := unix.Openat2(int(s), dir, &unix.OpenHow{
+		Flags:   unix.O_PATH | unix.O_DIRECTORY | unix.O_NOFOLLOW | unix.O_CLOEXEC,
+		Mode:    0,
+		Resolve: resolveFlags,
+	})
+	if err != nil {
+		return 0, err
+	}
+	return FS(subFD), nil
+}

internal/fixperms/fdfs/fdfs_test.go

@@ -0,0 +1,61 @@
+//go:build linux
+
+package fdfs
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestTOCTOUShenanigans(t *testing.T) {
+	path := "/tmp/TestTOCTOUShenanigans/foo"
+	if err := os.MkdirAll(path, 0o777); err != nil {
+		t.Fatalf("os.MkdirAll(%s, %o) = %v", path, 0o777, err)
+	}
+	fp := filepath.Join(path, "data")
+	if err := os.WriteFile(fp, []byte("innocent"), 0o666); err != nil {
+		t.Fatalf("os.WriteFile(%s, nil, 0o666) = %v", fp, err)
+	}
+
+	path2 := "/tmp/TestTOCTOUShenanigans/crimes"
+	if err := os.MkdirAll(path2, 0o777); err != nil {
+		t.Fatalf("os.MkdirAll(%s, %o) = %v", path2, 0o777, err)
+	}
+	fp2 := filepath.Join(path2, "data")
+	if err := os.WriteFile(fp2, []byte("guilty"), 0o666); err != nil {
+		t.Fatalf("os.WriteFile(%s, nil, 0o666) = %v", fp2, err)
+	}
+
+	// Do it in two steps, to simulate a trusted directory and an untrusted
+	// subpath.
+	fsys, err := DirFS("/tmp/TestTOCTOUShenanigans")
+	if err != nil {
+		t.Fatalf("DirFS(/tmp/TestTOCTOUShenanigans) error = %v", err)
+	}
+	defer fsys.Close()
+	fooFS, err := fsys.Sub("foo")
+	if err != nil {
+		t.Fatalf("DirFS(/tmp/TestTOCTOUShenanigans).Sub(foo) error = %v", err)
+	}
+	defer fooFS.Close()
+
+	// Replace foo with a symlink to crimes...
+	path3 := "/tmp/TestTOCTOUShenanigans/foo.bak"
+	if err := os.Rename(path, path3); err != nil {
+		t.Fatalf("os.Rename(%s, %s) = %v", path, path3, err)
+	}
+	if err := os.Symlink(path2, path); err != nil {
+		t.Fatalf("os.Symlink(%s, %s) = %v", path2, path, err)
+	}
+
+	// What do we get?
+	df, err := fs.ReadFile(fooFS, "data")
+	if err != nil {
+		t.Fatalf("fs.ReadFile(DirFS(%s), data) error = %v", path, err)
+	}
+	if got, want := string(df), "innocent"; got != want {
+		t.Fatalf("fs.ReadFile(DirFS(%s), data) contents = %q, want %q", path, got, want)
+	}
+}

internal/fixperms/fixer/fixer.go

@@ -0,0 +1,85 @@
+//go:buid linux
+
+package fixer
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/buildkite/elastic-ci-stack-for-aws/v6/internal/fixperms/fdfs"
+)
+
+// Main contains the higher-level operations of the permissions fixer.
+func Main(argv []string, baseDir, uname string) (string, int) {
+	if len(argv) != 4 {
+		return exitf(1, "Usage: %s AGENT_DIR ORG_DIR PIPELINE_DIR", argv[0])
+	}
+	for _, seg := range argv[1:] {
+		if seg != filepath.Clean(seg) {
+			return exitf(2, "Invalid argument %q", seg)
+		}
+		if seg == "." || seg == ".." || strings.ContainsRune(seg, '/') {
+			return exitf(2, "Invalid argument %q", seg)
+		}
+	}
+	subpath := filepath.Join(argv[1:]...)
+
+	// Get a file descriptor for the base builds directory.
+	bd, err := fdfs.DirFS(baseDir)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return exit0()
+		}
+		return exitf(3, "Couldn't open %s: %v", baseDir, err)
+	}
+	defer bd.Close()
+
+	// Get a file descriptor for the agentdir/orgdir/pipelinedir within the
+	// builds directory.
+	// openat2(2) flags ensures this is within the builds directory, and does
+	// not involve a symlink.
+	pd, err := bd.Sub(subpath)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return exit0()
+		}
+		return exitf(3, "Couldn't open %s: %v", subpath, err)
+	}
+	defer pd.Close()
+
+	// Get the uid and gid of buildkite-agent
+	agentUser, err := user.Lookup(uname)
+	if err != nil {
+		return exitf(4, "Couldn't look up buildkite-agent user: %v", err)
+	}
+	uid, err := strconv.Atoi(agentUser.Uid)
+	if err != nil {
+		return exitf(4, "buildkite-agent uid %q not an integer: %v", agentUser.Uid, err)
+	}
+	gid, err := strconv.Atoi(agentUser.Gid)
+	if err != nil {
+		return exitf(4, "buildkite-agent gid %q not an integer: %v", agentUser.Gid, err)
+	}
+
+	// fs.WalkDir to find everything within the directory.
+	// fchownat(2) to change the owner of the item.
+	// We allow symlinks here, but operate on the symlinks themselves.
+	if err := fs.WalkDir(pd, ".", func(path string, d fs.DirEntry, err error) error {
+		return pd.Lchown(path, uid, gid)
+	}); err != nil {
+		return exitf(5, "Couldn't recursively chown %s: %v", subpath, err)
+	}
+
+	return exit0()
+}
+
+func exit0() (string, int) { return "", 0 }
+
+func exitf(code int, f string, v ...any) (string, int) {
+	return fmt.Sprintf(f, v...), code
+}