Minor kind refactor to work within containers (#149)

.run/Run Migrations.run.xml

@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Run Migrations" type="ShConfigurationType">
+    <option name="SCRIPT_TEXT" value="mage runMigrations" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/usr/bin/zsh" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="false" />
+    <envs />
+    <method v="2" />
+  </configuration>
+</component>

.run/Start Dependencies.run.xml

@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Start Dependencies" type="ShConfigurationType">
+    <option name="SCRIPT_TEXT" value="" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/scripts/infrastructure-start.sh" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/usr/bin/zsh" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <envs />
+    <method v="2" />
+  </configuration>
+</component>

docker-compose.yaml

@@ -1,5 +1,3 @@
-version: "3.8"
-
 networks:
   kind:
     external: true
@@ -34,7 +32,7 @@ services:
       - kind
 
   pulsar:
-    image: ${PULSAR_IMAGE:-apachepulsar/pulsar:2.10.4}
+    image: ${PULSAR_IMAGE:-apachepulsar/pulsar:3.0.2}
     container_name: pulsar
     volumes:
       - ./developer/dependencies/pulsar.conf:/conf/pulsar.conf

e2e/setup/ingress-nginx.yaml

@@ -669,4 +669,4 @@ webhooks:
     - UPDATE
     resources:
     - ingresses
-  sideEffects: None
+  sideEffects: None

e2e/setup/kind.yaml

@@ -1,11 +1,13 @@
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 name: armada-test
+featureGates:
+  "KubeletInUserNamespace": true
 nodes:
 - role: worker
-  image: kindest/node:v1.24.7
+  image: kindest/node:v1.24.17
 - role: control-plane
-  image: kindest/node:v1.24.7
+  image: kindest/node:v1.24.17
   kubeadmConfigPatches:
   - |
     kind: InitConfiguration

magefiles/config.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"encoding/json"
+
+	"fmt"
+	"os"
+)
+
+type BuildConfig struct {
+	DockerRegistries map[string]string `json:"dockerRegistries"`
+}
+
+func getBuildConfig() (BuildConfig, error) {
+	configPath := os.Getenv("ARMADA_BUILD_CONFIG")
+	if configPath == "" {
+		return BuildConfig{}, nil
+	}
+
+	content, err := os.ReadFile(configPath)
+	if err != nil {
+		return BuildConfig{}, fmt.Errorf("error reading file: %w", err)
+	}
+
+	var config BuildConfig
+	err = json.Unmarshal(content, &config)
+	if err != nil {
+		return BuildConfig{}, fmt.Errorf("error parsing JSON: %w", err)
+	}
+
+	return config, nil
+}

magefiles/developer.go

@@ -49,12 +49,18 @@ func getComponentsList() []string {
 	return strings.Split(os.Getenv("ARMADA_COMPONENTS"), ",")
 }
 
-// Dependencies include pulsar, postgres (v1 and v2) as well as redis.
-func StartDependencies() error {
-	if onArm() {
-		os.Setenv("PULSAR_IMAGE", "richgross/pulsar:2.11.0")
+// Runs scheduler and lookout migrations
+func RunMigrations() error {
+	migrations := []string{
+		"scheduler-migration",
+		"lookoutv2-migration",
 	}
+	command := append([]string{"compose", "up", "-d"}, migrations...)
+	return dockerRun(command...)
+}
 
+// Starts armada infrastructure dependencies
+func StartDependencies() error {
 	command := append([]string{"compose", "up", "-d"}, dependencies...)
 	return dockerRun(command...)
 }

magefiles/kind.go

@@ -1,13 +1,14 @@
 package main
 
 import (
+	"fmt"
+	"github.com/magefile/mage/mg"
 	"os"
 	"path/filepath"
-	"runtime"
+	"regexp"
 	"strings"
 
 	semver "github.com/Masterminds/semver/v3"
-	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
 )
@@ -19,22 +20,12 @@ const (
 	KIND_NAME               = "armada-test"
 )
 
-func getImages() []string {
-	images := []string{
+func getImagesUsedInTestsOrControllers() []string {
+	return []string{
+		"nginx:1.21.6", // Used by ingress-controller
 		"alpine:3.18.3",
-		"nginx:1.21.6",
-		"registry.k8s.io/ingress-nginx/controller:v1.4.0",
-		"registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343",
-	}
-	// TODO: find suitable kubectl image for arm64
-	if !isAppleSilicon() {
-		images = append(images, "bitnami/kubectl:1.24.8")
+		"bitnami/kubectl:1.30",
 	}
-	return images
-}
-
-func isAppleSilicon() bool {
-	return runtime.GOOS == "darwin" && runtime.GOARCH == "arm64"
 }
 
 func kindBinary() string {
@@ -73,18 +64,6 @@ func kindCheck() error {
 	return constraintCheck(version, KIND_VERSION_CONSTRAINT, "kind")
 }
 
-// Images that need to be available in the Kind cluster,
-// e.g., images required for e2e tests.
-func kindGetImages() error {
-	for _, image := range getImages() {
-		if err := dockerRun("pull", image); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func kindInitCluster() error {
 	out, err := kindOutput("get", "clusters")
 	if err != nil {
@@ -103,23 +82,122 @@ func kindInitCluster() error {
 	return nil
 }
 
-func kindSetup() error {
-	mg.Deps(kindInitCluster, kindGetImages)
+func imagesFromFile(resourceYamlPath string) ([]string, error) {
+	content, err := os.ReadFile(resourceYamlPath)
+	if err != nil {
+		return nil, fmt.Errorf("error reading file: %w", err)
+	}
+
+	re := regexp.MustCompile(`(?m)image:\s*([^\s]+)`)
+	matches := re.FindAllStringSubmatch(string(content), -1)
+	if matches == nil {
+		return nil, nil
+	}
+
+	var images []string
+	for _, match := range matches {
+		if len(match) > 1 {
+			images = append(images, match[1])
+		}
+	}
+
+	return images, nil
+}
+
+func remapDockerRegistryIfRequired(image string, registries map[string]string) string {
+	for registryFrom, registryTo := range registries {
+		if strings.HasPrefix(image, registryFrom) {
+			return registryTo + strings.TrimPrefix(image, registryFrom)
+		}
+	}
+	return image
+}
+
+func remapDockerImagesInKubernetesManifest(filePath string, images []string, buildConfig BuildConfig) (string, error) {
+	if buildConfig.DockerRegistries == nil {
+		return filePath, nil
+	}
+
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return filePath, fmt.Errorf("error reading manifest: %w", err)
+	}
+
+	replacedContent := ""
+	for _, image := range images {
+		targetImage := remapDockerRegistryIfRequired(image, buildConfig.DockerRegistries)
+		if targetImage != image {
+			if replacedContent == "" {
+				replacedContent = string(content)
+			}
+
+			replacedContent = strings.ReplaceAll(replacedContent, image, targetImage)
+		}
+	}
+
+	if replacedContent == "" {
+		return filePath, nil
+	}
+
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		return filePath, fmt.Errorf("error creating temporary file: %w", err)
+	}
+	_, err = f.WriteString(replacedContent)
+	if err != nil {
+		return filePath, fmt.Errorf("error writing temporary file: %w", err)
+	}
+	return f.Name(), nil
+}
+
+func kindSetupExternalImages(buildConfig BuildConfig, images []string) error {
+	for _, image := range images {
+		image = remapDockerRegistryIfRequired(image, buildConfig.DockerRegistries)
+		if err := dockerRun("pull", image); err != nil {
+			return fmt.Errorf("error pulling image: %w", err)
+		}
 
-	for _, image := range getImages() {
 		err := kindRun("load", "docker-image", image, "--name", KIND_NAME)
 		if err != nil {
-			return err
+			return fmt.Errorf("error loading image to kind: %w", err)
 		}
 	}
-	// Resources to create in the Kind cluster.
+
+	return nil
+}
+
+func kindSetup() error {
+	mg.Deps(kindInitCluster)
+
+	buildConfig, err := getBuildConfig()
+
+	err = kindSetupExternalImages(buildConfig, getImagesUsedInTestsOrControllers())
+	if err != nil {
+		return nil
+	}
+
 	resources := []string{
 		"e2e/setup/ingress-nginx.yaml",
 		"e2e/setup/priorityclasses.yaml",
 		"e2e/setup/namespace-with-anonymous-user.yaml",
 	}
 	for _, f := range resources {
-		err := kubectlRun("apply", "-f", f, "--context", "kind-armada-test")
+		images, err := imagesFromFile(f)
+		if err != nil {
+			return err
+		}
+
+		err = kindSetupExternalImages(buildConfig, images)
+		if err != nil {
+			return err
+		}
+
+		file, err := remapDockerImagesInKubernetesManifest(f, images, buildConfig)
+		if err != nil {
+			return err
+		}
+
+		err = kubectlRun("apply", "-f", file, "--context", "kind-armada-test")
 		if err != nil {
 			return err
 		}

scripts/infrastructure-start.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/sh
+mage kind startDependencies

scripts/infrastructure-stop.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/sh
+mage stopDependencies kindTeardown