Add local registry support for k3d environment

Dockerfile

@@ -52,20 +52,18 @@ LABEL app=$COMPONENT
 
 CMD ["/app"]
 
-FROM builder as e2e
-
-# Copy common CA certificates from Builder image (installed by default with ca-certificates package)
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-
-COPY --from=builder /bin/$COMPONENT /app.test
+FROM builder as e2e-builder
 
 RUN apk add --no-cache 'git=>2.30' && \
-    go install github.com/onsi/ginkgo/ginkgo@latest
+    export CGO_ENABLED=0 && go install github.com/onsi/ginkgo/ginkgo@latest
 
-LABEL source=git@github.com:capactio/capact.git
-LABEL app=$COMPONENT
+FROM generic as e2e
+ARG COMPONENT
+
+COPY --from=builder /bin/$COMPONENT /app.test
+COPY --from=e2e-builder /go/bin/ginkgo /ginkgo
 
-CMD ["/go/bin/ginkgo", "-v", "-nodes=1", "/app.test" ]
+CMD ["/ginkgo", "-v", "-nodes=1", "/app.test" ]
 
 FROM alpine:3.13.5 as terraform-runner
 ARG COMPONENT

Makefile

@@ -7,8 +7,8 @@ export GOPROXY = https://proxy.golang.org
 # enable the BuildKit builder in the Docker CLI.
 export DOCKER_BUILDKIT = 1
 
-export DOCKER_REPOSITORY ?= ghcr.io/capactio
-export DOCKER_TAG ?= latest
+DOCKER_REPOSITORY ?= ghcr.io/capactio
+DOCKER_TAG ?= latest
 
 all: generate build-all-images test-unit test-lint ## Default: generate all, build all, test all and lint
 .PHONY: all

cmd/cli/cmd/environment/create/k3d.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	"capact.io/capact/internal/cli/capact"
 	"capact.io/capact/internal/cli/environment/create"
 	"capact.io/capact/internal/cli/printer"
 
@@ -18,34 +19,56 @@ import (
 func NewK3d() *cobra.Command {
 	var opts create.K3dOptions
 
+	status := printer.NewStatus(os.Stdout, "")
+
 	k3d := cluster.NewCmdClusterCreate()
 	k3d.Use = "k3d"
 	k3d.Args = cobra.NoArgs
 	k3d.Short = "Provision local k3d cluster"
+	// Needs to be `PersistentPreRunE` as the `PreRunE` is used by k3d to configure viper config.
 	k3d.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
 		spinnerFmt := printer.NewLogrusSpinnerFormatter(fmt.Sprintf("Creating cluster %s...", opts.Name))
 		logrus.SetFormatter(spinnerFmt)
-		return create.K3dSetDefaultConfig(cmd.Flags())
+
+		if err := create.K3dSetDefaultConfig(cmd.Flags(), opts); err != nil {
+			return err
+		}
+		if !opts.RegistryEnabled {
+			return nil
+		}
+		return create.LocalRegistry(cmd.Context(), status)
 	}
-	k3d.RunE = func(cmd *cobra.Command, _ []string) (err error) {
-		// Run k3d create cmd
+	k3d.RunE = func(cmd *cobra.Command, _ []string) error {
+		// 1. Create k3d cluster
 		k3d.Run(cmd, []string{opts.Name})
 
 		if opts.Wait == time.Duration(0) {
 			return nil
 		}
+		// 2. Wait for k3d cluster
 		ctx, cancel := context.WithTimeout(cmd.Context(), opts.Wait)
 		defer cancel()
-		return create.WaitForK3dReadyNodes(ctx, os.Stdout, opts.Name)
+		return create.WaitForK3dReadyNodes(ctx, status, opts.Name)
+	}
+	k3d.PostRunE = func(cmd *cobra.Command, args []string) (err error) {
+		if !opts.RegistryEnabled {
+			return nil
+		}
+		if err := create.RegistryConnWithNetwork(cmd.Context(), create.K3dDockerNetwork); err != nil {
+			return err
+		}
+
+		return capact.AddRegistryToHostsFile()
 	}
 
-	create.K3dRemoveWaitAndTimeoutFlags(k3d) // remove it so we use own `--wait` flag
+	create.K3dRemoveWaitAndTimeoutFlags(k3d) // remove it, so we use own `--wait` flag
 
 	// add `name` flag to have the same UX as we have for `kind` in the minimal scenario:
 	//   $ capact env create kind --name capact-dev --wait 10m
 	//   $ capact env create k3d  --name capact-dev --wait 10m
 	k3d.Flags().StringVar(&opts.Name, "name", create.DefaultClusterName, "Cluster name")
 	k3d.Flags().DurationVar(&opts.Wait, "wait", time.Duration(0), "Wait for control plane node to be ready")
+	k3d.Flags().BoolVar(&opts.RegistryEnabled, "enable-registry", false, "Create Capact local Docker registry and configure k3d environment to use it")
 
 	return k3d
 }

cmd/cli/cmd/environment/delete/k3d.go

@@ -13,25 +13,28 @@ import (
 
 // NewK3d returns a cobra.Command for creating k3d environment.
 func NewK3d() *cobra.Command {
-	var name string
+	var opts delete.K3dOptions
+
 	cmd := &cobra.Command{
 		Use:   "k3d",
 		Short: "Delete local k3d cluster",
 		Args:  cobra.NoArgs,
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			spinnerFmt := printer.NewLogrusSpinnerFormatter(fmt.Sprintf("Deleting cluster %q...", name))
+			spinnerFmt := printer.NewLogrusSpinnerFormatter(fmt.Sprintf("Deleting cluster %q...", opts.Name))
 			logrus.SetFormatter(spinnerFmt)
 		},
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			return delete.K3d(cmd.Context(), name)
+			return delete.K3d(cmd.Context(), opts)
 		},
 	}
 
 	// add `name` flag to have the same UX as we have for `kind` in the minimal scenario:
 	//   $ capact env delete kind --name capact-dev
 	//   $ capact env delete k3d  --name capact-dev
-	cmd.Flags().StringVar(&name, "name", create.DefaultClusterName, "Cluster name")
+	cmd.Flags().StringVar(&opts.Name, "name", create.DefaultClusterName, "Cluster name")
 	_ = cmd.RegisterFlagCompletionFunc("name", util.ValidArgsAvailableClusters)
 
+	cmd.Flags().BoolVar(&opts.RemoveRegistry, "remove-registry", true, "Remove registry")
+
 	return cmd
 }

cmd/cli/cmd/install.go

@@ -35,6 +35,10 @@ func NewInstall() *cobra.Command {
 			# Install Capact from local git repository. Needs to be run from the main directory
 			<cli> install --version @local`, cli.Name),
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := opts.Validate(); err != nil {
+				return err
+			}
+
 			k8sCfg, err := config.GetConfig()
 			if err != nil {
 				return errors.Wrap(err, "while creating k8s config")
@@ -57,6 +61,7 @@ func NewInstall() *cobra.Command {
 	flags.BoolVar(&opts.UpdateHostsFile, "update-hosts-file", true, "Updates /etc/hosts with entry for Capact GraphQL Gateway.")
 	flags.BoolVar(&opts.UpdateTrustedCerts, "update-trusted-certs", true, "Add Capact GraphQL Gateway certificate.")
 	flags.StringVar(&opts.Parameters.Override.HelmRepoURL, "helm-repo-url", capact.HelmRepoStable, fmt.Sprintf("Capact Helm chart repository URL. Use %s tag to select repository which holds the latest Helm chart versions.", capact.LatestVersionTag))
+	flags.BoolVar(&opts.LocalRegistryEnabled, "enable-registry", false, "If specified, Capact images are pushed to Capact local Docker registry.")
 	flags.StringSliceVar(&opts.Parameters.Override.CapactStringOverrides, "capact-overrides", []string{}, "Overrides for Capact component.")
 	flags.StringSliceVar(&opts.Parameters.Override.IngressStringOverrides, "ingress-controller-overrides", []string{}, "Overrides for Ingress controller component.")
 	flags.StringSliceVar(&opts.Parameters.Override.CertManagerStringOverrides, "cert-manager-overrides", []string{}, "Overrides for Cert Manager component.")

cmd/cli/docs/capact_environment_create_k3d.md

@@ -27,6 +27,7 @@ capact environment create k3d [flags]
       --agents-memory string                                           Memory limit imposed on the agents nodes [From docker]
       --api-port [HOST:]HOSTPORT                                       Specify the Kubernetes API server port exposed on the LoadBalancer (Format: [HOST:]HOSTPORT)
                                                                         - Example: `k3d cluster create --servers 3 --api-port 0.0.0.0:6550`
+      --enable-registry                                                Create Capact local Docker registry and configure k3d environment to use it
   -e, --env KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]                   Add environment variables to nodes (Format: KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]
                                                                         - Example: `k3d cluster create --agents 2 -e "HTTP_PROXY=my.proxy.com@server[0]" -e "SOME_KEY=SOME_VAL@server[0]"`
       --gpus string                                                    GPU devices to add to the cluster node containers ('all' to pass all GPUs) [From docker]

cmd/cli/docs/capact_environment_delete_k3d.md

@@ -13,8 +13,9 @@ capact environment delete k3d [flags]
 ### Options
 
 ```
-  -h, --help          help for k3d
-      --name string   Cluster name (default "dev-capact")
+  -h, --help              help for k3d
+      --name string       Cluster name (default "dev-capact")
+      --remove-registry   Remove registry (default true)
 ```
 
 ### Options inherited from parent commands

cmd/cli/docs/capact_install.md

@@ -33,6 +33,7 @@ capact install --version @local
       --build-image strings                    Local images names that should be build when using @local version. Takes comma-separated list. (default [argo-actions,argo-runner,e2e-test,gateway,hub-js,k8s-engine,populator])
       --capact-overrides strings               Overrides for Capact component.
       --cert-manager-overrides strings         Overrides for Cert Manager component.
+      --enable-registry                        If specified, Capact images are pushed to Capact local Docker registry.
       --environment string                     Capact environment. (default "kind")
       --helm-repo-url string                   Capact Helm chart repository URL. Use @latest tag to select repository which holds the latest Helm chart versions. (default "https://storage.googleapis.com/capactio-stable-charts")
   -h, --help                                   help for install

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2
 	github.com/docker/cli v20.10.6+incompatible
 	github.com/docker/docker v20.10.8+incompatible
+	github.com/docker/go-connections v0.4.0
 	github.com/evanphx/json-patch/v5 v5.5.0 // indirect
 	github.com/fatih/color v1.12.0
 	github.com/fatih/structs v1.1.0

hack/dev-cluster-create.sh

@@ -32,7 +32,6 @@ main() {
     capact::create_cluster
 
     export DOCKER_TAG=dev
-    export DOCKER_REPOSITORY="local"
     export PRINT_INSECURE_NOTES="true"
     shout "Installing Capact on development local cluster..."
     capact::install

hack/dev-cluster-update.sh

@@ -23,7 +23,6 @@ main() {
     shout "Updating development local cluster..."
 
     export DOCKER_TAG=dev-$RANDOM
-    export DOCKER_REPOSITORY="local"
     export BUILD_IMAGES="true"
     export REPO_DIR=$REPO_ROOT_DIR
     export CLUSTER_TYPE=${CLUSTER_TYPE:-"kind"}

hack/lib/utilities.sh

@@ -163,10 +163,14 @@ capact::create_cluster() {
     if [[ "${MULTINODE_CLUSTER:-"false"}" == "true" ]]; then
       CLUSTER_CONFIG_FLAG=--cluster-config="${REPO_DIR}/hack/cluster-config/kind/config-multinode.yaml"
     fi
+    if [[ "${DISABLE_K3D_REGISTRY:-"false"}" == "false" && "${CLUSTER_TYPE}" == "k3d" ]]; then
+      K3D_REGISTRY="--enable-registry"
+    fi
     # shellcheck disable=SC2086
     capact::cli env create ${CLUSTER_TYPE} --verbose \
       --name="${CLUSTER_NAME}" \
       ${CLUSTER_CONFIG_FLAG:-} \
+      ${K3D_REGISTRY:-} \
       --wait=5m
 }
 
@@ -182,13 +186,15 @@ capact::delete_cluster() {
 # Installs Capact charts. If they are already installed, it upgrades them.
 #
 # Required envs:
-#  - DOCKER_REPOSITORY
 #  - DOCKER_TAG
 #  - REPO_DIR
 #  - CAPACT_NAMESPACE
-#  - CAPACT_VERSION
 #  - CLUSTER_TYPE
 #  - CLUSTER_NAME
+# Optional envs:
+#  - CAPACT_VERSION
+#  - DOCKER_REPOSITORY
+#  - PRINT_INSECURE_NOTES
 #  - ENABLE_POPULATOR - if set to true then database populator will be enabled and it will populate database with manifests
 #  - USE_TEST_SETUP - if set to true, then a test policy is configured
 #  - INCREASE_RESOURCE_LIMITS - if set to true, then the components will use higher resource requests and limits
@@ -214,7 +220,6 @@ capact::install() {
     export CAPACT_OVERRIDES=${CAPACT_OVERRIDES:=""}
     export CAPACT_INSTALL_ADDITIONAL_OPTS=""
 
-    CAPACT_OVERRIDES+=",global.containerRegistry.path=${DOCKER_REPOSITORY}"
     CAPACT_OVERRIDES+=",global.containerRegistry.overrideTag=${DOCKER_TAG}"
     CAPACT_OVERRIDES+=",hub-public.populator.enabled=${ENABLE_POPULATOR}"
     CAPACT_OVERRIDES+=",engine.testSetup.enabled=${USE_TEST_SETUP}"
@@ -240,6 +245,10 @@ capact::install() {
       CAPACT_OVERRIDES+=",hub-public.populator.manifestsLocation.repository=${HUB_MANIFESTS_SOURCE_REPO_URL}"
     fi
 
+    if [ -n "${DOCKER_REPOSITORY:-}" ]; then
+      CAPACT_OVERRIDES+=",global.containerRegistry.path=${DOCKER_REPOSITORY}"
+    fi
+
     if [[ "${BUILD_IMAGES:-"true"}" == "false" ]]; then
       BUILD_IMAGES_FLAG=--build-image=""
     fi
@@ -248,6 +257,10 @@ capact::install() {
       CAPACT_INSTALL_ADDITIONAL_OPTS="${CAPACT_INSTALL_ADDITIONAL_OPTS} --helm-repo-url=${CAPACT_HELM_REPO_URL}"
     fi
 
+    if [[ "${DISABLE_K3D_REGISTRY:-"false"}" == "false" && "${CLUSTER_TYPE}" == "k3d" ]]; then
+      CAPACT_INSTALL_ADDITIONAL_OPTS="${CAPACT_INSTALL_ADDITIONAL_OPTS} --enable-registry"
+    fi
+
     # shellcheck disable=SC2086
     capact::cli install --verbose \
         --environment="${CLUSTER_TYPE}" \

internal/cli/capact/build.go

@@ -1,15 +1,17 @@
 package capact
 
 import (
+	"bytes"
 	"context"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
 	"sort"
 
-	"github.com/pkg/errors"
+	"capact.io/capact/internal/cli"
+	"capact.io/capact/internal/cli/printer"
 
+	"github.com/pkg/errors"
 	"k8s.io/utils/strings/slices"
 )
 
@@ -71,10 +73,9 @@ var Images = images{
 	},
 }
 
-func buildImage(ctx context.Context, w io.Writer, imgName string, img image, repository, version string) (string, error) {
+func buildImage(ctx context.Context, status *printer.Status, imgName string, img image, repository, version string) (string, error) {
 	// docker build --build-arg COMPONENT=$(APP) --target generic -t $(DOCKER_REPOSITORY)/$(APP):$(DOCKER_TAG)
 	imageTag := fmt.Sprintf("%s/%s:%s", repository, imgName, version)
-
 	// #nosec G204
 	cmd := exec.CommandContext(ctx, "docker",
 		"build",
@@ -99,27 +100,57 @@ func buildImage(ctx context.Context, w io.Writer, imgName string, img image, rep
 		}
 	}
 	cmd.Dir = img.Dir
-	cmd.Stdout = w
-	cmd.Stderr = w
-	err := cmd.Run()
-	if err != nil {
+	if err := runCMD(cmd, status, "Building image %s", imageTag); err != nil {
 		return "", err
 	}
+
 	return imageTag, nil
 }
 
 // BuildImages builds passed images setting passed repository and version
-func BuildImages(ctx context.Context, w io.Writer, repository, version string, names []string) ([]string, error) {
+func BuildImages(ctx context.Context, status *printer.Status, repository, version string, names []string) ([]string, error) {
 	var created []string
 
 	for _, image := range Images.All() {
-		if slices.Contains(names, image) {
-			imageTag, err := buildImage(ctx, w, image, Images[image], repository, version)
-			if err != nil {
-				return nil, errors.Wrapf(err, "while building image %s", image)
-			}
-			created = append(created, imageTag)
+		if !slices.Contains(names, image) {
+			continue
+		}
+		imageTag, err := buildImage(ctx, status, image, Images[image], repository, version)
+		if err != nil {
+			return nil, errors.Wrapf(err, "while building image %s", image)
 		}
+		created = append(created, imageTag)
 	}
 	return created, nil
 }
+
+// PushImages pushes passed images to a given registry
+func PushImages(ctx context.Context, status *printer.Status, names []string) error {
+	for _, image := range names {
+		// #nosec G204
+		cmd := exec.CommandContext(ctx, "docker", "push", image)
+
+		if err := runCMD(cmd, status, "Pushing %s", image); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func runCMD(cmd *exec.Cmd, status *printer.Status, stageFmt string, args ...interface{}) error {
+	if cli.VerboseMode.IsTracing() {
+		cmd.Stdout = status.Writer()
+		cmd.Stderr = status.Writer()
+		return cmd.Run()
+	}
+
+	var buff bytes.Buffer
+	status.Step(stageFmt, args...)
+	cmd.Stderr = &buff
+	err := cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "while running cmd [stderr: %s]", buff.String())
+	}
+
+	return nil
+}