refactor: disable local registry in some cases

e2e/tests/localregistry/localregistry.go

@@ -423,9 +423,6 @@ var _ = DevSpaceDescribe("localregistry", func() {
 		}
 		err = buildCmd.RunDefault(f)
 		framework.ExpectError(err)
-		gomega.Expect(output.String()).To(
-			gomega.ContainSubstring("unable to push image my-docker-username/helloworld-kaniko and only docker and buildkit builds support using a local registry"),
-		)
 		gomega.Expect(output.String()).To(
 			gomega.ContainSubstring("UNAUTHORIZED: authentication required"),
 		)

helper/cmd/proxycommands/configure.go

@@ -6,6 +6,7 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/loft-sh/devspace/helper/util/stderrlog"
@@ -17,12 +18,14 @@ var (
 	sshPrivateKeyPath = "/tmp/ssh_private_key"
 	sshPublicKeyPath  = "/tmp/ssh_public_key"
 	proxyCommandsPath = "/tmp/proxy_commands"
+	portPath          = "/tmp/port"
 )
 
 // ConfigureCmd holds the ssh cmd flags
 type ConfigureCmd struct {
 	PublicKey  string
 	PrivateKey string
+	Port       int
 	WorkingDir string
 
 	GitCredentials bool
@@ -42,6 +45,7 @@ func NewConfigureCmd() *cobra.Command {
 
 	configureCmd.Flags().StringVar(&cmd.PublicKey, "public-key", "", "Public key to use")
 	configureCmd.Flags().StringVar(&cmd.PrivateKey, "private-key", "", "Private key to use")
+	configureCmd.Flags().IntVar(&cmd.Port, "port", 0, "Port inside the container to connect to")
 	configureCmd.Flags().StringVar(&cmd.WorkingDir, "working-dir", "", "Working dir to use")
 	configureCmd.Flags().StringSliceVar(&cmd.Commands, "commands", []string{}, "Commands to overwrite")
 	configureCmd.Flags().BoolVar(&cmd.GitCredentials, "git-credentials", false, "If git credentials should get configured")
@@ -112,6 +116,14 @@ func (cmd *ConfigureCmd) Run(_ *cobra.Command, _ []string) error {
 		}
 	}
 
+	// now configure the port
+	if cmd.Port > 0 {
+		err = os.WriteFile(portPath, []byte(strconv.Itoa(cmd.Port)), 0644)
+		if err != nil {
+			return errors.Wrap(err, "write port file")
+		}
+	}
+
 	// now configure working dir
 	workingDir := cmd.WorkingDir
 	if workingDir == "" {

helper/cmd/proxycommands/run.go

@@ -50,19 +50,27 @@ func runProxyCommand(args []string) error {
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 	}
 
-	client, err := ssh.Dial("tcp", "localhost:10567", clientConfig)
+	// get port
+	port, err := os.ReadFile(portPath)
+	if err != nil {
+		return errors.Wrap(err, "read port")
+	}
+
+	// dial ssh
+	client, err := ssh.Dial("tcp", "localhost:"+string(port), clientConfig)
 	if err != nil {
 		return errors.Wrap(err, "dial ssh")
 	}
 	defer client.Close()
 
+	// create new session
 	session, err := client.NewSession()
 	if err != nil {
 		return errors.Wrap(err, "new session")
 	}
 	defer session.Close()
 
-	// check if we should use a pty#
+	// check if we should use a pty
 	var (
 		width  = 0
 		height = 0

pkg/devspace/build/build.go

@@ -7,8 +7,6 @@ import (
 	"strings"
 
 	"github.com/loft-sh/devspace/pkg/devspace/build/builder"
-	"github.com/loft-sh/devspace/pkg/devspace/build/builder/buildkit"
-	"github.com/loft-sh/devspace/pkg/devspace/build/builder/docker"
 	"github.com/loft-sh/devspace/pkg/devspace/build/registry"
 	"github.com/loft-sh/devspace/pkg/devspace/build/types"
 	"github.com/loft-sh/devspace/pkg/devspace/config/constants"
@@ -151,38 +149,34 @@ func (c *controller) Build(ctx devspacecontext.Context, images []string, options
 		imageCache.ImageName = imageName
 		imageCache.LocalRegistryImageName = ""
 
-		if registry.UseLocalRegistry(kubeClient, conf, options.SkipPush) && !registry.HasPushPermission(imageConf) {
-			if SupportsLocalRegistry(builder) {
-				// Not able to deploy a local registry without a valid kube context
-				if kubeClient == nil {
-					return fmt.Errorf("unable to push image %s and a valid kube context is not available", imageConf.Image)
-				}
+		if registry.UseLocalRegistry(kubeClient, conf, imageConf, builder, options.SkipPush) && !registry.HasPushPermission(imageConf) {
+			// Not able to deploy a local registry without a valid kube context
+			if kubeClient == nil {
+				return fmt.Errorf("unable to push image %s and a valid kube context is not available", imageConf.Image)
+			}
 
-				registryOptions := registry.NewDefaultOptions().
-					WithNamespace(kubeClient.Namespace()).
-					WithLocalRegistryConfig(conf.LocalRegistry)
+			registryOptions := registry.NewDefaultOptions().
+				WithNamespace(kubeClient.Namespace()).
+				WithLocalRegistryConfig(conf.LocalRegistry)
 
-				// Create and start a local registry if one isn't already running
-				localRegistry, err := registry.GetOrCreateLocalRegistry(ctx, registryOptions)
-				if err != nil {
-					return errors.Wrap(err, "get or create local registry")
-				}
+			// Create and start a local registry if one isn't already running
+			localRegistry, err := registry.GetOrCreateLocalRegistry(ctx, registryOptions)
+			if err != nil {
+				return errors.Wrap(err, "get or create local registry")
+			}
 
-				// Update cache for local registry use
-				imageCache.LocalRegistryImageName, err = localRegistry.RewriteImage(imageName)
-				if err != nil {
-					return errors.Wrap(err, "rewrite image")
-				}
-				ctx.Config().LocalCache().SetImageCache(imageConfigName, imageCache)
+			// Update cache for local registry use
+			imageCache.LocalRegistryImageName, err = localRegistry.RewriteImage(imageName)
+			if err != nil {
+				return errors.Wrap(err, "rewrite image")
+			}
+			ctx.Config().LocalCache().SetImageCache(imageConfigName, imageCache)
 
-				// Reset the builder for local registry usage
-				// TODO: refactor so this isn't necessary!
-				builder, err = c.createBuilder(ctx, imageConfigName, imageConf, imageTags, options)
-				if err != nil {
-					return errors.Wrap(err, "create builder")
-				}
-			} else {
-				ctx.Log().Warnf("unable to push image %s and only docker and buildkit builds support using a local registry", imageConf.Image)
+			// Reset the builder for local registry usage
+			// TODO: refactor so this isn't necessary!
+			builder, err = c.createBuilder(ctx, imageConfigName, imageConf, imageTags, options)
+			if err != nil {
+				return errors.Wrap(err, "create builder")
 			}
 		}
 
@@ -430,14 +424,3 @@ func (c *controller) waitForBuild(ctx devspacecontext.Context, errChan <-chan er
 
 	return nil
 }
-
-func SupportsLocalRegistry(builder builder.Interface) bool {
-	switch builder.(type) {
-	case *buildkit.Builder:
-		return true
-	case *docker.Builder:
-		return true
-	default:
-		return false
-	}
-}

pkg/devspace/build/registry/util.go

@@ -3,8 +3,11 @@ package registry
 import (
 	"context"
 	"fmt"
+	"github.com/loft-sh/devspace/pkg/devspace/build/builder"
+	"github.com/loft-sh/devspace/pkg/devspace/build/builder/kaniko"
 	"io"
 	"net/http"
+	"runtime"
 	"strings"
 
 	"github.com/docker/docker/pkg/jsonmessage"
@@ -120,22 +123,49 @@ func CopyImageToRemote(ctx context.Context, client dockerclient.Client, imageNam
 	return <-errChan
 }
 
-func UseLocalRegistry(client kubectl.Client, config *latest.Config, skipPush bool) bool {
+func UseLocalRegistry(client kubectl.Client, config *latest.Config, imageConfig *latest.Image, imageBuilder builder.Interface, skipPush bool) bool {
 	if skipPush {
 		return false
+	} else if client == nil {
+		return false
 	}
 
-	if client == nil {
+	// TODO: better check for arm64 architectures
+	if runtime.GOARCH != "amd64" {
 		return false
 	}
 
+	// check if builder is kaniko
+	if imageBuilder != nil {
+		_, ok := imageBuilder.(*kaniko.Builder)
+		if ok {
+			return false
+		}
+	}
+
+	// check if image looks weird like localhost / cluster.local
+	if imageConfig != nil {
+		if imageConfig.Kaniko != nil {
+			return false
+		} else if imageConfig.Custom != nil {
+			return false
+		} else if imageConfig.BuildKit != nil && imageConfig.BuildKit.InCluster != nil {
+			return false
+		}
+
+		imageWithoutPort := strings.Split(imageConfig.Image, ":")[0]
+		if imageWithoutPort == "" || imageWithoutPort == "localhost" || imageWithoutPort == "127.0.0.1" || strings.HasSuffix(imageWithoutPort, ".cluster.local") {
+			return false
+		}
+	}
+
+	// check if fallback
 	if !IsLocalRegistryFallback(config) {
 		return IsLocalRegistryEnabled(config)
 	}
 
-	context := client.CurrentContext()
-
 	// Determine if this is a vcluster
+	context := client.CurrentContext()
 	isVClusterContext := strings.Contains(context, "vcluster_")
 
 	// Determine if this is a local kubernetes cluster

pkg/devspace/build/registry/util_test.go

@@ -307,7 +307,7 @@ func TestUseLocalRegistry(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		actual := UseLocalRegistry(testCase.client, testCase.config, testCase.skipPush)
+		actual := UseLocalRegistry(testCase.client, testCase.config, nil, nil, testCase.skipPush)
 		assert.Equal(t, actual, testCase.expected, "Unexpected result in test case %s", testCase.name)
 	}
 }

pkg/devspace/services/proxycommands/commands.go

@@ -3,6 +3,7 @@ package proxycommands
 import (
 	"encoding/base64"
 	"fmt"
+	"strconv"
 	"strings"
 
 	sshpkg "github.com/gliderlabs/ssh"
@@ -20,22 +21,25 @@ import (
 var DefaultRemotePort = 10567
 
 // StartProxyCommands starts the reverse commands functionality
-func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, selector targetselector.TargetSelector, parent *tomb.Tomb) (retErr error) {
+func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
 	if ctx == nil || ctx.Config() == nil || ctx.Config().Config() == nil {
 		return fmt.Errorf("DevSpace config is nil")
 	}
 
 	// init done array is used to track when sync was initialized
 	initDoneArray := []chan struct{}{}
+	remotePort := DefaultRemotePort
 	loader.EachDevContainer(devPod, func(devContainer *latest.DevContainer) bool {
 		if len(devContainer.ProxyCommands) == 0 {
 			return true
 		}
 
+		oldRemotePort := remotePort
 		initDone := parent.NotifyGo(func() error {
-			return startProxyCommands(ctx, devContainer, devPod.Name, string(devContainer.Arch), devContainer.ProxyCommands, selector.WithContainer(devContainer.Container), parent)
+			return startProxyCommands(ctx, devPod.Name, string(devContainer.Arch), oldRemotePort, devContainer.ProxyCommands, selector.WithContainer(devContainer.Container), parent)
 		})
 		initDoneArray = append(initDoneArray, initDone)
+		remotePort++
 		return true
 	})
 
@@ -46,7 +50,7 @@ func StartProxyCommands(ctx devspacecontext.Context, devPod *latest.DevPod, sele
 	return nil
 }
 
-func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevContainer, name, arch string, reverseCommands []*latest.ProxyCommand, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
+func startProxyCommands(ctx devspacecontext.Context, name, arch string, remotePort int, reverseCommands []*latest.ProxyCommand, selector targetselector.TargetSelector, parent *tomb.Tomb) error {
 	if ctx.IsDone() {
 		return nil
 	}
@@ -59,11 +63,8 @@ func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevCon
 
 	defer ssh.GetInstance(ctx.Log()).ReleasePort(port)
 
-	// get remote port
-	defaultRemotePort := DefaultRemotePort
-
 	// start reverse port forwarding from that port
-	mapping := fmt.Sprintf("%d:%d", port, defaultRemotePort)
+	mapping := fmt.Sprintf("%d:%d", port, remotePort)
 	err = portforwarding.StartReversePortForwarding(ctx, name, arch, []*latest.PortMapping{
 		{
 			Port: mapping,
@@ -74,10 +75,10 @@ func startProxyCommands(ctx devspacecontext.Context, devContainer *latest.DevCon
 	}
 
 	// start ssh
-	return startLocalSSH(ctx, selector, devContainer, reverseCommands, fmt.Sprintf(":%d", port), parent)
+	return startLocalSSH(ctx, selector, reverseCommands, remotePort, fmt.Sprintf(":%d", port), parent)
 }
 
-func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSelector, devContainer *latest.DevContainer, reverseCommands []*latest.ProxyCommand, addr string, parent *tomb.Tomb) error {
+func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSelector, reverseCommands []*latest.ProxyCommand, remotePort int, addr string, parent *tomb.Tomb) error {
 	if ctx.IsDone() {
 		return nil
 	}
@@ -111,7 +112,7 @@ func startLocalSSH(ctx devspacecontext.Context, selector targetselector.TargetSe
 	}
 
 	// execute configure command in container
-	command := []string{inject.DevSpaceHelperContainerPath, "proxy-commands", "configure", "--public-key", base64.StdEncoding.EncodeToString([]byte(publicKey)), "--private-key", base64.StdEncoding.EncodeToString([]byte(privateKey))}
+	command := []string{inject.DevSpaceHelperContainerPath, "proxy-commands", "configure", "--port", strconv.Itoa(remotePort), "--public-key", base64.StdEncoding.EncodeToString([]byte(publicKey)), "--private-key", base64.StdEncoding.EncodeToString([]byte(privateKey))}
 	if len(commandsToReplace) > 0 {
 		command = append(command, "--commands", strings.Join(commandsToReplace, ","))
 	}