cli: get proxy API can query proxies from k8s-gateway (#9226)

* env: inject deployer image.tag

* use plugin registry from extensions

* fix import cycle in gloo translator

* WIP - supoort proxy cache

* supoort proxy source in request

* use settings instead of whole opts

* more api enhancements, cli todo

* introduce port-forward utility

* Cli uses new port forward utility

* expand get proxies request

* cleanup code

* undo changes unrelated to proxy endpoit api

* generated code

* define rest.config once

* check proxies ignores k8s gateway proxies without status

* wrap error to improve debugability

* fix glooctl check unit test

* improve helm tests

* add changelog

* use 1.12.0: after the grpc api was first introduced

* remove problematic helm tests

* cleanup channel management

* bad ref to readyCh

* new gloo verion, update default validation max bytes

* delete more helm tests that are out of date

* order inputs, add comments

* support getting all proxies

* rework port-forwarding

* fix compile issue, waitForStop

* Adding changelog file to new location

* Deleting changelog file from old location

* goimports

* go mod tidy

* improve error handling

* sync access to cmd

* generated code

* close requires write lock

* lock all operations

* always use CLI for port forwarding...for now

* fix up port-forwarding

* goimports

* use CLI for now, comment why API portforward API is faulty

* add comments around CLI API

* cleanup cmd.Ctx

* pass context. avoid cancelling prematurely

* use default options

* pass context ot function

* Update projects/gloo/cli/pkg/cmd/options/options.go

Co-authored-by: Jenny Shu <28537278+jenshu@users.noreply.github.com>

* update changelog

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: changelog-bot <changelog-bot>
Co-authored-by: Jenny Shu <28537278+jenshu@users.noreply.github.com>

changelog/v1.17.0-beta12/support-get-proxy-api.yaml

@@ -0,0 +1,20 @@
+changelog:
+  - type: BREAKING_CHANGE
+    issueLink: https://github.com/solo-io/solo-projects/issues/5874
+    resolvesIssue: false
+    description: >-
+      `glooctl get proxy` will not work if you have persisted Proxy CRs in etcD and you are querying
+      and older server version (1.16 and below). In general, we recommend that you keep your client and 
+      server versions in sync. You can verify the client/server versions you are currently running by calling `glooctl version`.
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/5874
+    resolvesIssue: false
+    description: >-
+      Update the ProxyEndpointServer API to support returning proxies produced by the K8s Gateway translator.
+      Update the CLI fix the `get proxy` API so that you can effectively query proxies.
+      Introduce a PortForwarder.
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/gloo/issues/6661
+    resolvesIssue: false
+    description: >-
+      Remove helm upgrade tests that upgrade from a version of Gloo (1.11) that is no longer in the support window.

docs/content/reference/cli/glooctl_get_proxy.md

@@ -17,7 +17,11 @@ glooctl get proxy [flags]
 ### Options
 
 ```
-  -h, --help   help for proxy
+      --all               get all proxies
+      --edge              include proxies produced from edge gateway resources
+  -h, --help              help for proxy
+      --kube              include proxies produced from k8s gateway resources
+      --proxy-ns string   namespace where proxies are persisted (default "gloo-system")
 ```
 
 ### Options inherited from parent commands

docs/content/static/content/osa_provided.md

@@ -3,6 +3,7 @@ Name|Version|License
 [semver/v3](https://github.com/Masterminds/semver)|v3.2.1|MIT License
 [Netflix/go-expect](https://github.com/Netflix/go-expect)|v0.0.0-20180928190340-9d1f4485533b|Apache License 2.0
 [avast/retry-go](https://github.com/avast/retry-go)|v2.4.3+incompatible|MIT License
+[retry-go/v4](https://github.com/avast/retry-go)|v4.3.3|MIT License
 [aws/aws-sdk-go](https://github.com/aws/aws-sdk-go)|v1.34.9|Apache License 2.0
 [census-instrumentation/opencensus-proto](https://github.com/census-instrumentation/opencensus-proto)|v0.2.0|Apache License 2.0
 [xds/go](https://github.com/cncf/xds)|v0.0.0-20230607035331-e9ce68804cb4|Apache License 2.0

go.mod

@@ -88,6 +88,7 @@ require (
 )
 
 require (
+	github.com/avast/retry-go/v4 v4.3.3
 	github.com/go-logr/zapr v1.2.4
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.1
@@ -116,7 +117,6 @@ require (
 	github.com/armon/go-metrics v0.3.11 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
-	github.com/avast/retry-go/v4 v4.3.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/bufbuild/protocompile v0.6.0 // indirect

pkg/cliutil/uri.go

@@ -10,9 +10,13 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/avast/retry-go/v4"
+	"github.com/solo-io/gloo/pkg/utils/kubeutils/portforward"
+
 	"github.com/hashicorp/go-multierror"
 	errors "github.com/rotisserie/eris"
 	corev1 "k8s.io/api/core/v1"
@@ -27,6 +31,15 @@ const (
 	defaultTimeout = 30 * time.Second
 )
 
+var (
+	defaultRetryOptions = []retry.Option{
+		retry.LastErrorOnly(true),
+		retry.Delay(100 * time.Millisecond),
+		retry.DelayType(retry.BackOffDelay),
+		retry.Attempts(5),
+	}
+)
+
 // GetResource identified by the given URI.
 // The URI can either be a http(s) address or a relative/absolute file path.
 func GetResource(uri string) (io.ReadCloser, error) {
@@ -178,41 +191,51 @@ func minikubeIp(clusterName string) (string, error) {
 }
 
 // PortForward call kubectl port-forward. Callers are expected to clean up the returned portFwd *exec.cmd after the port-forward is no longer needed.
-func PortForward(namespace string, resource string, localPort string, kubePort string, verbose bool) (*exec.Cmd, error) {
-
-	/** port-forward command **/
-
-	portFwd := exec.Command("kubectl", "port-forward", "-n", namespace,
-		resource, fmt.Sprintf("%s:%s", localPort, kubePort))
-
+// Deprecated: Prefer portforward.NewPortForwarder
+func PortForward(ctx context.Context, namespace string, resource string, localPort string, kubePort string, verbose bool) (portforward.PortForwarder, error) {
 	err := Initialize()
 	if err != nil {
 		return nil, err
 	}
 	logger := GetLogger()
 
-	portFwd.Stderr = io.MultiWriter(logger, os.Stderr)
+	outWriter := logger
+	errWriter := io.MultiWriter(logger, os.Stderr)
 	if verbose {
-		portFwd.Stdout = io.MultiWriter(logger, os.Stdout)
-	} else {
-		portFwd.Stdout = logger
+		outWriter = io.MultiWriter(logger, os.Stdout)
 	}
 
-	if err := portFwd.Start(); err != nil {
+	resourceTypeName := strings.Split(resource, "/") // ie. deployment/gloo
+	localPortInt, err := strconv.Atoi(localPort)
+	if err != nil {
+		return nil, err
+	}
+	remotePortInt, err := strconv.Atoi(kubePort)
+	if err != nil {
 		return nil, err
 	}
 
-	return portFwd, nil
+	portForwarder := portforward.NewPortForwarder(
+		portforward.WithResource(resourceTypeName[1], namespace, resourceTypeName[0]),
+		portforward.WithPorts(localPortInt, remotePortInt),
+		portforward.WithWriters(outWriter, errWriter),
+	)
 
+	err = portForwarder.Start(ctx, defaultRetryOptions...)
+	if err != nil {
+		return nil, err
+	}
+
+	return portForwarder, nil
 }
 
 // PortForwardGet call kubectl port-forward and make a GET request.
-// Callers are expected to clean up the returned portFwd *exec.cmd after the port-forward is no longer needed.
-func PortForwardGet(ctx context.Context, namespace string, resource string, localPort string, kubePort string, verbose bool, getPath string) (string, *exec.Cmd, error) {
+// Callers are expected to clean up the returned portforward.PortForwarder after the port-forward is no longer needed.
+// Deprecated: Prefer portforward.NewPortForwarder
+func PortForwardGet(ctx context.Context, namespace string, resource string, localPort string, kubePort string, verbose bool, getPath string) (string, portforward.PortForwarder, error) {
 
 	/** port-forward command **/
-
-	portFwd, err := PortForward(namespace, resource, localPort, kubePort, verbose)
+	portForwarder, err := PortForward(ctx, namespace, resource, localPort, kubePort, verbose)
 	if err != nil {
 		return "", nil, err
 	}
@@ -231,7 +254,7 @@ func PortForwardGet(ctx context.Context, namespace string, resource string, loca
 				return
 			default:
 			}
-			res, err := http.Get("http://localhost:" + localPort + getPath)
+			res, err := http.Get(fmt.Sprintf("http://%s/%s", portForwarder.Address(), strings.TrimPrefix(getPath, "/")))
 			if err != nil {
 				errs <- err
 				time.Sleep(retryInterval)
@@ -260,13 +283,9 @@ func PortForwardGet(ctx context.Context, namespace string, resource string, loca
 		case err := <-errs:
 			multiErr = multierror.Append(multiErr, err)
 		case res := <-result:
-			return res, portFwd, nil
+			return res, portForwarder, nil
 		case <-localCtx.Done():
-			if portFwd.Process != nil {
-				portFwd.Process.Kill()
-				portFwd.Process.Release()
-			}
-			return "", nil, errors.Errorf("timed out trying to connect to localhost during port-forward, errors: %v", multiErr)
+			return "", portForwarder, errors.Errorf("timed out trying to connect to localhost during port-forward, errors: %v", multiErr)
 		}
 	}
 

pkg/utils/kubeutils/names.go

@@ -0,0 +1,5 @@
+package kubeutils
+
+const (
+	GlooDeploymentName = "gloo"
+)

pkg/utils/kubeutils/pods.go

@@ -0,0 +1,83 @@
+package kubeutils
+
+import (
+	"context"
+
+	"k8s.io/client-go/rest"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// Inspired by: https://github.com/solo-io/gloo-mesh-enterprise/blob/main/pkg/utils/kubeutils/pods.go
+
+// GetPodsForDeployment gets all pods backing a deployment
+func GetPodsForDeployment(
+	ctx context.Context,
+	restConfig *rest.Config,
+	deploymentName string,
+	deploymentNamespace string,
+) ([]string, error) {
+	kubeClient, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	deployment, err := kubeClient.AppsV1().Deployments(deploymentNamespace).Get(ctx, deploymentName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	matchLabels := deployment.Spec.Selector.MatchLabels
+	listOptions := (&client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(matchLabels),
+		FieldSelector: fields.Set{"status.phase": "Running"}.AsSelector(),
+	}).AsListOptions()
+
+	podList, err := kubeClient.CoreV1().Pods(deploymentNamespace).List(ctx, *listOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	pods := make([]string, len(podList.Items))
+	for i := range podList.Items {
+		pods[i] = podList.Items[i].Name
+	}
+
+	return pods, nil
+}
+
+// GetPodsForService gets all pods backing a deployment
+func GetPodsForService(
+	ctx context.Context,
+	restConfig *rest.Config,
+	serviceName string,
+	serviceNamespace string,
+) ([]string, error) {
+	kubeClient, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	service, err := kubeClient.CoreV1().Services(serviceNamespace).Get(ctx, serviceName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	matchLabels := service.Spec.Selector
+	listOptions := (&client.ListOptions{LabelSelector: labels.SelectorFromSet(matchLabels)}).AsListOptions()
+
+	podList, err := kubeClient.CoreV1().Pods(serviceNamespace).List(ctx, *listOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	pods := make([]string, len(podList.Items))
+	for i := range podList.Items {
+		pods[i] = podList.Items[i].Name
+	}
+
+	return pods, nil
+}