feat: add metrics-adapter component to support centralized hpa

Signed-off-by: jwcesign <jiangwei115@huawei.com>

Makefile

@@ -17,7 +17,8 @@ TARGETS := karmada-aggregated-apiserver \
 			karmada-scheduler-estimator \
 			karmada-interpreter-webhook-example \
 			karmada-search \
-			karmada-operator
+			karmada-operator \
+			karmada-metrics-adapter
 
 CTL_TARGETS := karmadactl kubectl-karmada
 
@@ -122,6 +123,7 @@ endif
 	docker push ${REGISTRY}/karmada-aggregated-apiserver:${VERSION}
 	docker push ${REGISTRY}/karmada-search:${VERSION}
 	docker push ${REGISTRY}/karmada-operator:${VERSION}
+	docker push ${REGISTRY}/karmada-metrics-adapter:${VERSION}
 
 # Build and package binary
 #

artifacts/deploy/karmada-metrics-adapter-apiservice.yaml

@@ -0,0 +1,25 @@
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+  labels:
+    app: karmada-metrics-adapter
+    apiserver: "true"
+spec:
+  insecureSkipTLSVerify: true
+  group: metrics.k8s.io
+  groupPriorityMinimum: 2000
+  service:
+    name: karmada-metrics-adapter
+    namespace: karmada-system
+  version: v1beta1
+  versionPriority: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: karmada-metrics-adapter
+  namespace: karmada-system
+spec:
+  type: ExternalName
+  externalName: karmada-metrics-adapter.karmada-system.svc.cluster.local

artifacts/deploy/karmada-metrics-adapter.yaml

@@ -0,0 +1,84 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: karmada-metrics-adapter
+  namespace: karmada-system
+  labels:
+    app: karmada-metrics-adapter
+    apiserver: "true"
+spec:
+  selector:
+    matchLabels:
+      app: karmada-metrics-adapter
+      apiserver: "true"
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: karmada-metrics-adapter
+        apiserver: "true"
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: karmada-metrics-adapter
+          image: docker.io/karmada/karmada-metrics-adapter:latest
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
+            - name: kubeconfig
+              subPath: kubeconfig
+              mountPath: /etc/kubeconfig
+          command:
+            - /bin/karmada-metrics-adapter
+            - --kubeconfig=/etc/kubeconfig
+            - --authentication-kubeconfig=/etc/kubeconfig
+            - --authorization-kubeconfig=/etc/kubeconfig
+            - --client-ca-file=/etc/karmada/pki/ca.crt
+            - --audit-log-path=-
+            - --audit-log-maxage=0
+            - --audit-log-maxbackup=0
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 443
+              scheme: HTTPS
+            initialDelaySeconds: 1
+            periodSeconds: 3
+            timeoutSeconds: 15
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 443
+              scheme: HTTPS
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 15
+          resources:
+            requests:
+              cpu: 100m
+      volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
+        - name: kubeconfig
+          secret:
+            secretName: kubeconfig
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: karmada-metrics-adapter
+  namespace: karmada-system
+  labels:
+    app: karmada-metrics-adapter
+    apiserver: "true"
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 443
+  selector:
+    app: karmada-metrics-adapter

cmd/metrics-adapter/app/metrics-adapter.go

@@ -0,0 +1,62 @@
+package app
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/term"
+
+	"github.com/karmada-io/karmada/cmd/metrics-adapter/app/options"
+	"github.com/karmada-io/karmada/pkg/sharedcli"
+	"github.com/karmada-io/karmada/pkg/sharedcli/klogflag"
+	"github.com/karmada-io/karmada/pkg/version/sharedcommand"
+)
+
+// NewMetricsAdapterCommand creates a *cobra.Command object with default parameters
+func NewMetricsAdapterCommand(ctx context.Context) *cobra.Command {
+	opts := options.NewOptions()
+
+	cmd := &cobra.Command{
+		Use:  "karmada-metrics-adapter",
+		Long: `The karmada-metrics-adapter is a adapter to aggregate the metrics from member clusters.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := opts.Complete(); err != nil {
+				return err
+			}
+			if err := opts.Validate(); err != nil {
+				return err
+			}
+			if err := opts.Run(ctx); err != nil {
+				return err
+			}
+			return nil
+		},
+		Args: func(cmd *cobra.Command, args []string) error {
+			for _, arg := range args {
+				if len(arg) > 0 {
+					return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
+				}
+			}
+			return nil
+		},
+	}
+
+	fss := cliflag.NamedFlagSets{}
+
+	genericFlagSet := fss.FlagSet("generic")
+	opts.AddFlags(genericFlagSet)
+
+	// Set klog flags
+	logsFlagSet := fss.FlagSet("logs")
+	klogflag.Add(logsFlagSet)
+
+	cmd.AddCommand(sharedcommand.NewCmdVersion("karmada-metrics-adapter"))
+	cmd.Flags().AddFlagSet(genericFlagSet)
+	cmd.Flags().AddFlagSet(logsFlagSet)
+
+	cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
+	sharedcli.SetUsageAndHelpFunc(cmd, fss, cols)
+	return cmd
+}

cmd/metrics-adapter/app/options/options.go

@@ -0,0 +1,95 @@
+package options
+
+import (
+	"context"
+
+	"github.com/spf13/pflag"
+	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/custom-metrics-apiserver/pkg/cmd/options"
+	"sigs.k8s.io/metrics-server/pkg/api"
+
+	karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
+	informerfactory "github.com/karmada-io/karmada/pkg/generated/informers/externalversions"
+	generatedopenapi "github.com/karmada-io/karmada/pkg/generated/openapi"
+	"github.com/karmada-io/karmada/pkg/metricsadapter"
+	"github.com/karmada-io/karmada/pkg/version"
+)
+
+// Options contains everything necessary to create and run metrics-adapter.
+type Options struct {
+	CustomMetricsAdapterServerOptions *options.CustomMetricsAdapterServerOptions
+
+	KubeConfig string
+}
+
+// NewOptions builds a default metrics-adapter options.
+func NewOptions() *Options {
+	o := &Options{
+		CustomMetricsAdapterServerOptions: options.NewCustomMetricsAdapterServerOptions(),
+	}
+
+	return o
+}
+
+func (o *Options) Complete() error {
+	return nil
+}
+
+func (o *Options) AddFlags(fs *pflag.FlagSet) {
+	o.CustomMetricsAdapterServerOptions.AddFlags(fs)
+
+	fs.StringVar(&o.KubeConfig, "kubeconfig", o.KubeConfig, "Path to karmada control plane kubeconfig file.")
+}
+
+func (o *Options) Config() (*metricsadapter.MetricsServer, error) {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", o.KubeConfig)
+	if err != nil {
+		klog.Errorf("Unable to build restConfig: %v", err)
+		return nil, err
+	}
+
+	karmadaClient := karmadaclientset.NewForConfigOrDie(restConfig)
+	factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0)
+
+	metricsController := metricsadapter.NewMetricsController(restConfig, factory)
+	metricsAdapter := metricsadapter.NewMetricsAdapter(metricsController, o.CustomMetricsAdapterServerOptions)
+	metricsAdapter.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(api.Scheme))
+	metricsAdapter.OpenAPIConfig.Info.Title = "karmada-metrics-adapter"
+	metricsAdapter.OpenAPIConfig.Info.Version = "1.0.0"
+
+	server, err := metricsAdapter.Server()
+	if err != nil {
+		klog.Errorf("Unable to construct metrics adapter: %v", err)
+		return nil, err
+	}
+
+	err = server.GenericAPIServer.AddPostStartHook("start-karmada-informers", func(context genericapiserver.PostStartHookContext) error {
+		factory.Start(context.StopCh)
+		return nil
+	})
+	if err != nil {
+		klog.Errorf("Unable to add post hook: %v", err)
+		return nil, err
+	}
+
+	if err := api.Install(metricsAdapter, metricsAdapter.PodLister, metricsAdapter.NodeLister, server.GenericAPIServer, nil); err != nil {
+		klog.Errorf("unable to install resource metrics adapter: %v", err)
+		return nil, err
+	}
+
+	return metricsadapter.NewMetricsServer(metricsController, metricsAdapter), nil
+}
+
+func (o *Options) Run(ctx context.Context) error {
+	klog.Infof("karmada-metrics-adapter version: %s", version.Get())
+
+	metricsServer, err := o.Config()
+	if err != nil {
+		return err
+	}
+
+	return metricsServer.StartServer(ctx.Done())
+}

cmd/metrics-adapter/app/options/validation.go

@@ -0,0 +1,13 @@
+package options
+
+import (
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+func (o *Options) Validate() error {
+	var errs []error
+
+	errs = append(errs, o.CustomMetricsAdapterServerOptions.Validate()...)
+
+	return utilerrors.NewAggregate(errs)
+}

cmd/metrics-adapter/main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"os"
+
+	"k8s.io/component-base/cli"
+	_ "k8s.io/component-base/logs/json/register" // for JSON log format registration
+	controllerruntime "sigs.k8s.io/controller-runtime"
+
+	"github.com/karmada-io/karmada/cmd/metrics-adapter/app"
+)
+
+func main() {
+	ctx := controllerruntime.SetupSignalHandler()
+	cmd := app.NewMetricsAdapterCommand(ctx)
+	code := cli.Run(cmd)
+	os.Exit(code)
+}

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/prometheus/client_golang v1.14.0
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	github.com/vektra/mockery/v2 v2.10.0
 	github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64
 	go.uber.org/atomic v1.9.0
@@ -38,16 +38,19 @@ require (
 	k8s.io/code-generator v0.26.2
 	k8s.io/component-base v0.26.2
 	k8s.io/component-helpers v0.26.2
-	k8s.io/klog/v2 v2.80.1
+	k8s.io/klog/v2 v2.90.1
 	k8s.io/kube-aggregator v0.26.2
-	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280
+	k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d
 	k8s.io/kubectl v0.26.2
-	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
+	k8s.io/metrics v0.26.2
+	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
 	layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf
 	sigs.k8s.io/cluster-api v1.4.0
 	sigs.k8s.io/controller-runtime v0.14.5
+	sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230308103314-bd3192a29bc8
 	sigs.k8s.io/kind v0.17.0
 	sigs.k8s.io/mcs-api v0.1.0
+	sigs.k8s.io/metrics-server v0.6.1-0.20230509102056-1a23b5bd2e12
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -69,7 +72,7 @@ require (
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/fatih/camelcase v1.0.0 // indirect
@@ -80,8 +83,8 @@ require (
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.2.3 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.1 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gobuffalo/flect v1.0.2 // indirect
@@ -175,7 +178,7 @@ require (
 	k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
 	k8s.io/kms v0.26.2 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 )