Feat: init multicluster

apis/core.oam.dev/v1alpha1/envbinding_types.go

@@ -34,6 +34,9 @@ const (
 
 	// SingleClusterEngine represents single cluster ClusterManagerEngine
 	SingleClusterEngine ClusterManagementEngine = "single-cluster"
+
+	// ClusterGatewayEngine represents multi-cluster management solution with cluster-gateway
+	ClusterGatewayEngine ClusterManagementEngine = "cluster-gateway"
 )
 
 // EnvBindingPhase is a label for the condition of a EnvBinding at the current time
@@ -88,6 +91,7 @@ type AppTemplate struct {
 }
 
 // ClusterDecision recorded the mapping of environment and cluster
+// TODO we might need to rename ClusterDecision to EnvDecision as it is also associated with namespace
 type ClusterDecision struct {
 	Env       string `json:"env"`
 	Cluster   string `json:"cluster,omitempty"`

apis/core.oam.dev/v1alpha2/core_scope_types.go

@@ -58,6 +58,7 @@ type HealthScopeSpec struct {
 // AppReference records references of an application's components
 type AppReference struct {
 	AppName        string          `json:"appName,omitempty"`
+	ClusterName    string          `json:"clusterName,omitempty"`
 	CompReferences []CompReference `json:"compReferences,omitempty"`
 }
 

charts/vela-core/templates/_helpers.tpl

@@ -56,6 +56,11 @@ app.kubernetes.io/name: {{ include "kubevela.name" . }}-apiserver
 app.kubernetes.io/instance: {{ .Release.Name }}-apiserver
 {{- end -}}
 
+{{- define "kubevela-cluster-gateway.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "kubevela.name" . }}-cluster-gateway
+app.kubernetes.io/instance: {{ .Release.Name }}-cluster-gateway
+{{- end -}}
+
 {{/*
 Create the name of the service account to use
 */}}

charts/vela-core/templates/cluster-gateway.yaml

@@ -0,0 +1,84 @@
+{{ if .Values.multicluster.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-cluster-gateway
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "kubevela.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.multicluster.clusterGateway.replicaCount }}
+  selector:
+    matchLabels:
+    {{- include "kubevela-cluster-gateway.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+      {{- include "kubevela-cluster-gateway.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "kubevela.serviceAccountName" . }}
+      securityContext:
+      {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ include "kubevela.fullname" . }}-cluster-gateway
+          securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
+          args:
+            - "apiserver"
+            - "--secure-port={{ .Values.multicluster.clusterGateway.port }}"
+            - "--secret-namespace={{ .Release.Namespace }}"
+            - "--feature-gates=APIPriorityAndFairness=false"
+          image: {{ .Values.multicluster.clusterGateway.image.repository }}:{{ .Values.multicluster.clusterGateway.image.tag }}
+          imagePullPolicy: {{ .Values.multicluster.clusterGateway.image.pullPolicy }}
+          resources:
+          {{- toYaml .Values.multicluster.clusterGateway.resources | nindent 12 }}
+          ports:
+            - containerPort: {{ .Values.multicluster.clusterGateway.port }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{ end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-cluster-gateway-service
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+  {{- include "kubevela-cluster-gateway.selectorLabels" . | nindent 4 }}
+  ports:
+    - protocol: TCP
+      port: {{ .Values.multicluster.clusterGateway.port }}
+      targetPort: {{ .Values.multicluster.clusterGateway.port }}
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1alpha1.cluster.core.oam.dev
+  labels:
+    api: cluster-extension-apiserver
+    apiserver: "true"
+spec:
+  version: v1alpha1
+  group: cluster.core.oam.dev
+  groupPriorityMinimum: 2000
+  service:
+    name: {{ .Release.Name }}-cluster-gateway-service
+    namespace: {{ .Release.Namespace }}
+    port: {{ .Values.multicluster.clusterGateway.port }}
+  versionPriority: 10
+  insecureSkipTLSVerify: true

charts/vela-core/templates/kubevela-controller.yaml

@@ -129,6 +129,10 @@ spec:
             - "--application-revision-limit={{ .Values.applicationRevisionLimit }}"
             - "--definition-revision-limit={{ .Values.definitionRevisionLimit }}"
             - "--oam-spec-ver={{ .Values.OAMSpecVer }}"
+            {{ if .Values.multicluster.enabled }}
+            - "--multi-cluster-enabled"
+            - "--cluster-gateway-secret-namespace={{ .Release.Namespace }}"
+            {{ end }}
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
           imagePullPolicy: {{ quote .Values.image.pullPolicy }}
           resources:

charts/vela-core/values.yaml

@@ -108,4 +108,18 @@ apiServer:
   port: 8000
   replicaCount: 1
   Service:
-    type: ClusterIP
+    type: ClusterIP
+
+multicluster:
+  enabled: false
+  clusterGateway:
+    replicaCount: 1
+    port: 9443
+    image:
+      repository: oamdev/cluster-gateway
+      tag: latest
+      pullPolicy: Always
+    resource:
+      limits:
+        cpu: 100m
+        memory: 10Mi

cmd/core/main.go

@@ -32,14 +32,18 @@ import (
 
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 
+	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	standardcontroller "github.com/oam-dev/kubevela/pkg/controller"
 	commonconfig "github.com/oam-dev/kubevela/pkg/controller/common"
 	oamcontroller "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev"
+	"github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha1/envbinding"
 	oamv1alpha2 "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2"
 	"github.com/oam-dev/kubevela/pkg/controller/utils"
 	"github.com/oam-dev/kubevela/pkg/cue/packages"
+	"github.com/oam-dev/kubevela/pkg/multicluster"
 	"github.com/oam-dev/kubevela/pkg/oam"
 	"github.com/oam-dev/kubevela/pkg/oam/discoverymapper"
 	"github.com/oam-dev/kubevela/pkg/utils/common"
@@ -74,6 +78,7 @@ func main() {
 	var qps float64
 	var burst int
 	var pprofAddr string
+	var multiClusterEnabled bool
 
 	flag.BoolVar(&useWebhook, "use-webhook", false, "Enable Admission Webhook")
 	flag.StringVar(&certDir, "webhook-cert-dir", "/k8s-webhook-server/serving-certs", "Admission webhook cert/key dir.")
@@ -111,6 +116,8 @@ func main() {
 	flag.StringVar(&controllerArgs.OAMSpecVer, "oam-spec-ver", "v0.3", "oam-spec-ver is the oam spec version controller want to setup, available options: v0.2, v0.3, all")
 	flag.StringVar(&pprofAddr, "pprof-addr", "", "The address for pprof to use while exporting profiling results. The default value is empty which means do not expose it. Set it to address like :6666 to expose it.")
 	flag.BoolVar(&commonconfig.PerfEnabled, "perf-enabled", false, "Enable performance logging for controllers, disabled by default.")
+	flag.BoolVar(&multiClusterEnabled, "multi-cluster-enabled", false, "Enable multi-cluster, disabled by default.")
+	flag.StringVar(&envbinding.ClusterGatewaySecretNamespace, "cluster-gateway-secret-namespace", "vela-system", "The namespace where cluster-gateway use to store cluster secrets.")
 
 	flag.Parse()
 	// setup logging
@@ -164,6 +171,12 @@ func main() {
 	klog.InfoS("Vela-Core init", "definition namespace", oam.SystemDefinitonNamespace)
 
 	restConfig := ctrl.GetConfigOrDie()
+
+	// wrapper the round tripper by multi cluster rewriter
+	if multiClusterEnabled {
+		restConfig.Wrap(multicluster.NewSecretModeMultiClusterRoundTripper)
+	}
+
 	restConfig.UserAgent = kubevelaName + "/" + version.GitRevision
 	restConfig.QPS = float32(qps)
 	restConfig.Burst = burst
@@ -178,6 +191,7 @@ func main() {
 		CertDir:                 certDir,
 		HealthProbeBindAddress:  healthAddr,
 		SyncPeriod:              &syncPeriod,
+		ClientDisableCacheFor:   []client.Object{&v1beta1.ResourceTracker{}},
 	})
 	if err != nil {
 		klog.ErrorS(err, "Unable to create a controller manager")

go.mod

@@ -5,7 +5,6 @@ go 1.16
 require (
 	cuelang.org/go v0.2.2
 	github.com/AlecAivazis/survey/v2 v2.1.1
-	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
@@ -24,24 +23,21 @@ require (
 	github.com/gin-contrib/static v0.0.0-20200815103939-31fb0c56a3d1
 	github.com/gin-gonic/gin v1.7.0
 	github.com/go-logr/logr v0.4.0
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
-	github.com/go-openapi/spec v0.19.8 // indirect
-	github.com/go-openapi/swag v0.19.11 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/google/go-cmp v0.5.6
 	github.com/google/go-github/v32 v32.1.0
 	github.com/gosuri/uitable v0.0.4
 	github.com/hashicorp/hcl/v2 v2.9.1
 	github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174
 	github.com/imdario/mergo v0.3.12
-	github.com/klauspost/compress v1.10.5 // indirect
 	github.com/kyokomi/emoji v2.2.4+incompatible
 	github.com/labstack/echo/v4 v4.5.0
-	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/mholt/archiver/v3 v3.3.0
 	github.com/mitchellh/hashstructure/v2 v2.0.1
+	github.com/oam-dev/cluster-gateway v0.0.0-20210907072424-2f8720b116f8
 	github.com/oam-dev/terraform-config-inspect v0.0.0-20210418082552-fc72d929aa28
 	github.com/oam-dev/terraform-controller v0.1.18
-	github.com/olekukonko/tablewriter v0.0.4
+	github.com/olekukonko/tablewriter v0.0.5
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.14.0
 	github.com/openkruise/kruise-api v0.9.0
@@ -58,6 +54,7 @@ require (
 	github.com/wercker/stern v0.0.0-20190705090245-4fa46dd6987f
 	github.com/wonderflow/cert-manager-api v1.0.3
 	go.uber.org/zap v1.18.1
+	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
 	golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
@@ -74,7 +71,7 @@ require (
 	k8s.io/klog/v2 v2.8.0
 	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7
 	k8s.io/kubectl v0.21.0
-	k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471
+	k8s.io/utils v0.0.0-20210802155522-efc7438f0176
 	open-cluster-management.io/api v0.0.0-20210804091127-340467ff6239
 	rsc.io/letsencrypt v0.0.3 // indirect
 	sigs.k8s.io/controller-runtime v0.9.5