[feature] adapt k8s v1.22 version

.github/workflows/ci.yaml

@@ -129,8 +129,8 @@ jobs:
 
       - name: Install Required Commands
         run: |
-          go get sigs.k8s.io/kind@v0.11.1
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.20.7/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+          go get sigs.k8s.io/kind@v0.12.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.22.3/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
       - name: Build Images
         run: make docker-build
       - name: Local Up Openyurt Cluster With Kind

Makefile

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+KUBERNETESVERSION ?=v1.22
 TARGET_PLATFORMS ?= linux/amd64
 IMAGE_REPO ?= openyurt
 IMAGE_TAG ?= $(shell git describe --abbrev=0 --tags)
@@ -62,14 +63,14 @@ clean:
 #   - on centos env: make local-up-openyurt
 #   - on MACBook Pro M1: make local-up-openyurt TARGET_PLATFORMS=linux/arm64
 local-up-openyurt:
-	YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh
+	KUBERNETESVERSION=${KUBERNETESVERSION} YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh
 
 # Build all OpenYurt components images and then start up OpenYurt cluster on local machine based on a Kind cluster
 # And you can run the following command on different env by specify TARGET_PLATFORMS, default platform is linux/amd64
 #   - on centos env: make docker-build-and-up-openyurt
 #   - on MACBook Pro M1: make docker-build-and-up-openyurt TARGET_PLATFORMS=linux/arm64
 docker-build-and-up-openyurt: docker-build
-	YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh
+	KUBERNETESVERSION=${KUBERNETESVERSION} YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh
 
 e2e-tests:
 	bash hack/make-rules/run-e2e-tests.sh

cmd/yurt-controller-manager/app/controllermanager.go

@@ -182,7 +182,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 		MaxHeaderBytes: 1 << 20,
 	}
 
-	if _, err := apiserver.RunServer(insecureServer, listener, 0, stopCh); err != nil {
+	if _, _, err := apiserver.RunServer(insecureServer, listener, 0, stopCh); err != nil {
 		klog.Fatalf("error run http server: %v", err)
 		return err
 	}

cmd/yurt-controller-manager/app/core.go

@@ -56,12 +56,12 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er
 }
 
 func startYurtCSRApproverController(ctx ControllerContext) (http.Handler, bool, error) {
-	clientSet := ctx.ClientBuilder.ClientOrDie("csr-controller")
+	clientSet := ctx.ClientBuilder.ClientOrDie("yurt-csr-controller")
 	csrApprover, err := certificates.NewCSRApprover(clientSet, ctx.InformerFactory)
 	if err != nil {
 		return nil, false, err
 	}
-	go csrApprover.Run(certificates.YurtCSRApproverThreadiness, ctx.Stop)
+	go csrApprover.Run(2, ctx.Stop)
 
 	return nil, true, nil
 }

cmd/yurt-controller-manager/controller-manager.go

@@ -27,6 +27,9 @@ import (
 
 	"k8s.io/component-base/logs"
 
+	// for JSON log format registration
+	_ "k8s.io/component-base/logs/json/register"
+
 	// load all the prometheus client-go plugin
 	_ "k8s.io/component-base/metrics/prometheus/clientgo"
 

cmd/yurt-tunnel-server/app/start.go

@@ -105,15 +105,21 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 		go iptablesMgr.Run(stopCh, &wg)
 	}
 
-	// 2. create a certificate manager for the tunnel server and run the
-	// csr approver for both yurttunnel-server and yurttunnel-agent
+	// 2. create a certificate manager for the tunnel server
 	serverCertMgr, err := certmanager.NewYurttunnelServerCertManager(cfg.Client, cfg.SharedInformerFactory, cfg.CertDir, cfg.CertDNSNames, cfg.CertIPs, stopCh)
 	if err != nil {
 		return err
 	}
 	serverCertMgr.Start()
 
-	// 3. create handler wrappers
+	// 3. create a certificate manager for the tunnel proxy client
+	tunnelProxyCertMgr, err := certmanager.NewTunnelProxyClientCertManager(cfg.Client, cfg.CertDir)
+	if err != nil {
+		return err
+	}
+	tunnelProxyCertMgr.Start()
+
+	// 4. create handler wrappers
 	mInitializer := initializer.NewMiddlewareInitializer(cfg.SharedInformerFactory)
 	wrappers, err := wraphandler.InitHandlerWrappers(mInitializer)
 	if err != nil {
@@ -124,23 +130,28 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 	// after all of informers are configured completed, start the shared index informer
 	cfg.SharedInformerFactory.Start(stopCh)
 
-	// 4. waiting for the certificate is generated
+	// 5. waiting for the certificate is generated
 	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
 		// keep polling until the certificate is signed
-		if serverCertMgr.Current() != nil {
+		if serverCertMgr.Current() != nil && tunnelProxyCertMgr.Current() != nil {
 			return true, nil
 		}
 		klog.Infof("waiting for the master to sign the %s certificate", projectinfo.GetServerName())
 		return false, nil
 	}, stopCh)
 
-	// 5. generate the TLS configuration based on the latest certificate
-	tlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert)
+	// 6. generate the TLS configuration based on the latest certificate
+	tlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert, "server")
+	if err != nil {
+		return err
+	}
+
+	proxyClientTlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(tunnelProxyCertMgr, cfg.RootCert, "client")
 	if err != nil {
 		return err
 	}
 
-	// 6. start the server
+	// 7. start the server
 	ts := server.NewTunnelServer(
 		cfg.EgressSelectorEnabled,
 		cfg.InterceptorServerUDSFile,
@@ -149,13 +160,14 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 		cfg.ListenAddrForAgent,
 		cfg.ServerCount,
 		tlsCfg,
+		proxyClientTlsCfg,
 		wrappers,
 		cfg.ProxyStrategy)
 	if err := ts.Run(); err != nil {
 		return err
 	}
 
-	// 7. start meta server
+	// 8. start meta server
 	util.RunMetaServer(cfg.ListenMetaAddr)
 
 	<-stopCh

cmd/yurthub/app/start.go

@@ -125,7 +125,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
 	trace++
 
 	klog.Infof("%d. create tls config for secure servers ", trace)
-	cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.YurtHubProxyServerSecureDummyAddr, stopCh)
+	cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.YurtHubProxyServerSecureDummyAddr, stopCh)
 	if err != nil {
 		return fmt.Errorf("could not create tls config, %w", err)
 	}

config/setup/yurt-controller-manager.yaml

@@ -91,7 +91,8 @@ rules:
     resources:
       - signers
     resourceNames:
-      - "kubernetes.io/legacy-unknown"
+      - kubernetes.io/kube-apiserver-client
+      - kubernetes.io/kubelet-serving
     verbs:
       - approve
 ---

config/setup/yurt-tunnel-server.yaml

@@ -1,5 +1,42 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: tunnel-proxy-client
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/stats
+      - nodes/metrics
+      - nodes/log
+      - nodes/spec
+      - nodes/proxy
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - delete
+      - update
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tunnel-proxy-client
+subjects:
+  - kind: User
+    name: tunnel-server-proxy-client
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: tunnel-proxy-client
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   annotations:
     rbac.authorization.kubernetes.io/autoupdate: "true"
@@ -14,20 +51,6 @@ rules:
     - get
     - list
     - watch
-- apiGroups:
-    - certificates.k8s.io
-  resources:
-    - certificatesigningrequests/approval
-  verbs:
-    - update
-- apiGroups:
-    - certificates.k8s.io
-  resourceNames:
-    - kubernetes.io/legacy-unknown
-  resources:
-    - signers
-  verbs:
-    - approve
 - apiGroups:
   - ""
   resources:
@@ -40,6 +63,7 @@ rules:
   - ""
   resources:
   - nodes
+  - pods
   verbs:
   - list
   - watch
@@ -72,7 +96,7 @@ rules:
   - update
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: yurt-tunnel-server
 subjects:

config/yaml-template/yurt-controller-manager.yaml

@@ -89,7 +89,8 @@ rules:
   - apiGroups:
       - certificates.k8s.io
     resourceNames:
-      - kubernetes.io/legacy-unknown
+      - kubernetes.io/kube-apiserver-client
+      - kubernetes.io/kubelet-serving
     resources:
       - signers
     verbs:

config/yaml-template/yurt-tunnel-server.yaml

@@ -1,5 +1,42 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: tunnel-proxy-client
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/stats
+      - nodes/metrics
+      - nodes/log
+      - nodes/spec
+      - nodes/proxy
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - delete
+      - update
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tunnel-proxy-client
+subjects:
+  - kind: User
+    name: tunnel-server-proxy-client
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: tunnel-proxy-client
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   annotations:
     rbac.authorization.kubernetes.io/autoupdate: "true"
@@ -14,20 +51,6 @@ rules:
     - get
     - list
     - watch
-- apiGroups:
-    - certificates.k8s.io
-  resources:
-    - certificatesigningrequests/approval
-  verbs:
-    - update
-- apiGroups:
-    - certificates.k8s.io
-  resourceNames:
-    - kubernetes.io/legacy-unknown
-  resources:
-    - signers
-  verbs:
-    - approve
 - apiGroups:
     - ""
   resources:
@@ -40,6 +63,7 @@ rules:
     - ""
   resources:
     - nodes
+    - pods
   verbs:
     - list
     - watch