[feature] adapt k8s v1.22 version

.github/workflows/ci.yaml

@@ -129,8 +129,8 @@ jobs:
 
       - name: Install Required Commands
         run: |
-          go get sigs.k8s.io/kind@v0.11.1
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.20.7/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+          go get sigs.k8s.io/kind@v0.12.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.22.3/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
       - name: Build Images
         run: make docker-build
       - name: Local Up Openyurt Cluster With Kind

Makefile

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+KUBERNETESVERSION ?=v1.22
 TARGET_PLATFORMS ?= linux/amd64
 IMAGE_REPO ?= openyurt
 IMAGE_TAG ?= $(shell git describe --abbrev=0 --tags)
@@ -62,14 +63,14 @@ clean:
 #   - on centos env: make local-up-openyurt
 #   - on MACBook Pro M1: make local-up-openyurt TARGET_PLATFORMS=linux/arm64
 local-up-openyurt:
-	YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh
+	KUBERNETESVERSION=${KUBERNETESVERSION} YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh
 
 # Build all OpenYurt components images and then start up OpenYurt cluster on local machine based on a Kind cluster
 # And you can run the following command on different env by specify TARGET_PLATFORMS, default platform is linux/amd64
 #   - on centos env: make docker-build-and-up-openyurt
 #   - on MACBook Pro M1: make docker-build-and-up-openyurt TARGET_PLATFORMS=linux/arm64
 docker-build-and-up-openyurt: docker-build
-	YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh
+	KUBERNETESVERSION=${KUBERNETESVERSION} YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh
 
 e2e-tests:
 	bash hack/make-rules/run-e2e-tests.sh

cmd/yurt-controller-manager/app/controllermanager.go

@@ -182,7 +182,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 		MaxHeaderBytes: 1 << 20,
 	}
 
-	if _, err := apiserver.RunServer(insecureServer, listener, 0, stopCh); err != nil {
+	if _, _, err := apiserver.RunServer(insecureServer, listener, 0, stopCh); err != nil {
 		klog.Fatalf("error run http server: %v", err)
 		return err
 	}

cmd/yurt-controller-manager/app/core.go

@@ -56,12 +56,12 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er
 }
 
 func startYurtCSRApproverController(ctx ControllerContext) (http.Handler, bool, error) {
-	clientSet := ctx.ClientBuilder.ClientOrDie("csr-controller")
+	clientSet := ctx.ClientBuilder.ClientOrDie("yurt-csr-controller")
 	csrApprover, err := certificates.NewCSRApprover(clientSet, ctx.InformerFactory)
 	if err != nil {
 		return nil, false, err
 	}
-	go csrApprover.Run(certificates.YurtCSRApproverThreadiness, ctx.Stop)
+	go csrApprover.Run(2, ctx.Stop)
 
 	return nil, true, nil
 }

cmd/yurt-controller-manager/controller-manager.go

@@ -27,6 +27,9 @@ import (
 
 	"k8s.io/component-base/logs"
 
+	// for JSON log format registration
+	_ "k8s.io/component-base/logs/json/register"
+
 	// load all the prometheus client-go plugin
 	_ "k8s.io/component-base/metrics/prometheus/clientgo"
 

cmd/yurt-tunnel-server/app/start.go

@@ -105,14 +105,20 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 		go iptablesMgr.Run(stopCh, &wg)
 	}
 
-	// 2. create a certificate manager for the tunnel server and run the
-	// csr approver for both yurttunnel-server and yurttunnel-agent
+	// 2. create a certificate manager for the tunnel server
 	serverCertMgr, err := certmanager.NewYurttunnelServerCertManager(cfg.Client, cfg.SharedInformerFactory, cfg.CertDir, cfg.CertDNSNames, cfg.CertIPs, stopCh)
 	if err != nil {
 		return err
 	}
 	serverCertMgr.Start()
 
+	// 2. create a certificate manager for the tunnel proxy client
+	tunnelProxyCertMgr, err := certmanager.NewTunnelProxyClientCertManager(cfg.Client, cfg.CertDir)
+	if err != nil {
+		return err
+	}
+	tunnelProxyCertMgr.Start()
+
 	// 3. create handler wrappers
 	mInitializer := initializer.NewMiddlewareInitializer(cfg.SharedInformerFactory)
 	wrappers, err := wraphandler.InitHandlerWrappers(mInitializer)
@@ -127,15 +133,20 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 	// 4. waiting for the certificate is generated
 	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
 		// keep polling until the certificate is signed
-		if serverCertMgr.Current() != nil {
+		if serverCertMgr.Current() != nil && tunnelProxyCertMgr.Current() != nil {
 			return true, nil
 		}
 		klog.Infof("waiting for the master to sign the %s certificate", projectinfo.GetServerName())
 		return false, nil
 	}, stopCh)
 
 	// 5. generate the TLS configuration based on the latest certificate
-	tlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert)
+	tlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert, "server")
+	if err != nil {
+		return err
+	}
+
+	proxyClientTlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(tunnelProxyCertMgr, cfg.RootCert, "client")
 	if err != nil {
 		return err
 	}
@@ -149,6 +160,7 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 		cfg.ListenAddrForAgent,
 		cfg.ServerCount,
 		tlsCfg,
+		proxyClientTlsCfg,
 		wrappers,
 		cfg.ProxyStrategy)
 	if err := ts.Run(); err != nil {

cmd/yurthub/app/start.go

@@ -125,7 +125,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
 	trace++
 
 	klog.Infof("%d. create tls config for secure servers ", trace)
-	cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.YurtHubProxyServerSecureDummyAddr, stopCh)
+	cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.YurtHubProxyServerSecureDummyAddr, stopCh)
 	if err != nil {
 		return fmt.Errorf("could not create tls config, %w", err)
 	}

config/setup/yurt-controller-manager.yaml

@@ -91,7 +91,8 @@ rules:
     resources:
       - signers
     resourceNames:
-      - "kubernetes.io/legacy-unknown"
+      - kubernetes.io/kube-apiserver-client
+      - kubernetes.io/kubelet-serving
     verbs:
       - approve
 ---

config/setup/yurt-tunnel-server.yaml

@@ -1,5 +1,42 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: tunnel-proxy-client
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/stats
+      - nodes/metrics
+      - nodes/log
+      - nodes/spec
+      - nodes/proxy
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - delete
+      - update
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tunnel-proxy-client
+subjects:
+  - kind: User
+    name: tunnel-server-proxy-client
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: tunnel-proxy-client
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   annotations:
     rbac.authorization.kubernetes.io/autoupdate: "true"
@@ -14,20 +51,6 @@ rules:
     - get
     - list
     - watch
-- apiGroups:
-    - certificates.k8s.io
-  resources:
-    - certificatesigningrequests/approval
-  verbs:
-    - update
-- apiGroups:
-    - certificates.k8s.io
-  resourceNames:
-    - kubernetes.io/legacy-unknown
-  resources:
-    - signers
-  verbs:
-    - approve
 - apiGroups:
   - ""
   resources:
@@ -40,6 +63,7 @@ rules:
   - ""
   resources:
   - nodes
+  - pods
   verbs:
   - list
   - watch
@@ -72,7 +96,7 @@ rules:
   - update
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: yurt-tunnel-server
 subjects:

config/yaml-template/yurt-controller-manager.yaml

@@ -89,7 +89,8 @@ rules:
   - apiGroups:
       - certificates.k8s.io
     resourceNames:
-      - kubernetes.io/legacy-unknown
+      - kubernetes.io/kube-apiserver-client
+      - kubernetes.io/kubelet-serving
     resources:
       - signers
     verbs:

config/yaml-template/yurt-tunnel-server.yaml

@@ -1,5 +1,42 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: tunnel-proxy-client
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/stats
+      - nodes/metrics
+      - nodes/log
+      - nodes/spec
+      - nodes/proxy
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - delete
+      - update
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: tunnel-proxy-client
+subjects:
+  - kind: User
+    name: tunnel-server-proxy-client
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: tunnel-proxy-client
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   annotations:
     rbac.authorization.kubernetes.io/autoupdate: "true"
@@ -14,20 +51,6 @@ rules:
     - get
     - list
     - watch
-- apiGroups:
-    - certificates.k8s.io
-  resources:
-    - certificatesigningrequests/approval
-  verbs:
-    - update
-- apiGroups:
-    - certificates.k8s.io
-  resourceNames:
-    - kubernetes.io/legacy-unknown
-  resources:
-    - signers
-  verbs:
-    - approve
 - apiGroups:
     - ""
   resources:
@@ -40,6 +63,7 @@ rules:
     - ""
   resources:
     - nodes
+    - pods
   verbs:
     - list
     - watch

go.mod

@@ -3,60 +3,57 @@ module github.com/openyurtio/openyurt
 go 1.16
 
 require (
+	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/Microsoft/go-winio v0.4.15
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.579
 	github.com/daviddengcn/go-colortext v1.0.0
 	github.com/emicklei/go-restful v2.12.0+incompatible // indirect
-	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52 // indirect
-	github.com/golangplus/testing v1.0.0 // indirect
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.4
 	github.com/lithammer/dedent v1.1.0
+	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
 	github.com/opencontainers/selinux v1.10.0
 	github.com/openyurtio/yurt-app-manager-api v0.18.8
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
-	github.com/spf13/cobra v1.1.3
+	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
 	github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb
-	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
-	google.golang.org/grpc v1.27.1
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
+	google.golang.org/grpc v1.40.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.25
 	gopkg.in/square/go-jose.v2 v2.2.2
 	k8s.io/api v0.22.3
 	k8s.io/apimachinery v0.22.3
-	k8s.io/apiserver v0.20.11
+	k8s.io/apiserver v0.22.3
 	k8s.io/client-go v0.22.3
-	k8s.io/cluster-bootstrap v0.20.11
-	k8s.io/component-base v0.20.11
+	k8s.io/cluster-bootstrap v0.22.3
+	k8s.io/component-base v0.22.3
 	k8s.io/component-helpers v0.22.3
-	k8s.io/controller-manager v0.20.11
-	k8s.io/klog/v2 v2.30.0
-	k8s.io/kube-controller-manager v0.20.11
-	k8s.io/kubelet v0.20.11
-	k8s.io/system-validators v1.2.0
+	k8s.io/controller-manager v0.22.3
+	k8s.io/klog/v2 v2.9.0
+	k8s.io/kube-controller-manager v0.22.3
+	k8s.io/kubelet v0.22.3
+	k8s.io/system-validators v1.6.0
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
 	sigs.k8s.io/apiserver-network-proxy v0.0.15
+	sigs.k8s.io/yaml v1.3.0 // indirect
 
 )
 
 replace (
-	github.com/daviddengcn/go-colortext => github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd
-	github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.4.1
-	gopkg.in/cheggaaa/pb.v1 => gopkg.in/cheggaaa/pb.v1 v1.0.25
-	k8s.io/api => k8s.io/api v0.20.11
-	k8s.io/apimachinery => k8s.io/apimachinery v0.20.12-rc.0
-	k8s.io/apiserver => k8s.io/apiserver v0.20.11
-	k8s.io/client-go => k8s.io/client-go v0.20.11
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.11
-	k8s.io/component-base => k8s.io/component-base v0.20.11
-	k8s.io/component-helpers => k8s.io/component-helpers v0.22.3
+	k8s.io/api => k8s.io/api v0.22.3
+	k8s.io/apimachinery => k8s.io/apimachinery v0.22.3
+	k8s.io/apiserver => k8s.io/apiserver v0.22.3
+	k8s.io/client-go => k8s.io/client-go v0.22.3
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.22.3
+	k8s.io/component-base => k8s.io/component-base v0.22.3
 	k8s.io/klog/v2 => k8s.io/klog/v2 v2.9.0
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.11
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.22.3
 	sigs.k8s.io/apiserver-network-proxy => github.com/openyurtio/apiserver-network-proxy v1.18.8
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22
 )