增加托管集群部署

完善一些部署流程

.github/workflows/go.yml

@@ -0,0 +1,36 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.13
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+              dep ensure
+          fi
+      - name: Build
+        run: go build -v cmd/admin-controller/main.go
+
+#    - name: Test
+#      run: go test -v ./...

.gitignore

@@ -12,11 +12,8 @@
 *.out
 
 # Dependency directories (remove the comment below to include it)
-
-.github/
+vendor/
+bin/
 .idea/
 .vscode/
-tools/
-k8s/
-vendor/
-bin/
+k8s/

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.14.3 as builder
+FROM golang:1.13 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

Makefile

@@ -1,32 +1,40 @@
-
+VERSION ?= v0.1.0
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
+IMG_REG ?= registry.cn-hangzhou.aliyuncs.com/wtxue
+IMG_CTL := $(IMG_REG)/onkube-controller
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
 
+# This repo's root import path (under GOPATH).
+ROOT := github.com/wtxue/kube-on-kube-operator
+
+GO_VERSION := 1.14.4
+ARCH     ?= $(shell go env GOARCH)
+BUILD_DATE = $(shell date +'%Y-%m-%dT%H:%M:%SZ')
+COMMIT    = $(shell git rev-parse --short HEAD)
+GOENV    := CGO_ENABLED=0 GOOS=$(shell uname -s | tr A-Z a-z) GOARCH=$(ARCH) GOPROXY=https://goproxy.io,direct
+#GO       := $(GOENV) go build -mod=vendor
+GO       := $(GOENV) go build
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
 else
 GOBIN=$(shell go env GOBIN)
 endif
 
-all: manager
+all: build
 
 # Run tests
 test: generate fmt vet manifests
 	go test ./... -coverprofile cover.out
 
-# Build manager binary
-manager: generate fmt vet
-	go build -o bin/manager main.go
-
 # Run against the configured Kubernetes cluster in ~/.kube/config
 run: generate fmt vet manifests
 	go run ./main.go
 
 # generate crd spec and deepcopy
-crd: generate manifests
+crd: generate manifests pre-build
 	kustomize build config/crd > manifests/crds/crd.yaml
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
@@ -55,12 +63,19 @@ pre-build:
 #	go generate ./pkg/... ./cmd/...
 
 # Build the docker image
-docker-build: test
-	docker build . -t ${IMG}
+docker-build-controller:
+	docker run --rm -v "$$PWD":/go/src/${ROOT} -v ${GOPATH}/pkg/mod:/go/pkg/mod -w /go/src/${ROOT} golang:${GO_VERSION} make build-controller
+
+build: build-controller
+
+build-controller:
+	$(GO) -v -o bin/onkube-controller -ldflags "-s -w -X $(ROOT)/pkg/version.Release=$(VERSION) -X  $(ROOT)/pkg/version.Commit=$(COMMIT)   \
+	-X  $(ROOT)/pkg/version.BuildDate=$(BUILD_DATE)" cmd/admin-controller/main.go
 
 # Push the docker image
 docker-push:
-	docker push ${IMG}
+	docker build -t ${IMG_CTL}:${VERSION} -f ./docker/onkube/Dockerfile .
+	docker push ${IMG_CTL}:${VERSION}
 
 # find or download controller-gen
 # download controller-gen if necessary
@@ -71,7 +86,7 @@ ifeq (, $(shell which controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen

PROJECT

@@ -1,4 +1,4 @@
-domain: k8s.io
+domain: gostship.io
 repo: github.com/wtxue/kube-on-kube-operator
 resources:
 - group: devops
@@ -7,4 +7,4 @@ resources:
 - group: devops
   kind: Machine
   version: v1
-version: "3"
+version: "2"

README.md

@@ -5,10 +5,11 @@ kube-on-kube-operator 是一个自动化部署高可用kubernetes的operator
 # 特性
 
 - 云原生架构，crd+controller，采用声明式api描述一个集群的最终状态
+- 支持裸金属和master组件托管两种方式部署集群
 - 可以启用一个fake-cluster，解决裸金属第一次部署集群没有元集群问题
-- 无坑版100年集群证书
+- 无坑版100年集群证书，kubelet自动生成证书
 - 除kubelet外集群组件全部容器化部署，componentstatuses可以发现三个etcd
-- 支持flannel，metrics-server等直接一键部署
+- 支持coredns, flannel，metrics-server等 addons 模板化部署
 
 # 安装部署
 
@@ -23,7 +24,7 @@ $ ./init.sh
 
 # 进入项目根目录  运行 fake apiserver
 $ cd ..
-$ go run cmd/admin-controller/main.go fake -v 4 
+$ go run cmd/admin-controller/main.go fake --baseBinDir k8s/bin --rootDir k8s -v 4 
 
 # 运行正常后
 $ cat k8s/cfg/fake-kubeconfig.yaml
@@ -47,6 +48,7 @@ users:
 
 ## 运行
 
+本地运行
 ```bash
 # apply crd
 $ export KUBECONFIG=k8s/cfg/fake-kubeconfig.yaml && kubectl apply -f manifests/crds/
@@ -57,7 +59,58 @@ customresourcedefinition.apiextensions.k8s.io/machines.devops.k8s.io created
 # 运行
 $ go run cmd/admin-controller/main.go ctrl -v 4 --kubeconfig=k8s/cfg/fake-kubeconfig.yaml
 ```
+docker 运行
+```bash
+$ docker run --name fake-cluster -d --restart=always \
+   --net="host" \
+   --pid="host" \
+   -v /root/wtxue/k8s:/k8s \
+   registry.cn-hangzhou.aliyuncs.com/wtxue/onkube-controller:v0.1.0 \
+   onkube-controller fake -v 4
+
+$ docker run --name onkube-controller -d --restart=always \
+   --net="host" \
+   --pid="host" \
+   -v /root/wtxue/k8s:/k8s \
+   registry.cn-hangzhou.aliyuncs.com/wtxue/onkube-controller:v0.1.0  \
+   onkube-controller ctrl -v 4 --kubeconfig=/k8s/cfg/fake-kubeconfig.yaml
+
+
+```
+
+## 创建集群
+### 创建裸金属集群
+```bash
+# 设置 fake-cluster kubeconfig
+$ export KUBECONFIG=/root/wtxue/k8s/cfg/fake-kubeconfig.yaml
+
+# 创建集群cr
+$ kubectl apply -f ./manifests/example-cluster.yaml
+
+# 创建集群结点
+$ kubectl apply -f ./manifests/example-cluster-node.yaml
+```
+
+### 创建托管集群
+创建托管集群时，onkube-controller需要运行在真实集群上，这里使用上面创建裸金属集群 example-cluster, 注意一个namespace一个集群
+```bash
+# 设置 example-cluster kubeconfig
+$ export KUBECONFIG=/root/wtxue/k8s/cfg/example-cluster-kubeconfig.yaml
+
+# 这里演示直接本地运行，也可以deployment跑到集群上
+$ go run cmd/admin-controller/main.go ctrl -v 4 --kubeconfig=k8s/cfg/fake-kubeconfig.yaml
+
+# 创建 etcd 集群
+$ kubectl apply -f ./manifests/etcd-statefulset.yaml
+
+# 创建托管集群cr
+kubectl apply -f ./manifests/hosted-cluster.yaml
+
+# 创建托管集群结点
+kubectl apply -f ./manifests/hosted-cluster-node.yaml
+```
 
 # 计划
 
-- [x]  master组件托管
+- [x]  打通元集群及托管集群service网络，以支持聚合apiserver
+- [x]  支持 helm v3 部署 addons

cmd/admin-controller/app/app_option/option.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2020 wtxue.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package app_option
 
 import (

cmd/admin-controller/app/certs.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2020 wtxue.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package app
 
 import (

cmd/admin-controller/app/controller.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2020 wtxue.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package app
 
 import (

cmd/admin-controller/app/fake_apiserver.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2020 wtxue.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package app
 
 import (
@@ -59,7 +75,7 @@ func tryRun(opt *option.ApiServerOption, stopCh <-chan struct{}) error {
 			return err
 		}
 
-		err = CreateKubeConfigFile("k8s/cfg", "fake-kubeconfig.yaml", cfg)
+		err = CreateKubeConfigFile(opt.RootDir+"/cfg", "fake-kubeconfig.yaml", cfg)
 		if err != nil {
 			return err
 		}

cmd/admin-controller/app/root.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2020 wtxue.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package app
 
 import (

cmd/admin-controller/app/version.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2020 wtxue.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package app
 
 import (