Skip to content

Commit

Permalink
Merge pull request #1010 from pixiake/registry
Browse files Browse the repository at this point in the history
Support Kata and Node Feature Discovery
  • Loading branch information
ks-ci-bot committed Jan 20, 2022
2 parents af3e0e7 + 716c857 commit b451d94
Show file tree
Hide file tree
Showing 8 changed files with 1,028 additions and 1 deletion.
29 changes: 29 additions & 0 deletions apis/kubekey/v1alpha2/kubernetes_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package v1alpha2

import "k8s.io/apimachinery/pkg/runtime"

// Kubernetes contains the configuration for the cluster
type Kubernetes struct {
Type string `yaml:"type" json:"type,omitempty"`
Version string `yaml:"version" json:"version,omitempty"`
Expand All @@ -36,6 +37,8 @@ type Kubernetes struct {
EtcdBackupScriptDir string `yaml:"etcdBackupScript" json:"etcdBackupScript,omitempty"`
ContainerManager string `yaml:"containerManager" json:"containerManager,omitempty"`
ContainerRuntimeEndpoint string `yaml:"containerRuntimeEndpoint" json:"containerRuntimeEndpoint,omitempty"`
NodeFeatureDiscovery Kata `yaml:"nodeFeatureDiscovery" json:"nodeFeatureDiscovery,omitempty"`
Kata NodeFeatureDiscovery `yaml:"kata" json:"kata,omitempty"`
ApiServerArgs []string `yaml:"apiserverArgs" json:"apiserverArgs,omitempty"`
ControllerManagerArgs []string `yaml:"controllerManagerArgs" json:"controllerManagerArgs,omitempty"`
SchedulerArgs []string `yaml:"schedulerArgs" json:"schedulerArgs,omitempty"`
Expand All @@ -46,10 +49,36 @@ type Kubernetes struct {
KubeProxyConfiguration runtime.RawExtension `yaml:"kubeProxyConfiguration" json:"kubeProxyConfiguration,omitempty"`
}

// Kata contains the configuration for the kata in cluster
type Kata struct {
Enabled *bool `yaml:"enabled" json:"enabled,omitempty"`
}

// NodeFeatureDiscovery contains the configuration for the node-feature-discovery in cluster
type NodeFeatureDiscovery struct {
Enabled *bool `yaml:"enabled" json:"enabled,omitempty"`
}

// EnableNodelocaldns is used to determine whether to deploy nodelocaldns.
func (k *Kubernetes) EnableNodelocaldns() bool {
if k.Nodelocaldns == nil {
return true
}
return *k.Nodelocaldns
}

// EnableKataDeploy is used to determine whether to deploy kata.
func (k *Kubernetes) EnableKataDeploy() bool {
if k.Kata.Enabled == nil {
return false
}
return *k.Kata.Enabled
}

// EnableNodeFeatureDiscovery is used to determine whether to deploy node-feature-discovery.
func (k *Kubernetes) EnableNodeFeatureDiscovery() bool {
if k.NodeFeatureDiscovery.Enabled == nil {
return false
}
return *k.NodeFeatureDiscovery.Enabled
}
42 changes: 42 additions & 0 deletions apis/kubekey/v1alpha2/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions config/crd/bases/kubekey.kubesphere.io_clusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ spec:
type: object
type: array
kubernetes:
description: Kubernetes contains the configuration for the cluster
properties:
apiserverArgs:
items:
Expand Down Expand Up @@ -494,6 +495,13 @@ spec:
additionalProperties:
type: boolean
type: object
kata:
description: NodeFeatureDiscovery contains the configuration for
the node-feature-discovery in cluster
properties:
enabled:
type: boolean
type: object
keepBackupNumber:
type: integer
kubeProxyArgs:
Expand All @@ -514,6 +522,12 @@ spec:
type: integer
nodeCidrMaskSize:
type: integer
nodeFeatureDiscovery:
description: Kata contains the configuration for the kata in cluster
properties:
enabled:
type: boolean
type: object
nodelocaldns:
type: boolean
proxyMode:
Expand Down
5 changes: 4 additions & 1 deletion pkg/images/tasks.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,12 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s
// storage
"provisioner-localpv": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "openebs", Repo: "provisioner-localpv", Tag: "2.10.1", Group: kubekeyv1alpha2.Worker, Enable: false},
"linux-utils": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "openebs", Repo: "linux-utils", Tag: "2.10.0", Group: kubekeyv1alpha2.Worker, Enable: false},

// load balancer
"haproxy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "library", Repo: "haproxy", Tag: "2.3", Group: kubekeyv1alpha2.Worker, Enable: kubeConf.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()},
// kata-deploy
"kata-deploy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kata-deploy", Tag: "stable", Group: kubekeyv1alpha2.Worker, Enable: kubeConf.Cluster.Kubernetes.EnableKataDeploy()},
// node-feature-discovery
"node-feature-discovery": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "node-feature-discovery", Tag: "v0.10.0", Group: kubekeyv1alpha2.K8s, Enable: kubeConf.Cluster.Kubernetes.EnableNodeFeatureDiscovery()},
}

image = ImageList[name]
Expand Down
2 changes: 2 additions & 0 deletions pkg/pipelines/create_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/kubesphere/kubekey/pkg/container"
"github.com/kubesphere/kubekey/pkg/images"
"github.com/kubesphere/kubekey/pkg/kubernetes"
"github.com/kubesphere/kubekey/pkg/plugins"
"github.com/kubesphere/kubekey/pkg/plugins/dns"
"io/ioutil"
"path/filepath"
Expand Down Expand Up @@ -77,6 +78,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error {
&filesystem.ChownModule{},
&certs.AutoRenewCertsModule{},
&kubernetes.SaveKubeConfigModule{},
&plugins.DeployPluginsModule{},
&addons.AddonsModule{},
&storage.DeployLocalVolumeModule{Skip: !runtime.Arg.DeployLocalStorage && !runtime.Cluster.KubeSphere.Enabled},
&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
Expand Down
204 changes: 204 additions & 0 deletions pkg/plugins/kata.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package plugins

import (
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/core/action"
"github.com/kubesphere/kubekey/pkg/core/connector"
"github.com/kubesphere/kubekey/pkg/core/task"
"github.com/kubesphere/kubekey/pkg/core/util"
"github.com/kubesphere/kubekey/pkg/images"
"github.com/lithammer/dedent"
"github.com/pkg/errors"
"path/filepath"
"text/template"
)

// Kata Containers is an open source community working to build a secure container runtime with lightweight virtual
// machines that feel and perform like containers, but provide stronger workload isolation using hardware virtualization
// technology as a second layer of defense.

var (
KataDeploy = template.Must(template.New("kata-deploy.yaml").Parse(
dedent.Dedent(`---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kata-label-node
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: node-labeler
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kata-label-node-rb
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: node-labeler
subjects:
- kind: ServiceAccount
name: kata-label-node
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kata-deploy
namespace: kube-system
spec:
selector:
matchLabels:
name: kata-deploy
template:
metadata:
labels:
name: kata-deploy
spec:
serviceAccountName: kata-label-node
containers:
- name: kube-kata
image: {{ .KataDeployImage }}
imagePullPolicy: Always
lifecycle:
preStop:
exec:
command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"]
command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ]
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
privileged: false
volumeMounts:
- name: crio-conf
mountPath: /etc/crio/
- name: containerd-conf
mountPath: /etc/containerd/
- name: kata-artifacts
mountPath: /opt/kata/
- name: dbus
mountPath: /var/run/dbus
- name: systemd
mountPath: /run/systemd
- name: local-bin
mountPath: /usr/local/bin/
volumes:
- name: crio-conf
hostPath:
path: /etc/crio/
- name: containerd-conf
hostPath:
path: /etc/containerd/
- name: kata-artifacts
hostPath:
path: /opt/kata/
type: DirectoryOrCreate
- name: dbus
hostPath:
path: /var/run/dbus
- name: systemd
hostPath:
path: /run/systemd
- name: local-bin
hostPath:
path: /usr/local/bin/
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
---
kind: RuntimeClass
apiVersion: node.k8s.io/v1beta1
metadata:
name: kata-qemu
handler: kata-qemu
overhead:
podFixed:
memory: "160Mi"
cpu: "250m"
---
kind: RuntimeClass
apiVersion: node.k8s.io/v1beta1
metadata:
name: kata-clh
handler: kata-clh
overhead:
podFixed:
memory: "130Mi"
cpu: "250m"
---
kind: RuntimeClass
apiVersion: node.k8s.io/v1beta1
metadata:
name: kata-fc
handler: kata-fc
overhead:
podFixed:
memory: "130Mi"
cpu: "250m"
`)))
)

func DeployKataTasks(d *DeployPluginsModule) []task.Interface {
generateKataDeployManifests := &task.RemoteTask{
Name: "GenerateKataDeployManifests",
Desc: "Generate kata-deploy manifests",
Hosts: d.Runtime.GetHostsByRole(common.Master),
Prepare: new(common.OnlyFirstMaster),
Action: &action.Template{
Template: KataDeploy,
Data: util.Data{
"KataDeployImage": images.GetImage(d.Runtime, d.KubeConf, "kata-deploy").ImageName(),
},
Dst: filepath.Join(common.KubeAddonsDir, KataDeploy.Name()),
},
Parallel: false,
}

deployKata := &task.RemoteTask{
Name: "ApplyKataDeployManifests",
Desc: "Apply kata-deploy manifests",
Hosts: d.Runtime.GetHostsByRole(common.Master),
Prepare: new(common.OnlyFirstMaster),
Action: new(ApplyKataDeployManifests),
}

return []task.Interface{
generateKataDeployManifests,
deployKata,
}
}

type ApplyKataDeployManifests struct {
common.KubeAction
}

func (a *ApplyKataDeployManifests) Execute(runtime connector.Runtime) error {
if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/kata-deploy.yaml", true); err != nil {
return errors.Wrap(errors.WithStack(err), "apply kata-deploy manifests failed")
}
return nil
}

0 comments on commit b451d94

Please sign in to comment.