Skip to content

Commit

Permalink
[dhctl] Prevent restart deckhouse when bootstrap was restarted
Browse files Browse the repository at this point in the history
  • Loading branch information
@name212 @konstantin-axenov
name212 authored and konstantin-axenov committed Nov 15, 2021
1 parent 6819830 commit 8c1da08
Show file tree
Hide file tree
Showing 3 changed files with 227 additions and 72 deletions.
92 changes: 61 additions & 31 deletions dhctl/pkg/kubernetes/actions/deckhouse/install.go
View file
Expand Up @@ -69,8 +69,53 @@ func (c *Config) IsRegistryAccessRequired() bool {
return c.Registry.DockerCfg != ""
}

func deckhouseDeploymentFromConfig(cfg *Config) *appsv1.Deployment {
return manifests.DeckhouseDeployment(cfg.GetImage(), cfg.LogLevel, cfg.Bundle, cfg.IsRegistryAccessRequired())
func prepareDeckhouseDeploymentForUpdate(kubeCl *client.KubernetesClient, cfg *Config, manifestForUpdate *appsv1.Deployment) (*appsv1.Deployment, error) {
resDeployment := manifestForUpdate
err := retry.NewSilentLoop("get deployment", 10, 3*time.Second).Run(func() error {
currentManifestInCluster, err := kubeCl.AppsV1().Deployments(manifestForUpdate.GetNamespace()).Get(context.TODO(), manifestForUpdate.GetName(), metav1.GetOptions{})
if err != nil {
return err
}

// Parametrize existing Deployment manifest to prevent redundant restarting
// of deckhouse's Pod if params are not changed between dhctl executions.
// deployTime is a 'write once' parameter, so it is preserved for this check.
//
// It helps to reduce wait time on bootstrap process restarting,
// and prevents a race condition when deckhouse's Pod is scheduled
// on the non-approved node, so the bootstrap process never finishes.
deployTime := manifests.GetDeckhouseDeployTime(currentManifestInCluster)
params := deckhouseDeploymentParamsFromCfg(cfg)
params.DeployTime = deployTime
resDeployment = manifests.ParametrizeDeckhouseDeployment(currentManifestInCluster.DeepCopy(), params)

return nil
})

return resDeployment, err
}

func controllerDeploymentTask(kubeCl *client.KubernetesClient, cfg *Config) actions.ManifestTask {
return actions.ManifestTask{
Name: `Deployment "deckhouse"`,
Manifest: func() interface{} {
return CreateDeckhouseDeploymentManifest(cfg)
},
CreateFunc: func(manifest interface{}) error {
_, err := kubeCl.AppsV1().Deployments("d8-system").Create(context.TODO(), manifest.(*appsv1.Deployment), metav1.CreateOptions{})
return err
},
UpdateFunc: func(manifest interface{}) error {
preparedManifest, err := prepareDeckhouseDeploymentForUpdate(kubeCl, cfg, manifest.(*appsv1.Deployment))
if err != nil {
return err
}

_, err = kubeCl.AppsV1().Deployments("d8-system").Update(context.TODO(), preparedManifest, metav1.UpdateOptions{})

return err
},
}
}

func CreateDeckhouseManifests(kubeCl *client.KubernetesClient, cfg *Config) error {
Expand Down Expand Up @@ -291,20 +336,7 @@ func CreateDeckhouseManifests(kubeCl *client.KubernetesClient, cfg *Config) erro
})
}

tasks = append(tasks, actions.ManifestTask{
Name: `Deployment "deckhouse"`,
Manifest: func() interface{} {
return deckhouseDeploymentFromConfig(cfg)
},
CreateFunc: func(manifest interface{}) error {
_, err := kubeCl.AppsV1().Deployments("d8-system").Create(context.TODO(), manifest.(*appsv1.Deployment), metav1.CreateOptions{})
return err
},
UpdateFunc: func(manifest interface{}) error {
_, err := kubeCl.AppsV1().Deployments("d8-system").Update(context.TODO(), manifest.(*appsv1.Deployment), metav1.UpdateOptions{})
return err
},
})
tasks = append(tasks, controllerDeploymentTask(kubeCl, cfg))

return log.Process("default", "Create Manifests", func() error {
for _, task := range tasks {
Expand Down Expand Up @@ -346,26 +378,24 @@ func WaitForReadiness(kubeCl *client.KubernetesClient) error {
}

func CreateDeckhouseDeployment(kubeCl *client.KubernetesClient, cfg *Config) error {
task := actions.ManifestTask{
Name: `Deployment "deckhouse"`,
Manifest: func() interface{} {
return manifests.DeckhouseDeployment(cfg.GetImage(), cfg.LogLevel, cfg.Bundle, cfg.IsRegistryAccessRequired())
},
CreateFunc: func(manifest interface{}) error {
_, err := kubeCl.AppsV1().Deployments("d8-system").Create(context.TODO(), manifest.(*appsv1.Deployment), metav1.CreateOptions{})
return err
},
UpdateFunc: func(manifest interface{}) error {
_, err := kubeCl.AppsV1().Deployments("d8-system").Update(context.TODO(), manifest.(*appsv1.Deployment), metav1.UpdateOptions{})
return err
},
}
task := controllerDeploymentTask(kubeCl, cfg)

return log.Process("default", "Create Deployment", task.CreateOrUpdate)
}

func deckhouseDeploymentParamsFromCfg(cfg *Config) manifests.DeckhouseDeploymentParams {
return manifests.DeckhouseDeploymentParams{
Registry: cfg.GetImage(),
LogLevel: cfg.LogLevel,
Bundle: cfg.Bundle,
IsSecureRegistry: cfg.IsRegistryAccessRequired(),
}
}

func CreateDeckhouseDeploymentManifest(cfg *Config) *appsv1.Deployment {
return manifests.DeckhouseDeployment(cfg.GetImage(), cfg.LogLevel, cfg.Bundle, cfg.IsRegistryAccessRequired())
params := deckhouseDeploymentParamsFromCfg(cfg)

return manifests.DeckhouseDeployment(params)
}

func WaitForKubernetesAPI(kubeCl *client.KubernetesClient) error {
Expand Down
160 changes: 122 additions & 38 deletions dhctl/pkg/kubernetes/actions/manifests/manifests.go
View file
Expand Up @@ -36,10 +36,115 @@ import (
const (
deckhouseRegistrySecretName = "deckhouse-registry"
deckhouseRegistryVolumeName = "registrysecret"

deployTimeEnvVarName = "KUBERNETES_DEPLOYED"
deployTimeEnvVarFormat = time.RFC3339
)

type DeckhouseDeploymentParams struct {
Bundle string
Registry string
LogLevel string
DeployTime time.Time
IsSecureRegistry bool
}

func GetDeckhouseDeployTime(deployment *appsv1.Deployment) time.Time {
deployTime := time.Time{}
for i, env := range deployment.Spec.Template.Spec.Containers[0].Env {
if env.Name != deployTimeEnvVarName {
continue
}

timeAsString := deployment.Spec.Template.Spec.Containers[0].Env[i].Value
t, err := time.Parse(deployTimeEnvVarFormat, timeAsString)
if err == nil {
deployTime = t
}

break
}

return deployTime
}

func ParametrizeDeckhouseDeployment(deployment *appsv1.Deployment, params DeckhouseDeploymentParams) *appsv1.Deployment {
deployment.Spec.Template.Spec.Containers[0].Image = params.Registry

deployTime := params.DeployTime
if deployTime.IsZero() {
deployTime = time.Now()
}

for i, env := range deployment.Spec.Template.Spec.Containers[0].Env {
switch env.Name {
case "LOG_LEVEL":
deployment.Spec.Template.Spec.Containers[0].Env[i].Value = params.LogLevel
case "DECKHOUSE_BUNDLE":
deployment.Spec.Template.Spec.Containers[0].Env[i].Value = params.Bundle
case deployTimeEnvVarName:
deployment.Spec.Template.Spec.Containers[0].Env[i].Value = deployTime.Format(deployTimeEnvVarFormat)
}
}

if params.IsSecureRegistry {
deployment.Spec.Template.Spec.ImagePullSecrets = []apiv1.LocalObjectReference{
{Name: deckhouseRegistrySecretName},
}

// set volume

volumeToSet := apiv1.Volume{
Name: deckhouseRegistryVolumeName,
VolumeSource: apiv1.VolumeSource{
Secret: &apiv1.SecretVolumeSource{SecretName: deckhouseRegistrySecretName},
},
}

volumeIndx := -1

for i := range deployment.Spec.Template.Spec.Volumes {
if deployment.Spec.Template.Spec.Volumes[i].Name == deckhouseRegistryVolumeName {
volumeIndx = i
break
}
}

if volumeIndx < 0 {
deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, volumeToSet)
} else {
deployment.Spec.Template.Spec.Volumes[volumeIndx] = volumeToSet
}

// set volume mount

volumeMountToSet := apiv1.VolumeMount{
Name: deckhouseRegistryVolumeName,
MountPath: "/etc/registrysecret",
ReadOnly: true,
}

volumeMountIndx := -1

for i := range deployment.Spec.Template.Spec.Containers[0].VolumeMounts {
if deployment.Spec.Template.Spec.Containers[0].VolumeMounts[i].Name == deckhouseRegistryVolumeName {
volumeMountIndx = i
break
}
}

if volumeMountIndx < 0 {
deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, volumeMountToSet)
} else {
deployment.Spec.Template.Spec.Containers[0].VolumeMounts[volumeMountIndx] = volumeMountToSet
}
}

return deployment
}

//nolint:funlen
func DeckhouseDeployment(registry, logLevel, bundle string, isSecureRegistry bool) *appsv1.Deployment {
func DeckhouseDeployment(params DeckhouseDeploymentParams) *appsv1.Deployment {
deckhouseDeployment := `
kind: Deployment
apiVersion: apps/v1
Expand Down Expand Up @@ -105,6 +210,11 @@ spec:
value: "yes"
- name: KUBERNETES_DEPLOYED
value: PLACEHOLDER
volumeMounts:
- mountPath: /tmp
name: tmp
- mountPath: /.kube
name: kube
ports:
- containerPort: 9650
name: self
Expand All @@ -126,48 +236,22 @@ spec:
node-role.kubernetes.io/master: ""
tolerations:
- operator: Exists
volumes:
- emptyDir:
medium: Memory
name: tmp
- emptyDir:
medium: Memory
name: kube
`

var deployment appsv1.Deployment
_ = yaml.Unmarshal([]byte(deckhouseDeployment), &deployment)

deployment.Spec.Template.Spec.Containers[0].Image = registry

for i, env := range deployment.Spec.Template.Spec.Containers[0].Env {
switch env.Name {
case "LOG_LEVEL":
deployment.Spec.Template.Spec.Containers[0].Env[i].Value = logLevel
case "DECKHOUSE_BUNDLE":
deployment.Spec.Template.Spec.Containers[0].Env[i].Value = bundle
case "KUBERNETES_DEPLOYED":
deployment.Spec.Template.Spec.Containers[0].Env[i].Value = time.Now().Format(time.RFC3339)
}
}

if isSecureRegistry {
deployment.Spec.Template.Spec.ImagePullSecrets = []apiv1.LocalObjectReference{
{Name: deckhouseRegistrySecretName},
}

deployment.Spec.Template.Spec.Volumes = []apiv1.Volume{
{
Name: deckhouseRegistryVolumeName,
VolumeSource: apiv1.VolumeSource{
Secret: &apiv1.SecretVolumeSource{SecretName: deckhouseRegistrySecretName},
},
},
}

deployment.Spec.Template.Spec.Containers[0].VolumeMounts = []apiv1.VolumeMount{
{
Name: deckhouseRegistryVolumeName,
MountPath: "/etc/registrysecret",
ReadOnly: true,
},
}
err := yaml.Unmarshal([]byte(deckhouseDeployment), &deployment)
if err != nil {
panic(err)
}

return &deployment
return ParametrizeDeckhouseDeployment(&deployment, params)
}

func DeckhouseNamespace(name string) *apiv1.Namespace {
Expand Down
47 changes: 44 additions & 3 deletions dhctl/pkg/kubernetes/actions/manifests/manifests_test.go
View file
Expand Up @@ -18,15 +18,23 @@ import (
"fmt"
"strings"
"testing"
"time"

"github.com/stretchr/testify/require"
"sigs.k8s.io/yaml"
)

func Test_struct_vs_unmarshal(t *testing.T) {
depl1 := DeckhouseDeployment("registry.example.com/deckhouse:master", "debug", "default", true)
params := DeckhouseDeploymentParams{
Registry: "registry.example.com/deckhouse:master",
LogLevel: "debug",
Bundle: "default",
IsSecureRegistry: true,
}

depl1 := DeckhouseDeployment(params)

depl2 := DeckhouseDeployment("registry.example.com/deckhouse:master", "debug", "default", true)
// depl2 := generateDeckhouseDeployment2("registry.example.com/deckhouse:master", "debug", "default", true)
depl2 := DeckhouseDeployment(params)

depl1Yaml, err := yaml.Marshal(depl1)
if err != nil {
Expand Down Expand Up @@ -73,3 +81,36 @@ func Test_struct_vs_unmarshal(t *testing.T) {

fmt.Printf("%d lines are differ\n", diff)
}

func Test_DeployTime(t *testing.T) {
paramsGet := func() DeckhouseDeploymentParams {
return DeckhouseDeploymentParams{
Registry: "registry.example.com/deckhouse:master",
LogLevel: "debug",
Bundle: "default",
IsSecureRegistry: true,
}
}

t.Run("set non zero deploy time if DeployTime param does not pass", func(t *testing.T) {
p := paramsGet()

depl := DeckhouseDeployment(p)
tm := GetDeckhouseDeployTime(depl)

require.False(t, tm.IsZero())
})

t.Run("set same deploy time as DeployTime from param if it present", func(t *testing.T) {
expectTime, _ := time.Parse(time.RFC822, "02 Jan 06 15:04 MST")

p := paramsGet()
p.DeployTime = expectTime

depl := DeckhouseDeployment(p)
tm := GetDeckhouseDeployTime(depl)

require.False(t, tm.IsZero())
require.Equal(t, tm.UnixNano(), expectTime.UnixNano())
})
}

0 comments on commit 8c1da08

Please sign in to comment.