From 422cfad1dae94658d281d78e958a45e275fc7e60 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 6 Jan 2021 09:38:03 +0200 Subject: [PATCH] Add containerd config file to Flatcar based instances --- nodeup/pkg/model/containerd.go | 87 +++++++++++-------- nodeup/pkg/model/containerd_test.go | 14 +-- .../containerdbuilder/flatcar/cluster.yaml | 39 +++++++++ .../containerdbuilder/flatcar/tasks.yaml | 20 +++++ 4 files changed, 120 insertions(+), 40 deletions(-) create mode 100644 nodeup/pkg/model/tests/containerdbuilder/flatcar/cluster.yaml create mode 100644 nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml diff --git a/nodeup/pkg/model/containerd.go b/nodeup/pkg/model/containerd.go index 8583e91e80896..e1409a9097403 100644 --- a/nodeup/pkg/model/containerd.go +++ b/nodeup/pkg/model/containerd.go @@ -51,16 +51,14 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error { switch b.Distribution { case distributions.DistributionFlatcar: klog.Infof("Detected Flatcar; won't install containerd") - if err := b.buildContainerOSConfigurationDropIn(c); err != nil { - return err + if b.Cluster.Spec.ContainerRuntime == "containerd" { + b.buildSystemdServiceOverrideFlatcar(c) + b.buildConfigFile(c) } return nil - case distributions.DistributionContainerOS: klog.Infof("Detected ContainerOS; won't install containerd") - if err := b.buildContainerOSConfigurationDropIn(c); err != nil { - return err - } + b.buildSystemdServiceOverrideContainerOS(c) return nil } @@ -75,19 +73,7 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error { } // Add config file - { - containerdConfigOverride := "" - if b.Cluster.Spec.Containerd != nil { - containerdConfigOverride = fi.StringValue(b.Cluster.Spec.Containerd.ConfigOverride) - } - - t := &nodetasks.File{ - Path: "/etc/containerd/config-kops.toml", - Contents: fi.NewStringResource(containerdConfigOverride), - Type: nodetasks.FileType_File, - } - c.AddTask(t) - } + b.buildConfigFile(c) // Add binaries from assets if b.Cluster.Spec.ContainerRuntime == "containerd" { @@ -129,7 +115,7 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error { } c.AddTask(b.buildSystemdService(sv)) - if err := b.buildSysconfig(c); err != nil { + if err := b.buildSysconfigFile(c); err != nil { return err } @@ -191,41 +177,58 @@ func (b *ContainerdBuilder) buildSystemdService(sv semver.Version) *nodetasks.Se return service } -// buildContainerOSConfigurationDropIn is responsible for configuring the containerd daemon options -func (b *ContainerdBuilder) buildContainerOSConfigurationDropIn(c *fi.ModelBuilderContext) error { +// buildSystemdServiceOverrideContainerOS is responsible for overriding the containerd service for ContainerOS +func (b *ContainerdBuilder) buildSystemdServiceOverrideContainerOS(c *fi.ModelBuilderContext) { lines := []string{ "[Service]", - "EnvironmentFile=/etc/sysconfig/containerd", "EnvironmentFile=/etc/environment", "TasksMax=infinity", } contents := strings.Join(lines, "\n") c.AddTask(&nodetasks.File{ - AfterFiles: []string{"/etc/sysconfig/containerd"}, - Path: "/etc/systemd/system/containerd.service.d/10-kops.conf", - Contents: fi.NewStringResource(contents), - Type: nodetasks.FileType_File, + Path: "/etc/systemd/system/containerd.service.d/10-kops.conf", + Contents: fi.NewStringResource(contents), + Type: nodetasks.FileType_File, OnChangeExecute: [][]string{ {"systemctl", "daemon-reload"}, {"systemctl", "restart", "containerd.service"}, // We need to restart kops-configuration service since nodeup needs to load images - // into containerd with the new config. Restart is on the background because - // kops-configuration is of type 'one-shot' so the restart command will wait for - // nodeup to finish executing + // into containerd with the new config. We restart in the background because + // kops-configuration is of type "one-shot", so the restart command will wait for + // nodeup to finish executing. {"systemctl", "restart", "kops-configuration.service", "&"}, }, }) +} - if err := b.buildSysconfig(c); err != nil { - return err +// buildSystemdServiceOverrideFlatcar is responsible for overriding the containerd service for Flatcar +func (b *ContainerdBuilder) buildSystemdServiceOverrideFlatcar(c *fi.ModelBuilderContext) { + lines := []string{ + "[Service]", + "Environment=CONTAINERD_CONFIG=/etc/containerd/config-kops.toml", + "EnvironmentFile=/etc/environment", } + contents := strings.Join(lines, "\n") - return nil + c.AddTask(&nodetasks.File{ + Path: "/etc/systemd/system/containerd.service.d/10-kops.conf", + Contents: fi.NewStringResource(contents), + Type: nodetasks.FileType_File, + OnChangeExecute: [][]string{ + {"systemctl", "daemon-reload"}, + {"systemctl", "restart", "containerd.service"}, + // We need to restart kops-configuration service since nodeup needs to load images + // into containerd with the new config. We restart in the background because + // kops-configuration is of type "one-shot", so the restart command will wait for + // nodeup to finish executing. + {"systemctl", "restart", "kops-configuration.service", "&"}, + }, + }) } -// buildSysconfig is responsible for extracting the containerd configuration and writing the sysconfig file -func (b *ContainerdBuilder) buildSysconfig(c *fi.ModelBuilderContext) error { +// buildSysconfigFile is responsible for creating the containerd sysconfig file +func (b *ContainerdBuilder) buildSysconfigFile(c *fi.ModelBuilderContext) error { var containerd kops.ContainerdConfig if b.Cluster.Spec.Containerd != nil { containerd = *b.Cluster.Spec.Containerd @@ -250,6 +253,20 @@ func (b *ContainerdBuilder) buildSysconfig(c *fi.ModelBuilderContext) error { return nil } +// buildConfigFile is responsible for creating the containerd configuration file +func (b *ContainerdBuilder) buildConfigFile(c *fi.ModelBuilderContext) { + containerdConfigOverride := "" + if b.Cluster.Spec.Containerd != nil { + containerdConfigOverride = fi.StringValue(b.Cluster.Spec.Containerd.ConfigOverride) + } + + c.AddTask(&nodetasks.File{ + Path: "/etc/containerd/config-kops.toml", + Contents: fi.NewStringResource(containerdConfigOverride), + Type: nodetasks.FileType_File, + }) +} + // skipInstall determines if kops should skip the installation and configuration of containerd func (b *ContainerdBuilder) skipInstall() bool { d := b.Cluster.Spec.Containerd diff --git a/nodeup/pkg/model/containerd_test.go b/nodeup/pkg/model/containerd_test.go index 2eba07f53b1eb..d5470c715573d 100644 --- a/nodeup/pkg/model/containerd_test.go +++ b/nodeup/pkg/model/containerd_test.go @@ -29,15 +29,19 @@ import ( ) func TestContainerdBuilder_Docker_19_03_13(t *testing.T) { - runContainerdBuilderTest(t, "from_docker_19.03.11") + runContainerdBuilderTest(t, "from_docker_19.03.11", distributions.DistributionUbuntu2004) } func TestContainerdBuilder_Docker_19_03_14(t *testing.T) { - runContainerdBuilderTest(t, "from_docker_19.03.14") + runContainerdBuilderTest(t, "from_docker_19.03.14", distributions.DistributionUbuntu2004) } func TestContainerdBuilder_Simple(t *testing.T) { - runContainerdBuilderTest(t, "simple") + runContainerdBuilderTest(t, "simple", distributions.DistributionUbuntu2004) +} + +func TestContainerdBuilder_Flatcar(t *testing.T) { + runContainerdBuilderTest(t, "flatcar", distributions.DistributionFlatcar) } func TestContainerdBuilder_SkipInstall(t *testing.T) { @@ -123,7 +127,7 @@ func TestContainerdBuilder_BuildFlags(t *testing.T) { } } -func runContainerdBuilderTest(t *testing.T, key string) { +func runContainerdBuilderTest(t *testing.T, key string, distro distributions.Distribution) { basedir := path.Join("tests/containerdbuilder/", key) nodeUpModelContext, err := BuildNodeupModelContext(basedir) @@ -132,7 +136,7 @@ func runContainerdBuilderTest(t *testing.T, key string) { return } - nodeUpModelContext.Distribution = distributions.DistributionUbuntu1604 + nodeUpModelContext.Distribution = distro nodeUpModelContext.Assets = fi.NewAssetStore("") nodeUpModelContext.Assets.AddForTest("containerd", "usr/local/bin/containerd", "testing containerd content") diff --git a/nodeup/pkg/model/tests/containerdbuilder/flatcar/cluster.yaml b/nodeup/pkg/model/tests/containerdbuilder/flatcar/cluster.yaml new file mode 100644 index 0000000000000..b867850212329 --- /dev/null +++ b/nodeup/pkg/model/tests/containerdbuilder/flatcar/cluster.yaml @@ -0,0 +1,39 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + containerRuntime: containerd + containerd: + version: 1.4.3 + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: events + kubernetesVersion: v1.19.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a diff --git a/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml b/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml new file mode 100644 index 0000000000000..9b11f2b83c356 --- /dev/null +++ b/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml @@ -0,0 +1,20 @@ +contents: "" +path: /etc/containerd/config-kops.toml +type: file +--- +contents: |- + [Service] + Environment=CONTAINERD_CONFIG=/etc/containerd/config-kops.toml + EnvironmentFile=/etc/environment +onChangeExecute: +- - systemctl + - daemon-reload +- - systemctl + - restart + - containerd.service +- - systemctl + - restart + - kops-configuration.service + - '&' +path: /etc/systemd/system/containerd.service.d/10-kops.conf +type: file