Fix ubuntu dist-upgrade to be noninteractive (#282)

kubermatic · Jul 23, 2018 · aa2d906 · aa2d906
1 parent 2e0d2f0
commit aa2d906
Show file tree

Hide file tree

Showing 9 changed files with 216 additions and 37 deletions.
diff --git a/pkg/controller/machine.go b/pkg/controller/machine.go
@@ -464,7 +464,10 @@ func (c *Controller) ensureInstanceExistsForMachine(prov cloud.Provider, machine
 	}
 
 	cacheKey := string(machine.UID) + machine.ResourceVersion
-	if !c.validationCache[cacheKey] {
+	c.validationCacheMutex.Lock()
+	validationSuccess := c.validationCache[cacheKey]
+	c.validationCacheMutex.Unlock()
+	if !validationSuccess {
 		if err := c.validateMachine(prov, machine); err != nil {
 			if _, errNested := c.updateMachineError(machine, machinev1alpha1.InvalidConfigurationMachineError, err.Error()); errNested != nil {
 				return fmt.Errorf("failed to update machine error after failed validation: %v", errNested)

diff --git a/pkg/userdata/ubuntu/testdata/docker-1.13-dist-upgrade-on-boot-aws.golden b/pkg/userdata/ubuntu/testdata/docker-1.13-dist-upgrade-on-boot-aws.golden
@@ -21,15 +21,30 @@ write_files:
   content: deb http://apt.kubernetes.io/ kubernetes-xenial main
 
 - path: "/usr/local/bin/setup"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
     mkdir -p /opt/bin
     apt-key add /opt/docker.asc
     apt-key add /opt/kubernetes.asc
     apt-get update
-    apt-get dist-upgrade -y
+
+    # If something failed during package installation but one of docker/kubeadm/kubelet was already installed
+    # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if
+    # yes put them on hold
+    set +e
+    which docker && apt-mark hold docker docker-ce
+    which kubelet && apt-mark hold kubelet
+    which kubeadm && apt-mark hold kubeadm
+
+    # When docker is started from within the apt installation it fails with a
+    # 'no sockets found via socket activation: make sure the service was started by systemd'
+    # Apparently the package is broken in a way that it gets started without its dependencies, manually starting
+    # it works fine thought
+    which docker && systemctl start docker
+    set -e
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y
     if [[ -e /var/run/reboot-required ]]; then
       reboot
     fi
@@ -45,7 +60,8 @@ write_files:
     # specified, installation of the kube packages fails
     export CNI_PKG=''
 
-    apt-get install -y curl \
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \
+      curl \
       ca-certificates \
       ceph-common \
       cifs-utils \
@@ -70,7 +86,7 @@ write_files:
     cp /etc/default/kubelet-overwrite /etc/default/kubelet
 
     systemctl enable --now docker
-    systemctl enable --now kubelet
+    systemctl enable kubelet
 
     if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
       kubeadm join \
@@ -168,7 +184,7 @@ write_files:
     -----END PGP PUBLIC KEY BLOCK-----
 
 - path: "/usr/local/bin/supervise.sh"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
@@ -183,12 +199,14 @@ write_files:
       --cluster-dns=10.10.10.10 --cluster-domain=cluster.local
 
 - path: "/etc/systemd/system/kubelet.service.d/20-extra.conf"
+  permissions: "0644"
   content: |
     [Service]
     EnvironmentFile=/etc/default/kubelet
 
 
 - path: "/etc/systemd/system/setup.service"
+  permissions: "0644"
   content: |
     [Install]
     WantedBy=multi-user.target

diff --git a/pkg/userdata/ubuntu/testdata/docker-1.13-kubelet-1.11-aws.golden b/pkg/userdata/ubuntu/testdata/docker-1.13-kubelet-1.11-aws.golden
@@ -21,14 +21,29 @@ write_files:
   content: deb http://apt.kubernetes.io/ kubernetes-xenial main
 
 - path: "/usr/local/bin/setup"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
     mkdir -p /opt/bin
     apt-key add /opt/docker.asc
     apt-key add /opt/kubernetes.asc
     apt-get update
+
+    # If something failed during package installation but one of docker/kubeadm/kubelet was already installed
+    # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if
+    # yes put them on hold
+    set +e
+    which docker && apt-mark hold docker docker-ce
+    which kubelet && apt-mark hold kubelet
+    which kubeadm && apt-mark hold kubeadm
+
+    # When docker is started from within the apt installation it fails with a
+    # 'no sockets found via socket activation: make sure the service was started by systemd'
+    # Apparently the package is broken in a way that it gets started without its dependencies, manually starting
+    # it works fine thought
+    which docker && systemctl start docker
+    set -e
     if [[ -e /var/run/reboot-required ]]; then
       reboot
     fi
@@ -44,7 +59,8 @@ write_files:
     # specified, installation of the kube packages fails
     export CNI_PKG=''
 
-    apt-get install -y curl \
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \
+      curl \
       ca-certificates \
       ceph-common \
       cifs-utils \
@@ -69,7 +85,7 @@ write_files:
     cp /etc/default/kubelet-overwrite /etc/default/kubelet
 
     systemctl enable --now docker
-    systemctl enable --now kubelet
+    systemctl enable kubelet
 
     if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
       kubeadm join \
@@ -167,7 +183,7 @@ write_files:
     -----END PGP PUBLIC KEY BLOCK-----
 
 - path: "/usr/local/bin/supervise.sh"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
@@ -183,6 +199,7 @@ write_files:
 
 
 - path: "/etc/systemd/system/setup.service"
+  permissions: "0644"
   content: |
     [Install]
     WantedBy=multi-user.target

diff --git a/pkg/userdata/ubuntu/testdata/docker-17.03-openstack-kubelet-v-version-prefix.golden b/pkg/userdata/ubuntu/testdata/docker-17.03-openstack-kubelet-v-version-prefix.golden
@@ -21,15 +21,30 @@ write_files:
   content: deb http://apt.kubernetes.io/ kubernetes-xenial main
 
 - path: "/usr/local/bin/setup"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
     mkdir -p /opt/bin
     apt-key add /opt/docker.asc
     apt-key add /opt/kubernetes.asc
     apt-get update
-    apt-get dist-upgrade -y
+
+    # If something failed during package installation but one of docker/kubeadm/kubelet was already installed
+    # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if
+    # yes put them on hold
+    set +e
+    which docker && apt-mark hold docker docker-ce
+    which kubelet && apt-mark hold kubelet
+    which kubeadm && apt-mark hold kubeadm
+
+    # When docker is started from within the apt installation it fails with a
+    # 'no sockets found via socket activation: make sure the service was started by systemd'
+    # Apparently the package is broken in a way that it gets started without its dependencies, manually starting
+    # it works fine thought
+    which docker && systemctl start docker
+    set -e
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y
     if [[ -e /var/run/reboot-required ]]; then
       reboot
     fi
@@ -45,7 +60,8 @@ write_files:
     # specified, installation of the kube packages fails
     export CNI_PKG=''
 
-    apt-get install -y curl \
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \
+      curl \
       ca-certificates \
       ceph-common \
       cifs-utils \
@@ -70,7 +86,7 @@ write_files:
     cp /etc/default/kubelet-overwrite /etc/default/kubelet
 
     systemctl enable --now docker
-    systemctl enable --now kubelet
+    systemctl enable kubelet
 
     if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
       kubeadm join \
@@ -168,7 +184,7 @@ write_files:
     -----END PGP PUBLIC KEY BLOCK-----
 
 - path: "/usr/local/bin/supervise.sh"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
@@ -183,12 +199,14 @@ write_files:
       --cluster-dns=10.10.10.10 --cluster-domain=cluster.local
 
 - path: "/etc/systemd/system/kubelet.service.d/20-extra.conf"
+  permissions: "0644"
   content: |
     [Service]
     EnvironmentFile=/etc/default/kubelet
 
 
 - path: "/etc/systemd/system/setup.service"
+  permissions: "0644"
   content: |
     [Install]
     WantedBy=multi-user.target

diff --git a/pkg/userdata/ubuntu/testdata/docker-17.03-openstack-multiple-dns.golden b/pkg/userdata/ubuntu/testdata/docker-17.03-openstack-multiple-dns.golden
@@ -21,15 +21,30 @@ write_files:
   content: deb http://apt.kubernetes.io/ kubernetes-xenial main
 
 - path: "/usr/local/bin/setup"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
     mkdir -p /opt/bin
     apt-key add /opt/docker.asc
     apt-key add /opt/kubernetes.asc
     apt-get update
-    apt-get dist-upgrade -y
+
+    # If something failed during package installation but one of docker/kubeadm/kubelet was already installed
+    # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if
+    # yes put them on hold
+    set +e
+    which docker && apt-mark hold docker docker-ce
+    which kubelet && apt-mark hold kubelet
+    which kubeadm && apt-mark hold kubeadm
+
+    # When docker is started from within the apt installation it fails with a
+    # 'no sockets found via socket activation: make sure the service was started by systemd'
+    # Apparently the package is broken in a way that it gets started without its dependencies, manually starting
+    # it works fine thought
+    which docker && systemctl start docker
+    set -e
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y
     if [[ -e /var/run/reboot-required ]]; then
       reboot
     fi
@@ -45,7 +60,8 @@ write_files:
     # specified, installation of the kube packages fails
     export CNI_PKG=''
 
-    apt-get install -y curl \
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \
+      curl \
       ca-certificates \
       ceph-common \
       cifs-utils \
@@ -70,7 +86,7 @@ write_files:
     cp /etc/default/kubelet-overwrite /etc/default/kubelet
 
     systemctl enable --now docker
-    systemctl enable --now kubelet
+    systemctl enable kubelet
 
     if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
       kubeadm join \
@@ -168,7 +184,7 @@ write_files:
     -----END PGP PUBLIC KEY BLOCK-----
 
 - path: "/usr/local/bin/supervise.sh"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
@@ -183,12 +199,14 @@ write_files:
       --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 --cluster-domain=cluster.local
 
 - path: "/etc/systemd/system/kubelet.service.d/20-extra.conf"
+  permissions: "0644"
   content: |
     [Service]
     EnvironmentFile=/etc/default/kubelet
 
 
 - path: "/etc/systemd/system/setup.service"
+  permissions: "0644"
   content: |
     [Install]
     WantedBy=multi-user.target

diff --git a/pkg/userdata/ubuntu/testdata/docker-17.03-openstack-overwrite-cloud-config.golden b/pkg/userdata/ubuntu/testdata/docker-17.03-openstack-overwrite-cloud-config.golden
@@ -23,15 +23,30 @@ write_files:
   content: deb http://apt.kubernetes.io/ kubernetes-xenial main
 
 - path: "/usr/local/bin/setup"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
     mkdir -p /opt/bin
     apt-key add /opt/docker.asc
     apt-key add /opt/kubernetes.asc
     apt-get update
-    apt-get dist-upgrade -y
+
+    # If something failed during package installation but one of docker/kubeadm/kubelet was already installed
+    # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if
+    # yes put them on hold
+    set +e
+    which docker && apt-mark hold docker docker-ce
+    which kubelet && apt-mark hold kubelet
+    which kubeadm && apt-mark hold kubeadm
+
+    # When docker is started from within the apt installation it fails with a
+    # 'no sockets found via socket activation: make sure the service was started by systemd'
+    # Apparently the package is broken in a way that it gets started without its dependencies, manually starting
+    # it works fine thought
+    which docker && systemctl start docker
+    set -e
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y
     if [[ -e /var/run/reboot-required ]]; then
       reboot
     fi
@@ -47,7 +62,8 @@ write_files:
     # specified, installation of the kube packages fails
     export CNI_PKG=''
 
-    apt-get install -y curl \
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \
+      curl \
       ca-certificates \
       ceph-common \
       cifs-utils \
@@ -72,7 +88,7 @@ write_files:
     cp /etc/default/kubelet-overwrite /etc/default/kubelet
 
     systemctl enable --now docker
-    systemctl enable --now kubelet
+    systemctl enable kubelet
 
     if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
       kubeadm join \
@@ -170,7 +186,7 @@ write_files:
     -----END PGP PUBLIC KEY BLOCK-----
 
 - path: "/usr/local/bin/supervise.sh"
-  permissions: "0777"
+  permissions: "0755"
   content: |
     #!/bin/bash
     set -xeuo pipefail
@@ -185,12 +201,14 @@ write_files:
       --cluster-dns=10.10.10.10 --cluster-domain=cluster.local
 
 - path: "/etc/systemd/system/kubelet.service.d/20-extra.conf"
+  permissions: "0644"
   content: |
     [Service]
     EnvironmentFile=/etc/default/kubelet
 
 
 - path: "/etc/systemd/system/setup.service"
+  permissions: "0644"
   content: |
     [Install]
     WantedBy=multi-user.target