feat: configurable unattended-upgrades (#4614)

docs/topics/clusterdefinitions.md

@@ -863,6 +863,7 @@ A cluster can have 0 to 12 agent pool profiles. Agent Pool Profiles are used for
 | adminUsername                    | yes      | Describes the username to be used on all linux clusters                                  |
 | ssh.publicKeys[].keyData         | yes      | The public SSH key used for authenticating access to all Linux nodes in the cluster      |
 | secrets                          | no       | Specifies an array of key vaults to pull secrets from and what secrets to pull from each |
+| enableUnattendedUpgrades | no (will be required in a future release)      | Configure each Linux node VM (including control plane node VMs) to run `/usr/bin/unattended-upgrade` in the background according to a daily schedule. If enabled, the default `unattended-upgrades` package configuration will be used as provided by the Ubuntu distro version running on the VM. More information [here](https://help.ubuntu.com/community/AutomaticSecurityUpdates). By default, `enableUnattendedUpgrades` is set to `true`. We encourage you to declare an explicit configuration for `enableUnattendedUpgrades` and a warning will be logged if you do not. In a future release of `aks-engine`, we may default this to `false`, requiring the user to self-maintain OS package and security updates. |
 | runUnattendedUpgradesOnBootstrap | no       | Invoke an unattended-upgrade when each Linux node VM comes online for the first time. In practice this is accomplished by performing an `apt-get update`, followed by a manual invocation of `/usr/bin/unattended-upgrade`, to fetch updated apt configuration, and install all package updates provided by the unattended-upgrade facility, respectively. Defaults to true for public Azure clouds, and to false for Azure Stack Hub and other non-public, custom cloud environments. |
 | customSearchDomain.name          | no       | describes the search domain to be used on all linux clusters                             |
 | customSearchDomain.realmUser     | no       | describes the realm user with permissions to update dns registries on Windows Server DNS |

examples/e2e-tests/kubernetes/release/default/definition.json

@@ -98,6 +98,7 @@
       }
     ],
     "linuxProfile": {
+      "enableUnattendedUpgrades": false,
       "adminUsername": "azureuser",
       "ssh": {
         "publicKeys": [

parts/k8s/cloud-init/artifacts/cse_config.sh

@@ -394,7 +394,7 @@ ensureKubelet() {
   if [[ -n ${MASTER_NODE} ]]; then
     systemctlEnableAndStart kubelet || exit {{GetCSEErrorCode "ERR_KUBELET_START_FAIL"}}
   else
-{{- if not RunUnattendedUpgrades}}
+{{- if not RunUnattendedUpgradesOnBootstrap}}
     systemctlEnableAndStart kubelet || exit {{GetCSEErrorCode "ERR_KUBELET_START_FAIL"}}
 {{else}}
     systemctl_enable 100 5 30 kubelet || exit {{GetCSEErrorCode "ERR_KUBELET_START_FAIL"}}
@@ -405,7 +405,7 @@ ensureKubelet() {
   if [[ -n ${MASTER_NODE} ]]; then
     systemctlEnableAndStart kubelet-monitor || exit {{GetCSEErrorCode "ERR_KUBELET_START_FAIL"}}
   else
-  {{- if not RunUnattendedUpgrades}}
+  {{- if not RunUnattendedUpgradesOnBootstrap}}
     systemctlEnableAndStart kubelet-monitor || exit {{GetCSEErrorCode "ERR_KUBELET_START_FAIL"}}
   {{else}}
     systemctl_enable 100 5 30 kubelet-monitor || exit {{GetCSEErrorCode "ERR_KUBELET_START_FAIL"}}

parts/k8s/cloud-init/artifacts/cse_main.sh

@@ -271,7 +271,9 @@ fi
 {{end}}
 
 {{- /* re-enable unattended upgrades */}}
+{{- if EnableUnattendedUpgrades}}
 rm -f /etc/apt/apt.conf.d/99periodic
+{{- end}}
 
 {{- if not IsAzureStackCloud}}
 if [[ $OS == $UBUNTU_OS_NAME ]]; then
@@ -280,7 +282,7 @@ fi
 {{end}}
 
 {{- if not HasBlockOutboundInternet}}
-    {{- if RunUnattendedUpgrades}}
+    {{- if RunUnattendedUpgradesOnBootstrap}}
 apt_get_update && unattended_upgrade
     {{- end}}
 {{- end}}
@@ -292,7 +294,7 @@ if [ -f /var/run/reboot-required ]; then
     aptmarkWALinuxAgent unhold &
   fi
 else
-{{- if RunUnattendedUpgrades}}
+{{- if RunUnattendedUpgradesOnBootstrap}}
   if [[ -z ${MASTER_NODE} ]]; then
     systemctl_restart 100 5 30 kubelet
     systemctl_restart 100 5 30 kubelet-monitor

pkg/api/const.go

@@ -223,6 +223,10 @@ const (
 	StandardVMType = "standard"
 	// DefaultRunUnattendedUpgradesOnBootstrap sets the default configuration for running a blocking unattended-upgrade on Linux VMs as part of CSE
 	DefaultRunUnattendedUpgradesOnBootstrap = true
+	// DefaultEnableUnattendedUpgrades sets the default configuration for running unattended-upgrade on a regular schedule in the background
+	DefaultEnableUnattendedUpgrades = true
+	// DefaultEnableUnattendedUpgradesAzureStack sets the default configuration for running unattended-upgrade on a regular schedule in the background for Azure Stack Hub
+	DefaultEnableUnattendedUpgradesAzureStack = true
 	// DefaultEth0MTU is the default MTU configuration for eth0 Linux interfaces
 	DefaultEth0MTU = 1500
 )

pkg/api/defaults.go

@@ -703,6 +703,13 @@ func (p *Properties) setLinuxProfileDefaults() {
 	if !p.IsAzureStackCloud() && p.LinuxProfile.RunUnattendedUpgradesOnBootstrap == nil {
 		p.LinuxProfile.RunUnattendedUpgradesOnBootstrap = to.BoolPtr(DefaultRunUnattendedUpgradesOnBootstrap)
 	}
+	if p.LinuxProfile.EnableUnattendedUpgrades == nil {
+		if p.IsAzureStackCloud() {
+			p.LinuxProfile.EnableUnattendedUpgrades = to.BoolPtr(DefaultEnableUnattendedUpgradesAzureStack)
+		} else {
+			p.LinuxProfile.EnableUnattendedUpgrades = to.BoolPtr(DefaultEnableUnattendedUpgrades)
+		}
+	}
 	if p.OrchestratorProfile.IsAzureCNI() && p.LinuxProfile.Eth0MTU == 0 {
 		p.LinuxProfile.Eth0MTU = DefaultEth0MTU
 	}

pkg/api/types.go

@@ -140,6 +140,7 @@ type LinuxProfile struct {
 	CustomNodesDNS                   *CustomNodesDNS     `json:"CustomNodesDNS,omitempty"`
 	IsSSHKeyAutoGenerated            *bool               `json:"isSSHKeyAutoGenerated,omitempty"`
 	RunUnattendedUpgradesOnBootstrap *bool               `json:"runUnattendedUpgradesOnBootstrap,omitempty"`
+	EnableUnattendedUpgrades         *bool               `json:"enableUnattendedUpgrades,omitempty"`
 	Eth0MTU                          int                 `json:"eth0MTU,omitempty"`
 }
 

pkg/api/vlabs/types.go

@@ -140,6 +140,7 @@ type LinuxProfile struct {
 	CustomSearchDomain               *CustomSearchDomain `json:"customSearchDomain,omitempty"`
 	CustomNodesDNS                   *CustomNodesDNS     `json:"customNodesDNS,omitempty"`
 	RunUnattendedUpgradesOnBootstrap *bool               `json:"runUnattendedUpgradesOnBootstrap,omitempty"`
+	EnableUnattendedUpgrades         *bool               `json:"enableUnattendedUpgrades,omitempty"`
 	Eth0MTU                          int                 `json:"eth0MTU,omitempty"`
 }
 

pkg/api/vlabs/validate.go

@@ -650,6 +650,9 @@ func (a *Properties) validateLinuxProfile() error {
 			return errors.New("KeyData in LinuxProfile.SSH.PublicKeys cannot be empty string")
 		}
 	}
+	if a.LinuxProfile.EnableUnattendedUpgrades == nil {
+		log.Warnf("linuxProfile.enableUnattendedUpgrades configuration was not declared, your cluster nodes will be configured to run unattended-upgrade by default")
+	}
 	return validateKeyVaultSecrets(a.LinuxProfile.Secrets, false)
 }
 

pkg/api/vlabs/validate_test.go

@@ -998,6 +998,24 @@ func ExampleProperties_validateAddons() {
 	// level=warning msg="The Azure CNI networkmonitor addon has been deprecated, it will be marked as disabled"
 }
 
+func ExampleProperties_validateLinuxProfile() {
+	log.SetOutput(os.Stdout)
+	log.SetFormatter(&log.TextFormatter{
+		DisableColors:    true,
+		DisableTimestamp: true,
+	})
+	p := Properties{
+		LinuxProfile: &LinuxProfile{},
+	}
+
+	if err := p.validateLinuxProfile(); err != nil {
+		fmt.Printf("error in validateLinuxProfile: %s", err)
+	}
+
+	// Output:
+	// level=warning msg="linuxProfile.enableUnattendedUpgrades configuration was not declared, your cluster nodes will be configured to run unattended-upgrade by default"
+}
+
 func Test_Properties_ValidateNetworkPlugin(t *testing.T) {
 	p := &Properties{}
 	p.OrchestratorProfile = &OrchestratorProfile{}

pkg/engine/template_generator.go

@@ -802,12 +802,18 @@ version = 2
 		"GetLinuxCSELogPath": func() string {
 			return linuxCSELogPath
 		},
-		"RunUnattendedUpgrades": func() bool {
+		"RunUnattendedUpgradesOnBootstrap": func() bool {
 			if cs.Properties.LinuxProfile != nil {
 				return to.Bool(cs.Properties.LinuxProfile.RunUnattendedUpgradesOnBootstrap)
 			}
 			return false
 		},
+		"EnableUnattendedUpgrades": func() bool {
+			if cs.Properties.LinuxProfile != nil {
+				return to.Bool(cs.Properties.LinuxProfile.EnableUnattendedUpgrades)
+			}
+			return false
+		},
 		"GetEth0MTU": func() int {
 			if cs.Properties.LinuxProfile != nil {
 				return cs.Properties.LinuxProfile.Eth0MTU