chore: add logs, remove hostname from event telemetry

parts/linux/cloud-init/artifacts/secure-tls-bootstrap.sh

@@ -28,9 +28,9 @@ logs_to_events() {
     ret=$?
     local endTime=$(date +"%F %T.%3N")
 
-    msg_string=$(jq -n --arg Status "Succeeded" --arg Hostname "$(uname -n)" '{Status: $Status, Hostname: $Hostname}')
+    msg_string=$(jq -n --arg Status "Succeeded" '{Status: $Status}')
     if [ "$ret" != "0" ]; then
-        msg_string=$(jq -n --arg Status "Failed" --arg Hostname "$(uname -n)" --arg LogTail "$(tail -n 10 $LOG_FILE_PATH)" '{Status: $Status, Hostname: $Hostname, LogTail: $LogTail}')
+        msg_string=$(jq -n --arg Status "Failed" --arg LogTail "$(tail -n 10 $LOG_FILE_PATH)" '{Status: $Status, LogTail: $LogTail}')
     fi
 
     json_string=$( jq -n \
@@ -82,7 +82,7 @@ bootstrap() {
          --kubeconfig="$KUBECONFIG_PATH" \
          --log-file="$LOG_FILE_PATH"
 
-        [ $? -eq 0 ] && break
+        [ $? -eq 0 ] && echo "secure TLS bootstrapping succeeded, generated kubeconfig is at ${KUBECONFIG_PATH}" && break
 
         sleep $RETRY_WAIT_SECONDS
     done

parts/linux/cloud-init/artifacts/start-kubelet.sh

@@ -16,14 +16,16 @@ setKubeletTLSBootstrapFlags() {
   if [ -f "${KUBECONFIG_FILE}" ]; then
     # if we have a kubeconfig at this point, we can remove the bootstrap-kubeconfig if present
     # to ensure that no bootstrap tokens are left on disk when not needed
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
     # if we don't have a kubeconfig but we do have a bootstrap-kubeconfig, have kubelet
     # use it to request its own certificate at runtime
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }
 

pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData

@@ -298,7 +298,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/6yTv27bMBDGdz7Flc3QDgSLji082I1sGFGtwJKnthAo5+QQZSiDpLwEevcikhJTf7tkknD3++67O+k+fuCZ1DwT9pEQiw4YlgWc5RlzIRUhd4dVEAZJ+iParbebdL0Ng3QdLjfxgt48Tya/MUor6ouT5XYX7G9HtZ3clDTdH3bJ9mdjMFqgQ/TLbPbR4X7E3Yv3JAPWh15WdVdmqNAlYbwqCmedEee1Eif76TM8E4AXmbeYBb8Iw5XM+N9GVz+Phc7liQCsoiiJk/3yPv2vLnt1Y50Kr20mYZxeq7VTMI+FmU4oIQAyh1/Acmhn95qpKPz5Du4RNQEAME8tNtl9RRsQXWk0fCEAuexZzGg7Zu8yHzA2tr6hZIyiTfvV7NcnhJfWcFUchapPq60Iv+spGEMtMoXMormgeYvq4gGZEhkq6/9zu+g2SMPlKgjjir7Bl8XX63uhyidkZ1WepGYP0iw4umPtajQ6tLwhGsC2upuZbfaRwXWPAJ0Tnsx3LnRAeafYz9XBfwEAAP//9FFrmqsEAAA=
+    H4sIAAAAAAAA/6xUTY/aMBC9+1dM0z20Uq1UPbbiAN2A0KawgnBqq8hhx2CtsZHtcEH894okyzohH5eeQDPvzXtvDPPxQ5gJFWbM7gmx6IBiruEojsiZkIQ8bSZRHCXpz+ViOp+l03kcpdN4PFuPgodzZ/M7DYJL4JOT8XwRrR5bubVeFzVdbRbJ/Fcp0DqghmiOma2Wm+cWda/eoNxhfdB1VU95hhJdEq8nWjvrDDtOJdvZT5/hTACuNG8xo/DETChFFr6WvOJzqxUXOwIwWS6TdbIaP6eDvOxNjdYmvNlM4nX6Pq1KQT0s9DgJCAEQHH4D5VBl98xcAvj7A9weFQEAwO1eQ+BNFhaOBi0qBxlybRCsY8YJtYNK6wsYPOjTtXLLAd4ELiQCc/Bw7tzIJSjEzaHyOAhElxsFXwkAF418PdzBpEq74bQWXVGgtO3ZwOnepMC1Ac6kzNj2tUzzX565y87QMrkgl97fPiFhbk0o9ZbJ4rBU2vCnME8pKpZJpBbNCc2tqvQLUskylNb/xy2Wj1EajydRvL4EN/Bp9O39u5b5AelR5juh6IswoxDdtlA1Ch3asESUAFvxHnqW2ITc3bYWQO2AdfZr9+kO5R2iZq8o/gsAAP//bsF4nqkFAAA=
 
 - path: /opt/azure/containers/kubelet.sh
   permissions: "0755"

pkg/agent/testdata/AKSUbuntu1604+Containerd/line301.sh

@@ -14,12 +14,14 @@ setKubeletTLSBootstrapFlags() {
   KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig"
 
   if [ -f "${KUBECONFIG_FILE}" ]; then
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }
 

pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData

@@ -298,7 +298,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/6yTv27bMBDGdz7Flc3QDgSLji082I1sGFGtwJKnthAo5+QQZSiDpLwEevcikhJTf7tkknD3++67O+k+fuCZ1DwT9pEQiw4YlgWc5RlzIRUhd4dVEAZJ+iParbebdL0Ng3QdLjfxgt48Tya/MUor6ouT5XYX7G9HtZ3clDTdH3bJ9mdjMFqgQ/TLbPbR4X7E3Yv3JAPWh15WdVdmqNAlYbwqCmedEee1Eif76TM8E4AXmbeYBb8Iw5XM+N9GVz+Phc7liQCsoiiJk/3yPv2vLnt1Y50Kr20mYZxeq7VTMI+FmU4oIQAyh1/Acmhn95qpKPz5Du4RNQEAME8tNtl9RRsQXWk0fCEAuexZzGg7Zu8yHzA2tr6hZIyiTfvV7NcnhJfWcFUchapPq60Iv+spGEMtMoXMormgeYvq4gGZEhkq6/9zu+g2SMPlKgjjir7Bl8XX63uhyidkZ1WepGYP0iw4umPtajQ6tLwhGsC2upuZbfaRwXWPAJ0Tnsx3LnRAeafYz9XBfwEAAP//9FFrmqsEAAA=
+    H4sIAAAAAAAA/6xUTY/aMBC9+1dM0z20Uq1UPbbiAN2A0KawgnBqq8hhx2CtsZHtcEH894okyzohH5eeQDPvzXtvDPPxQ5gJFWbM7gmx6IBiruEojsiZkIQ8bSZRHCXpz+ViOp+l03kcpdN4PFuPgodzZ/M7DYJL4JOT8XwRrR5bubVeFzVdbRbJ/Fcp0DqghmiOma2Wm+cWda/eoNxhfdB1VU95hhJdEq8nWjvrDDtOJdvZT5/hTACuNG8xo/DETChFFr6WvOJzqxUXOwIwWS6TdbIaP6eDvOxNjdYmvNlM4nX6Pq1KQT0s9DgJCAEQHH4D5VBl98xcAvj7A9weFQEAwO1eQ+BNFhaOBi0qBxlybRCsY8YJtYNK6wsYPOjTtXLLAd4ELiQCc/Bw7tzIJSjEzaHyOAhElxsFXwkAF418PdzBpEq74bQWXVGgtO3ZwOnepMC1Ac6kzNj2tUzzX565y87QMrkgl97fPiFhbk0o9ZbJ4rBU2vCnME8pKpZJpBbNCc2tqvQLUskylNb/xy2Wj1EajydRvL4EN/Bp9O39u5b5AelR5juh6IswoxDdtlA1Ch3asESUAFvxHnqW2ITc3bYWQO2AdfZr9+kO5R2iZq8o/gsAAP//bsF4nqkFAAA=
 
 - path: /opt/azure/containers/kubelet.sh
   permissions: "0755"

pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line301.sh

@@ -14,12 +14,14 @@ setKubeletTLSBootstrapFlags() {
   KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig"
 
   if [ -f "${KUBECONFIG_FILE}" ]; then
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }
 

pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData

@@ -298,7 +298,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/6yTv27bMBDGdz7Flc3QDgSLji082I1sGFGtwJKnthAo5+QQZSiDpLwEevcikhJTf7tkknD3++67O+k+fuCZ1DwT9pEQiw4YlgWc5RlzIRUhd4dVEAZJ+iParbebdL0Ng3QdLjfxgt48Tya/MUor6ouT5XYX7G9HtZ3clDTdH3bJ9mdjMFqgQ/TLbPbR4X7E3Yv3JAPWh15WdVdmqNAlYbwqCmedEee1Eif76TM8E4AXmbeYBb8Iw5XM+N9GVz+Phc7liQCsoiiJk/3yPv2vLnt1Y50Kr20mYZxeq7VTMI+FmU4oIQAyh1/Acmhn95qpKPz5Du4RNQEAME8tNtl9RRsQXWk0fCEAuexZzGg7Zu8yHzA2tr6hZIyiTfvV7NcnhJfWcFUchapPq60Iv+spGEMtMoXMormgeYvq4gGZEhkq6/9zu+g2SMPlKgjjir7Bl8XX63uhyidkZ1WepGYP0iw4umPtajQ6tLwhGsC2upuZbfaRwXWPAJ0Tnsx3LnRAeafYz9XBfwEAAP//9FFrmqsEAAA=
+    H4sIAAAAAAAA/6xUTY/aMBC9+1dM0z20Uq1UPbbiAN2A0KawgnBqq8hhx2CtsZHtcEH894okyzohH5eeQDPvzXtvDPPxQ5gJFWbM7gmx6IBiruEojsiZkIQ8bSZRHCXpz+ViOp+l03kcpdN4PFuPgodzZ/M7DYJL4JOT8XwRrR5bubVeFzVdbRbJ/Fcp0DqghmiOma2Wm+cWda/eoNxhfdB1VU95hhJdEq8nWjvrDDtOJdvZT5/hTACuNG8xo/DETChFFr6WvOJzqxUXOwIwWS6TdbIaP6eDvOxNjdYmvNlM4nX6Pq1KQT0s9DgJCAEQHH4D5VBl98xcAvj7A9weFQEAwO1eQ+BNFhaOBi0qBxlybRCsY8YJtYNK6wsYPOjTtXLLAd4ELiQCc/Bw7tzIJSjEzaHyOAhElxsFXwkAF418PdzBpEq74bQWXVGgtO3ZwOnepMC1Ac6kzNj2tUzzX565y87QMrkgl97fPiFhbk0o9ZbJ4rBU2vCnME8pKpZJpBbNCc2tqvQLUskylNb/xy2Wj1EajydRvL4EN/Bp9O39u5b5AelR5juh6IswoxDdtlA1Ch3asESUAFvxHnqW2ITc3bYWQO2AdfZr9+kO5R2iZq8o/gsAAP//bsF4nqkFAAA=
 
 - path: /opt/azure/containers/kubelet.sh
   permissions: "0755"

pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line301.sh

@@ -14,12 +14,14 @@ setKubeletTLSBootstrapFlags() {
   KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig"
 
   if [ -f "${KUBECONFIG_FILE}" ]; then
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }
 

pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData

@@ -298,7 +298,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/6yTv27bMBDGdz7Flc3QDgSLji082I1sGFGtwJKnthAo5+QQZSiDpLwEevcikhJTf7tkknD3++67O+k+fuCZ1DwT9pEQiw4YlgWc5RlzIRUhd4dVEAZJ+iParbebdL0Ng3QdLjfxgt48Tya/MUor6ouT5XYX7G9HtZ3clDTdH3bJ9mdjMFqgQ/TLbPbR4X7E3Yv3JAPWh15WdVdmqNAlYbwqCmedEee1Eif76TM8E4AXmbeYBb8Iw5XM+N9GVz+Phc7liQCsoiiJk/3yPv2vLnt1Y50Kr20mYZxeq7VTMI+FmU4oIQAyh1/Acmhn95qpKPz5Du4RNQEAME8tNtl9RRsQXWk0fCEAuexZzGg7Zu8yHzA2tr6hZIyiTfvV7NcnhJfWcFUchapPq60Iv+spGEMtMoXMormgeYvq4gGZEhkq6/9zu+g2SMPlKgjjir7Bl8XX63uhyidkZ1WepGYP0iw4umPtajQ6tLwhGsC2upuZbfaRwXWPAJ0Tnsx3LnRAeafYz9XBfwEAAP//9FFrmqsEAAA=
+    H4sIAAAAAAAA/6xUTY/aMBC9+1dM0z20Uq1UPbbiAN2A0KawgnBqq8hhx2CtsZHtcEH894okyzohH5eeQDPvzXtvDPPxQ5gJFWbM7gmx6IBiruEojsiZkIQ8bSZRHCXpz+ViOp+l03kcpdN4PFuPgodzZ/M7DYJL4JOT8XwRrR5bubVeFzVdbRbJ/Fcp0DqghmiOma2Wm+cWda/eoNxhfdB1VU95hhJdEq8nWjvrDDtOJdvZT5/hTACuNG8xo/DETChFFr6WvOJzqxUXOwIwWS6TdbIaP6eDvOxNjdYmvNlM4nX6Pq1KQT0s9DgJCAEQHH4D5VBl98xcAvj7A9weFQEAwO1eQ+BNFhaOBi0qBxlybRCsY8YJtYNK6wsYPOjTtXLLAd4ELiQCc/Bw7tzIJSjEzaHyOAhElxsFXwkAF418PdzBpEq74bQWXVGgtO3ZwOnepMC1Ac6kzNj2tUzzX565y87QMrkgl97fPiFhbk0o9ZbJ4rBU2vCnME8pKpZJpBbNCc2tqvQLUskylNb/xy2Wj1EajydRvL4EN/Bp9O39u5b5AelR5juh6IswoxDdtlA1Ch3asESUAFvxHnqW2ITc3bYWQO2AdfZr9+kO5R2iZq8o/gsAAP//bsF4nqkFAAA=
 
 - path: /opt/azure/containers/kubelet.sh
   permissions: "0755"

pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line301.sh

@@ -14,12 +14,14 @@ setKubeletTLSBootstrapFlags() {
   KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig"
 
   if [ -f "${KUBECONFIG_FILE}" ]; then
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }
 

pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData

@@ -298,7 +298,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/6yTv27bMBDGdz7Flc3QDgSLji082I1sGFGtwJKnthAo5+QQZSiDpLwEevcikhJTf7tkknD3++67O+k+fuCZ1DwT9pEQiw4YlgWc5RlzIRUhd4dVEAZJ+iParbebdL0Ng3QdLjfxgt48Tya/MUor6ouT5XYX7G9HtZ3clDTdH3bJ9mdjMFqgQ/TLbPbR4X7E3Yv3JAPWh15WdVdmqNAlYbwqCmedEee1Eif76TM8E4AXmbeYBb8Iw5XM+N9GVz+Phc7liQCsoiiJk/3yPv2vLnt1Y50Kr20mYZxeq7VTMI+FmU4oIQAyh1/Acmhn95qpKPz5Du4RNQEAME8tNtl9RRsQXWk0fCEAuexZzGg7Zu8yHzA2tr6hZIyiTfvV7NcnhJfWcFUchapPq60Iv+spGEMtMoXMormgeYvq4gGZEhkq6/9zu+g2SMPlKgjjir7Bl8XX63uhyidkZ1WepGYP0iw4umPtajQ6tLwhGsC2upuZbfaRwXWPAJ0Tnsx3LnRAeafYz9XBfwEAAP//9FFrmqsEAAA=
+    H4sIAAAAAAAA/6xUTY/aMBC9+1dM0z20Uq1UPbbiAN2A0KawgnBqq8hhx2CtsZHtcEH894okyzohH5eeQDPvzXtvDPPxQ5gJFWbM7gmx6IBiruEojsiZkIQ8bSZRHCXpz+ViOp+l03kcpdN4PFuPgodzZ/M7DYJL4JOT8XwRrR5bubVeFzVdbRbJ/Fcp0DqghmiOma2Wm+cWda/eoNxhfdB1VU95hhJdEq8nWjvrDDtOJdvZT5/hTACuNG8xo/DETChFFr6WvOJzqxUXOwIwWS6TdbIaP6eDvOxNjdYmvNlM4nX6Pq1KQT0s9DgJCAEQHH4D5VBl98xcAvj7A9weFQEAwO1eQ+BNFhaOBi0qBxlybRCsY8YJtYNK6wsYPOjTtXLLAd4ELiQCc/Bw7tzIJSjEzaHyOAhElxsFXwkAF418PdzBpEq74bQWXVGgtO3ZwOnepMC1Ac6kzNj2tUzzX565y87QMrkgl97fPiFhbk0o9ZbJ4rBU2vCnME8pKpZJpBbNCc2tqvQLUskylNb/xy2Wj1EajydRvL4EN/Bp9O39u5b5AelR5juh6IswoxDdtlA1Ch3asESUAFvxHnqW2ITc3bYWQO2AdfZr9+kO5R2iZq8o/gsAAP//bsF4nqkFAAA=
 
 - path: /opt/azure/containers/kubelet.sh
   permissions: "0755"

pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line301.sh

@@ -14,12 +14,14 @@ setKubeletTLSBootstrapFlags() {
   KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig"
 
   if [ -f "${KUBECONFIG_FILE}" ]; then
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }
 

pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData

@@ -298,7 +298,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/6yTv27bMBDGdz7Flc3QDgSLji082I1sGFGtwJKnthAo5+QQZSiDpLwEevcikhJTf7tkknD3++67O+k+fuCZ1DwT9pEQiw4YlgWc5RlzIRUhd4dVEAZJ+iParbebdL0Ng3QdLjfxgt48Tya/MUor6ouT5XYX7G9HtZ3clDTdH3bJ9mdjMFqgQ/TLbPbR4X7E3Yv3JAPWh15WdVdmqNAlYbwqCmedEee1Eif76TM8E4AXmbeYBb8Iw5XM+N9GVz+Phc7liQCsoiiJk/3yPv2vLnt1Y50Kr20mYZxeq7VTMI+FmU4oIQAyh1/Acmhn95qpKPz5Du4RNQEAME8tNtl9RRsQXWk0fCEAuexZzGg7Zu8yHzA2tr6hZIyiTfvV7NcnhJfWcFUchapPq60Iv+spGEMtMoXMormgeYvq4gGZEhkq6/9zu+g2SMPlKgjjir7Bl8XX63uhyidkZ1WepGYP0iw4umPtajQ6tLwhGsC2upuZbfaRwXWPAJ0Tnsx3LnRAeafYz9XBfwEAAP//9FFrmqsEAAA=
+    H4sIAAAAAAAA/6xUTY/aMBC9+1dM0z20Uq1UPbbiAN2A0KawgnBqq8hhx2CtsZHtcEH894okyzohH5eeQDPvzXtvDPPxQ5gJFWbM7gmx6IBiruEojsiZkIQ8bSZRHCXpz+ViOp+l03kcpdN4PFuPgodzZ/M7DYJL4JOT8XwRrR5bubVeFzVdbRbJ/Fcp0DqghmiOma2Wm+cWda/eoNxhfdB1VU95hhJdEq8nWjvrDDtOJdvZT5/hTACuNG8xo/DETChFFr6WvOJzqxUXOwIwWS6TdbIaP6eDvOxNjdYmvNlM4nX6Pq1KQT0s9DgJCAEQHH4D5VBl98xcAvj7A9weFQEAwO1eQ+BNFhaOBi0qBxlybRCsY8YJtYNK6wsYPOjTtXLLAd4ELiQCc/Bw7tzIJSjEzaHyOAhElxsFXwkAF418PdzBpEq74bQWXVGgtO3ZwOnepMC1Ac6kzNj2tUzzX565y87QMrkgl97fPiFhbk0o9ZbJ4rBU2vCnME8pKpZJpBbNCc2tqvQLUskylNb/xy2Wj1EajydRvL4EN/Bp9O39u5b5AelR5juh6IswoxDdtlA1Ch3asESUAFvxHnqW2ITc3bYWQO2AdfZr9+kO5R2iZq8o/gsAAP//bsF4nqkFAAA=
 
 - path: /opt/azure/containers/kubelet.sh
   permissions: "0755"

pkg/agent/testdata/AKSUbuntu1604+Docker/line301.sh

@@ -14,12 +14,14 @@ setKubeletTLSBootstrapFlags() {
   KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig"
 
   if [ -f "${KUBECONFIG_FILE}" ]; then
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }
 

pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CustomData

@@ -298,7 +298,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/6yTv27bMBDGdz7Flc3QDgSLji082I1sGFGtwJKnthAo5+QQZSiDpLwEevcikhJTf7tkknD3++67O+k+fuCZ1DwT9pEQiw4YlgWc5RlzIRUhd4dVEAZJ+iParbebdL0Ng3QdLjfxgt48Tya/MUor6ouT5XYX7G9HtZ3clDTdH3bJ9mdjMFqgQ/TLbPbR4X7E3Yv3JAPWh15WdVdmqNAlYbwqCmedEee1Eif76TM8E4AXmbeYBb8Iw5XM+N9GVz+Phc7liQCsoiiJk/3yPv2vLnt1Y50Kr20mYZxeq7VTMI+FmU4oIQAyh1/Acmhn95qpKPz5Du4RNQEAME8tNtl9RRsQXWk0fCEAuexZzGg7Zu8yHzA2tr6hZIyiTfvV7NcnhJfWcFUchapPq60Iv+spGEMtMoXMormgeYvq4gGZEhkq6/9zu+g2SMPlKgjjir7Bl8XX63uhyidkZ1WepGYP0iw4umPtajQ6tLwhGsC2upuZbfaRwXWPAJ0Tnsx3LnRAeafYz9XBfwEAAP//9FFrmqsEAAA=
+    H4sIAAAAAAAA/6xUTY/aMBC9+1dM0z20Uq1UPbbiAN2A0KawgnBqq8hhx2CtsZHtcEH894okyzohH5eeQDPvzXtvDPPxQ5gJFWbM7gmx6IBiruEojsiZkIQ8bSZRHCXpz+ViOp+l03kcpdN4PFuPgodzZ/M7DYJL4JOT8XwRrR5bubVeFzVdbRbJ/Fcp0DqghmiOma2Wm+cWda/eoNxhfdB1VU95hhJdEq8nWjvrDDtOJdvZT5/hTACuNG8xo/DETChFFr6WvOJzqxUXOwIwWS6TdbIaP6eDvOxNjdYmvNlM4nX6Pq1KQT0s9DgJCAEQHH4D5VBl98xcAvj7A9weFQEAwO1eQ+BNFhaOBi0qBxlybRCsY8YJtYNK6wsYPOjTtXLLAd4ELiQCc/Bw7tzIJSjEzaHyOAhElxsFXwkAF418PdzBpEq74bQWXVGgtO3ZwOnepMC1Ac6kzNj2tUzzX565y87QMrkgl97fPiFhbk0o9ZbJ4rBU2vCnME8pKpZJpBbNCc2tqvQLUskylNb/xy2Wj1EajydRvL4EN/Bp9O39u5b5AelR5juh6IswoxDdtlA1Ch3asESUAFvxHnqW2ITc3bYWQO2AdfZr9+kO5R2iZq8o/gsAAP//bsF4nqkFAAA=
 
 - path: /opt/azure/containers/kubelet.sh
   permissions: "0755"

pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/line301.sh

@@ -14,12 +14,14 @@ setKubeletTLSBootstrapFlags() {
   KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig"
 
   if [ -f "${KUBECONFIG_FILE}" ]; then
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }
 

pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CustomData

@@ -297,7 +297,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/6yTv27bMBDGdz7Flc3QDgSLji082I1sGFGtwJKnthAo5+QQZSiDpLwEevcikhJTf7tkknD3++67O+k+fuCZ1DwT9pEQiw4YlgWc5RlzIRUhd4dVEAZJ+iParbebdL0Ng3QdLjfxgt48Tya/MUor6ouT5XYX7G9HtZ3clDTdH3bJ9mdjMFqgQ/TLbPbR4X7E3Yv3JAPWh15WdVdmqNAlYbwqCmedEee1Eif76TM8E4AXmbeYBb8Iw5XM+N9GVz+Phc7liQCsoiiJk/3yPv2vLnt1Y50Kr20mYZxeq7VTMI+FmU4oIQAyh1/Acmhn95qpKPz5Du4RNQEAME8tNtl9RRsQXWk0fCEAuexZzGg7Zu8yHzA2tr6hZIyiTfvV7NcnhJfWcFUchapPq60Iv+spGEMtMoXMormgeYvq4gGZEhkq6/9zu+g2SMPlKgjjir7Bl8XX63uhyidkZ1WepGYP0iw4umPtajQ6tLwhGsC2upuZbfaRwXWPAJ0Tnsx3LnRAeafYz9XBfwEAAP//9FFrmqsEAAA=
+    H4sIAAAAAAAA/6xUTY/aMBC9+1dM0z20Uq1UPbbiAN2A0KawgnBqq8hhx2CtsZHtcEH894okyzohH5eeQDPvzXtvDPPxQ5gJFWbM7gmx6IBiruEojsiZkIQ8bSZRHCXpz+ViOp+l03kcpdN4PFuPgodzZ/M7DYJL4JOT8XwRrR5bubVeFzVdbRbJ/Fcp0DqghmiOma2Wm+cWda/eoNxhfdB1VU95hhJdEq8nWjvrDDtOJdvZT5/hTACuNG8xo/DETChFFr6WvOJzqxUXOwIwWS6TdbIaP6eDvOxNjdYmvNlM4nX6Pq1KQT0s9DgJCAEQHH4D5VBl98xcAvj7A9weFQEAwO1eQ+BNFhaOBi0qBxlybRCsY8YJtYNK6wsYPOjTtXLLAd4ELiQCc/Bw7tzIJSjEzaHyOAhElxsFXwkAF418PdzBpEq74bQWXVGgtO3ZwOnepMC1Ac6kzNj2tUzzX565y87QMrkgl97fPiFhbk0o9ZbJ4rBU2vCnME8pKpZJpBbNCc2tqvQLUskylNb/xy2Wj1EajydRvL4EN/Bp9O39u5b5AelR5juh6IswoxDdtlA1Ch3asESUAFvxHnqW2ITc3bYWQO2AdfZr9+kO5R2iZq8o/gsAAP//bsF4nqkFAAA=
 
 - path: /opt/azure/containers/kubelet.sh
   permissions: "0755"

pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/line300.sh

@@ -14,12 +14,14 @@ setKubeletTLSBootstrapFlags() {
   KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig"
 
   if [ -f "${KUBECONFIG_FILE}" ]; then
+    echo "kubeconfig is present before starting kubelet, removing bootstrap kubeconfig file at ${BOOTSTRAP_KUBECONFIG_FILE}"
     rm -f "${BOOTSTRAP_KUBECONFIG_FILE}"
     return 0
   fi
 
   if [ -f "${BOOTSTRAP_KUBECONFIG_FILE}" ]; then
-    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
+    echo "kubeconfig is not present before starting kubelet, setting --bootstrap-kubeconfig to ${BOOTSTRAP_KUBECONFIG_FILE} for fallback"
+    KUBELET_TLS_BOOTSTRAP_FLAGS="--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig ${BOOTSTRAP_KUBECONFIG_FILE}"
   fi
 }