Use DaemonSet as a proxy to collect logs from CI

hack/log-dump/log-dump-daemonset.yaml

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: log-dump-node
+spec:
+  selector:
+    matchLabels:
+      app: log-dump-node
+  template:
+    metadata:
+      labels:
+        app: log-dump-node
+    spec:
+      containers:
+      - name: log-dump-node
+        image: fedora # A base image that has 'journalctl' binary
+        args:
+        - sleep
+        - "3600"
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: runlog
+          mountPath: /run/log
+      nodeSelector:
+        kubernetes.io/os: linux
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Equal
+        value: "true"
+      - effect: NoExecute
+        operator: Exists
+      - effect: NoSchedule
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: runlog
+        hostPath:
+          path: /run/log

hack/log-dump/log-dump.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
+cd "${REPO_ROOT}" || exit 1
+
+# shellcheck source=../hack/ensure-kind.sh
+source "${REPO_ROOT}/hack/ensure-kind.sh"
+# shellcheck source=../hack/ensure-kubectl.sh
+source "${REPO_ROOT}/hack/ensure-kubectl.sh"
+
+export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
+mkdir -p "${ARTIFACTS}/management-cluster" "${ARTIFACTS}/workload-cluster"
+
+export KUBECONFIG="${KUBECONFIG:-${PWD}/kubeconfig}"
+
+get_node_name() {
+    local -r pod_name="${1}"
+    echo "$(kubectl get pod "${pod_name}" -ojsonpath={.spec.nodeName})"
+}
+
+dump_mgmt_cluster_logs() {
+    # Assume the first kind cluster is the management cluster
+    local -r mgmt_cluster_name="$(kind get clusters | head -n 1)"
+    if [[ -z mgmt_cluster_name ]]; then
+        return
+    fi
+
+    kind get kubeconfig --name "${mgmt_cluster_name}" > ${PWD}/kind.kubeconfig
+    local -r kubectl_kind="kubectl --kubeconfig=${PWD}/kind.kubeconfig"
+
+    local -r resources=(
+        "clusters"
+        "azureclusters"
+        "machines"
+        "azuremachines"
+        "kubeadmconfigs"
+        "machinedeployments"
+        "azuremachinetemplates"
+        "kubeadmconfigtemplates"
+        "machinesets"
+        "kubeadmcontrolplanes"
+        "machinepools"
+        "azuremachinepools"
+    )
+    mkdir -p "${ARTIFACTS}/management-cluster/resources"
+    for resource in "${resources[@]}"; do
+        ${kubectl_kind} get --all-namespaces "${resource}" -oyaml > "${ARTIFACTS}/management-cluster/resources/${resource}.log" || true
+    done
+
+    {
+        echo "images in docker"
+        docker images
+        echo "images in bootstrap cluster using kubectl CLI"
+        (${kubectl_kind} get pods --all-namespaces -ojson \
+        | jq --raw-output '.items[].spec.containers[].image' | sort)
+        echo "images in deployed cluster using kubectl CLI"
+        (${kubectl_kind} get pods --all-namespaces -ojson \
+        | jq --raw-output '.items[].spec.containers[].image' | sort)
+    } > "${ARTIFACTS}/management-cluster/images.info"
+
+    {
+        echo "kind cluster-info"
+        ${kubectl_kind} cluster-info dump
+    } > "${ARTIFACTS}/management-cluster/kind-cluster.info"
+
+    kind export logs --name="${mgmt_cluster_name}" "${ARTIFACTS}/management-cluster"
+}
+
+dump_workload_cluster_logs() {
+    echo "Deploying log-dump-daemonset"
+    kubectl apply -f "${REPO_ROOT}/hack/log-dump/log-dump-daemonset.yaml"
+    kubectl wait pod -l app=log-dump-node --for=condition=Ready --timeout=5m
+
+    local -r log_dump_pods=( $(kubectl get pod -l app=log-dump-node -ojsonpath={.items[*].metadata.name}) )
+    local -r log_dump_commands=(
+        "journalctl --output=short-precise -u kubelet > kubelet.log"
+        "journalctl --output=short-precise -u containerd > containerd.log"
+        "journalctl --output=short-precise -k > kern.log"
+        "journalctl --output=short-precise > journal.log"
+        "cat /var/log/cloud-init.log > cloud-init.log"
+        "cat /var/log/cloud-init-output.log > cloud-init-output.log"
+        "tar -cf - var/log/pods --ignore-failed-read | tar xf - --strip-components=2 -C . var/log/pods/kube-system*"
+    )
+
+    for log_dump_pod in "${log_dump_pods[@]}"; do
+        local node_name="$(get_node_name "${log_dump_pod}")"
+
+        local log_dump_dir="${ARTIFACTS}/workload-cluster/${node_name}"
+        mkdir -p "${log_dump_dir}"
+        pushd "${log_dump_dir}" &> /dev/null
+        for cmd in "${log_dump_commands[@]}"; do
+            bash -c "kubectl exec ${log_dump_pod} -- ${cmd}" &
+        done
+
+        popd &> /dev/null
+        echo "Exported logs for node \"${node_name}\""
+    done
+
+    wait
+}
+
+cleanup() {
+    kubectl delete -f "${REPO_ROOT}/hack/log-dump/log-dump-daemonset.yaml"
+    source "${REPO_ROOT}/hack/log-dump/redact.sh"
+}
+
+trap cleanup EXIT
+
+echo "================ DUMPING LOGS FOR MANAGEMENT CLUSTER ================"
+dump_mgmt_cluster_logs
+
+echo "================ DUMPING LOGS FOR WORKLOAD CLUSTER ================"
+dump_workload_cluster_logs

hack/log-dump/redact.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Verify the required Environment Variables are present.
+: "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}"
+: "${AZURE_TENANT_ID:?Environment variable empty or not defined.}"
+: "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}"
+: "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}"
+
+echo "================ REDACTING LOGS ================"
+log_files=( $(find ${ARTIFACTS} -type f) )
+# TODO(chewong): add ${AZURE_STANDARD_JSON_B64} and ${AZURE_VMSS_JSON_B64}
+# to redact_vars when #653 is merged
+redact_vars=(
+    "${AZURE_CLIENT_ID}"
+    "${AZURE_CLIENT_SECRET}"
+    "${AZURE_SUBSCRIPTION_ID}"
+    "${AZURE_TENANT_ID}"
+    "$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')"
+    "$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')"
+    "$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')"
+    "$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -d '\n')"
+)
+
+for log_file in "${log_files[@]}"; do
+    for redact_var in "${redact_vars[@]}"; do
+        # LC_CTYPE=C and LANG=C will prevent "illegal byte sequence" error from sed
+        # Uncomment the following command when using Mac OS
+        # LC_CTYPE=C LANG=C sed -i "" "s|${redact_var}|===REDACTED===|g" "${log_file}"
+        LC_CTYPE=C LANG=C sed -i "s|${redact_var}|===REDACTED===|g" "${log_file}"
+    done
+done
+
+echo "All sensitive variables are redacted"

scripts/ci-e2e.sh

@@ -73,4 +73,6 @@ export JOB_NAME="${JOB_NAME:-"cluster-api-provider-azure-e2e"}"
 make test-e2e
 test_status="${?}"
 
+source "${REPO_ROOT}/hack/log-dump/redact.sh"
+
 exit "${test_status}"

scripts/ci-entrypoint.sh

@@ -127,10 +127,6 @@ run_upstream_e2e_tests() {
     unset KUBERNETES_CONFORMANCE_TEST
 }
 
-get_logs() {
-    kubectl logs deploy/capz-controller-manager -n capz-system manager > "${ARTIFACTS}/logs/capz-manager.log" || true
-}
-
 # cleanup all resources we use
 cleanup() {
     timeout 600 kubectl \
@@ -144,16 +140,15 @@ cleanup() {
 
 on_exit() {
     unset KUBECONFIG
-    get_logs
+    source "${REPO_ROOT}/hack/log-dump/log-dump.sh"
     # cleanup
     if [[ -z "${SKIP_CLEANUP:-}" ]]; then
         cleanup
     fi
 }
 
 trap on_exit EXIT
-ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
-mkdir -p "${ARTIFACTS}/logs"
+export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
 
 # create cluster
 if [[ -z "${SKIP_CREATE_WORKLOAD_CLUSTER:-}" ]]; then