OADP-1801: Include restic and velero binaries in must-gather image, and gather various pieces of version information.

must-gather/Dockerfile

@@ -1,3 +1,16 @@
+FROM quay.io/konveyor/builder:latest as konveyor-builder
+ARG RESTIC_BRANCH=konveyor-0.15.0
+ARG VELERO_BRANCH=konveyor-dev
+WORKDIR /build
+RUN curl --location --output velero.tgz https://github.com/openshift/velero/archive/refs/heads/${VELERO_BRANCH}.tar.gz && \
+    tar -xzvf velero.tgz && cd velero-${VELERO_BRANCH} && \
+    CGO_ENABLED=0 GOOS=linux go build -a -mod=mod -ldflags '-extldflags "-static" -X github.com/vmware-tanzu/velero/pkg/buildinfo.Version=${VELERO_BRANCH}' -o /velero github.com/vmware-tanzu/velero/cmd/velero && \
+    cd .. && rm -rf velero.tgz velero-${VELERO_BRANCH} && \
+    curl --location --output restic.tgz https://github.com/openshift/restic/archive/refs/heads/${RESTIC_BRANCH}.tar.gz && \
+    tar -xzvf restic.tgz && cd restic-${RESTIC_BRANCH} && \
+    CGO_ENABLED=0 GOOS=linux go build -a -mod=mod -ldflags '-extldflags "-static"' -o /restic github.com/restic/restic/cmd/restic && \
+    cd .. && rm -rf restic.tgz restic-${RESTIC_BRANCH}
+
 FROM registry.access.redhat.com/ubi8/go-toolset:1.17.10 as gobuilder
 
 RUN go install -v github.com/google/pprof@latest
@@ -10,8 +23,11 @@ RUN microdnf -y install rsync tar gzip graphviz findutils
 
 COPY --from=gobuilder /opt/app-root/src/go/bin/pprof /usr/bin/pprof
 COPY --from=builder /usr/bin/oc /usr/bin/oc
+COPY --from=konveyor-builder /velero /usr/bin/velero
+COPY --from=konveyor-builder /restic /usr/bin/restic
 
 COPY collection-scripts/* /usr/bin/
+COPY collection-scripts/debug/* /usr/bin/
 COPY collection-scripts/logs/* /usr/bin/
 COPY collection-scripts/time_window_gather /usr/bin/
 

must-gather/collection-scripts/debug/tail_failed_pods

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+skip_tls="${1:-false}"
+
+for pod in $(oc get pods --insecure-skip-tls-verify=${skip_tls} -n openshift-adp -o jsonpath='{.items[?(.status.containerStatuses[0].lastState.terminated.reason=="Error")].metadata.name}'); do
+    echo "***"
+    echo "* Last logs from failed pod openshift-adp/${pod}:"
+    oc logs -n openshift-adp --insecure-skip-tls-verify=${skip_tls} $pod --tail=10
+    echo -e "\n\n\n"
+done

must-gather/collection-scripts/gather

@@ -36,6 +36,10 @@ else
   echo "Essential-only must-gather was requested. Skipping prometheus metrics collection"
 fi
 
+# Gather version information
+echo "Gathering version information for top-level debug file"
+/usr/bin/gather_versions "/must-gather/clusters/${clusterID}/oadp-debug-info" ${skip_tls} &
+
 # Waits for gather_crs, gather_logs, gather_metrics running in background
 echo "Waiting for background gather tasks to finish"
 wait

must-gather/collection-scripts/gather_versions

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+debug_info="${1:-oadp-debug-info}"
+skip_tls="${2:-false}"
+
+mkdir -p $(dirname ${debug_info})
+
+# Write information section to debug file, and fill with output of given command
+function log_command() {
+    heading=$1
+    command=$2
+    echo "***" >> ${debug_info}
+    echo "* $heading" >> ${debug_info}
+    $command 2>&1 >> ${debug_info}
+    echo -e "\n\n" >> ${debug_info}
+}
+
+# Search for given operator in all available namespaces, and save information
+function log_operator() {
+    display_name=$1
+    operator_name=$2
+
+    # Find all operators with matching name in all namespaces
+    namespaces=$(oc get subscriptions.operators.coreos.com --all-namespaces --insecure-skip-tls-verify=${skip_tls} \
+                -o jsonpath='{.items[?(.metadata.name=="'"${operator_name}"'")].metadata.namespace}')
+    if [ -z "${namespaces}" ]; then # No matches, mark as not found
+        log_command "${display_name} information" "echo ${operator_name} operator not found"
+    else # Loop through matching operators and save information to debug file
+        for namespace in ${namespaces}; do
+            log_command "${display_name} version information in namespace ${namespace}" \
+                        "oc get subscriptions.operators.coreos.com ${operator_name} -n ${namespace} --insecure-skip-tls-verify=${skip_tls} -o yaml"
+        done
+    fi
+}
+
+# Search for named deployments in all namespaces
+function log_deployment() {
+    display_name=$1
+    deployment_name=$2
+
+    namespaces=$(oc get deployment --all-namespaces --insecure-skip-tls-verify=${skip_tls} \
+                 -o jsonpath='{.items[?(.metadata.name=="'"${deployment_name}"'")].metadata.namespace}')
+    if [ -z "${namespaces}" ]; then
+        log_command "${display_name} information" "echo ${deployment_name} deployment not found"
+    else
+        for namespace in ${namespaces}; do
+            log_command "${display_name} deployment information in namespace ${namespace}" \
+                        "oc get deployment ${deployment_name} -n ${namespace} --insecure-skip-tls-verify=${skip_tls} -o yaml"
+        done
+    fi
+}
+
+# OpenShift version
+log_command "OpenShift version information"\
+            "oc version --insecure-skip-tls-verify=${skip_tls} -o yaml"
+
+# Red Hat OADP operator information
+log_operator "OADP" "redhat-oadp-operator"
+
+# Community OADP operator
+log_operator "Community OADP operator" "oadp-operator"
+
+# Volsync operator information
+log_operator "Volsync" "volsync-product"
+
+# Restic version
+if [ -z "${skip_tls}" ]; then
+    log_command "Restic version" "restic version"
+else
+    log_command "Restic version" "restic version --insecure-tls"
+fi
+
+# Velero versions
+log_command "Velero client version" "velero version --client-only"
+log_deployment "Velero" "velero"
+
+# All storage classes
+storageclasses=$(oc get storageclass --insecure-skip-tls-verify=${skip_tls} -o jsonpath='{range .items[*]}{.metadata.name}{" "}')
+if [ -z "${storageclasses}" -o "${storageclasses}" == " " ]; then
+    log_command "StorageClass" "echo No StorageClasses found in cluster"
+else
+    log_command "StorageClasses" "oc get storageclasses --insecure-skip-tls-verify=${skip_tls} -o yaml"
+fi
+
+# CSI volume snapshot classes
+volumesnapshotclasses=$(oc get volumesnapshotclass --insecure-skip-tls-verify=${skip_tls} -o jsonpath='{range .items[?(.metadata.annotations.velero\.io/csi-volumesnapshot-class == "true")]}{.metadata.name}{" "}')
+if [ -z "${volumesnapshotclasses}" -o "${volumesnapshotclasses}" == " " ]; then
+    log_command "CSI VolumeSnapshotClasses" "echo No CSI VolumeSnapshotClasses found in cluster"
+else
+    while read volumesnapshotclass; do
+        log_command "VolumeSnapshotClass ${volumesnapshotclass}" "oc get volumesnapshotclass --insecure-skip-tls-verify=${skip_tls} ${volumesnapshotclass} -o yaml"
+    done <<< "${volumesnapshotclasses}"
+fi
+
+
+# DataProtectionApplications
+dpas=$(oc get dpa --all-namespaces --insecure-skip-tls-verify=${skip_tls} -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.namespace}')
+if [ -z "${dpas}" -o "${dpas}" == " " ]; then
+    log_command "DataProtectionApplication CRs" "echo No DPAs found in cluster"
+else
+    while read dpa namespace; do
+        log_command "DataProtectionApplication ${namespace}/${dpa}" "oc get dpa --insecure-skip-tls-verify=${skip_tls} -n ${namespace} ${dpa} -o yaml"
+    done <<< "${dpas}"
+fi