Skip to content

Commit

Permalink
OADP-1801: Include restic and velero binaries in must-gather image, a…
Browse files Browse the repository at this point in the history
…nd gather various pieces of version information. (#994)

* Build restic and velero for must-gather image.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Clean up build steps to avoid running out of space.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Record versions of OADP and volsync operators.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Respect skip_tls option in gather_versions.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Move oadp-debug-info to cluster subdirectory.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Get restic version in must-gather.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Get velero client and deployment versions.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Get storage classes (short) and DPA details.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Add debug script directory and a small example.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Create directory up front, avoid whole DPA CR.

DPAs are already logged by gather_crs.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Save full YAML for DPAs and StorageClasses.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Remove default SC, add CSI VSC gathering.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Add defaults for debug file and skip_tls.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* Add missing --insecure-skip-tls-verify arguments.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

* One more missing --insecure-skip-tls-verify.

Signed-off-by: Matthew Arnold <marnold@redhat.com>

---------

Signed-off-by: Matthew Arnold <marnold@redhat.com>
  • Loading branch information
mrnold committed May 6, 2023
1 parent 93e44a8 commit c007cd9
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 0 deletions.
16 changes: 16 additions & 0 deletions must-gather/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,16 @@
FROM quay.io/konveyor/builder:latest as konveyor-builder
ARG RESTIC_BRANCH=konveyor-0.15.0
ARG VELERO_BRANCH=konveyor-dev
WORKDIR /build
RUN curl --location --output velero.tgz https://github.com/openshift/velero/archive/refs/heads/${VELERO_BRANCH}.tar.gz && \
tar -xzvf velero.tgz && cd velero-${VELERO_BRANCH} && \
CGO_ENABLED=0 GOOS=linux go build -a -mod=mod -ldflags '-extldflags "-static" -X github.com/vmware-tanzu/velero/pkg/buildinfo.Version=${VELERO_BRANCH}' -o /velero github.com/vmware-tanzu/velero/cmd/velero && \
cd .. && rm -rf velero.tgz velero-${VELERO_BRANCH} && \
curl --location --output restic.tgz https://github.com/openshift/restic/archive/refs/heads/${RESTIC_BRANCH}.tar.gz && \
tar -xzvf restic.tgz && cd restic-${RESTIC_BRANCH} && \
CGO_ENABLED=0 GOOS=linux go build -a -mod=mod -ldflags '-extldflags "-static"' -o /restic github.com/restic/restic/cmd/restic && \
cd .. && rm -rf restic.tgz restic-${RESTIC_BRANCH}

FROM registry.access.redhat.com/ubi8/go-toolset:1.17.10 as gobuilder

RUN go install -v github.com/google/pprof@latest
Expand All @@ -10,8 +23,11 @@ RUN microdnf -y install rsync tar gzip graphviz findutils

COPY --from=gobuilder /opt/app-root/src/go/bin/pprof /usr/bin/pprof
COPY --from=builder /usr/bin/oc /usr/bin/oc
COPY --from=konveyor-builder /velero /usr/bin/velero
COPY --from=konveyor-builder /restic /usr/bin/restic

COPY collection-scripts/* /usr/bin/
COPY collection-scripts/debug/* /usr/bin/
COPY collection-scripts/logs/* /usr/bin/
COPY collection-scripts/time_window_gather /usr/bin/

Expand Down
10 changes: 10 additions & 0 deletions must-gather/collection-scripts/debug/tail_failed_pods
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/bash

skip_tls="${1:-false}"

for pod in $(oc get pods --insecure-skip-tls-verify=${skip_tls} -n openshift-adp -o jsonpath='{.items[?(.status.containerStatuses[0].lastState.terminated.reason=="Error")].metadata.name}'); do
echo "***"
echo "* Last logs from failed pod openshift-adp/${pod}:"
oc logs -n openshift-adp --insecure-skip-tls-verify=${skip_tls} $pod --tail=10
echo -e "\n\n\n"
done
4 changes: 4 additions & 0 deletions must-gather/collection-scripts/gather
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ else
echo "Essential-only must-gather was requested. Skipping prometheus metrics collection"
fi

# Gather version information
echo "Gathering version information for top-level debug file"
/usr/bin/gather_versions "/must-gather/clusters/${clusterID}/oadp-debug-info" ${skip_tls} &

# Waits for gather_crs, gather_logs, gather_metrics running in background
echo "Waiting for background gather tasks to finish"
wait
Expand Down
104 changes: 104 additions & 0 deletions must-gather/collection-scripts/gather_versions
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
#!/bin/bash

debug_info="${1:-oadp-debug-info}"
skip_tls="${2:-false}"

mkdir -p $(dirname ${debug_info})

# Write information section to debug file, and fill with output of given command
function log_command() {
heading=$1
command=$2
echo "***" >> ${debug_info}
echo "* $heading" >> ${debug_info}
$command 2>&1 >> ${debug_info}
echo -e "\n\n" >> ${debug_info}
}

# Search for given operator in all available namespaces, and save information
function log_operator() {
display_name=$1
operator_name=$2

# Find all operators with matching name in all namespaces
namespaces=$(oc get subscriptions.operators.coreos.com --all-namespaces --insecure-skip-tls-verify=${skip_tls} \
-o jsonpath='{.items[?(.metadata.name=="'"${operator_name}"'")].metadata.namespace}')
if [ -z "${namespaces}" ]; then # No matches, mark as not found
log_command "${display_name} information" "echo ${operator_name} operator not found"
else # Loop through matching operators and save information to debug file
for namespace in ${namespaces}; do
log_command "${display_name} version information in namespace ${namespace}" \
"oc get subscriptions.operators.coreos.com ${operator_name} -n ${namespace} --insecure-skip-tls-verify=${skip_tls} -o yaml"
done
fi
}

# Search for named deployments in all namespaces
function log_deployment() {
display_name=$1
deployment_name=$2

namespaces=$(oc get deployment --all-namespaces --insecure-skip-tls-verify=${skip_tls} \
-o jsonpath='{.items[?(.metadata.name=="'"${deployment_name}"'")].metadata.namespace}')
if [ -z "${namespaces}" ]; then
log_command "${display_name} information" "echo ${deployment_name} deployment not found"
else
for namespace in ${namespaces}; do
log_command "${display_name} deployment information in namespace ${namespace}" \
"oc get deployment ${deployment_name} -n ${namespace} --insecure-skip-tls-verify=${skip_tls} -o yaml"
done
fi
}

# OpenShift version
log_command "OpenShift version information"\
"oc version --insecure-skip-tls-verify=${skip_tls} -o yaml"

# Red Hat OADP operator information
log_operator "OADP" "redhat-oadp-operator"

# Community OADP operator
log_operator "Community OADP operator" "oadp-operator"

# Volsync operator information
log_operator "Volsync" "volsync-product"

# Restic version
if [ -z "${skip_tls}" ]; then
log_command "Restic version" "restic version"
else
log_command "Restic version" "restic version --insecure-tls"
fi

# Velero versions
log_command "Velero client version" "velero version --client-only"
log_deployment "Velero" "velero"

# All storage classes
storageclasses=$(oc get storageclass --insecure-skip-tls-verify=${skip_tls} -o jsonpath='{range .items[*]}{.metadata.name}{" "}')
if [ -z "${storageclasses}" -o "${storageclasses}" == " " ]; then
log_command "StorageClass" "echo No StorageClasses found in cluster"
else
log_command "StorageClasses" "oc get storageclasses --insecure-skip-tls-verify=${skip_tls} -o yaml"
fi

# CSI volume snapshot classes
volumesnapshotclasses=$(oc get volumesnapshotclass --insecure-skip-tls-verify=${skip_tls} -o jsonpath='{range .items[?(.metadata.annotations.velero\.io/csi-volumesnapshot-class == "true")]}{.metadata.name}{" "}')
if [ -z "${volumesnapshotclasses}" -o "${volumesnapshotclasses}" == " " ]; then
log_command "CSI VolumeSnapshotClasses" "echo No CSI VolumeSnapshotClasses found in cluster"
else
while read volumesnapshotclass; do
log_command "VolumeSnapshotClass ${volumesnapshotclass}" "oc get volumesnapshotclass --insecure-skip-tls-verify=${skip_tls} ${volumesnapshotclass} -o yaml"
done <<< "${volumesnapshotclasses}"
fi


# DataProtectionApplications
dpas=$(oc get dpa --all-namespaces --insecure-skip-tls-verify=${skip_tls} -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.namespace}')
if [ -z "${dpas}" -o "${dpas}" == " " ]; then
log_command "DataProtectionApplication CRs" "echo No DPAs found in cluster"
else
while read dpa namespace; do
log_command "DataProtectionApplication ${namespace}/${dpa}" "oc get dpa --insecure-skip-tls-verify=${skip_tls} -n ${namespace} ${dpa} -o yaml"
done <<< "${dpas}"
fi

0 comments on commit c007cd9

Please sign in to comment.