./velero backup get
NAME STATUS ERRORS WARNINGS CREATED EXPIRES STORAGE LOCATION SELECTOR
backup-21-07-09-16-19-31 PartiallyFailed 8 0 2021-07-09 16:26:43 -0700 PDT 361d default <none>
./velero backup describe backup-21-07-09-16-19-31 --details
Restic Backups:
Completed:
zen/cpdbr-vol-mnt-d9b74fd9f-wsb48: data-redis-ha-server-0-vol, data-redis-ha-server-1-vol, data-redis-ha-server-2-vol, database-storage-wdp-couchdb-0-vol, database-storage-wdp-couchdb-1-vol, database-storage-wdp-couchdb-2-vol, datadir-zen-metastoredb-0-vol, datadir-zen-metastoredb-1-vol, datadir-zen-metastoredb-2-vol, elasticsearch-master-backups-vol, elasticsearch-master-elasticsearch-master-0-vol, elasticsearch-master-elasticsearch-master-1-vol, elasticsearch-master-elasticsearch-master-2-vol, volumes-vol-inst-pvc-vol
Failed:
zen/cpdbr-vol-mnt-d9b74fd9f-wsb48: data-dsx-influxdb-0-vol, data-rabbitmq-ha-0-vol, data-rabbitmq-ha-1-vol, data-rabbitmq-ha-2-vol, volumes-vol-inst2-pvc-vol
zen/ibm-nginx-dc95986b6-5fkfn: user-home-mount
./velero backup logs backup-21-07-09-16-19-31 > backup-21-07-09-16-19-31.log
time="2021-07-09T23:28:01Z" level=error msg="Error backing up item" backup=oadp-operator/backup-21-07-09-16-19-31 error="pod volume backup failed: error running restic backup, stderr={\"message_type\":\"error\",\"error\":{\"Op\":\"open\",\"Path\":\".erlang.cookie\",\"Err\":13},\"during\":\"archival\",\"item\":\"/host_pods/c04bd831-8d1a-4f83-ba02-7e04c418119f/volumes/kubernetes.io~nfs/pvc-0fa0256c-8e6b-4b15-8f73-ea2e0e63452e/.erlang.cookie\"}\nWarning: failed to read all source data during backup\n: exit status 3" error.file="/go/src/github.com/vmware-tanzu/velero/pkg/restic/backupper.go:179" error.function="github.com/vmware-tanzu/velero/pkg/restic.(*backupper).BackupPodVolumes" logSource="pkg/backup/backup.go:431" name=cpdbr-vol-mnt-d9b74fd9f-wsb48
oc describe podvolumebackup backup-21-07-09-16-19-31-gxs2d -n oadp-operator
Name: backup-21-07-09-16-19-31-gxs2d
Namespace: oadp-operator
Labels: velero.io/backup-name=backup-21-07-09-16-19-31
velero.io/backup-uid=1ecedec5-3f4b-45a2-9f4b-ef8950d55e51
velero.io/pvc-uid=0fa0256c-8e6b-4b15-8f73-ea2e0e63452e
Annotations: velero.io/pvc-name: data-rabbitmq-ha-0
API Version: velero.io/v1
Kind: PodVolumeBackup
Metadata:
Creation Timestamp: 2021-07-09T23:27:17Z
Generate Name: backup-21-07-09-16-19-31-
Generation: 3
Managed Fields:
API Version: velero.io/v1
Fields Type: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.:
f:velero.io/pvc-name:
f:generateName:
f:labels:
.:
f:velero.io/backup-name:
f:velero.io/backup-uid:
f:velero.io/pvc-uid:
f:ownerReferences:
.:
k:{"uid":"1ecedec5-3f4b-45a2-9f4b-ef8950d55e51"}:
.:
f:apiVersion:
f:controller:
f:kind:
f:name:
f:uid:
f:spec:
.:
f:backupStorageLocation:
f:node:
f:pod:
.:
f:kind:
f:name:
f:namespace:
f:uid:
f:repoIdentifier:
f:tags:
.:
f:backup:
f:backup-uid:
f:ns:
f:pod:
f:pod-uid:
f:pvc-uid:
f:volume:
f:volume:
f:status:
.:
f:progress:
Manager: velero-server
Operation: Update
Time: 2021-07-09T23:27:17Z
API Version: velero.io/v1
Fields Type: FieldsV1
fieldsV1:
f:status:
f:completionTimestamp:
f:message:
f:phase:
f:startTimestamp:
Manager: restic-server
Operation: Update
Time: 2021-07-09T23:27:20Z
Owner References:
API Version: velero.io/v1
Controller: true
Kind: Backup
Name: backup-21-07-09-16-19-31
UID: 1ecedec5-3f4b-45a2-9f4b-ef8950d55e51
Resource Version: 70022615
Self Link: /apis/velero.io/v1/namespaces/oadp-operator/podvolumebackups/backup-21-07-09-16-19-31-gxs2d
UID: fc82c147-aa9e-4059-a869-8a001c0e7530
Spec:
Backup Storage Location: default
Node: worker5.ws4616-nfs.cp.fyre.ibm.com
Pod:
Kind: Pod
Name: cpdbr-vol-mnt-d9b74fd9f-wsb48
Namespace: zen
UID: c04bd831-8d1a-4f83-ba02-7e04c418119f
Repo Identifier: s3:http://minio-velero.apps.ws4616-nfs.cp.fyre.ibm.com/velero/restic/zen
Tags:
Backup: backup-21-07-09-16-19-31
Backup - UID: 1ecedec5-3f4b-45a2-9f4b-ef8950d55e51
Ns: zen
Pod: cpdbr-vol-mnt-d9b74fd9f-wsb48
Pod - UID: c04bd831-8d1a-4f83-ba02-7e04c418119f
Pvc - UID: 0fa0256c-8e6b-4b15-8f73-ea2e0e63452e
Volume: data-rabbitmq-ha-0-vol
Volume: data-rabbitmq-ha-0-vol
Status:
Completion Timestamp: 2021-07-09T23:27:20Z
Message: error running restic backup, stderr={"message_type":"error","error":{"Op":"open","Path":".erlang.cookie","Err":13},"during":"archival","item":"/host_pods/c04bd831-8d1a-4f83-ba02-7e04c418119f/volumes/kubernetes.io~nfs/pvc-0fa0256c-8e6b-4b15-8f73-ea2e0e63452e/.erlang.cookie"}
Warning: failed to read all source data during backup
: exit status 3
Phase: Failed
Progress:
Start Timestamp: 2021-07-09T23:27:18Z
Events: <none>
oc rsh rabbitmq-ha-0
ls -al /var/lib/rabbitmq
total 4
drwxrwxrwx. 3 nobody nobody 42 Jul 13 14:23 .
drwxr-xr-x. 1 root root 22 Jun 18 08:36 ..
-rw-------. 1 1000610000 nobody 33 Jul 13 14:23 .erlang.cookie
drwxr-xr-x. 4 1000610000 nobody 327 Jul 13 14:24 mnesia
cat /etc/exports
/data 10.22.36.233(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.37.233(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.38.233(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.39.233(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.40.233(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.41.233(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.42.233(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.43.233(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.46.232(rw,sync,no_wdelay,root_squash,insecure)
/data 10.22.47.232(rw,sync,no_wdelay,root_squash,insecure)
backup-21-07-09-16-19-31.log
restic-64r9q.log
