/
export.go
135 lines (113 loc) · 3.25 KB
/
export.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
package drive
import (
"context"
"strings"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/metrics"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
)
func NewExportCollection(
baseDir string,
backingCollection []data.RestoreCollection,
backupVersion int,
stats *metrics.ExportStats,
) export.Collectioner {
return export.BaseCollection{
BaseDir: baseDir,
BackingCollection: backingCollection,
BackupVersion: backupVersion,
Stream: streamItems,
Stats: stats,
}
}
// streamItems streams the streamItems in the backingCollection into the export stream chan
func streamItems(
ctx context.Context,
drc []data.RestoreCollection,
backupVersion int,
cec control.ExportConfig,
ch chan<- export.Item,
stats *metrics.ExportStats,
) {
defer close(ch)
errs := fault.New(false)
for _, rc := range drc {
for item := range rc.Items(ctx, errs) {
itemUUID := item.ID()
if isMetadataFile(itemUUID, backupVersion) {
continue
}
name, err := getItemName(ctx, itemUUID, backupVersion, rc)
if err != nil {
ch <- export.Item{
ID: itemUUID,
Error: err,
}
continue
}
stats.UpdateResourceCount(path.FilesCategory)
body := metrics.ReaderWithStats(item.ToReader(), path.FilesCategory, stats)
ch <- export.Item{
ID: itemUUID,
Name: name,
Body: body,
Error: err,
}
}
items, recovered := errs.ItemsAndRecovered()
// Return all the items that we failed to source from the persistence layer
for _, err := range items {
ch <- export.Item{
ID: err.ID,
Error: &err,
}
}
for _, err := range recovered {
ch <- export.Item{
Error: err,
}
}
}
}
// isMetadataFile is used to determine if a path corresponds to a
// metadata file. This is OneDrive specific logic and depends on the
// version of the backup unlike metadata.isMetadataFile which only has
// to be concerned about the current version.
func isMetadataFile(id string, backupVersion int) bool {
if backupVersion < version.OneDrive1DataAndMetaFiles {
return false
}
return strings.HasSuffix(id, metadata.MetaFileSuffix) ||
strings.HasSuffix(id, metadata.DirMetaFileSuffix)
}
// getItemName is used to get the name of the item.
// How we get the name depends on the version of the backup.
func getItemName(
ctx context.Context,
id string,
backupVersion int,
fin data.FetchItemByNamer,
) (string, error) {
if backupVersion < version.OneDrive1DataAndMetaFiles {
return id, nil
}
if backupVersion < version.OneDrive5DirMetaNoName {
return strings.TrimSuffix(id, metadata.DataFileSuffix), nil
}
if strings.HasSuffix(id, metadata.DataFileSuffix) {
trimmedName := strings.TrimSuffix(id, metadata.DataFileSuffix)
metaName := trimmedName + metadata.MetaFileSuffix
meta, err := FetchAndReadMetadata(ctx, fin, metaName)
if err != nil {
return "", clues.WrapWC(ctx, err, "getting metadata")
}
return meta.FileName, nil
}
return "", clues.NewWC(ctx, "invalid item id")
}