/
metadata_collection.go
181 lines (150 loc) · 4.2 KB
/
metadata_collection.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
package graph
import (
"bytes"
"context"
"encoding/json"
"io"
"time"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
var (
_ data.BackupCollection = &MetadataCollection{}
_ data.Item = &metadataItem{}
)
// MetadataCollection in a simple collection that assumes all items to be
// returned are already resident in-memory and known when the collection is
// created. This collection has no logic for lazily fetching item data.
type MetadataCollection struct {
fullPath path.Path
items []metadataItem
statusUpdater support.StatusUpdater
counter *count.Bus
}
// MetadataCollectionEntry describes a file that should get added to a metadata
// collection. The Data value will be encoded into json as part of a
// transformation into a MetadataItem.
type MetadataCollectionEntry struct {
fileName string
data any
}
func NewMetadataEntry(fileName string, mData any) MetadataCollectionEntry {
return MetadataCollectionEntry{fileName, mData}
}
func (mce MetadataCollectionEntry) toMetadataItem() (metadataItem, error) {
if len(mce.fileName) == 0 {
return metadataItem{}, clues.New("missing metadata filename")
}
if mce.data == nil {
return metadataItem{}, clues.New("missing metadata")
}
buf := &bytes.Buffer{}
encoder := json.NewEncoder(buf)
if err := encoder.Encode(mce.data); err != nil {
return metadataItem{}, clues.Wrap(err, "serializing metadata")
}
item, err := data.NewPrefetchedItem(
io.NopCloser(buf),
mce.fileName,
time.Now())
if err != nil {
return metadataItem{}, clues.Stack(err)
}
return metadataItem{
Item: item,
size: int64(buf.Len()),
}, nil
}
// MakeMetadataCollection creates a metadata collection that has a file
// containing all the provided metadata as a single json object. Returns
// nil if the map does not have any entries.
func MakeMetadataCollection(
pathPrefix path.Path,
metadata []MetadataCollectionEntry,
statusUpdater support.StatusUpdater,
counter *count.Bus,
) (data.BackupCollection, error) {
if len(metadata) == 0 {
return nil, nil
}
items := make([]metadataItem, 0, len(metadata))
for _, md := range metadata {
item, err := md.toMetadataItem()
if err != nil {
return nil, err
}
items = append(items, item)
}
coll := NewMetadataCollection(pathPrefix, items, statusUpdater, counter)
return coll, nil
}
func NewMetadataCollection(
p path.Path,
items []metadataItem,
statusUpdater support.StatusUpdater,
counter *count.Bus,
) *MetadataCollection {
return &MetadataCollection{
fullPath: p,
items: items,
statusUpdater: statusUpdater,
counter: counter,
}
}
func (md MetadataCollection) FullPath() path.Path {
return md.fullPath
}
// TODO(ashmrtn): Fill in with previous path once the Controller compares old
// and new folder hierarchies.
func (md MetadataCollection) PreviousPath() path.Path {
return nil
}
// TODO(ashmrtn): Fill in once the Controller compares old and new folder
// hierarchies.
func (md MetadataCollection) State() data.CollectionState {
return data.NewState
}
func (md MetadataCollection) DoNotMergeItems() bool {
return false
}
func (md MetadataCollection) Items(
ctx context.Context,
_ *fault.Bus, // not used, just here for interface compliance
) <-chan data.Item {
res := make(chan data.Item)
go func() {
totalBytes := int64(0)
defer func() {
// Need to report after the collection is created because otherwise
// statusUpdater may not have accounted for the fact that this collection
// will be running.
status := support.CreateStatus(
ctx,
support.Backup,
1,
support.CollectionMetrics{
Objects: len(md.items),
Successes: len(md.items),
Bytes: totalBytes,
},
md.fullPath.Folder(false))
md.counter.Add(count.MetadataItems, int64(len(md.items)))
md.statusUpdater(status)
}()
defer close(res)
for _, item := range md.items {
md.counter.Add(count.MetadataBytes, item.size)
totalBytes += item.size
res <- item
}
}()
return res
}
type metadataItem struct {
data.Item
size int64
}