-
Notifications
You must be signed in to change notification settings - Fork 44
/
layer.go
337 lines (287 loc) · 10.7 KB
/
layer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
package image
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/sylabs/squashfs"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/stereoscope/internal/bus"
"github.com/anchore/stereoscope/internal/log"
"github.com/anchore/stereoscope/pkg/event"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
)
const SingularitySquashFSLayer = "application/vnd.sylabs.sif.layer.v1.squashfs"
// Layer represents a single layer within a container image.
type Layer struct {
// layer is the raw layer metadata and content provider from the GCR lib
layer v1.Layer
// indexedContent provides index access to the cached and unzipped layer tar
indexedContent *file.TarIndex
// Metadata contains select layer attributes
Metadata LayerMetadata
// Tree is a filetree that represents the structure of the layer tar contents ("diff tree")
Tree filetree.Reader
// SquashedTree is a filetree that represents the combination of this layers diff tree and all diff trees
// in lower layers relative to this one.
SquashedTree filetree.Reader
// fileCatalog contains all file metadata for all files in all layers (not just this layer)
fileCatalog *FileCatalog
SquashedSearchContext filetree.Searcher
SearchContext filetree.Searcher
}
// NewLayer provides a new, unread layer object.
func NewLayer(layer v1.Layer) *Layer {
return &Layer{
layer: layer,
}
}
func (l *Layer) uncompressedTarCache(uncompressedLayersCacheDir string) (string, error) {
if uncompressedLayersCacheDir == "" {
return "", fmt.Errorf("no cache directory given")
}
tarPath := path.Join(uncompressedLayersCacheDir, l.Metadata.Digest+".tar")
if _, err := os.Stat(tarPath); !os.IsNotExist(err) {
return tarPath, nil
}
rawReader, err := l.layer.Uncompressed()
if err != nil {
return "", err
}
fh, err := os.Create(tarPath)
if err != nil {
return "", fmt.Errorf("unable to create layer cache dir=%q : %w", tarPath, err)
}
if _, err := io.Copy(fh, rawReader); err != nil {
return "", fmt.Errorf("unable to populate layer cache dir=%q : %w", tarPath, err)
}
return tarPath, nil
}
// Read parses information from the underlying layer tar into this struct. This includes layer metadata, the layer
// file tree, and the layer squash tree.
func (l *Layer) Read(catalog *FileCatalog, idx int, uncompressedLayersCacheDir string) error {
mediaType, err := l.layer.MediaType()
if err != nil {
return err
}
tree := filetree.New()
l.Tree = tree
l.fileCatalog = catalog
switch mediaType {
case types.OCILayer,
types.OCIUncompressedLayer,
types.OCIRestrictedLayer,
types.OCIUncompressedRestrictedLayer,
types.OCILayerZStd,
types.DockerLayer,
types.DockerForeignLayer,
types.DockerUncompressedLayer:
err := l.readStandardImageLayer(idx, uncompressedLayersCacheDir, tree)
if err != nil {
return err
}
case SingularitySquashFSLayer:
err := l.readSingularityImageLayer(idx, tree)
if err != nil {
return err
}
default:
return fmt.Errorf("unknown layer media type: %+v", mediaType)
}
l.SearchContext = filetree.NewSearchContext(l.Tree, l.fileCatalog.Index)
return nil
}
func (l *Layer) readStandardImageLayer(idx int, uncompressedLayersCacheDir string, tree *filetree.FileTree) error {
var err error
l.Metadata, err = newLayerMetadata(l.layer, idx)
monitor := trackReadProgress(l.Metadata)
if err != nil {
return err
}
log.Debugf("layer metadata: index=%+v digest=%+v mediaType=%+v",
l.Metadata.Index,
l.Metadata.Digest,
l.Metadata.MediaType)
tarFilePath, err := l.uncompressedTarCache(uncompressedLayersCacheDir)
if err != nil {
return err
}
l.indexedContent, err = file.NewTarIndex(
tarFilePath,
layerTarIndexer(tree, l.fileCatalog, &l.Metadata.Size, l, monitor),
)
if err != nil {
return fmt.Errorf("failed to read layer=%q tar : %w", l.Metadata.Digest, err)
}
monitor.SetCompleted()
return nil
}
func (l *Layer) readSingularityImageLayer(idx int, tree *filetree.FileTree) error {
var err error
l.Metadata, err = newLayerMetadata(l.layer, idx)
if err != nil {
return err
}
log.Debugf("layer metadata: index=%+v digest=%+v mediaType=%+v",
l.Metadata.Index,
l.Metadata.Digest,
l.Metadata.MediaType)
monitor := trackReadProgress(l.Metadata)
r, err := l.layer.Uncompressed()
if err != nil {
return fmt.Errorf("failed to read layer=%q: %w", l.Metadata.Digest, err)
}
// defer r.Close() // TODO: if we close this here, we can't read file contents after we return.
// Walk the more efficient walk if we're blessed with an io.ReaderAt.
if ra, ok := r.(io.ReaderAt); ok {
err = file.WalkSquashFS(ra, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor))
} else {
err = file.WalkSquashFSFromReader(r, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor))
}
if err != nil {
return fmt.Errorf("failed to walk layer=%q: %w", l.Metadata.Digest, err)
}
monitor.SetCompleted()
return nil
}
// OpenPath reads the file contents for the given path from the underlying layer blob, relative to the layers "diff tree".
// An error is returned if there is no file at the given path and layer or the read operation cannot continue.
func (l *Layer) OpenPath(path file.Path) (io.ReadCloser, error) {
return fetchReaderByPath(l.Tree, l.fileCatalog, path)
}
// OpenPathFromSquash reads the file contents for the given path from the underlying layer blob, relative to the layers squashed file tree.
// An error is returned if there is no file at the given path and layer or the read operation cannot continue.
func (l *Layer) OpenPathFromSquash(path file.Path) (io.ReadCloser, error) {
return fetchReaderByPath(l.SquashedTree, l.fileCatalog, path)
}
// FileContents reads the file contents for the given path from the underlying layer blob, relative to the layers "diff tree".
// An error is returned if there is no file at the given path and layer or the read operation cannot continue.
// Deprecated: use OpenPath() instead.
func (l *Layer) FileContents(path file.Path) (io.ReadCloser, error) {
return fetchReaderByPath(l.Tree, l.fileCatalog, path)
}
// FileContentsFromSquash reads the file contents for the given path from the underlying layer blob, relative to the layers squashed file tree.
// An error is returned if there is no file at the given path and layer or the read operation cannot continue.
// Deprecated: use OpenPathFromSquash() instead.
func (l *Layer) FileContentsFromSquash(path file.Path) (io.ReadCloser, error) {
return fetchReaderByPath(l.SquashedTree, l.fileCatalog, path)
}
// FilesByMIMEType returns file references for files that match at least one of the given MIME types relative to each layer tree.
// Deprecated: use SearchContext().SearchByMIMEType() instead.
func (l *Layer) FilesByMIMEType(mimeTypes ...string) ([]file.Reference, error) {
var refs []file.Reference
refVias, err := l.SearchContext.SearchByMIMEType(mimeTypes...)
if err != nil {
return nil, err
}
for _, refVia := range refVias {
if refVia.HasReference() {
refs = append(refs, *refVia.Reference)
}
}
return refs, nil
}
// FilesByMIMETypeFromSquash returns file references for files that match at least one of the given MIME types relative to the squashed file tree representation.
// Deprecated: use SquashedSearchContext().SearchByMIMEType() instead.
func (l *Layer) FilesByMIMETypeFromSquash(mimeTypes ...string) ([]file.Reference, error) {
var refs []file.Reference
refVias, err := l.SquashedSearchContext.SearchByMIMEType(mimeTypes...)
if err != nil {
return nil, err
}
for _, refVia := range refVias {
if refVia.HasReference() {
refs = append(refs, *refVia.Reference)
}
}
return refs, nil
}
func layerTarIndexer(ft filetree.Writer, fileCatalog *FileCatalog, size *int64, layerRef *Layer, monitor *progress.Manual) file.TarIndexVisitor {
builder := filetree.NewBuilder(ft, fileCatalog.Index)
return func(index file.TarIndexEntry) error {
var err error
var entry = index.ToTarFileEntry()
var contents = index.Open()
defer func() {
if err := contents.Close(); err != nil {
log.Warnf("unable to close file while indexing layer: %+v", err)
}
}()
metadata := file.NewMetadata(entry.Header, contents)
// note: the tar header name is independent of surrounding structure, for example, there may be a tar header entry
// for /some/path/to/file.txt without any entries to constituent paths (/some, /some/path, /some/path/to ).
// This is ok, and the FileTree will account for this by automatically adding directories for non-existing
// constituent paths. If later there happens to be a tar header entry for an already added constituent path
// the FileNode will be updated with the new file.Reference. If there is no tar header entry for constituent
// paths the FileTree is still structurally consistent (all paths can be iterated even though there may not have
// been a tar header entry for part of the given path).
//
// In summary: the set of all FileTrees can have NON-leaf nodes that don't exist in the FileCatalog, but
// the FileCatalog should NEVER have entries that don't appear in one (or more) FileTree(s).
ref, err := builder.Add(metadata)
if err != nil {
return err
}
if size != nil {
*(size) += metadata.Size()
}
fileCatalog.addImageReferences(ref.ID(), layerRef, index.Open)
if monitor != nil {
monitor.Increment()
}
return nil
}
}
func squashfsVisitor(ft filetree.Writer, fileCatalog *FileCatalog, size *int64, layerRef *Layer, monitor *progress.Manual) file.SquashFSVisitor {
builder := filetree.NewBuilder(ft, fileCatalog.Index)
return func(fsys fs.FS, path string, _ fs.DirEntry) error {
ff, err := fsys.Open(path)
if err != nil {
return err
}
defer ff.Close()
f, ok := ff.(*squashfs.File)
if !ok {
return errors.New("unexpected file type from squashfs")
}
metadata, err := file.NewMetadataFromSquashFSFile(path, f)
if err != nil {
return err
}
fileReference, err := builder.Add(metadata)
if err != nil {
return err
}
if size != nil {
*(size) += metadata.Size()
}
fileCatalog.addImageReferences(fileReference.ID(), layerRef, func() io.ReadCloser {
r, err := fsys.Open(path)
if err != nil {
// The file.Opener interface doesn't give us a way to return an error, and callers
// don't seem to handle a nil return. So, return a zero-byte reader.
log.Debug(err)
return io.NopCloser(bytes.NewReader(nil)) // TODO
}
return r
})
monitor.Increment()
return nil
}
}
func trackReadProgress(metadata LayerMetadata) *progress.Manual {
p := &progress.Manual{}
bus.Publish(partybus.Event{
Type: event.ReadLayer,
Source: metadata,
Value: progress.Monitorable(p),
})
return p
}