-
Notifications
You must be signed in to change notification settings - Fork 176
/
manifests.go
959 lines (914 loc) · 39.4 KB
/
manifests.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
package manifests
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/containers/common/internal"
"github.com/containers/common/pkg/manifests"
"github.com/containers/common/pkg/retry"
"github.com/containers/common/pkg/supplemented"
cp "github.com/containers/image/v5/copy"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/manifest"
ocilayout "github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/pkg/compression"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/signature/signer"
is "github.com/containers/image/v5/storage"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
"github.com/containers/storage/pkg/ioutils"
"github.com/containers/storage/pkg/lockfile"
digest "github.com/opencontainers/go-digest"
imgspec "github.com/opencontainers/image-spec/specs-go"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
const (
defaultMaxRetries = 3
)
const (
instancesData = "instances.json"
artifactsData = "artifacts.json"
pushingArtifactsSubdirectory = "referenced-artifacts"
)
// LookupReferenceFunc return an image reference based on the specified one.
// The returned reference can return custom ImageSource or ImageDestination
// objects which intercept or filter blobs, manifests, and signatures as
// they are read and written.
type LookupReferenceFunc func(ref types.ImageReference) (types.ImageReference, error)
// ErrListImageUnknown is returned when we attempt to create an image reference
// for a List that has not yet been saved to an image.
var ErrListImageUnknown = errors.New("unable to determine which image holds the manifest list")
type artifactsDetails struct {
Manifests map[digest.Digest]string `json:"manifests,omitempty"` // artifact (and other?) manifest digests → manifest contents
Files map[digest.Digest][]string `json:"files,omitempty"` // artifact (and other?) manifest digests → file paths (mainly for display)
Configs map[digest.Digest]digest.Digest `json:"config,omitempty"` // artifact (and other?) manifest digests → referenced config digests
Layers map[digest.Digest][]digest.Digest `json:"layers,omitempty"` // artifact (and other?) manifest digests → referenced layer digests
Detached map[digest.Digest]string `json:"detached,omitempty"` // "config" and "layer" (and other?) digests in (usually artifact) manifests → file paths
Blobs map[digest.Digest][]byte `json:"blobs,omitempty"` // "config" and "layer" (and other?) manifest digests → inlined blob contents
}
type list struct {
manifests.List
instances map[digest.Digest]string // instance manifest digests → image references
artifacts artifactsDetails
}
// List is a manifest list or image index, either created using Create(), or
// loaded from local storage using LoadFromImage().
type List interface {
manifests.List
SaveToImage(store storage.Store, imageID string, names []string, mimeType string) (string, error)
Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error)
Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error)
Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error)
AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error)
InstanceByFile(file string) (digest.Digest, error)
Files(instanceDigest digest.Digest) ([]string, error)
}
// PushOptions includes various settings which are needed for pushing the
// manifest list and its instances.
type PushOptions struct {
Store storage.Store
SystemContext *types.SystemContext // github.com/containers/image/types.SystemContext
ImageListSelection cp.ImageListSelection // set to either CopySystemImage, CopyAllImages, or CopySpecificImages
Instances []digest.Digest // instances to copy if ImageListSelection == CopySpecificImages
ReportWriter io.Writer // will be used to log the writing of the list and any blobs
Signers []*signer.Signer // if non-empty, asks for signatures to be added during the copy using the provided signers.
SignBy string // fingerprint of GPG key to use to sign images
SignPassphrase string // passphrase to use when signing with the key ID from SignBy.
SignBySigstorePrivateKeyFile string // if non-empty, asks for a signature to be added during the copy, using a sigstore private key file at the provided path.
SignSigstorePrivateKeyPassphrase []byte // passphrase to use when signing with SignBySigstorePrivateKeyFile.
RemoveSignatures bool // true to discard signatures in images
ManifestType string // the format to use when saving the list - possible options are oci, v2s1, and v2s2
SourceFilter LookupReferenceFunc // filter the list source
AddCompression []string // add existing instances with requested compression algorithms to manifest list
ForceCompressionFormat bool // force push with requested compression ignoring the blobs which can be reused.
// Maximum number of retries with exponential backoff when facing
// transient network errors. Default 3.
MaxRetries *uint
// RetryDelay used for the exponential back off of MaxRetries.
RetryDelay *time.Duration
}
// Create creates a new list containing information about the specified image,
// computing its manifest's digest, and retrieving OS and architecture
// information from its configuration blob. Returns the new list, and the
// instanceDigest for the initial image.
func Create() List {
return &list{
List: manifests.Create(),
instances: make(map[digest.Digest]string),
artifacts: artifactsDetails{
Manifests: make(map[digest.Digest]string),
Files: make(map[digest.Digest][]string),
Configs: make(map[digest.Digest]digest.Digest),
Layers: make(map[digest.Digest][]digest.Digest),
Detached: make(map[digest.Digest]string),
Blobs: make(map[digest.Digest][]byte),
},
}
}
// LoadFromImage reads the manifest list or image index, and additional
// information about where the various instances that it contains live, from an
// image record with the specified ID in local storage.
func LoadFromImage(store storage.Store, image string) (string, List, error) {
img, err := store.Image(image)
if err != nil {
return "", nil, fmt.Errorf("locating image %q for loading manifest list: %w", image, err)
}
manifestBytes, err := store.ImageBigData(img.ID, storage.ImageDigestManifestBigDataNamePrefix)
if err != nil {
return "", nil, fmt.Errorf("locating image %q for loading manifest list: %w", image, err)
}
manifestList, err := manifests.FromBlob(manifestBytes)
if err != nil {
return "", nil, err
}
list := &list{
List: manifestList,
instances: make(map[digest.Digest]string),
artifacts: artifactsDetails{
Manifests: make(map[digest.Digest]string),
Files: make(map[digest.Digest][]string),
Configs: make(map[digest.Digest]digest.Digest),
Layers: make(map[digest.Digest][]digest.Digest),
Detached: make(map[digest.Digest]string),
Blobs: make(map[digest.Digest][]byte),
},
}
instancesBytes, err := store.ImageBigData(img.ID, instancesData)
if err != nil {
return "", nil, fmt.Errorf("locating image %q for loading instance list: %w", image, err)
}
if err := json.Unmarshal(instancesBytes, &list.instances); err != nil {
return "", nil, fmt.Errorf("decoding instance list for image %q: %w", image, err)
}
artifactsBytes, err := store.ImageBigData(img.ID, artifactsData)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return "", nil, fmt.Errorf("locating image %q for loading instance list: %w", image, err)
}
artifactsBytes = []byte("{}")
}
if err := json.Unmarshal(artifactsBytes, &list.artifacts); err != nil {
return "", nil, fmt.Errorf("decoding artifact list for image %q: %w", image, err)
}
list.instances[""] = img.ID
return img.ID, list, nil
}
// SaveToImage saves the manifest list or image index as the manifest of an
// Image record with the specified names in local storage, generating a random
// image ID if none is specified. It also stores information about where the
// images whose manifests are included in the list can be found.
func (l *list) SaveToImage(store storage.Store, imageID string, names []string, mimeType string) (string, error) {
manifestBytes, err := l.List.Serialize(mimeType)
if err != nil {
return "", err
}
instancesBytes, err := json.Marshal(&l.instances)
if err != nil {
return "", err
}
artifactsBytes, err := json.Marshal(&l.artifacts)
if err != nil {
return "", err
}
manifestDigest, err := manifest.Digest(manifestBytes)
if err != nil {
return "", err
}
imageOptions := &storage.ImageOptions{
BigData: []storage.ImageBigDataOption{
{Key: storage.ImageDigestManifestBigDataNamePrefix, Data: manifestBytes, Digest: manifestDigest},
{Key: instancesData, Data: instancesBytes},
{Key: artifactsData, Data: artifactsBytes},
},
}
img, err := store.CreateImage(imageID, names, "", "", imageOptions)
if err != nil {
if imageID != "" && errors.Is(err, storage.ErrDuplicateID) {
for _, bd := range imageOptions.BigData {
digester := manifest.Digest
if !strings.HasPrefix(bd.Key, storage.ImageDigestManifestBigDataNamePrefix) {
digester = nil
}
err := store.SetImageBigData(imageID, bd.Key, bd.Data, digester)
if err != nil {
return "", fmt.Errorf("saving manifest list to image %q: %w", imageID, err)
}
}
return imageID, nil
}
return "", err
}
l.instances[""] = img.ID
return img.ID, nil
}
// Files returns the list of files associated with a particular artifact
// instance in the image index, primarily for display purposes.
func (l *list) Files(instanceDigest digest.Digest) ([]string, error) {
filesList, ok := l.artifacts.Files[instanceDigest]
if ok {
return slices.Clone(filesList), nil
}
return nil, nil
}
// instanceByFile returns the instanceDigest of the first manifest in the index
// which refers to the named file. The name will be passed to filepath.Abs()
// before searching for an instance which references it.
func (l *list) InstanceByFile(file string) (digest.Digest, error) {
if parsedDigest, err := digest.Parse(file); err == nil {
// nice try, but that's already a digest!
return parsedDigest, nil
}
abs, err := filepath.Abs(file)
if err != nil {
return "", err
}
for instanceDigest, files := range l.artifacts.Files {
for _, file := range files {
if file == abs {
return instanceDigest, nil
}
}
}
return "", os.ErrNotExist
}
// Reference returns an image reference for the composite image being built
// in the list, or an error if the list has never been saved to a local image.
func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) {
if l.instances[""] == "" {
return nil, fmt.Errorf("building reference to list, appears to have not been saved first: %w", ErrListImageUnknown)
}
s, err := is.Transport.ParseStoreReference(store, l.instances[""])
if err != nil {
return nil, fmt.Errorf("creating ImageReference from image %q: %w", l.instances[""], err)
}
references := make([]types.ImageReference, 0, len(l.instances))
whichInstances := make([]digest.Digest, 0, len(l.instances))
switch multiple {
case cp.CopyAllImages, cp.CopySystemImage:
for instance := range l.instances {
if instance != "" {
whichInstances = append(whichInstances, instance)
}
}
case cp.CopySpecificImages:
for instance := range l.instances {
if slices.Contains(instances, instance) {
whichInstances = append(whichInstances, instance)
}
}
}
if len(l.artifacts.Manifests) > 0 {
img, err := is.Transport.GetImage(s)
if err != nil {
return nil, fmt.Errorf("locating image %s: %w", transports.ImageName(s), err)
}
imgDirectory, err := store.ImageDirectory(img.ID)
if err != nil {
return nil, fmt.Errorf("locating per-image directory for %s: %w", img.ID, err)
}
tmp, err := os.MkdirTemp(imgDirectory, pushingArtifactsSubdirectory)
if err != nil {
return nil, err
}
subdir := 0
for artifactManifestDigest, contents := range l.artifacts.Manifests {
// create the blobs directory
subdir++
tmp := filepath.Join(tmp, strconv.Itoa(subdir))
blobsDir := filepath.Join(tmp, "blobs", artifactManifestDigest.Algorithm().String())
if err := os.MkdirAll(blobsDir, 0o700); err != nil {
return nil, fmt.Errorf("creating directory for blobs: %w", err)
}
// write the artifact manifest
if err := os.WriteFile(filepath.Join(blobsDir, artifactManifestDigest.Encoded()), []byte(contents), 0o644); err != nil {
return nil, fmt.Errorf("writing artifact manifest as blob: %w", err)
}
// symlink all of the referenced files and write the inlined blobs into the blobs directory
var referencedBlobDigests []digest.Digest
var symlinkedFiles []string
if referencedConfigDigest, ok := l.artifacts.Configs[artifactManifestDigest]; ok {
referencedBlobDigests = append(referencedBlobDigests, referencedConfigDigest)
}
referencedBlobDigests = append(referencedBlobDigests, l.artifacts.Layers[artifactManifestDigest]...)
for _, referencedBlobDigest := range referencedBlobDigests {
referencedFile, knownFile := l.artifacts.Detached[referencedBlobDigest]
referencedBlob, knownBlob := l.artifacts.Blobs[referencedBlobDigest]
if !knownFile && !knownBlob {
return nil, fmt.Errorf(`internal error: no file or blob with artifact "config" or "layer" digest %q recorded`, referencedBlobDigest)
}
expectedLayerBlobPath := filepath.Join(blobsDir, referencedBlobDigest.Encoded())
if _, err := os.Lstat(expectedLayerBlobPath); err == nil {
// did this one already
continue
} else if knownFile {
if err := os.Symlink(referencedFile, expectedLayerBlobPath); err != nil {
return nil, err
}
symlinkedFiles = append(symlinkedFiles, referencedFile)
} else if knownBlob {
if err := os.WriteFile(expectedLayerBlobPath, referencedBlob, 0o600); err != nil {
return nil, err
}
}
}
// write the index that refers to this one artifact image
tag := "latest"
indexFile := filepath.Join(tmp, "index.json")
index := v1.Index{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
MediaType: v1.MediaTypeImageIndex,
Manifests: []v1.Descriptor{{
MediaType: v1.MediaTypeImageManifest,
Digest: artifactManifestDigest,
Size: int64(len(contents)),
Annotations: map[string]string{
v1.AnnotationRefName: tag,
},
}},
}
indexBytes, err := json.Marshal(&index)
if err != nil {
return nil, fmt.Errorf("encoding image index for OCI layout: %w", err)
}
if err := os.WriteFile(indexFile, indexBytes, 0o644); err != nil {
return nil, fmt.Errorf("writing image index for OCI layout: %w", err)
}
// write the layout file
layoutFile := filepath.Join(tmp, "oci-layout")
if err := os.WriteFile(layoutFile, []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644); err != nil {
return nil, fmt.Errorf("writing oci-layout file: %w", err)
}
// build the reference to this artifact image's oci layout
ref, err := ocilayout.NewReference(tmp, tag)
if err != nil {
return nil, fmt.Errorf("creating ImageReference for artifact with files %q: %w", symlinkedFiles, err)
}
references = append(references, ref)
}
}
for _, instance := range whichInstances {
imageName := l.instances[instance]
ref, err := alltransports.ParseImageName(imageName)
if err != nil {
return nil, fmt.Errorf("creating ImageReference from image %q: %w", imageName, err)
}
references = append(references, ref)
}
return supplemented.Reference(s, references, multiple, instances), nil
}
// Push saves the manifest list and whichever blobs are needed to a destination location.
func (l *list) Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error) {
// Load the system signing policy.
pushPolicy, err := signature.DefaultPolicy(options.SystemContext)
if err != nil {
return nil, "", fmt.Errorf("obtaining default signature policy: %w", err)
}
// Override the settings for local storage to make sure that we can always read the source "image".
pushPolicy.Transports[is.Transport.Name()] = storageAllowedPolicyScopes
policyContext, err := signature.NewPolicyContext(pushPolicy)
if err != nil {
return nil, "", fmt.Errorf("creating new signature policy context: %w", err)
}
defer func() {
if err2 := policyContext.Destroy(); err2 != nil {
logrus.Errorf("Destroying signature policy context: %v", err2)
}
}()
// If we were given a media type that corresponds to a multiple-images
// type, reset it to a valid corresponding single-image type, since we
// already expect the image library to infer the list type from the
// image type that we're telling it to force.
singleImageManifestType := options.ManifestType
switch singleImageManifestType {
case v1.MediaTypeImageIndex:
singleImageManifestType = v1.MediaTypeImageManifest
case manifest.DockerV2ListMediaType:
singleImageManifestType = manifest.DockerV2Schema2MediaType
}
// Build a source reference for our list and grab bag full of blobs.
src, err := l.Reference(options.Store, options.ImageListSelection, options.Instances)
if err != nil {
return nil, "", err
}
if options.SourceFilter != nil {
if src, err = options.SourceFilter(src); err != nil {
return nil, "", err
}
}
compressionVariants, err := prepareAddWithCompression(options.AddCompression)
if err != nil {
return nil, "", err
}
copyOptions := &cp.Options{
ImageListSelection: options.ImageListSelection,
Instances: options.Instances,
SourceCtx: options.SystemContext,
DestinationCtx: options.SystemContext,
ReportWriter: options.ReportWriter,
RemoveSignatures: options.RemoveSignatures,
Signers: options.Signers,
SignBy: options.SignBy,
SignPassphrase: options.SignPassphrase,
SignBySigstorePrivateKeyFile: options.SignBySigstorePrivateKeyFile,
SignSigstorePrivateKeyPassphrase: options.SignSigstorePrivateKeyPassphrase,
ForceManifestMIMEType: singleImageManifestType,
EnsureCompressionVariantsExist: compressionVariants,
ForceCompressionFormat: options.ForceCompressionFormat,
}
retryOptions := retry.Options{}
retryOptions.MaxRetry = defaultMaxRetries
if options.MaxRetries != nil {
retryOptions.MaxRetry = int(*options.MaxRetries)
}
if options.RetryDelay != nil {
retryOptions.Delay = *options.RetryDelay
}
// Copy whatever we were asked to copy.
var manifestDigest digest.Digest
f := func() error {
opts := copyOptions
var manifestBytes []byte
var digest digest.Digest
var err error
if manifestBytes, err = cp.Image(ctx, policyContext, dest, src, opts); err == nil {
if digest, err = manifest.Digest(manifestBytes); err == nil {
manifestDigest = digest
}
}
return err
}
err = retry.IfNecessary(ctx, f, &retryOptions)
return nil, manifestDigest, err
}
func prepareAddWithCompression(variants []string) ([]cp.OptionCompressionVariant, error) {
res := []cp.OptionCompressionVariant{}
for _, name := range variants {
algo, err := compression.AlgorithmByName(name)
if err != nil {
return nil, fmt.Errorf("requested algorithm %s is not supported for replication: %w", name, err)
}
res = append(res, cp.OptionCompressionVariant{Algorithm: algo})
}
return res, nil
}
// Add adds information about the specified image to the list, computing the
// image's manifest's digest, retrieving OS and architecture information from
// the image's configuration, and recording the image's reference so that it
// can be found at push-time. Returns the instanceDigest for the image. If
// the reference points to an image list, either all instances are added (if
// "all" is true), or the instance which matches "sys" (if "all" is false) will
// be added.
func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error) {
src, err := ref.NewImageSource(ctx, sys)
if err != nil {
return "", fmt.Errorf("setting up to read manifest and configuration from %q: %w", transports.ImageName(ref), err)
}
defer src.Close()
type instanceInfo struct {
instanceDigest *digest.Digest
OS, Architecture, OSVersion, Variant string
Features, OSFeatures, Annotations []string
Size int64
ConfigInfo types.BlobInfo
ArtifactType string
}
var instanceInfos []instanceInfo
var manifestDigest digest.Digest
primaryManifestBytes, primaryManifestType, err := src.GetManifest(ctx, nil)
if err != nil {
return "", fmt.Errorf("reading manifest from %q: %w", transports.ImageName(ref), err)
}
if manifest.MIMETypeIsMultiImage(primaryManifestType) {
lists, err := manifests.FromBlob(primaryManifestBytes)
if err != nil {
return "", fmt.Errorf("parsing manifest list in %q: %w", transports.ImageName(ref), err)
}
if all {
for i, instance := range lists.OCIv1().Manifests {
platform := instance.Platform
if platform == nil {
platform = &v1.Platform{}
}
instanceDigest := instance.Digest
instanceInfo := instanceInfo{
instanceDigest: &instanceDigest,
OS: platform.OS,
Architecture: platform.Architecture,
OSVersion: platform.OSVersion,
Variant: platform.Variant,
Features: append([]string{}, lists.Docker().Manifests[i].Platform.Features...),
OSFeatures: append([]string{}, platform.OSFeatures...),
Size: instance.Size,
ArtifactType: instance.ArtifactType,
}
instanceInfos = append(instanceInfos, instanceInfo)
}
} else {
list, err := manifest.ListFromBlob(primaryManifestBytes, primaryManifestType)
if err != nil {
return "", fmt.Errorf("parsing manifest list in %q: %w", transports.ImageName(ref), err)
}
instanceDigest, err := list.ChooseInstance(sys)
if err != nil {
return "", fmt.Errorf("selecting image from manifest list in %q: %w", transports.ImageName(ref), err)
}
added := false
for i, instance := range lists.OCIv1().Manifests {
if instance.Digest != instanceDigest {
continue
}
platform := instance.Platform
if platform == nil {
platform = &v1.Platform{}
}
instanceInfo := instanceInfo{
instanceDigest: &instanceDigest,
OS: platform.OS,
Architecture: platform.Architecture,
OSVersion: platform.OSVersion,
Variant: platform.Variant,
Features: append([]string{}, lists.Docker().Manifests[i].Platform.Features...),
OSFeatures: append([]string{}, platform.OSFeatures...),
Size: instance.Size,
ArtifactType: instance.ArtifactType,
}
instanceInfos = append(instanceInfos, instanceInfo)
added = true
}
if !added {
instanceInfo := instanceInfo{
instanceDigest: &instanceDigest,
}
instanceInfos = append(instanceInfos, instanceInfo)
}
}
} else {
instanceInfo := instanceInfo{
instanceDigest: nil,
}
if primaryManifestType == v1.MediaTypeImageManifest {
if m, err := manifest.OCI1FromManifest(primaryManifestBytes); err == nil {
instanceInfo.ArtifactType = m.ArtifactType
}
}
instanceInfos = append(instanceInfos, instanceInfo)
}
knownConfigTypes := []string{manifest.DockerV2Schema2ConfigMediaType, v1.MediaTypeImageConfig}
for _, instanceInfo := range instanceInfos {
manifestBytes, manifestType, err := src.GetManifest(ctx, instanceInfo.instanceDigest)
if err != nil {
return "", fmt.Errorf("reading manifest from %q, instance %q: %w", transports.ImageName(ref), instanceInfo.instanceDigest, err)
}
instanceManifest, err := manifest.FromBlob(manifestBytes, manifestType)
if err != nil {
return "", fmt.Errorf("parsing manifest from %q, instance %q: %w", transports.ImageName(ref), instanceInfo.instanceDigest, err)
}
instanceInfo.ConfigInfo = instanceManifest.ConfigInfo()
hasPlatformConfig := instanceInfo.ArtifactType == "" && slices.Contains(knownConfigTypes, instanceInfo.ConfigInfo.MediaType)
needToParsePlatformConfig := (instanceInfo.OS == "" || instanceInfo.Architecture == "")
if hasPlatformConfig && needToParsePlatformConfig {
img, err := image.FromUnparsedImage(ctx, sys, image.UnparsedInstance(src, instanceInfo.instanceDigest))
if err != nil {
return "", fmt.Errorf("reading configuration blob from %q: %w", transports.ImageName(ref), err)
}
config, err := img.OCIConfig(ctx)
if err != nil {
return "", fmt.Errorf("reading info about config blob from %q: %w", transports.ImageName(ref), err)
}
if instanceInfo.OS == "" {
instanceInfo.OS = config.OS
instanceInfo.OSVersion = config.OSVersion
if config.OSFeatures != nil {
instanceInfo.OSFeatures = slices.Clone(config.OSFeatures)
}
}
if instanceInfo.Architecture == "" {
instanceInfo.Architecture = config.Architecture
instanceInfo.Variant = config.Variant
}
}
if instanceInfo.instanceDigest == nil {
manifestDigest, err = manifest.Digest(manifestBytes)
if err != nil {
return "", fmt.Errorf("computing digest of manifest from %q: %w", transports.ImageName(ref), err)
}
instanceInfo.instanceDigest = &manifestDigest
instanceInfo.Size = int64(len(manifestBytes))
} else if manifestDigest == "" {
manifestDigest = *instanceInfo.instanceDigest
}
err = l.List.AddInstance(*instanceInfo.instanceDigest, instanceInfo.Size, manifestType, instanceInfo.OS, instanceInfo.Architecture, instanceInfo.OSVersion, instanceInfo.OSFeatures, instanceInfo.Variant, instanceInfo.Features, instanceInfo.Annotations)
if err != nil {
return "", fmt.Errorf("adding instance with digest %q: %w", *instanceInfo.instanceDigest, err)
}
if _, ok := l.instances[*instanceInfo.instanceDigest]; !ok {
l.instances[*instanceInfo.instanceDigest] = transports.ImageName(ref)
}
}
return manifestDigest, nil
}
// AddArtifactOptions contains options which control the contents of the
// artifact manifest that AddArtifact will create and add to the image index.
// This should provide for all of the ways to construct a manifest outlined in
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage
// * no blobs → set ManifestArtifactType
// * blobs, no configuration → set ManifestArtifactType and possibly LayerMediaType, and provide file names
// * blobs and configuration → set ManifestArtifactType, possibly LayerMediaType, and ConfigDescriptor, and provide file names
//
// The older style of describing artifacts:
// * leave ManifestArtifactType blank
// * specify a zero-length application/vnd.oci.image.config.v1+json config blob
// * set LayerMediaType to a custom type
//
// When reading data produced elsewhere, note that newer tooling will produce
// manifests with ArtifactType set. If the manifest's ArtifactType is not set,
// consumers should consult the config descriptor's MediaType.
type AddArtifactOptions struct {
ManifestArtifactType *string // overall type of the artifact manifest. default: "application/vnd.unknown.artifact.v1"
Platform v1.Platform // default: add to the index without platform information
ConfigDescriptor *v1.Descriptor // default: a descriptor for an explicitly empty config blob
ConfigFile string // path to config contents, recorded if ConfigDescriptor.Size != 0 and ConfigDescriptor.Data is not set
LayerMediaType *string // default: mime.TypeByExtension() if basename contains ".", else http.DetectContentType()
Annotations map[string]string // optional, default is none
SubjectReference types.ImageReference // optional
ExcludeTitles bool // don't add "org.opencontainers.image.title" annotations set to file base names
}
// AddArtifact creates an artifact manifest describing the specified file or
// files, then adds them to the specified image index. Returns the
// instanceDigest for the artifact manifest.
// The caller could craft the manifest themselves and use Add() to add it to
// the image index and get the same end-result, but this should save them some
// work.
func (l *list) AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error) {
// If we were given a subject, build a descriptor for it first, since
// it might be remote, and anything else we do before looking at it
// might have to get thrown away if we can't get to it for whatever
// reason.
var subject *v1.Descriptor
if options.SubjectReference != nil {
subjectReference, err := options.SubjectReference.NewImageSource(ctx, sys)
if err != nil {
return "", fmt.Errorf("setting up to read manifest and configuration from subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
defer subjectReference.Close()
subjectManifestBytes, subjectManifestType, err := subjectReference.GetManifest(ctx, nil)
if err != nil {
return "", fmt.Errorf("reading manifest from subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
subjectManifestDigest, err := manifest.Digest(subjectManifestBytes)
if err != nil {
return "", fmt.Errorf("digesting manifest of subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
var subjectArtifactType string
if !manifest.MIMETypeIsMultiImage(subjectManifestType) {
var subjectManifest v1.Manifest
if json.Unmarshal(subjectManifestBytes, &subjectManifest) == nil {
subjectArtifactType = subjectManifest.ArtifactType
}
}
subject = &v1.Descriptor{
MediaType: subjectManifestType,
ArtifactType: subjectArtifactType,
Digest: subjectManifestDigest,
Size: int64(len(subjectManifestBytes)),
}
}
// Build up the layers list piece by piece.
var layers []v1.Descriptor
fileDigests := make(map[string]digest.Digest)
if len(files) == 0 {
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage
// says that we should have at least one layer listed, even if it's just a placeholder
layers = append(layers, v1.DescriptorEmptyJSON)
}
for _, file := range files {
if err := func() error {
// Open the file so that we can digest it.
absFile, err := filepath.Abs(file)
if err != nil {
return fmt.Errorf("converting %q to an absolute path: %w", file, err)
}
f, err := os.Open(absFile)
if err != nil {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
defer f.Close()
// Hang on to a copy of the first 512 bytes, but digest the whole thing.
digester := digest.Canonical.Digester()
writeCounter := ioutils.NewWriteCounter(digester.Hash())
var detectableData bytes.Buffer
_, err = io.CopyN(writeCounter, io.TeeReader(f, &detectableData), 512)
if err != nil && !errors.Is(err, io.EOF) {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
if err == nil {
if _, err := io.Copy(writeCounter, f); err != nil {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
}
fileDigests[absFile] = digester.Digest()
// If one wasn't specified, figure out what the MediaType should be.
title := filepath.Base(absFile)
layerMediaType := options.LayerMediaType
if layerMediaType == nil {
if index := strings.LastIndex(title, "."); index != -1 {
// File's basename has an extension, try to use a shortcut.
tmp := mime.TypeByExtension(title[index:])
if tmp != "" {
layerMediaType = &tmp
}
}
if layerMediaType == nil {
// File's basename has no extension or didn't map to a type, look at the contents we saved.
tmp := http.DetectContentType(detectableData.Bytes())
layerMediaType = &tmp
}
if layerMediaType != nil {
// Strip off any parameters, since we only want the type name.
if parsedMediaType, _, err := mime.ParseMediaType(*layerMediaType); err == nil {
layerMediaType = &parsedMediaType
}
}
}
// Build the descriptor for the layer.
descriptor := v1.Descriptor{
MediaType: *layerMediaType,
Digest: fileDigests[absFile],
Size: writeCounter.Count,
}
// OCI annotations are usually applied at the image manifest as a whole,
// but tools like oras (https://oras.land/) also apply them to blob
// descriptors. AnnotationTitle is used as a suggestion for the name
// to give to a blob if it's being stored as a file, and we default
// to adding one based on its original name.
if !options.ExcludeTitles {
descriptor.Annotations = map[string]string{
v1.AnnotationTitle: title,
}
}
layers = append(layers, descriptor)
return nil
}(); err != nil {
return "", err
}
}
// Unless we were told what this is, use the default that ORAS uses.
artifactType := "application/vnd.unknown.artifact.v1"
if options.ManifestArtifactType != nil {
artifactType = *options.ManifestArtifactType
}
// Unless we were explicitly told otherwise, default to an empty config blob.
configDescriptor := internal.DeepCopyDescriptor(&v1.DescriptorEmptyJSON)
if options.ConfigDescriptor != nil {
configDescriptor = internal.DeepCopyDescriptor(options.ConfigDescriptor)
}
if options.ConfigFile != "" {
if options.ConfigDescriptor == nil { // i.e., we assigned the default mediatype
configDescriptor.MediaType = v1.MediaTypeImageConfig
}
configDescriptor.Data = nil
configDescriptor.Digest = "" // to be figured out below
configDescriptor.Size = -1 // to be figured out below
}
configFilePath := ""
if configDescriptor.Size != 0 {
if len(configDescriptor.Data) == 0 {
if options.ConfigFile == "" {
return "", errors.New("needed config data file, but none was provided")
}
filePath, err := filepath.Abs(options.ConfigFile)
if err != nil {
return "", fmt.Errorf("recording artifact config data file %q: %w", options.ConfigFile, err)
}
digester := digest.Canonical.Digester()
counter := ioutils.NewWriteCounter(digester.Hash())
if err := func() error {
f, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("reading artifact config data file %q: %w", options.ConfigFile, err)
}
defer f.Close()
if _, err := io.Copy(counter, f); err != nil {
return fmt.Errorf("digesting artifact config data file %q: %w", options.ConfigFile, err)
}
return nil
}(); err != nil {
return "", err
}
configDescriptor.Data = nil
configDescriptor.Size = counter.Count
configDescriptor.Digest = digester.Digest()
configFilePath = filePath
} else {
decoder := bytes.NewReader(configDescriptor.Data)
digester := digest.Canonical.Digester()
counter := ioutils.NewWriteCounter(digester.Hash())
if _, err := io.Copy(counter, decoder); err != nil {
return "", fmt.Errorf("digesting inlined artifact config data: %w", err)
}
configDescriptor.Size = counter.Count
configDescriptor.Digest = digester.Digest()
}
} else {
configDescriptor.Data = nil
configDescriptor.Digest = digest.Canonical.FromString("")
}
// Construct the manifest.
artifactManifest := v1.Manifest{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
MediaType: v1.MediaTypeImageManifest,
ArtifactType: artifactType,
Config: *configDescriptor,
Layers: layers,
Subject: subject,
}
// Add in annotations, more or less exactly as specified.
if options.Annotations != nil {
artifactManifest.Annotations = maps.Clone(options.Annotations)
}
// Encode and save the data we care about.
artifactManifestBytes, err := json.Marshal(artifactManifest)
if err != nil {
return "", fmt.Errorf("marshalling the artifact manifest: %w", err)
}
artifactManifestDigest, err := manifest.Digest(artifactManifestBytes)
if err != nil {
return "", fmt.Errorf("digesting the artifact manifest: %w", err)
}
l.artifacts.Manifests[artifactManifestDigest] = string(artifactManifestBytes)
l.artifacts.Layers[artifactManifestDigest] = nil
l.artifacts.Configs[artifactManifestDigest] = artifactManifest.Config.Digest
if configFilePath != "" {
l.artifacts.Detached[artifactManifest.Config.Digest] = configFilePath
l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], configFilePath)
} else {
l.artifacts.Blobs[artifactManifest.Config.Digest] = slices.Clone(artifactManifest.Config.Data)
}
for filePath, fileDigest := range fileDigests {
l.artifacts.Layers[artifactManifestDigest] = append(l.artifacts.Layers[artifactManifestDigest], fileDigest)
l.artifacts.Detached[fileDigest] = filePath
l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], filePath)
}
for _, layer := range layers {
if len(layer.Data) != 0 {
l.artifacts.Blobs[layer.Digest] = slices.Clone(layer.Data)
l.artifacts.Layers[artifactManifestDigest] = append(l.artifacts.Layers[artifactManifestDigest], layer.Digest)
}
}
// Add this artifact manifest to the image index.
if err := l.AddInstance(artifactManifestDigest, int64(len(artifactManifestBytes)), artifactManifest.MediaType, options.Platform.OS, options.Platform.Architecture, options.Platform.OSVersion, options.Platform.OSFeatures, options.Platform.Variant, nil, nil); err != nil {
return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err)
}
// Set the artifact type in the image index entry if we have one, since AddInstance() didn't do that for us.
if artifactManifest.ArtifactType != "" {
if err := l.List.SetArtifactType(&artifactManifestDigest, artifactManifest.ArtifactType); err != nil {
return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err)
}
}
return artifactManifestDigest, nil
}
// Remove filters out any instances in the list which match the specified digest.
func (l *list) Remove(instanceDigest digest.Digest) error {
err := l.List.Remove(instanceDigest)
if err == nil {
delete(l.instances, instanceDigest)
}
return err
}
// LockerForImage returns a Locker for a given image record. It's recommended
// that processes which use LoadFromImage() to load a list from an image and
// then use that list's SaveToImage() method to save a modified version of the
// list to that image record use this lock to avoid accidentally wiping out
// changes that another process is also attempting to make.
func LockerForImage(store storage.Store, image string) (lockfile.Locker, error) { // nolint:staticcheck
img, err := store.Image(image)
if err != nil {
return nil, fmt.Errorf("locating image %q for locating lock: %w", image, err)
}
d := digest.NewDigestFromEncoded(digest.Canonical, img.ID)
if err := d.Validate(); err != nil {
return nil, fmt.Errorf("coercing image ID for %q into a digest: %w", image, err)
}
return store.GetDigestLock(d)
}