From 936070d18d226fa9215892900cf670ab17f05fe5 Mon Sep 17 00:00:00 2001 From: Radu Popescu Date: Fri, 29 Apr 2022 16:38:11 +0200 Subject: [PATCH] Add support for manifest lists --- ducc/docker-api/util.go | 17 +++++ ducc/lib/conversion.go | 6 +- ducc/lib/image.go | 155 ++++++++++++++++++++++++++++++---------- 3 files changed, 138 insertions(+), 40 deletions(-) diff --git a/ducc/docker-api/util.go b/ducc/docker-api/util.go index 2009065af7..9e981bdefe 100644 --- a/ducc/docker-api/util.go +++ b/ducc/docker-api/util.go @@ -27,6 +27,23 @@ type Manifest struct { Layers []Layer } +type ManifestList struct { + SchemaVersion int + MediaType string + Manifests []ManifestListItem +} + +type ManifestListItem struct { + MediaType string + Size int + Digest string + Platform struct { + Architecture string + OS string + Variant *string + } +} + type ThinImageLayer struct { Digest string `json:"digest"` Url string `json:"url,omitempty"` diff --git a/ducc/lib/conversion.go b/ducc/lib/conversion.go index 23d073765b..be27c8bbc7 100644 --- a/ducc/lib/conversion.go +++ b/ducc/lib/conversion.go @@ -318,11 +318,11 @@ func convertInputOutput(inputImage *Image, repo string, convertAgain, forceDownl } manifestPath := filepath.Join("/", "cvmfs", repo, ".metadata", inputImage.GetSimpleName(), "manifest.json") alreadyConverted := AlreadyConverted(manifestPath, manifest.Config.Digest) - l.Log().WithFields(log.Fields{"alreadyConverted": alreadyConverted}).Info( - "Already converted the image, skipping.") if alreadyConverted == ConversionMatch { - if convertAgain == false { + if !convertAgain { + l.Log().WithFields(log.Fields{"alreadyConverted": alreadyConverted}).Info( + "Already converted the image, skipping.") return nil } } diff --git a/ducc/lib/image.go b/ducc/lib/image.go index 4cf486005c..5c30729ba6 100644 --- a/ducc/lib/image.go +++ b/ducc/lib/image.go @@ -189,24 +189,89 @@ func (img *Image) PrintImage(machineFriendly, csv_header bool) { } } -func (img *Image) GetManifest() (da.Manifest, error) { - if img.Manifest != nil { - return *img.Manifest, nil - } - bytes, err := img.getByteManifest() +func (img *Image) fetchManifest() (*da.Manifest, error) { + bytes, err := img.getByteManifest(nil) if err != nil { - return da.Manifest{}, err + return nil, err } + l.Log().Infof("RADU Manifest response: %v\n", string(bytes)) + var manifest da.Manifest err = json.Unmarshal(bytes, &manifest) if err != nil { - return manifest, err + return nil, err } if reflect.DeepEqual(da.Manifest{}, manifest) { - return manifest, fmt.Errorf("got empty manifest") + return nil, fmt.Errorf("got empty manifest") } + img.Manifest = &manifest - return manifest, nil + return &manifest, nil +} + +func (img *Image) fetchManifestList() (*da.Manifest, error) { + bytes1, err := img.getByteManifestList() + if err != nil { + return nil, err + } + + var manifestList da.ManifestList + err = json.Unmarshal(bytes1, &manifestList) + if err != nil { + return nil, err + } + if reflect.DeepEqual(da.ManifestList{}, manifestList) { + return nil, fmt.Errorf("got empty manifest list") + } + + var manifestReference string + if len(manifestList.Manifests) == 1 { + manifestReference = manifestList.Manifests[0].Digest + } else { + // TODO: In case of a manifest list with multiple architectures, default to amd64 + // TODO: Support multi-arch images + for _, v := range manifestList.Manifests { + if v.Platform.Architecture == "amd64" { + manifestReference = v.Digest + } + } + } + + bytes2, err := img.getByteManifest(&manifestReference) + if err != nil { + return nil, err + } + + var manifest da.Manifest + err = json.Unmarshal(bytes2, &manifest) + if err != nil { + return nil, err + } + if reflect.DeepEqual(da.Manifest{}, manifest) { + return nil, fmt.Errorf("got empty manifest") + } + + img.Manifest = &manifest + return &manifest, nil +} + +func (img *Image) GetManifest() (da.Manifest, error) { + if img.Manifest != nil { + return *img.Manifest, nil + } + + // First try to fetch a simple manifest + manifest, err := img.fetchManifest() + if err != nil { + // If the first fetch fails, try to fetch from a manifest list + manifest, err := img.fetchManifestList() + if err != nil { + return da.Manifest{}, fmt.Errorf("could not retrieve manifest") + } + return *manifest, nil + } + + return *manifest, nil } func (img *Image) GetOCIImage() (config image.Image, err error) { @@ -416,37 +481,17 @@ func (i *Image) GetPublicSymlinkPath() string { return filepath.Join(i.Registry, i.Repository+":"+i.GetSimpleReference()) } -func (img *Image) getByteManifest() ([]byte, error) { +func (img *Image) getByteManifestList() ([]byte, error) { url := img.GetManifestUrl() + return makeGetRequest(url, map[string]string{"Accept": "application/vnd.docker.distribution.manifest.list.v2+json"}) +} - token, err := firstRequestForAuth(url) - if err != nil { - l.LogE(err).Error("Error in getting the authentication token") - return nil, err - } - - client := &http.Client{} - req, err := http.NewRequest("GET", url, nil) - if err != nil { - l.LogE(err).Error("Impossible to create a HTTP request") - return nil, err - } - - req.Header.Set("Authorization", token) - req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json") - - resp, err := client.Do(req) - if err != nil { - l.LogE(err).Error("Error in making the HTTP request") - return nil, err - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - l.LogE(err).Error("Error in reading the second http response") - return nil, err +func (img *Image) getByteManifest(reference *string) ([]byte, error) { + url := img.GetManifestUrl() + if reference != nil { + url = strings.Join([]string{url, *reference}, "/") } - return body, nil + return makeGetRequest(url, map[string]string{"Accept": "application/vnd.docker.distribution.manifest.v2+json"}) } func GetAuthToken(url string, credentials []Credentials) (token string, err error) { @@ -986,3 +1031,39 @@ func (img *Image) CreateSneakyChainStructure(CVMFSRepo string) (err error, lastC } return } + +func makeGetRequest(url string, headers map[string]string) ([]byte, error) { + token, err := firstRequestForAuth(url) + if err != nil { + l.LogE(err).Error("Error in getting the authentication token") + return nil, err + } + + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + if err != nil { + l.LogE(err).Error("Impossible to create a HTTP request") + return nil, err + } + + req.Header.Set("Authorization", token) + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := client.Do(req) + if err != nil { + l.LogE(err).Error("Error in making the HTTP request") + return nil, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + l.LogE(err).Error("Error in reading the second http response") + return nil, err + } + + l.Log().Infof("RADU: URL: %v, Response: %v\n", url, string(body)) + + return body, nil +}