Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[release/2.8] Add code to handle pagination of parts. Fixes max layer size of 10GB bug #3893

Merged
merged 2 commits into from May 9, 2023
Merged

[release/2.8] Add code to handle pagination of parts. Fixes max layer size of 10GB bug #3893

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1d52366
Merge pull request #2815 from bainsy88/issue_2814
dmcgowan Mar 16, 2021
22a8050
fix(ci): use go install instead of go get
davidspek May 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Expand Up @@ -31,7 +31,7 @@ jobs:
run: |
sudo apt-get -q update
sudo -E apt-get -yq --no-install-suggests --no-install-recommends install python2-minimal
cd /tmp && go get -u github.com/vbatts/git-validation
cd /tmp && go install github.com/vbatts/git-validation@latest
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW; this one was already included in 2392893, so should dissolve when rebasing.


- name: Build
working-directory: ./src/github.com/docker/distribution
Expand All @@ -47,4 +47,4 @@ jobs:

- uses: codecov/codecov-action@v1
with:
directory: ./src/github.com/docker/distribution
directory: ./src/github.com/docker/distribution
23 changes: 16 additions & 7 deletions registry/storage/driver/s3-aws/s3.go
View file
Open in desktop
Expand Up @@ -549,9 +549,9 @@ func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.Read

// Writer returns a FileWriter which will store the content written to it
// at the location designated by "path" after the call to Commit.
func (d *driver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
func (d *driver) Writer(ctx context.Context, path string, appendParam bool) (storagedriver.FileWriter, error) {
key := d.s3Path(path)
if !append {
if !appendParam {
// TODO (brianbland): cancel other uploads at this path
resp, err := d.S3.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
Bucket: aws.String(d.Bucket),
Expand All @@ -574,7 +574,7 @@ func (d *driver) Writer(ctx context.Context, path string, append bool) (storaged
if err != nil {
return nil, parseError(path, err)
}

var allParts []*s3.Part
for _, multi := range resp.Uploads {
if key != *multi.Key {
continue
Expand All @@ -587,11 +587,20 @@ func (d *driver) Writer(ctx context.Context, path string, append bool) (storaged
if err != nil {
return nil, parseError(path, err)
}
var multiSize int64
for _, part := range resp.Parts {
multiSize += *part.Size
allParts = append(allParts, resp.Parts...)
for *resp.IsTruncated {
resp, err = d.S3.ListParts(&s3.ListPartsInput{
Bucket: aws.String(d.Bucket),
Key: aws.String(key),
UploadId: multi.UploadId,
PartNumberMarker: resp.NextPartNumberMarker,
})
if err != nil {
return nil, parseError(path, err)
}
allParts = append(allParts, resp.Parts...)
}
return d.newWriter(key, *multi.UploadId, resp.Parts), nil
return d.newWriter(key, *multi.UploadId, allParts), nil
}
return nil, storagedriver.PathNotFoundError{Path: path}
}
Expand Down