Skip to content

Commit

Permalink
s3: add S3 IPv6 support with option "dualstack_endpoint" (bool)
Browse files Browse the repository at this point in the history
dualstack_endpoint=true enables IPv6 DNS lookup for S3 endpoints
in s3.go, add Options.DualstackEndpoint to support IPv6 on S3
  • Loading branch information
tonymet committed Dec 4, 2023
1 parent 298c13e commit 15e9e76
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 0 deletions.
11 changes: 11 additions & 0 deletions backend/s3/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -2219,6 +2219,13 @@ If it is set then rclone will use v2 authentication.
Use this only if v4 signatures don't work, e.g. pre Jewel/v10 CEPH.`,
Default: false,
Advanced: true,
}, {
Name: "use_dual_stack",
Help: `If true use AWS S3 dual-stack endpoint (IPv6 support).
See [AWS Docs on Dualstack Endpoints(https://docs.aws.amazon.com/AmazonS3/latest/userguide/dual-stack-endpoints.html)`,
Default: false,
Advanced: true,
}, {
Name: "use_accelerate_endpoint",
Provider: "AWS",
Expand Down Expand Up @@ -2628,6 +2635,7 @@ type Options struct {
Region string `config:"region"`
Endpoint string `config:"endpoint"`
STSEndpoint string `config:"sts_endpoint"`
UseDualStack bool `config:"use_dual_stack"`
LocationConstraint string `config:"location_constraint"`
ACL string `config:"acl"`
BucketACL string `config:"bucket_acl"`
Expand Down Expand Up @@ -2956,6 +2964,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
r.addService("sts", opt.STSEndpoint)
awsConfig.WithEndpointResolver(r)
}
if opt.UseDualStack {
awsConfig.UseDualStackEndpoint = endpoints.DualStackEndpointStateEnabled
}

// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
awsSessionOpts := session.Options{
Expand Down
32 changes: 32 additions & 0 deletions backend/s3/s3_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,23 @@
package s3

import (
"context"
"net/http"
"strings"
"testing"

"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
)

func SetupS3Test(t *testing.T) (context.Context, *Options, *http.Client) {
ctx, opt := context.Background(), new(Options)
opt.Provider = "AWS"
client := getClient(ctx, opt)
return ctx, opt, client
}

// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
Expand Down Expand Up @@ -39,6 +49,28 @@ func TestIntegration2(t *testing.T) {
})
}

func TestAWSDualStackOption(t *testing.T) {
{
// test enabled
ctx, opt, client := SetupS3Test(t)
opt.UseDualStack = true
s3Conn, _, _ := s3Connection(ctx, opt, client)
if !strings.Contains(s3Conn.Endpoint, "dualstack") {
t.Errorf("dualstack failed got: %s, wanted: dualstack", s3Conn.Endpoint)
t.Fail()
}
}
{
// test default case
ctx, opt, client := SetupS3Test(t)
s3Conn, _, _ := s3Connection(ctx, opt, client)
if strings.Contains(s3Conn.Endpoint, "dualstack") {
t.Errorf("dualstack failed got: %s, NOT wanted: dualstack", s3Conn.Endpoint)
t.Fail()
}
}
}

func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
Expand Down

0 comments on commit 15e9e76

Please sign in to comment.