Skip to content

Commit

Permalink
fix encoding with mc client
Browse files Browse the repository at this point in the history 
using the mc client file encoding were wrong
see rclone/gofakes3#2 for details
  • Loading branch information
@individual-it
individual-it committed Jun 22, 2023
1 parent b56c920 commit f8e9255
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 57 deletions.
149 changes: 114 additions & 35 deletions cmd/serve/s3/s3_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,19 @@
package s3

import (
"bytes"
"context"
"encoding/hex"
"fmt"
"github.com/minio/minio-go/v7"

Check failure on line 11 in cmd/serve/s3/s3_test.go

View workflow job for this annotation

GitHub Actions / lint 

File is not `goimports`-ed (goimports)
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/rclone/rclone/fs/object"
"io"
"math/rand"
"net/url"
"os"
"os/exec"
"path"
"strings"
"testing"
"time"
Expand All @@ -29,30 +36,45 @@ const (
endpoint = "localhost:0"
)

// Configure and serve the server
func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
keyid = RandString(16)
keysec = RandString(16)
serveropt := &Options{
HTTP: httplib.DefaultCfg(),
pathBucketMode: true,
hashName: "",
hashType: hash.None,
authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)},
}

serveropt.HTTP.ListenAddr = []string{endpoint}
w, _ := newServer(context.Background(), f, serveropt)
router := w.Router()

w.Bind(router)
w.Serve()
testURL = w.Server.URLs()[0]

return
}

func RandString(n int) string {
src := rand.New(rand.NewSource(time.Now().UnixNano()))
b := make([]byte, (n+1)/2)

if _, err := src.Read(b); err != nil {
panic(err)
}

return hex.EncodeToString(b)[:n]
}

// TestS3 runs the s3 server then runs the unit tests for the
// s3 remote against it.
func TestS3(t *testing.T) {
// Configure and start the server
start := func(f fs.Fs) (configmap.Simple, func()) {
//httpflags.Opt.ListenAddr = endpoint
keyid := RandString(16)
keysec := RandString(16)
serveropt := &Options{
HTTP: httplib.DefaultCfg(),
pathBucketMode: true,
hashName: "",
hashType: hash.None,
authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)},
}

serveropt.HTTP.ListenAddr = []string{endpoint}
w, err := newServer(context.Background(), f, serveropt)
router := w.Router()
assert.NoError(t, err)

w.Bind(router)
w.Serve()
testURL := w.Server.URLs()[0]
testURL, keyid, keysec := serveS3(f)
// Config for the backend we'll use to connect to the server
config := configmap.Simple{
"type": "s3",
Expand All @@ -66,21 +88,10 @@ func TestS3(t *testing.T) {
return config, func() {}
}

Run(t, "s3", start)
}

func RandString(n int) string {
src := rand.New(rand.NewSource(time.Now().UnixNano()))
b := make([]byte, (n+1)/2)

if _, err := src.Read(b); err != nil {
panic(err)
}

return hex.EncodeToString(b)[:n]
RunS3UnitTests(t, "s3", start)
}

func Run(t *testing.T, name string, start servetest.StartFn) {
func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
fstest.Initialise()
ci := fs.GetConfig(context.Background())
ci.DisableFeatures = append(ci.DisableFeatures, "Metadata")
Expand All @@ -106,7 +117,7 @@ func Run(t *testing.T, name string, start servetest.StartFn) {
require.NoError(t, os.Chdir(cwd))
}()

// Run the backend tests with an on the fly remote
// RunS3UnitTests the backend tests with an on the fly remote
args := []string{"test"}
if testing.Verbose() {
args = append(args, "-v")
Expand All @@ -127,11 +138,79 @@ func Run(t *testing.T, name string, start servetest.StartFn) {
cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v)
}

// Run the test
// RunS3UnitTests the test
out, err := cmd.CombinedOutput()
if len(out) != 0 {
t.Logf("\n----------\n%s----------\n", string(out))
}
assert.NoError(t, err, "Running "+name+" integration tests")
}

// tests using the minio client
func TestEncodingWithMinioClient(t *testing.T) {
cases := []struct {
description string
bucket string
path string
filename string
expected string
}{
{
description: "weird file in bucket root",
bucket: "mybucket",
path: "",
filename: " file with w€r^d ch@r \\#~+§4%&'. txt ",
},
{
description: "weird file inside a weird folder",
bucket: "mybucket",
path: "ä#/नेपाल&/?/",
filename: " file with w€r^d ch@r \\#~+§4%&'. txt ",
},
}

for _, tt := range cases {
t.Run(tt.description, func(t *testing.T) {
fstest.Initialise()
f, _, clean, err := fstest.RandomRemote()
assert.NoError(t, err)
defer clean()
err = f.Mkdir(context.Background(), path.Join(tt.bucket, tt.path))
assert.NoError(t, err)

buf := bytes.NewBufferString("contents")
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)

obji := object.NewStaticObjectInfo(
path.Join(tt.bucket, tt.path, tt.filename),
time.Now(),
int64(buf.Len()),
true,
nil,
nil,
)
_, err = f.Put(context.Background(), in, obji)
assert.NoError(t, err)

endpoint, keyid, keysec := serveS3(f)
testURL, _ := url.Parse(endpoint)
minioClient, err := minio.New(testURL.Host, &minio.Options{
Creds: credentials.NewStaticV4(keyid, keysec, ""),
Secure: false,
})
assert.NoError(t, err)

buckets, err := minioClient.ListBuckets(context.Background())
assert.NoError(t, err)
assert.Equal(t, buckets[0].Name, tt.bucket)
objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
Recursive: true,
})
for object := range objects {
assert.Equal(t, path.Join(tt.path, tt.filename), object.Key)
}
})
}

}
29 changes: 19 additions & 10 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd
github.com/Mikubill/gofakes3 v0.0.3-0.20230316054032-723f1854b42f
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700
github.com/Unknwon/goconfig v1.0.0
github.com/a8m/tree v0.0.0-20230208161321-36ae24ddad15
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3
Expand All @@ -18,7 +18,7 @@ require (
github.com/anacrolix/log v0.13.1
github.com/artyom/mtab v1.0.0
github.com/atotto/clipboard v0.1.4
github.com/aws/aws-sdk-go v1.44.246
github.com/aws/aws-sdk-go v1.44.256
github.com/buengese/sgzip v0.1.1
github.com/colinmarc/hdfs/v2 v2.3.0
github.com/coreos/go-semver v0.3.1
Expand Down Expand Up @@ -52,7 +52,7 @@ require (
github.com/rfjakob/eme v1.1.2
github.com/rivo/uniseg v0.4.4
github.com/shirou/gopsutil/v3 v3.23.3
github.com/sirupsen/logrus v1.9.0
github.com/sirupsen/logrus v1.9.2
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
Expand All @@ -64,12 +64,12 @@ require (
github.com/yunify/qingstor-sdk-go/v3 v3.2.0
go.etcd.io/bbolt v1.3.7
goftp.io/server v1.0.0-rc1
golang.org/x/crypto v0.7.0
golang.org/x/net v0.8.0
golang.org/x/crypto v0.9.0
golang.org/x/net v0.10.0
golang.org/x/oauth2 v0.6.0
golang.org/x/sync v0.1.0
golang.org/x/sys v0.7.0
golang.org/x/text v0.8.0
golang.org/x/sys v0.8.0
golang.org/x/text v0.9.0
golang.org/x/time v0.3.0
google.golang.org/api v0.115.0
gopkg.in/yaml.v2 v2.4.0
Expand All @@ -87,6 +87,7 @@ require (
github.com/cloudflare/circl v1.3.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/geoffgarside/ber v1.1.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
Expand All @@ -106,20 +107,27 @@ require (
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jlaffaye/ftp v0.1.1-0.20230214004652-d84bf4be2b6e // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.57 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
Expand All @@ -133,12 +141,13 @@ require (
github.com/zeebo/blake3 v0.2.3 // indirect
github.com/zeebo/errs v1.3.0 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/tools v0.6.0 // indirect
golang.org/x/tools v0.8.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect
google.golang.org/grpc v1.54.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
storj.io/drpc v0.0.32 // indirect
Expand All @@ -152,5 +161,5 @@ require (
github.com/google/go-querystring v1.1.0 // indirect
github.com/pkg/xattr v0.4.9
golang.org/x/mobile v0.0.0-20230301163155-e0f57694e12c
golang.org/x/term v0.7.0
golang.org/x/term v0.8.0
)

0 comments on commit f8e9255

Please sign in to comment.