From 36e922204e8399ff62bffd84c4de117370e71d3a Mon Sep 17 00:00:00 2001 From: abyss-w Date: Tue, 3 Aug 2021 11:10:26 +0800 Subject: [PATCH 1/4] feat: Implement CreateLink and setup linker test in go-service-oss --- .gitignore | 6 +++- generated.go | 72 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 6 ++-- go.sum | 15 ++++----- service.toml | 2 +- storage.go | 33 ++++++++++++++++++++ tests/storage_test.go | 7 +++++ utils.go | 1 + 8 files changed, 128 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 9113e8a..63f68ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ coverage.* bin/ -Makefile.env \ No newline at end of file +Makefile.env + +# Jetbrain IDE +.idea +*.iml \ No newline at end of file diff --git a/generated.go b/generated.go index 655c2b0..8d0aafd 100644 --- a/generated.go +++ b/generated.go @@ -464,6 +464,7 @@ func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *Stor var ( _ Appender = &Storage{} _ Direr = &Storage{} + _ Linker = &Storage{} _ Multiparter = &Storage{} _ Storager = &Storage{} ) @@ -544,6 +545,7 @@ type DefaultStoragePairs struct { Create []Pair CreateAppend []Pair CreateDir []Pair + CreateLink []Pair CreateMultipart []Pair Delete []Pair List []Pair @@ -725,6 +727,29 @@ func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, return result, nil } +// pairStorageCreateLink is the parsed struct +type pairStorageCreateLink struct { + pairs []Pair +} + +// parsePairStorageCreateLink will parse Pair slice into *pairStorageCreateLink +func (s *Storage) parsePairStorageCreateLink(opts []Pair) (pairStorageCreateLink, error) { + result := pairStorageCreateLink{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCreateLink{}, services.PairUnsupportedError{Pair: v} + } + } + + // Check required pairs. + + return result, nil +} + // pairStorageCreateMultipart is the parsed struct type pairStorageCreateMultipart struct { pairs []Pair @@ -1300,6 +1325,53 @@ func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs . return s.createDir(ctx, path, opt) } +// CreateLink Will create a link object. +// +// # Behavior +// +// - `path` and `target` COULD be relative or absolute path. +// - If `target` not exists, CreateLink will still create a link object to path. +// - If `path` exists: +// - If `path` is a symlink object, CreateLink will remove the symlink object and create a new link object to path. +// - If `path` is not a symlink object, CreateLink will return an ErrObjectModeInvalid error when the service does not support overwrite. +// - A link object COULD be returned in `Stat` or `List`. +// - CreateLink COULD implement virtual_link feature when service without native support. +// - Users SHOULD enable this feature by themselves. +// +// This function will create a context by default. +func (s *Storage) CreateLink(path string, target string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateLinkWithContext(ctx, path, target, pairs...) +} + +// CreateLinkWithContext Will create a link object. +// +// # Behavior +// +// - `path` and `target` COULD be relative or absolute path. +// - If `target` not exists, CreateLink will still create a link object to path. +// - If `path` exists: +// - If `path` is a symlink object, CreateLink will remove the symlink object and create a new link object to path. +// - If `path` is not a symlink object, CreateLink will return an ErrObjectModeInvalid error when the service does not support overwrite. +// - A link object COULD be returned in `Stat` or `List`. +// - CreateLink COULD implement virtual_link feature when service without native support. +// - Users SHOULD enable this feature by themselves. +func (s *Storage) CreateLinkWithContext(ctx context.Context, path string, target string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = s.formatError("create_link", err, path, target) + }() + + pairs = append(pairs, s.defaultPairs.CreateLink...) + var opt pairStorageCreateLink + + opt, err = s.parsePairStorageCreateLink(pairs) + if err != nil { + return + } + + return s.createLink(ctx, path, target, opt) +} + // CreateMultipart will create a new multipart. // // ## Behavior diff --git a/go.mod b/go.mod index 61a83f2..2babbee 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.14 require ( github.com/aliyun/aliyun-oss-go-sdk v2.1.9+incompatible github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect - github.com/beyondstorage/go-endpoint v1.0.1 - github.com/beyondstorage/go-integration-test/v4 v4.2.0 - github.com/beyondstorage/go-storage/v4 v4.4.0 + github.com/beyondstorage/go-endpoint v1.1.0 + github.com/beyondstorage/go-integration-test/v4 v4.3.0 + github.com/beyondstorage/go-storage/v4 v4.4.1-0.20210730075750-6e541b87ea46 github.com/google/uuid v1.3.0 github.com/satori/go.uuid v1.2.0 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect diff --git a/go.sum b/go.sum index 1432f9f..3403337 100644 --- a/go.sum +++ b/go.sum @@ -4,13 +4,13 @@ github.com/aliyun/aliyun-oss-go-sdk v2.1.9+incompatible h1:mO8fA9l5cQ7r0D2v3Wrib github.com/aliyun/aliyun-oss-go-sdk v2.1.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= -github.com/beyondstorage/go-endpoint v1.0.1 h1:F8x2dGLMu9je6g7zPbKoxCXDlug97K26SeCx7KEHgyg= -github.com/beyondstorage/go-endpoint v1.0.1/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM= -github.com/beyondstorage/go-integration-test/v4 v4.2.0 h1:h2+SLmlDqjfBg+NzVcDr6VCmcD7I2xG+mqMzDlaCG+0= -github.com/beyondstorage/go-integration-test/v4 v4.2.0/go.mod h1:jLyYWSGUjQRH7U1HdaLbXE5sxBgqrtK73q+Q7PGIuSs= -github.com/beyondstorage/go-storage/v4 v4.3.0/go.mod h1:0fdcRCzLKMQe7Ve4zPlyTGgoPYwuINiV79Gx9tCt9tQ= -github.com/beyondstorage/go-storage/v4 v4.4.0 h1:sWURraKFjNR4qpwthr45cAGOIx6EOLrrJcz6su4Je30= +github.com/beyondstorage/go-endpoint v1.1.0 h1:cpjmQdrAMyaLoT161NIFU/eXcsuMI3xViycid5/mBZg= +github.com/beyondstorage/go-endpoint v1.1.0/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM= +github.com/beyondstorage/go-integration-test/v4 v4.3.0 h1:WZ95f78RKlHpvft8zHcMaoa2aaTF/jzlzINhMD0EMHY= +github.com/beyondstorage/go-integration-test/v4 v4.3.0/go.mod h1:HKgzemQZpxoHBL49JYEUnLTb5eteUhzcvmmPL7EDT/Y= github.com/beyondstorage/go-storage/v4 v4.4.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw= +github.com/beyondstorage/go-storage/v4 v4.4.1-0.20210730075750-6e541b87ea46 h1:sUmtn3cWgpjetIv/lSzZ6AA9GnzvxPjalTvRY1ZDzOg= +github.com/beyondstorage/go-storage/v4 v4.4.1-0.20210730075750-6e541b87ea46/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= @@ -20,11 +20,9 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -63,7 +61,6 @@ golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgm golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/service.toml b/service.toml index a36ea7f..b9d7f46 100644 --- a/service.toml +++ b/service.toml @@ -8,7 +8,7 @@ optional = ["service_features", "default_service_pairs", "endpoint", "http_clien [namespace.storage] features = ["virtual_dir"] -implement = ["appender", "direr", "multiparter"] +implement = ["appender", "direr", "multiparter", "linker"] [namespace.storage.new] required = ["name"] diff --git a/storage.go b/storage.go index 416a972..c6064e2 100644 --- a/storage.go +++ b/storage.go @@ -157,6 +157,25 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre return } +func (s *Storage) createLink(ctx context.Context, path string, target string, opt pairStorageCreateLink) (o *Object, err error) { + rt := s.getAbsPath(target) + rp := s.getAbsPath(path) + + o = s.newObject(true) + o.ID = rp + o.Path = path + o.SetLinkTarget("/" + rt) + o.Mode |= ModeLink + + // oss `symlink` supports `overwrite`, so we don't need to check if path exists. + err = s.bucket.PutSymlink(rp, rt) + if err != nil { + return nil, err + } + + return +} + func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStorageCreateMultipart) (o *Object, err error) { rp := s.getAbsPath(path) @@ -454,6 +473,20 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { rp := s.getAbsPath(path) + if symlink, err := s.bucket.GetSymlink(rp); err == nil { + // The path is a symlink. + o = s.newObject(true) + o.ID = rp + o.Path = path + + target := symlink.Get(oss.HTTPHeaderOssSymlinkTarget) + o.SetLinkTarget("/" + target) + + o.Mode |= ModeLink + + return o, nil + } + if opt.HasMultipartID { _, err = s.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{ Bucket: s.bucket.BucketName, diff --git a/tests/storage_test.go b/tests/storage_test.go index 0b0776c..ff0d883 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -34,3 +34,10 @@ func TestDirer(t *testing.T) { } tests.TestDirer(t, setupTest(t)) } + +func TestLinker(t *testing.T) { + if os.Getenv("STORAGE_OSS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_OSS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestLinker(t, setupTest(t)) +} diff --git a/utils.go b/utils.go index ff70c71..c264903 100644 --- a/utils.go +++ b/utils.go @@ -43,6 +43,7 @@ type Storage struct { typ.UnimplementedAppender typ.UnimplementedMultiparter typ.UnimplementedDirer + typ.UnimplementedLinker } // String implements Storager.String From 7df90d5666254ada8c0d88f61ef6eb5257849c22 Mon Sep 17 00:00:00 2001 From: abyss-w Date: Tue, 3 Aug 2021 15:43:58 +0800 Subject: [PATCH 2/4] Add some comments and change the order of API call --- storage.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/storage.go b/storage.go index c6064e2..3cbbc04 100644 --- a/storage.go +++ b/storage.go @@ -161,18 +161,20 @@ func (s *Storage) createLink(ctx context.Context, path string, target string, op rt := s.getAbsPath(target) rp := s.getAbsPath(path) - o = s.newObject(true) - o.ID = rp - o.Path = path - o.SetLinkTarget("/" + rt) - o.Mode |= ModeLink - // oss `symlink` supports `overwrite`, so we don't need to check if path exists. err = s.bucket.PutSymlink(rp, rt) if err != nil { return nil, err } + o = s.newObject(true) + o.ID = rp + o.Path = path + // oss does not have an absolute path, so when we call `getAbsPath`, it will remove the prefix `/`. + // To ensure that the path matches the one the user gets, we should re-add `/` here. + o.SetLinkTarget("/" + rt) + o.Mode |= ModeLink + return } From ca8f65773082d68a75b3341b0bcea852efd7bbc0 Mon Sep 17 00:00:00 2001 From: abyss-w Date: Wed, 4 Aug 2021 15:22:43 +0800 Subject: [PATCH 3/4] Add symlink check to the list --- utils.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils.go b/utils.go index c264903..0f629c8 100644 --- a/utils.go +++ b/utils.go @@ -255,7 +255,11 @@ func (s *Storage) formatFileObject(v oss.ObjectProperties) (o *typ.Object, err e o = s.newObject(false) o.ID = v.Key o.Path = s.getRelPath(v.Key) - o.Mode |= typ.ModeRead + if v.Type == "Symlink" { + o.Mode |= typ.ModeLink + } else { + o.Mode |= typ.ModeRead + } o.SetContentLength(v.Size) o.SetLastModified(v.LastModified) From c5460f54fc1531ae0b3b37bf5c51c9f8e934d71b Mon Sep 17 00:00:00 2001 From: abyss-w Date: Wed, 4 Aug 2021 16:21:34 +0800 Subject: [PATCH 4/4] Remove unneeded assignment --- utils.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils.go b/utils.go index 0f629c8..8c2e10e 100644 --- a/utils.go +++ b/utils.go @@ -264,10 +264,6 @@ func (s *Storage) formatFileObject(v oss.ObjectProperties) (o *typ.Object, err e o.SetContentLength(v.Size) o.SetLastModified(v.LastModified) - if v.Type != "" { - o.SetContentType(v.Type) - } - // OSS advise us don't use Etag as Content-MD5. // // ref: https://help.aliyun.com/document_detail/31965.html