diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1f6e16cfc..c7f78d5af 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -53,6 +53,10 @@ updates: schedule: interval: daily - package-ecosystem: gomod - directory: "/services/memory" - schedule: - interval: daily + directory: "/services/memory" + schedule: + interval: daily + - package-ecosystem: gomod + directory: "/services/minio" + schedule: + interval: daily diff --git a/.github/workflows/services-test-minio.yml b/.github/workflows/services-test-minio.yml new file mode 100644 index 000000000..67c40a41e --- /dev/null +++ b/.github/workflows/services-test-minio.yml @@ -0,0 +1,45 @@ +name: "Services Test Minio" + +on: + push: + paths: + - 'services/minio/**' + pull_request: + paths: + - 'services/minio/**' + +jobs: + integration_test: + name: "Integration Test" + runs-on: ubuntu-latest + + services: + minio: + image: wktk/minio-server + ports: + - 9000:9000 + env: + MINIO_ACCESS_KEY: "minioadmin" + MINIO_SECRET_KEY: "minioadmin" + + strategy: + matrix: + go: [ "1.15", "1.16" ] + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Test + env: + STORAGE_MINIO_INTEGRATION_TEST: "on" + STORAGE_MINIO_CREDENTIAL: "hmac:minioadmin:minioadmin" + STORAGE_MINIO_NAME: "test-bucket" + STORAGE_MINIO_ENDPOINT: "http:127.0.0.1:9000" + working-directory: services/minio + run: make integration_test diff --git a/services/minio/.gitignore b/services/minio/.gitignore new file mode 100644 index 000000000..022355ca0 --- /dev/null +++ b/services/minio/.gitignore @@ -0,0 +1,4 @@ +coverage.* +bin/ +.idea +Makefile.env diff --git a/services/minio/CHANGELOG.md b/services/minio/CHANGELOG.md new file mode 100644 index 000000000..188529fc0 --- /dev/null +++ b/services/minio/CHANGELOG.md @@ -0,0 +1,29 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/) +and this project adheres to [Semantic Versioning](https://semver.org/). + +## [v0.2.0] - 2021-09-13 + +### Changed + +- ci: Enable auto merge for Dependabot +- docs: Update README (#25) + +### Fixed + +- fix: Regenerate code + +### Upgraded + +- build(deps): Bump github.com/minio/minio-go/v7 from 7.0.12 to 7.0.13 (#23) + +## v0.1.0 - 2021-07-26 + +### Added + +- Implement MinIO Servicer, Storager, Copier, Reacher. + +[v0.2.0]: https://github.com/beyondstorage/go-service-minio/compare/v0.1.0...v0.2.0 diff --git a/services/minio/Makefile b/services/minio/Makefile new file mode 100644 index 000000000..f92dccd49 --- /dev/null +++ b/services/minio/Makefile @@ -0,0 +1,43 @@ +SHELL := /bin/bash + +-include Makefile.env + +.PHONY: all check format lint build test generate tidy + +help: + @echo "Please use \`make \` where is one of" + @echo " check to do static check" + @echo " build to create bin directory and build" + @echo " generate to generate code" + @echo " test to run test" + +check: vet + +format: + go fmt ./... + +vet: + go vet ./... + +generate: + @echo "generate code" + go generate ./... + go fmt ./... + +build: tidy generate check + go build ./... + +test: + go test -race -coverprofile=coverage.txt -covermode=atomic -v . + go tool cover -html="coverage.txt" -o "coverage.html" + +integration_test: + go test -race -count=1 -covermode=atomic -v ./tests + +tidy: + go mod tidy + go mod verify + +clean: + @echo "clean generated files" + find . -type f -name 'generated.go' -delete diff --git a/services/minio/Makefile.env.example b/services/minio/Makefile.env.example new file mode 100644 index 000000000..cd42c5d59 --- /dev/null +++ b/services/minio/Makefile.env.example @@ -0,0 +1,4 @@ +export STORAGE_MINIO_INTEGRATION_TEST=on +export STORAGE_MINIO_CREDENTIAL=hmac:access_key:secret_key +export STORAGE_MINIO_NAME=bucketname +export STORAGE_MINIO_ENDPOINT=http:host:port \ No newline at end of file diff --git a/services/minio/README.md b/services/minio/README.md new file mode 100644 index 000000000..d9ae2d7c0 --- /dev/null +++ b/services/minio/README.md @@ -0,0 +1,37 @@ +[![Services Test Minio](https://github.com/beyondstorage/go-storage/actions/workflows/services-test-minio.yml/badge.svg)](https://github.com/beyondstorage/go-storage/actions/workflows/services-test-minio.yml) + +# minio + +[MinIO](https://min.io/) is an open source cloud-native high-performance object storage service. +This project will use minio's native SDK to implement [go-storage](https://github.com/beyondstorage/go-storage/), +enabling users to manipulate data on minio servers through a unified interface. + +## Install + +```go +go get go.beyondstorage.io/services/minio +``` + +## Usage + +```go +import ( + "log" + + _ "go.beyondstorage.io/services/minio" + "go.beyondstorage.io/v5/services" +) + +func main() { + store, err := services.NewStoragerFromString("minio:///?credential=hmac::&endpoint=https::") + if err != nil { + log.Fatal(err) + } + + // Write data from io.Reader into hello.txt + n, err := store.Write("hello.txt", r, length) +} +``` + +- See more examples in [go-storage-example](https://github.com/beyondstorage/go-storage-example). +- Read [more docs](https://beyondstorage.io/docs/go-storage/services/minio) about go-service-minio. diff --git a/services/minio/doc.go b/services/minio/doc.go new file mode 100644 index 000000000..f1ea4ac83 --- /dev/null +++ b/services/minio/doc.go @@ -0,0 +1,6 @@ +/* +Package minio provided support for minio by go-storage. +*/ +package minio + +//go:generate go run -tags tools go.beyondstorage.io/v5/cmd/definitions service.toml diff --git a/services/minio/generated.go b/services/minio/generated.go new file mode 100644 index 000000000..a10181d12 --- /dev/null +++ b/services/minio/generated.go @@ -0,0 +1,902 @@ +// Code generated by go generate via cmd/definitions; DO NOT EDIT. +package minio + +import ( + "context" + "io" + "net/http" + "strings" + "time" + + . "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/httpclient" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +var ( + _ Storager + _ services.ServiceError + _ httpclient.Options + _ time.Duration + _ http.Request + _ Error +) + +// Type is the type for minio +const Type = "minio" + +// ObjectSystemMetadata stores system metadata for object. +type ObjectSystemMetadata struct { + StorageClass string +} + +// GetObjectSystemMetadata will get ObjectSystemMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectServiceMetadata is read only and should not be modified. +func GetObjectSystemMetadata(o *Object) ObjectSystemMetadata { + sm, ok := o.GetSystemMetadata() + if ok { + return sm.(ObjectSystemMetadata) + } + return ObjectSystemMetadata{} +} + +// setObjectSystemMetadata will set ObjectSystemMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectSystemMetadata(o *Object, sm ObjectSystemMetadata) { + o.SetSystemMetadata(sm) +} + +// StorageSystemMetadata stores system metadata for object. +type StorageSystemMetadata struct { + StorageClass string +} + +// GetStorageSystemMetadata will get StorageSystemMetadata from Storage. +// +// - This function should not be called by service implementer. +// - The returning StorageServiceMetadata is read only and should not be modified. +func GetStorageSystemMetadata(s *StorageMeta) StorageSystemMetadata { + sm, ok := s.GetSystemMetadata() + if ok { + return sm.(StorageSystemMetadata) + } + return StorageSystemMetadata{} +} + +// setStorageSystemMetadata will set StorageSystemMetadata into Storage. +// +// - This function should only be called once, please make sure all data has been written before set. +func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) { + s.SetSystemMetadata(sm) +} + +// WithDefaultServicePairs will apply default_service_pairs value to Options. +func WithDefaultServicePairs(v DefaultServicePairs) Pair { + return Pair{Key: "default_service_pairs", Value: v} +} + +// WithDefaultStoragePairs will apply default_storage_pairs value to Options. +func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { + return Pair{Key: "default_storage_pairs", Value: v} +} + +// WithEnableVirtualDir will apply enable_virtual_dir value to Options. +// +// virtual_dir feature is designed for a service that doesn't have native dir support but wants to +// provide simulated operations. +// +// - If this feature is disabled (the default behavior), the service will behave like it doesn't have +// any dir support. +// - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, +// list, delete, and so on. +// +// This feature was introduced in GSP-109. +func WithEnableVirtualDir() Pair { + return Pair{Key: "enable_virtual_dir", Value: true} +} + +// WithServiceFeatures will apply service_features value to Options. +func WithServiceFeatures(v ServiceFeatures) Pair { + return Pair{Key: "service_features", Value: v} +} + +// WithStorageClass will apply storage_class value to Options. +func WithStorageClass(v string) Pair { + return Pair{Key: "storage_class", Value: v} +} + +// WithStorageFeatures will apply storage_features value to Options. +func WithStorageFeatures(v StorageFeatures) Pair { + return Pair{Key: "storage_features", Value: v} +} + +var pairMap = map[string]string{"content_md5": "string", "content_type": "string", "context": "context.Context", "continuation_token": "string", "credential": "string", "default_content_type": "string", "default_io_callback": "func([]byte)", "default_service_pairs": "DefaultServicePairs", "default_storage_pairs": "DefaultStoragePairs", "enable_virtual_dir": "bool", "endpoint": "string", "expire": "time.Duration", "http_client_options": "*httpclient.Options", "interceptor": "Interceptor", "io_callback": "func([]byte)", "list_mode": "ListMode", "location": "string", "multipart_id": "string", "name": "string", "object_mode": "ObjectMode", "offset": "int64", "service_features": "ServiceFeatures", "size": "int64", "storage_class": "string", "storage_features": "StorageFeatures", "work_dir": "string"} +var _ Servicer = &Service{} + +type ServiceFeatures struct { +} + +// pairServiceNew is the parsed struct +type pairServiceNew struct { + pairs []Pair + + // Required pairs + HasCredential bool + Credential string + HasEndpoint bool + Endpoint string + // Optional pairs + HasDefaultServicePairs bool + DefaultServicePairs DefaultServicePairs + HasServiceFeatures bool + ServiceFeatures ServiceFeatures + // Enable features +} + +// parsePairServiceNew will parse Pair slice into *pairServiceNew +func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { + result := + pairServiceNew{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "credential": + if result.HasCredential { + continue + } + result.HasCredential = true + result.Credential = v.Value.(string) + case "endpoint": + if result.HasEndpoint { + continue + } + result.HasEndpoint = true + result.Endpoint = v.Value.(string) + case "default_service_pairs": + if result.HasDefaultServicePairs { + continue + } + result.HasDefaultServicePairs = true + result.DefaultServicePairs = v.Value.(DefaultServicePairs) + case "service_features": + if result.HasServiceFeatures { + continue + } + result.HasServiceFeatures = true + result.ServiceFeatures = v.Value.(ServiceFeatures) + } + } + // Enable features + + // Default pairs + + if !result.HasCredential { + return pairServiceNew{}, services.PairRequiredError{Keys: []string{"credential"}} + } + if !result.HasEndpoint { + return pairServiceNew{}, services.PairRequiredError{Keys: []string{"endpoint"}} + } + return result, nil +} + +// DefaultServicePairs is default pairs for specific action +type DefaultServicePairs struct { + Create []Pair + Delete []Pair + Get []Pair + List []Pair +} +type pairServiceCreate struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Service) parsePairServiceCreate(opts []Pair) (pairServiceCreate, error) { + result := + pairServiceCreate{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairServiceDelete struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Service) parsePairServiceDelete(opts []Pair) (pairServiceDelete, error) { + result := + pairServiceDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairServiceGet struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Service) parsePairServiceGet(opts []Pair) (pairServiceGet, error) { + result := + pairServiceGet{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairServiceGet{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairServiceList struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Service) parsePairServiceList(opts []Pair) (pairServiceList, error) { + result := + pairServiceList{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairServiceList{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} +func (s *Service) Create(name string, pairs ...Pair) (store Storager, err error) { + ctx := context.Background() + return s.CreateWithContext(ctx, name, pairs...) +} +func (s *Service) CreateWithContext(ctx context.Context, name string, pairs ...Pair) (store Storager, err error) { + defer func() { + err = + s.formatError("create", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Create...) + var opt pairServiceCreate + + opt, err = s.parsePairServiceCreate(pairs) + if err != nil { + return + } + return s.create(ctx, name, opt) +} +func (s *Service) Delete(name string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.DeleteWithContext(ctx, name, pairs...) +} +func (s *Service) DeleteWithContext(ctx context.Context, name string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("delete", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Delete...) + var opt pairServiceDelete + + opt, err = s.parsePairServiceDelete(pairs) + if err != nil { + return + } + return s.delete(ctx, name, opt) +} +func (s *Service) Get(name string, pairs ...Pair) (store Storager, err error) { + ctx := context.Background() + return s.GetWithContext(ctx, name, pairs...) +} +func (s *Service) GetWithContext(ctx context.Context, name string, pairs ...Pair) (store Storager, err error) { + defer func() { + err = + s.formatError("get", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Get...) + var opt pairServiceGet + + opt, err = s.parsePairServiceGet(pairs) + if err != nil { + return + } + return s.get(ctx, name, opt) +} +func (s *Service) List(pairs ...Pair) (sti *StoragerIterator, err error) { + ctx := context.Background() + return s.ListWithContext(ctx, pairs...) +} +func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *StoragerIterator, err error) { + defer func() { + err = + s.formatError("list", err, "") + }() + + pairs = append(pairs, s.defaultPairs.List...) + var opt pairServiceList + + opt, err = s.parsePairServiceList(pairs) + if err != nil { + return + } + return s.list(ctx, opt) +} + +var ( + _ Copier = &Storage{} + _ Reacher = &Storage{} + _ Storager = &Storage{} +) + +type StorageFeatures struct { // virtual_dir feature is designed for a service that doesn't have native dir support but wants to + // provide simulated operations. + // + // - If this feature is disabled (the default behavior), the service will behave like it doesn't have + // any dir support. + // - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, + // list, delete, and so on. + // + // This feature was introduced in GSP-109. + VirtualDir bool +} + +// pairStorageNew is the parsed struct +type pairStorageNew struct { + pairs []Pair + + // Required pairs + HasName bool + Name string + // Optional pairs + HasDefaultContentType bool + DefaultContentType string + HasDefaultIoCallback bool + DefaultIoCallback func([]byte) + HasDefaultStoragePairs bool + DefaultStoragePairs DefaultStoragePairs + HasStorageFeatures bool + StorageFeatures StorageFeatures + HasWorkDir bool + WorkDir string + // Enable features + hasEnableVirtualDir bool + EnableVirtualDir bool +} + +// parsePairStorageNew will parse Pair slice into *pairStorageNew +func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { + result := + pairStorageNew{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "name": + if result.HasName { + continue + } + result.HasName = true + result.Name = v.Value.(string) + case "default_content_type": + if result.HasDefaultContentType { + continue + } + result.HasDefaultContentType = true + result.DefaultContentType = v.Value.(string) + case "default_io_callback": + if result.HasDefaultIoCallback { + continue + } + result.HasDefaultIoCallback = true + result.DefaultIoCallback = v.Value.(func([]byte)) + case "default_storage_pairs": + if result.HasDefaultStoragePairs { + continue + } + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs = v.Value.(DefaultStoragePairs) + case "storage_features": + if result.HasStorageFeatures { + continue + } + result.HasStorageFeatures = true + result.StorageFeatures = v.Value.(StorageFeatures) + case "work_dir": + if result.HasWorkDir { + continue + } + result.HasWorkDir = true + result.WorkDir = v.Value.(string) + case "enable_virtual_dir": + if result.hasEnableVirtualDir { + continue + } + result.hasEnableVirtualDir = true + result.EnableVirtualDir = true + } + } + // Enable features + if result.hasEnableVirtualDir { + result.HasStorageFeatures = true + result.StorageFeatures.VirtualDir = true + } + // Default pairs + if result.HasDefaultContentType { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithContentType(result.DefaultContentType)) + } + if result.HasDefaultIoCallback { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.Read = append(result.DefaultStoragePairs.Read, WithIoCallback(result.DefaultIoCallback)) + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithIoCallback(result.DefaultIoCallback)) + } + if !result.HasName { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"name"}} + } + return result, nil +} + +// DefaultStoragePairs is default pairs for specific action +type DefaultStoragePairs struct { + Copy []Pair + Create []Pair + Delete []Pair + List []Pair + Metadata []Pair + Reach []Pair + Read []Pair + Stat []Pair + Write []Pair +} +type pairStorageCopy struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageCopy(opts []Pair) (pairStorageCopy, error) { + result := + pairStorageCopy{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCopy{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreate struct { + pairs []Pair + // Required pairs + // Optional pairs + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) { + result := + pairStorageCreate{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageDelete struct { + pairs []Pair + // Required pairs + // Optional pairs + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) { + result := + pairStorageDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageList struct { + pairs []Pair + // Required pairs + // Optional pairs + HasListMode bool + ListMode ListMode +} + +func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { + result := + pairStorageList{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "list_mode": + if result.HasListMode { + continue + } + result.HasListMode = true + result.ListMode = v.Value.(ListMode) + default: + return pairStorageList{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageMetadata struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageMetadata(opts []Pair) (pairStorageMetadata, error) { + result := + pairStorageMetadata{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageReach struct { + pairs []Pair + // Required pairs + // Optional pairs + HasExpire bool + Expire time.Duration +} + +func (s *Storage) parsePairStorageReach(opts []Pair) (pairStorageReach, error) { + result := + pairStorageReach{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "expire": + if result.HasExpire { + continue + } + result.HasExpire = true + result.Expire = v.Value.(time.Duration) + default: + return pairStorageReach{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageRead struct { + pairs []Pair + // Required pairs + // Optional pairs + HasIoCallback bool + IoCallback func([]byte) + HasOffset bool + Offset int64 + HasSize bool + Size int64 +} + +func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { + result := + pairStorageRead{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + case "offset": + if result.HasOffset { + continue + } + result.HasOffset = true + result.Offset = v.Value.(int64) + case "size": + if result.HasSize { + continue + } + result.HasSize = true + result.Size = v.Value.(int64) + default: + return pairStorageRead{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageStat struct { + pairs []Pair + // Required pairs + // Optional pairs + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { + result := + pairStorageStat{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageStat{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageWrite struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasIoCallback bool + IoCallback func([]byte) + HasStorageClass bool + StorageClass string +} + +func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { + result := + pairStorageWrite{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_md5": + if result.HasContentMd5 { + continue + } + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + case "content_type": + if result.HasContentType { + continue + } + result.HasContentType = true + result.ContentType = v.Value.(string) + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + default: + return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} +func (s *Storage) Copy(src string, dst string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.CopyWithContext(ctx, src, dst, pairs...) +} +func (s *Storage) CopyWithContext(ctx context.Context, src string, dst string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("copy", err, src, dst) + }() + + pairs = append(pairs, s.defaultPairs.Copy...) + var opt pairStorageCopy + + opt, err = s.parsePairStorageCopy(pairs) + if err != nil { + return + } + return s.copy(ctx, strings.ReplaceAll(src, "\\", "/"), strings.ReplaceAll(dst, "\\", "/"), opt) +} +func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { + pairs = append(pairs, s.defaultPairs.Create...) + var opt pairStorageCreate + + // Ignore error while handling local functions. + opt, _ = s.parsePairStorageCreate(pairs) + return s.create(path, opt) +} +func (s *Storage) Delete(path string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.DeleteWithContext(ctx, path, pairs...) +} +func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("delete", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Delete...) + var opt pairStorageDelete + + opt, err = s.parsePairStorageDelete(pairs) + if err != nil { + return + } + return s.delete(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) List(path string, pairs ...Pair) (oi *ObjectIterator, err error) { + ctx := context.Background() + return s.ListWithContext(ctx, path, pairs...) +} +func (s *Storage) ListWithContext(ctx context.Context, path string, pairs ...Pair) (oi *ObjectIterator, err error) { + defer func() { + err = + s.formatError("list", err, path) + }() + + pairs = append(pairs, s.defaultPairs.List...) + var opt pairStorageList + + opt, err = s.parsePairStorageList(pairs) + if err != nil { + return + } + return s.list(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Metadata(pairs ...Pair) (meta *StorageMeta) { + pairs = append(pairs, s.defaultPairs.Metadata...) + var opt pairStorageMetadata + + // Ignore error while handling local functions. + opt, _ = s.parsePairStorageMetadata(pairs) + return s.metadata(opt) +} +func (s *Storage) Reach(path string, pairs ...Pair) (url string, err error) { + ctx := context.Background() + return s.ReachWithContext(ctx, path, pairs...) +} +func (s *Storage) ReachWithContext(ctx context.Context, path string, pairs ...Pair) (url string, err error) { + defer func() { + err = + s.formatError("reach", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Reach...) + var opt pairStorageReach + + opt, err = s.parsePairStorageReach(pairs) + if err != nil { + return + } + return s.reach(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Read(path string, w io.Writer, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.ReadWithContext(ctx, path, w, pairs...) +} +func (s *Storage) ReadWithContext(ctx context.Context, path string, w io.Writer, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("read", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Read...) + var opt pairStorageRead + + opt, err = s.parsePairStorageRead(pairs) + if err != nil { + return + } + return s.read(ctx, strings.ReplaceAll(path, "\\", "/"), w, opt) +} +func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.StatWithContext(ctx, path, pairs...) +} +func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("stat", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Stat...) + var opt pairStorageStat + + opt, err = s.parsePairStorageStat(pairs) + if err != nil { + return + } + return s.stat(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Write(path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteWithContext(ctx, path, r, size, pairs...) +} +func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("write", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Write...) + var opt pairStorageWrite + + opt, err = s.parsePairStorageWrite(pairs) + if err != nil { + return + } + return s.write(ctx, strings.ReplaceAll(path, "\\", "/"), r, size, opt) +} +func init() { + services.RegisterServicer(Type, NewServicer) + services.RegisterStorager(Type, NewStorager) + services.RegisterSchema(Type, pairMap) +} diff --git a/services/minio/go.mod b/services/minio/go.mod new file mode 100644 index 000000000..99b7a5d92 --- /dev/null +++ b/services/minio/go.mod @@ -0,0 +1,11 @@ +module go.beyondstorage.io/services/minio + +go 1.15 + +require ( + github.com/google/uuid v1.3.0 + github.com/minio/minio-go/v7 v7.0.14 + go.beyondstorage.io/credential v1.0.0 + go.beyondstorage.io/endpoint v1.2.0 + go.beyondstorage.io/v5 v5.0.0 +) diff --git a/services/minio/go.sum b/services/minio/go.sum new file mode 100644 index 000000000..32ba10d24 --- /dev/null +++ b/services/minio/go.sum @@ -0,0 +1,150 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Xuanwo/gg v0.2.0 h1:axbZmA0qmidh3s9PA86GqvBXVQ3o7Bbpf0aImGtlimA= +github.com/Xuanwo/gg v0.2.0/go.mod h1:0fLiiSxR87u2UA0ZNZiKZXuz3jnJdbDHWtU2xpdcH3s= +github.com/Xuanwo/go-bufferpool v0.2.0 h1:DXzqJD9lJufXbT/03GrcEvYOs4gXYUj9/g5yi6Q9rUw= +github.com/Xuanwo/go-bufferpool v0.2.0/go.mod h1:Mle++9GGouhOwGj52i9PJLNAPmW2nb8PWBP7JJzNCzk= +github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= +github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= +github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.14 h1:T7cw8P586gVwEEd0y21kTYtloD576XZgP62N8pE130s= +github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.6 h1:lH+Snxmzl92r1jww8/jYPqKkhs3C9AF4LunzU56ZZr4= +github.com/smartystreets/goconvey v1.6.6/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.beyondstorage.io/credential v1.0.0 h1:xJ7hBXmeUE0+rbW+RYZSz4KgHpXvc9g7oQ56f8dXdBk= +go.beyondstorage.io/credential v1.0.0/go.mod h1:7KAYievVw4a8u/eLZmnQt65Z91n84sMQj3LFbt8Xous= +go.beyondstorage.io/endpoint v1.2.0 h1:/7mgKquTykeqJ9op82hso2+WQfECeywGd/Lda1N3tF4= +go.beyondstorage.io/endpoint v1.2.0/go.mod h1:oZ7Z7HZ7mAo337JBLjuCF/DM66HVEUu6+hw68c3UcLs= +go.beyondstorage.io/v5 v5.0.0 h1:k9Axfgbt+oZXoDwSBVCl1XANHSL4rkNTGP2Lz9YdJe0= +go.beyondstorage.io/v5 v5.0.0/go.mod h1:3wV9gCQnqu7tD/3LMeo2yimUKIeTSHpTc6wHSb0yY20= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/minio/iterator.go b/services/minio/iterator.go new file mode 100644 index 000000000..768822f35 --- /dev/null +++ b/services/minio/iterator.go @@ -0,0 +1,27 @@ +package minio + +import ( + "strconv" + + "github.com/minio/minio-go/v7" +) + +type storagePageStatus struct { + buckets []minio.BucketInfo +} + +func (i *storagePageStatus) ContinuationToken() string { + return "" +} + +type objectPageStatus struct { + bufferSize int + counter int + options minio.ListObjectsOptions + + objChan <-chan minio.ObjectInfo +} + +func (i *objectPageStatus) ContinuationToken() string { + return strconv.Itoa(i.counter) +} diff --git a/services/minio/service.go b/services/minio/service.go new file mode 100644 index 000000000..3cf931bec --- /dev/null +++ b/services/minio/service.go @@ -0,0 +1,59 @@ +package minio + +import ( + "context" + + "github.com/minio/minio-go/v7" + + ps "go.beyondstorage.io/v5/pairs" + . "go.beyondstorage.io/v5/types" +) + +func (s *Service) create(ctx context.Context, name string, opt pairServiceCreate) (store Storager, err error) { + st, err := s.newStorage(ps.WithName(name)) + if err != nil { + return nil, err + } + err = s.service.MakeBucket(ctx, name, minio.MakeBucketOptions{}) + if err != nil { + return nil, err + } + return st, nil +} + +func (s *Service) delete(ctx context.Context, name string, opt pairServiceDelete) (err error) { + err = s.service.RemoveBucket(ctx, name) + if err != nil { + return err + } + return nil +} + +func (s *Service) get(ctx context.Context, name string, opt pairServiceGet) (store Storager, err error) { + st, err := s.newStorage(ps.WithName(name)) + if err != nil { + return nil, err + } + return st, nil +} + +func (s *Service) list(ctx context.Context, opt pairServiceList) (sti *StoragerIterator, err error) { + input := &storagePageStatus{} + return NewStoragerIterator(ctx, s.nextStoragePage, input), nil +} + +func (s *Service) nextStoragePage(ctx context.Context, page *StoragerPage) (err error) { + input := page.Status.(*storagePageStatus) + input.buckets, err = s.service.ListBuckets(ctx) + if err != nil { + return err + } + for _, v := range input.buckets { + store, err := s.newStorage(ps.WithName(v.Name)) + if err != nil { + return err + } + page.Data = append(page.Data, store) + } + return IterateDone +} diff --git a/services/minio/service.toml b/services/minio/service.toml new file mode 100644 index 000000000..2a6a93c7c --- /dev/null +++ b/services/minio/service.toml @@ -0,0 +1,41 @@ +name = "minio" + +[namespace.service] + +[namespace.service.new] +required = ["credential", "endpoint"] + +[namespace.storage] +implement = ["copier", "reacher"] +features = ["virtual_dir"] + +[namespace.storage.new] +required = ["name"] +optional = ["work_dir"] + +[namespace.storage.op.create] +optional = ["object_mode"] + +[namespace.storage.op.delete] +optional = ["object_mode"] + +[namespace.storage.op.list] +optional = ["list_mode"] + +[namespace.storage.op.read] +optional = ["offset", "io_callback", "size"] + +[namespace.storage.op.stat] +optional = ["object_mode"] + +[namespace.storage.op.write] +optional = ["content_md5", "content_type", "io_callback", "storage_class"] + +[namespace.storage.op.reach] +optional = ["expire"] + +[pairs.storage_class] +type = "string" + +[infos.object.meta.storage-class] +type = "string" \ No newline at end of file diff --git a/services/minio/storage.go b/services/minio/storage.go new file mode 100644 index 000000000..c4aeb7dbd --- /dev/null +++ b/services/minio/storage.go @@ -0,0 +1,202 @@ +package minio + +import ( + "context" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/iowrap" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +const defaultListObjectBufferSize = 100 + +func (s *Storage) copy(ctx context.Context, src string, dst string, opt pairStorageCopy) (err error) { + srcOpts := minio.CopySrcOptions{ + Bucket: s.bucket, + Object: s.getAbsPath(src), + } + dstOpts := minio.CopyDestOptions{ + Bucket: s.bucket, + Object: s.getAbsPath(dst), + } + _, err = s.client.CopyObject(ctx, dstOpts, srcOpts) + return err +} + +func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { + rp := s.getAbsPath(path) + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + return + } + rp += "/" + o = s.newObject(true) + o.Mode = ModeDir + } else { + o = s.newObject(false) + o.Mode = ModeRead + } + o.ID = rp + o.Path = path + return o +} + +func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { + rp := s.getAbsPath(path) + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + rp += "/" + } + err = s.client.RemoveObject(ctx, s.bucket, rp, minio.RemoveObjectOptions{}) + return err +} + +func (s *Storage) list(ctx context.Context, path string, opt pairStorageList) (oi *ObjectIterator, err error) { + rp := s.getAbsPath(path) + options := minio.ListObjectsOptions{} + if !opt.HasListMode || opt.ListMode.IsPrefix() { + options.Recursive = true + } else if opt.ListMode.IsDir() { + if !strings.HasSuffix(rp, "/") { + rp += "/" + } + } else { + return nil, services.ListModeInvalidError{Actual: opt.ListMode} + } + options.Prefix = rp + input := &objectPageStatus{ + bufferSize: defaultListObjectBufferSize, + options: options, + } + return NewObjectIterator(ctx, s.nextObjectPage, input), nil +} + +func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { + meta = NewStorageMeta() + meta.Name = s.bucket + meta.WorkDir = s.workDir + return meta +} + +func (s *Storage) nextObjectPage(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + if input.objChan == nil { + input.objChan = s.client.ListObjects(ctx, s.bucket, input.options) + } + for i := 0; i < input.bufferSize; i++ { + v, ok := <-input.objChan + if !ok { + return IterateDone + } + if v.Err != nil { + return v.Err + } + o, err := s.formatFileObject(v) + if err != nil { + return err + } + page.Data = append(page.Data, o) + input.counter++ + } + return nil +} + +func (s *Storage) reach(ctx context.Context, path string, opt pairStorageReach) (url_ string, err error) { + rp := s.getAbsPath(path) + var expire = time.Hour * 1 + if opt.HasExpire { + expire = opt.Expire + } + URL, err := s.client.PresignedGetObject(ctx, s.bucket, rp, expire, url.Values{}) + if err != nil { + return "", err + } + return URL.String(), nil +} + +func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairStorageRead) (n int64, err error) { + rp := s.getAbsPath(path) + output, err := s.client.GetObject(ctx, s.bucket, rp, minio.GetObjectOptions{}) + if err != nil { + return 0, err + } + defer func() { + cerr := output.Close() + if cerr != nil { + err = cerr + } + }() + if opt.HasOffset { + _, err = output.Seek(opt.Offset, 0) + if err != nil { + return 0, err + } + } + var rc io.ReadCloser = output + if opt.HasSize { + rc = iowrap.LimitReadCloser(rc, opt.Size) + } + if opt.HasIoCallback { + rc = iowrap.CallbackReadCloser(rc, opt.IoCallback) + } + return io.Copy(w, rc) +} + +func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { + rp := s.getAbsPath(path) + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + rp += "/" + } + output, err := s.client.StatObject(ctx, s.bucket, rp, minio.StatObjectOptions{}) + if err != nil { + return nil, err + } + o, err = s.formatFileObject(output) + if err != nil { + return nil, err + } + // Object.Path is either the absolute path or the relative path based on the working directory depends on the input. + o.Path = path + return +} + +func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { + // According to GSP-751, we should allow the user to pass in a nil io.Reader. + // ref: https://github.com/beyondstorage/go-storage/blob/master/docs/rfcs/751-write-empty-file-behavior.md + if r == nil && size != 0 { + return 0, fmt.Errorf("reader is nil but size is not 0") + } + + rp := s.getAbsPath(path) + r = io.LimitReader(r, size) + if opt.HasIoCallback { + r = iowrap.CallbackReader(r, opt.IoCallback) + } + options := minio.PutObjectOptions{} + if opt.HasContentType { + options.ContentType = opt.ContentType + } + if opt.HasStorageClass { + options.StorageClass = opt.StorageClass + } + _, err = s.client.PutObject(ctx, s.bucket, rp, r, size, options) + if err != nil { + return 0, err + } + return size, nil +} diff --git a/services/minio/tests/README.md b/services/minio/tests/README.md new file mode 100644 index 000000000..405bce421 --- /dev/null +++ b/services/minio/tests/README.md @@ -0,0 +1,32 @@ +## How run integration tests + +### Run tests locally + +Copy example files and update corresponding values. + +```shell +cp Makefile.env.exmaple Makefile.env +``` + +Run tests + +```shell +make integration_test +``` + +### Run tests in CI + +Set following environment variables: + +```shell +export STORAGE_MINIO_INTEGRATION_TEST=on +export STORAGE_MINIO_CREDENTIAL=hmac:access_key:secret_key +export STORAGE_MINIO_NAME=bucketname +export STORAGE_MINIO_ENDPOINT=http:host:port +``` + +Run tests + +```shell +make integration_test +``` \ No newline at end of file diff --git a/services/minio/tests/storage_test.go b/services/minio/tests/storage_test.go new file mode 100644 index 000000000..a76a5a96e --- /dev/null +++ b/services/minio/tests/storage_test.go @@ -0,0 +1,22 @@ +package tests + +import ( + "os" + "testing" + + "go.beyondstorage.io/v5/tests" +) + +func TestStorage(t *testing.T) { + if os.Getenv("STORAGE_MINIO_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_MINIO_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestStorager(t, setupTest(t)) +} + +func TestCopier(t *testing.T) { + if os.Getenv("STORAGE_MINIO_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_MINIO_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestCopier(t, setupTest(t)) +} diff --git a/services/minio/tests/utils_test.go b/services/minio/tests/utils_test.go new file mode 100644 index 000000000..a05f5f603 --- /dev/null +++ b/services/minio/tests/utils_test.go @@ -0,0 +1,54 @@ +package tests + +import ( + "os" + "testing" + + "github.com/google/uuid" + + minio "go.beyondstorage.io/services/minio" + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/types" +) + +func setupTest(t *testing.T) types.Storager { + t.Log("Setup test for minio") + + srv, err := minio.NewServicer( + ps.WithCredential(os.Getenv("STORAGE_MINIO_CREDENTIAL")), + ps.WithEndpoint(os.Getenv("STORAGE_MINIO_ENDPOINT")), + ) + if err != nil { + t.Errorf("new servicer: %v", err) + } + + bucketName := os.Getenv("STORAGE_MINIO_NAME") + + _, err = srv.Create(bucketName) + if err != nil { + t.Errorf("create storager: %v", err) + } + + store, err := minio.NewStorager( + ps.WithCredential(os.Getenv("STORAGE_MINIO_CREDENTIAL")), + ps.WithEndpoint(os.Getenv("STORAGE_MINIO_ENDPOINT")), + ps.WithName(bucketName), + ps.WithWorkDir("/"+uuid.New().String()), + ) + if err != nil { + t.Errorf("new storager: %v", err) + } + + t.Cleanup(func() { + err = store.Delete("") + if err != nil { + t.Errorf("cleanup: %v", err) + } + + err = srv.Delete(bucketName) + if err != nil { + t.Errorf("cleanup: %v", err) + } + }) + return store +} diff --git a/services/minio/tools.go b/services/minio/tools.go new file mode 100644 index 000000000..a61584d72 --- /dev/null +++ b/services/minio/tools.go @@ -0,0 +1,8 @@ +//go:build tools +// +build tools + +package minio + +import ( + _ "go.beyondstorage.io/v5/cmd/definitions" +) diff --git a/services/minio/utils.go b/services/minio/utils.go new file mode 100644 index 000000000..8d5c21eac --- /dev/null +++ b/services/minio/utils.go @@ -0,0 +1,264 @@ +package minio + +import ( + "fmt" + "net/http" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "go.beyondstorage.io/credential" + "go.beyondstorage.io/endpoint" + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/services" + "go.beyondstorage.io/v5/types" +) + +// Service is the minio service. +type Service struct { + service *minio.Client + + defaultPairs DefaultServicePairs + features ServiceFeatures + + types.UnimplementedServicer +} + +func (s *Service) String() string { + return fmt.Sprintf("Servicer minio") +} + +// Storage is the example client. +type Storage struct { + client *minio.Client + + bucket string + workDir string + + defaultPairs DefaultStoragePairs + features StorageFeatures + + types.UnimplementedStorager + types.UnimplementedCopier + types.UnimplementedReacher +} + +// String implements Storager.String +func (s *Storage) String() string { + return fmt.Sprintf( + "Storager minio {Name: %s, WorkDir: %s}", + s.bucket, s.workDir, + ) +} + +func New(pairs ...types.Pair) (types.Servicer, types.Storager, error) { + return newServicerAndStorager(pairs...) +} + +func NewServicer(pairs ...types.Pair) (types.Servicer, error) { + return newServicer(pairs...) +} + +// NewStorager will create Storager only. +func NewStorager(pairs ...types.Pair) (types.Storager, error) { + _, store, err := newServicerAndStorager(pairs...) + return store, err +} + +func newServicer(pairs ...types.Pair) (srv *Service, err error) { + defer func() { + if err != nil { + err = services.InitError{Op: "new_servicer", Type: Type, Err: formatError(err), Pairs: pairs} + } + }() + + srv = &Service{} + + opt, err := parsePairServiceNew(pairs) + if err != nil { + return nil, err + } + + cp, err := credential.Parse(opt.Credential) + if err != nil { + return nil, err + } + if cp.Protocol() != credential.ProtocolHmac { + return nil, services.PairUnsupportedError{Pair: ps.WithCredential(opt.Credential)} + } + ak, sk := cp.Hmac() + + ep, err := endpoint.Parse(opt.Endpoint) + if err != nil { + return nil, err + } + + var host string + var port int + var secure bool + switch ep.Protocol() { + case endpoint.ProtocolHTTP: + _, host, port = ep.HTTP() + secure = false + case endpoint.ProtocolHTTPS: + _, host, port = ep.HTTPS() + secure = true + default: + return nil, services.PairUnsupportedError{Pair: ps.WithEndpoint(opt.Endpoint)} + } + url := fmt.Sprintf("%s:%d", host, port) + + srv.service, err = minio.New(url, &minio.Options{ + Creds: credentials.NewStaticV4(ak, sk, ""), + Secure: secure, + }) + if err != nil { + return nil, err + } + + if opt.HasDefaultServicePairs { + srv.defaultPairs = opt.DefaultServicePairs + } + if opt.HasServiceFeatures { + srv.features = opt.ServiceFeatures + } + + return +} + +func newServicerAndStorager(pairs ...types.Pair) (srv *Service, store *Storage, err error) { + srv, err = newServicer(pairs...) + if err != nil { + return + } + + store, err = srv.newStorage(pairs...) + if err != nil { + err = services.InitError{Op: "new_storager", Type: Type, Err: formatError(err), Pairs: pairs} + return nil, nil, err + } + return srv, store, nil +} + +func formatError(err error) error { + if _, ok := err.(services.InternalError); ok { + return err + } + + e, ok := err.(minio.ErrorResponse) + if ok { + switch e.Code { + case "AccessDenied": + return fmt.Errorf("%w, %v", services.ErrPermissionDenied, err) + case "NoSuchKey": + return fmt.Errorf("%w, %v", services.ErrObjectNotExist, err) + case "InternalError": + return fmt.Errorf("%w, %v", services.ErrServiceInternal, err) + } + + switch e.StatusCode { + case http.StatusTooManyRequests: + return fmt.Errorf("%w, %v", services.ErrRequestThrottled, err) + case http.StatusServiceUnavailable: + return fmt.Errorf("%w, %v", services.ErrRequestThrottled, err) + } + } + + return fmt.Errorf("%w, %v", services.ErrUnexpected, err) +} + +func (s *Service) newStorage(pairs ...types.Pair) (st *Storage, err error) { + opt, err := parsePairStorageNew(pairs) + if err != nil { + return nil, err + } + + store := &Storage{ + client: s.service, + bucket: opt.Name, + workDir: "/", + } + + if opt.HasWorkDir { + if !strings.HasSuffix(opt.WorkDir, "/") { + opt.WorkDir += "/" + } + store.workDir = opt.WorkDir + } + if opt.HasDefaultStoragePairs { + store.defaultPairs = opt.DefaultStoragePairs + } + if opt.HasStorageFeatures { + store.features = opt.StorageFeatures + } + + return store, nil +} + +func (s *Service) formatError(op string, err error, name string) error { + if err == nil { + return nil + } + + return services.ServiceError{ + Op: op, + Err: formatError(err), + Servicer: s, + Name: name, + } +} + +func (s *Storage) formatError(op string, err error, path ...string) error { + if err == nil { + return nil + } + + return services.StorageError{ + Op: op, + Err: formatError(err), + Storager: s, + Path: path, + } +} + +// getAbsPath will calculate object storage's abs path +func (s *Storage) getAbsPath(path string) string { + if strings.HasPrefix(path, s.workDir) { + return strings.TrimPrefix(path, "/") + } + prefix := strings.TrimPrefix(s.workDir, "/") + return prefix + path +} + +// getRelPath will get object storage's rel path. +func (s *Storage) getRelPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return strings.TrimPrefix(path, prefix) +} + +func (s *Storage) formatFileObject(v minio.ObjectInfo) (o *types.Object, err error) { + o = s.newObject(true) + if v.ETag == "" { + o.Mode |= types.ModeDir + } else { + o.Mode |= types.ModeRead + } + + o.SetID(v.Key) + o.SetPath(s.getRelPath(v.Key)) + o.SetEtag(v.ETag) + o.SetContentLength(v.Size) + o.SetContentType(v.ContentType) + o.SetLastModified(v.LastModified) + o.SetUserMetadata(v.UserMetadata) + o.SetSystemMetadata(ObjectSystemMetadata{ + StorageClass: v.StorageClass, + }) + + return +} + +func (s *Storage) newObject(done bool) *types.Object { + return types.NewObject(s, done) +}