From 4ad950a8e3d79942e422becb6838b92749ec551b Mon Sep 17 00:00:00 2001 From: Gaius Date: Mon, 27 Jun 2022 17:30:04 +0800 Subject: [PATCH] feat: object storage add filter field (#1412) * feat: object storage add filter field Signed-off-by: Gaius * feat: object storage add logger Signed-off-by: Gaius * feat: GetObjectMetadata return isExist Signed-off-by: Gaius --- api/manager/docs.go | 136 +++++++++++++------ api/manager/swagger.json | 136 +++++++++++++------ api/manager/swagger.yaml | 92 +++++++++---- client/config/peerhost.go | 4 + client/config/peerhost_darwin.go | 1 + client/config/peerhost_linux.go | 1 + client/config/peerhost_test.go | 1 + client/config/testdata/config/daemon.yaml | 1 + client/daemon/objectstorage/objectstorage.go | 113 ++++++++++----- client/daemon/objectstorage/types.go | 11 +- pkg/objectstorage/objectstorage.go | 2 +- pkg/objectstorage/oss.go | 18 ++- pkg/objectstorage/s3.go | 22 +-- 13 files changed, 386 insertions(+), 152 deletions(-) diff --git a/api/manager/docs.go b/api/manager/docs.go index 44913ae4611..f5cd7c98e72 100644 --- a/api/manager/docs.go +++ b/api/manager/docs.go @@ -3692,9 +3692,15 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "download_rate_limit": { "type": "integer" }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3713,6 +3719,9 @@ const docTemplate = `{ "state": { "type": "string" }, + "updated_at": { + "type": "string" + }, "url": { "type": "string" }, @@ -3724,52 +3733,22 @@ const docTemplate = `{ } } }, - "model.Assertion": { + "model.Config": { "type": "object", "properties": { - "key": { + "bio": { "type": "string" }, - "policy": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "policyMap": { - "type": "object", - "additionalProperties": { - "type": "integer" - } + "created_at": { + "type": "string" }, - "rm": {}, - "tokens": { - "type": "array", - "items": { - "type": "string" - } + "id": { + "type": "integer" }, - "value": { - "type": "string" - } - } - }, - "model.AssertionMap": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/model.Assertion" - } - }, - "model.Config": { - "type": "object", - "properties": { - "bio": { + "name": { "type": "string" }, - "name": { + "updated_at": { "type": "string" }, "user_id": { @@ -3793,6 +3772,12 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "result": { "$ref": "#/definitions/model.JSONMap" }, @@ -3817,6 +3802,9 @@ const docTemplate = `{ "type": { "type": "string" }, + "updated_at": { + "type": "string" + }, "user_id": { "type": "integer" } @@ -3834,20 +3822,35 @@ const docTemplate = `{ "client_secret": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, "redirect_url": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, "model.Scheduler": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "host_name": { "type": "string" }, + "id": { + "type": "integer" + }, "idc": { "type": "string" }, @@ -3868,6 +3871,9 @@ const docTemplate = `{ }, "state": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, @@ -3886,6 +3892,12 @@ const docTemplate = `{ "config": { "$ref": "#/definitions/model.JSONMap" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "is_default": { "type": "boolean" }, @@ -3909,6 +3921,9 @@ const docTemplate = `{ "items": { "$ref": "#/definitions/model.SeedPeerCluster" } + }, + "updated_at": { + "type": "string" } } }, @@ -3918,6 +3933,12 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3926,6 +3947,9 @@ const docTemplate = `{ "items": { "$ref": "#/definitions/model.SecurityRule" } + }, + "updated_at": { + "type": "string" } } }, @@ -3935,9 +3959,15 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "domain": { "type": "string" }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3949,18 +3979,27 @@ const docTemplate = `{ "items": { "$ref": "#/definitions/model.SecurityGroup" } + }, + "updated_at": { + "type": "string" } } }, "model.SeedPeer": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "download_port": { "type": "integer" }, "host_name": { "type": "string" }, + "id": { + "type": "integer" + }, "idc": { "type": "string" }, @@ -3987,6 +4026,9 @@ const docTemplate = `{ }, "type": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, @@ -4002,6 +4044,12 @@ const docTemplate = `{ "config": { "$ref": "#/definitions/model.JSONMap" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "is_default": { "type": "boolean" }, @@ -4025,6 +4073,9 @@ const docTemplate = `{ }, "security_group_id": { "type": "integer" + }, + "updated_at": { + "type": "string" } } }, @@ -4037,9 +4088,15 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "email": { "type": "string" }, + "id": { + "type": "integer" + }, "location": { "type": "string" }, @@ -4051,6 +4108,9 @@ const docTemplate = `{ }, "state": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, diff --git a/api/manager/swagger.json b/api/manager/swagger.json index 41a4ac06652..7e303913021 100644 --- a/api/manager/swagger.json +++ b/api/manager/swagger.json @@ -3685,9 +3685,15 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "download_rate_limit": { "type": "integer" }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3706,6 +3712,9 @@ "state": { "type": "string" }, + "updated_at": { + "type": "string" + }, "url": { "type": "string" }, @@ -3717,52 +3726,22 @@ } } }, - "model.Assertion": { + "model.Config": { "type": "object", "properties": { - "key": { + "bio": { "type": "string" }, - "policy": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "policyMap": { - "type": "object", - "additionalProperties": { - "type": "integer" - } + "created_at": { + "type": "string" }, - "rm": {}, - "tokens": { - "type": "array", - "items": { - "type": "string" - } + "id": { + "type": "integer" }, - "value": { - "type": "string" - } - } - }, - "model.AssertionMap": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/model.Assertion" - } - }, - "model.Config": { - "type": "object", - "properties": { - "bio": { + "name": { "type": "string" }, - "name": { + "updated_at": { "type": "string" }, "user_id": { @@ -3786,6 +3765,12 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "result": { "$ref": "#/definitions/model.JSONMap" }, @@ -3810,6 +3795,9 @@ "type": { "type": "string" }, + "updated_at": { + "type": "string" + }, "user_id": { "type": "integer" } @@ -3827,20 +3815,35 @@ "client_secret": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, "redirect_url": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, "model.Scheduler": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "host_name": { "type": "string" }, + "id": { + "type": "integer" + }, "idc": { "type": "string" }, @@ -3861,6 +3864,9 @@ }, "state": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, @@ -3879,6 +3885,12 @@ "config": { "$ref": "#/definitions/model.JSONMap" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "is_default": { "type": "boolean" }, @@ -3902,6 +3914,9 @@ "items": { "$ref": "#/definitions/model.SeedPeerCluster" } + }, + "updated_at": { + "type": "string" } } }, @@ -3911,6 +3926,12 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3919,6 +3940,9 @@ "items": { "$ref": "#/definitions/model.SecurityRule" } + }, + "updated_at": { + "type": "string" } } }, @@ -3928,9 +3952,15 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "domain": { "type": "string" }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3942,18 +3972,27 @@ "items": { "$ref": "#/definitions/model.SecurityGroup" } + }, + "updated_at": { + "type": "string" } } }, "model.SeedPeer": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "download_port": { "type": "integer" }, "host_name": { "type": "string" }, + "id": { + "type": "integer" + }, "idc": { "type": "string" }, @@ -3980,6 +4019,9 @@ }, "type": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, @@ -3995,6 +4037,12 @@ "config": { "$ref": "#/definitions/model.JSONMap" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "is_default": { "type": "boolean" }, @@ -4018,6 +4066,9 @@ }, "security_group_id": { "type": "integer" + }, + "updated_at": { + "type": "string" } } }, @@ -4030,9 +4081,15 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "email": { "type": "string" }, + "id": { + "type": "integer" + }, "location": { "type": "string" }, @@ -4044,6 +4101,9 @@ }, "state": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, diff --git a/api/manager/swagger.yaml b/api/manager/swagger.yaml index 9a59e13c1ef..4716cd5e219 100644 --- a/api/manager/swagger.yaml +++ b/api/manager/swagger.yaml @@ -4,8 +4,12 @@ definitions: properties: bio: type: string + created_at: + type: string download_rate_limit: type: integer + id: + type: integer name: type: string scheduler_clusters: @@ -18,6 +22,8 @@ definitions: type: array state: type: string + updated_at: + type: string url: type: string user: @@ -25,38 +31,18 @@ definitions: user_id: type: integer type: object - model.Assertion: - properties: - key: - type: string - policy: - items: - items: - type: string - type: array - type: array - policyMap: - additionalProperties: - type: integer - type: object - rm: {} - tokens: - items: - type: string - type: array - value: - type: string - type: object - model.AssertionMap: - additionalProperties: - $ref: '#/definitions/model.Assertion' - type: object model.Config: properties: bio: type: string + created_at: + type: string + id: + type: integer name: type: string + updated_at: + type: string user_id: type: integer value: @@ -71,6 +57,10 @@ definitions: $ref: '#/definitions/model.JSONMap' bio: type: string + created_at: + type: string + id: + type: integer result: $ref: '#/definitions/model.JSONMap' scheduler_clusters: @@ -87,6 +77,8 @@ definitions: type: string type: type: string + updated_at: + type: string user_id: type: integer type: object @@ -98,15 +90,25 @@ definitions: type: string client_secret: type: string + created_at: + type: string + id: + type: integer name: type: string redirect_url: type: string + updated_at: + type: string type: object model.Scheduler: properties: + created_at: + type: string host_name: type: string + id: + type: integer idc: type: string ip: @@ -121,6 +123,8 @@ definitions: type: integer state: type: string + updated_at: + type: string type: object model.SchedulerCluster: properties: @@ -132,6 +136,10 @@ definitions: $ref: '#/definitions/model.JSONMap' config: $ref: '#/definitions/model.JSONMap' + created_at: + type: string + id: + type: integer is_default: type: boolean jobs: @@ -148,24 +156,36 @@ definitions: items: $ref: '#/definitions/model.SeedPeerCluster' type: array + updated_at: + type: string type: object model.SecurityGroup: properties: bio: type: string + created_at: + type: string + id: + type: integer name: type: string security_rules: items: $ref: '#/definitions/model.SecurityRule' type: array + updated_at: + type: string type: object model.SecurityRule: properties: bio: type: string + created_at: + type: string domain: type: string + id: + type: integer name: type: string proxy_domain: @@ -174,13 +194,19 @@ definitions: items: $ref: '#/definitions/model.SecurityGroup' type: array + updated_at: + type: string type: object model.SeedPeer: properties: + created_at: + type: string download_port: type: integer host_name: type: string + id: + type: integer idc: type: string ip: @@ -199,6 +225,8 @@ definitions: type: string type: type: string + updated_at: + type: string type: object model.SeedPeerCluster: properties: @@ -208,6 +236,10 @@ definitions: type: string config: $ref: '#/definitions/model.JSONMap' + created_at: + type: string + id: + type: integer is_default: type: boolean jobs: @@ -224,6 +256,8 @@ definitions: $ref: '#/definitions/model.JSONMap' security_group_id: type: integer + updated_at: + type: string type: object model.User: properties: @@ -231,8 +265,12 @@ definitions: type: string bio: type: string + created_at: + type: string email: type: string + id: + type: integer location: type: string name: @@ -241,6 +279,8 @@ definitions: type: string state: type: string + updated_at: + type: string type: object objectstorage.BucketMetadata: properties: diff --git a/client/config/peerhost.go b/client/config/peerhost.go index 4da8d972991..b2c21f8b74a 100644 --- a/client/config/peerhost.go +++ b/client/config/peerhost.go @@ -366,6 +366,10 @@ type UploadOption struct { type ObjectStorageOption struct { // Enable object storage. Enable bool `mapstructure:"enable" yaml:"enable"` + // Filter is used to generate a unique Task ID by + // filtering unnecessary query params in the URL, + // it is separated by & character. + Filter string `mapstructure:"filter" yaml:"filter"` // ListenOption is object storage service listener. ListenOption `yaml:",inline" mapstructure:",squash"` } diff --git a/client/config/peerhost_darwin.go b/client/config/peerhost_darwin.go index eb46840790d..38f73ff8f72 100644 --- a/client/config/peerhost_darwin.go +++ b/client/config/peerhost_darwin.go @@ -120,6 +120,7 @@ var peerHostConfig = DaemonOption{ }, ObjectStorage: ObjectStorageOption{ Enable: false, + Filter: "Expires&Signature&ns", ListenOption: ListenOption{ Security: SecurityOption{ Insecure: true, diff --git a/client/config/peerhost_linux.go b/client/config/peerhost_linux.go index f67215491bf..dbca09616ad 100644 --- a/client/config/peerhost_linux.go +++ b/client/config/peerhost_linux.go @@ -119,6 +119,7 @@ var peerHostConfig = DaemonOption{ }, ObjectStorage: ObjectStorageOption{ Enable: false, + Filter: "Expires&Signature&ns", ListenOption: ListenOption{ Security: SecurityOption{ Insecure: true, diff --git a/client/config/peerhost_test.go b/client/config/peerhost_test.go index 7ce74632f24..b1b68dd5e94 100644 --- a/client/config/peerhost_test.go +++ b/client/config/peerhost_test.go @@ -328,6 +328,7 @@ func TestPeerHostOption_Load(t *testing.T) { }, ObjectStorage: ObjectStorageOption{ Enable: true, + Filter: "Expires&Signature&ns", ListenOption: ListenOption{ Security: SecurityOption{ Insecure: true, diff --git a/client/config/testdata/config/daemon.yaml b/client/config/testdata/config/daemon.yaml index 4fe6510c91b..aa5afd8f6a7 100644 --- a/client/config/testdata/config/daemon.yaml +++ b/client/config/testdata/config/daemon.yaml @@ -72,6 +72,7 @@ upload: objectStorage: enable: true + filter: Expires&Signature&ns security: insecure: true caCert: caCert diff --git a/client/daemon/objectstorage/objectstorage.go b/client/daemon/objectstorage/objectstorage.go index c748e7378ce..3dc432d3a85 100644 --- a/client/daemon/objectstorage/objectstorage.go +++ b/client/daemon/objectstorage/objectstorage.go @@ -57,6 +57,8 @@ const ( AsyncWriteBack // Ephemeral only writes the object to the dfdaemon. + // It is only provided for creating temporary objects between peers, + // and users are not allowed to use this mode. Ephemeral ) @@ -174,13 +176,46 @@ func (o *objectStorage) getObject(ctx *gin.Context) { return } + var query GetObjectQuery + if err := ctx.ShouldBindQuery(&query); err != nil { + ctx.JSON(http.StatusUnprocessableEntity, gin.H{"errors": err.Error()}) + return + } + var ( - urlMeta *base.UrlMeta + bucketName = params.ID + objectKey = strings.TrimPrefix(params.ObjectKey, "/") + filter = query.Filter artifactRange *clientutil.Range ranges []clientutil.Range err error ) + // Initialize filter field. + urlMeta := &base.UrlMeta{Filter: o.config.ObjectStorage.Filter} + if filter != "" { + urlMeta.Filter = filter + } + + client, err := o.client() + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) + return + } + + meta, isExist, err := client.GetObjectMetadata(ctx, bucketName, objectKey) + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) + return + } + + if !isExist { + ctx.JSON(http.StatusNotFound, gin.H{"errors": http.StatusText(http.StatusNotFound)}) + return + } + + urlMeta.Digest = meta.Digest + // Parse http range header. rangeHeader := ctx.GetHeader(headers.Range) if len(rangeHeader) > 0 { @@ -195,24 +230,15 @@ func (o *objectStorage) getObject(ctx *gin.Context) { urlMeta.Range = strings.TrimLeft(rangeHeader, "bytes=") } - client, err := o.client() - if err != nil { - ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) - return - } - - meta, err := client.GetObjectMetadata(ctx, params.ID, params.ObjectKey) + signURL, err := client.GetSignURL(ctx, bucketName, objectKey, objectstorage.MethodGet, defaultSignExpireTime) if err != nil { ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) return } - urlMeta.Digest = meta.Digest - signURL, err := client.GetSignURL(ctx, params.ID, params.ObjectKey, objectstorage.MethodGet, defaultSignExpireTime) - if err != nil { - ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) - return - } + taskID := idgen.TaskID(signURL, urlMeta) + log := logger.WithTaskID(taskID) + log.Infof("get object %s meta: %s %#v", objectKey, signURL, urlMeta) reader, attr, err := o.peerTaskManager.StartStreamTask(ctx, &peer.StreamTaskRequest{ URL: signURL, @@ -233,6 +259,7 @@ func (o *objectStorage) getObject(ctx *gin.Context) { } } + log.Infof("object content length is %d and content type is %s", contentLength, attr[headers.ContentType]) ctx.DataFromReader(http.StatusOK, contentLength, attr[headers.ContentType], reader, nil) } @@ -250,7 +277,13 @@ func (o *objectStorage) destroyObject(ctx *gin.Context) { return } - if err := client.DeleteObject(ctx, params.ID, params.ObjectKey); err != nil { + var ( + bucketName = params.ID + objectKey = strings.TrimPrefix(params.ObjectKey, "/") + ) + + logger.Infof("destroy object %s in bucket %s", objectKey, bucketName) + if err := client.DeleteObject(ctx, bucketName, objectKey); err != nil { ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) return } @@ -273,8 +306,13 @@ func (o *objectStorage) createObject(ctx *gin.Context) { return } - bucketName := params.ID - objectKey := form.Key + var ( + bucketName = params.ID + objectKey = form.Key + mode = form.Mode + filter = form.Filter + fileHeader = form.File + ) client, err := o.client() if err != nil { @@ -289,7 +327,7 @@ func (o *objectStorage) createObject(ctx *gin.Context) { } // If it is temporary storage, whether the data exists in the backend is not considered. - if isExist && form.Mode != Ephemeral { + if isExist && mode != Ephemeral { ctx.JSON(http.StatusConflict, gin.H{"errors": http.StatusText(http.StatusConflict)}) return } @@ -300,20 +338,31 @@ func (o *objectStorage) createObject(ctx *gin.Context) { return } - dgst := o.md5FromFileHeader(form.File) - urlMeta := &base.UrlMeta{Digest: dgst.String()} + // Initialize url meta. + urlMeta := &base.UrlMeta{Filter: o.config.ObjectStorage.Filter} + dgst := o.md5FromFileHeader(fileHeader) + urlMeta.Digest = dgst.String() + if filter != "" { + urlMeta.Filter = filter + } + + // Initialize task id and peer id. taskID := idgen.TaskID(signURL, urlMeta) peerID := o.peerIDGenerator.PeerID() + log := logger.WithTaskAndPeerID(taskID, peerID) + log.Infof("upload object %s meta: %s %#v", objectKey, signURL, urlMeta) // Import object to local storage. - if err := o.importObjectToLocalStorage(ctx, taskID, peerID, form.File); err != nil { + log.Infof("import object %s to local storage", objectKey) + if err := o.importObjectToLocalStorage(ctx, taskID, peerID, fileHeader); err != nil { log.Error(err) ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) return } - // Announce task information to scheduler. + // Announce peer information to scheduler. + log.Info("announce peer to scheduler") if err := o.peerTaskManager.AnnouncePeerTask(ctx, storage.PeerTaskMetadata{ TaskID: taskID, PeerID: peerID, @@ -324,52 +373,52 @@ func (o *objectStorage) createObject(ctx *gin.Context) { } // Handle task for backend. - switch form.Mode { + switch mode { case Ephemeral: ctx.Status(http.StatusOK) return case WriteBack: // Import object to seed peer. go func() { - if err := o.importObjectToSeedPeers(context.Background(), bucketName, objectKey, Ephemeral, form.File); err != nil { + log.Infof("import object %s to seed peer", objectKey) + if err := o.importObjectToSeedPeers(context.Background(), bucketName, objectKey, Ephemeral, fileHeader); err != nil { log.Errorf("import object %s to seed peer failed: %s", objectKey, err) } - log.Infof("import object %s to seed peer", objectKey) }() // Import object to object storage. - if err := o.importObjectToBackend(ctx, bucketName, objectKey, dgst, form.File, client); err != nil { + log.Infof("import object %s to bucket %s", objectKey, bucketName) + if err := o.importObjectToBackend(ctx, bucketName, objectKey, dgst, fileHeader, client); err != nil { log.Error(err) ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) return } - log.Infof("import object %s to bucket %s", objectKey, bucketName) ctx.Status(http.StatusOK) return case AsyncWriteBack: // Import object to seed peer. go func() { - if err := o.importObjectToSeedPeers(context.Background(), bucketName, objectKey, Ephemeral, form.File); err != nil { + log.Infof("import object %s to seed peer", objectKey) + if err := o.importObjectToSeedPeers(context.Background(), bucketName, objectKey, Ephemeral, fileHeader); err != nil { log.Errorf("import object %s to seed peer failed: %s", objectKey, err) } - log.Infof("import object %s to seed peer", objectKey) }() // Import object to object storage. go func() { - if err := o.importObjectToBackend(context.Background(), bucketName, objectKey, dgst, form.File, client); err != nil { + log.Infof("import object %s to bucket %s", objectKey, bucketName) + if err := o.importObjectToBackend(context.Background(), bucketName, objectKey, dgst, fileHeader, client); err != nil { log.Errorf("import object %s to bucket %s failed: %s", objectKey, bucketName, err.Error()) return } - log.Infof("import object %s to bucket %s", objectKey, bucketName) }() ctx.Status(http.StatusOK) return } - ctx.JSON(http.StatusUnprocessableEntity, gin.H{"errors": fmt.Sprintf("unknow mode %d", form.Mode)}) + ctx.JSON(http.StatusUnprocessableEntity, gin.H{"errors": fmt.Sprintf("unknow mode %d", mode)}) return } diff --git a/client/daemon/objectstorage/types.go b/client/daemon/objectstorage/types.go index 93beb5f159e..ef5df36570f 100644 --- a/client/daemon/objectstorage/types.go +++ b/client/daemon/objectstorage/types.go @@ -28,7 +28,12 @@ type CreateObjectParams struct { } type CreateObjectRequset struct { - Key string `form:"key" binding:"required"` - Mode uint `form:"mode,default=0" binding:"omitempty,gte=0,lte=2"` - File *multipart.FileHeader `form:"file" binding:"required"` + Key string `form:"key" binding:"required"` + Mode uint `form:"mode,default=0" binding:"omitempty,gte=0,lte=2"` + Filter string `form:"filter" binding:"omitempty"` + File *multipart.FileHeader `form:"file" binding:"required"` +} + +type GetObjectQuery struct { + Filter string `form:"filter" binding:"omitempty"` } diff --git a/pkg/objectstorage/objectstorage.go b/pkg/objectstorage/objectstorage.go index 17e6c9f11c7..af471f5b335 100644 --- a/pkg/objectstorage/objectstorage.go +++ b/pkg/objectstorage/objectstorage.go @@ -73,7 +73,7 @@ type ObjectStorage interface { ListBucketMetadatas(ctx context.Context) ([]*BucketMetadata, error) // GetObjectMetadata returns metadata of object. - GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, error) + GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, bool, error) // GetOject returns data of object. GetOject(ctx context.Context, bucketName, objectKey string) (io.ReadCloser, error) diff --git a/pkg/objectstorage/oss.go b/pkg/objectstorage/oss.go index 62b3244f548..bf0273a9917 100644 --- a/pkg/objectstorage/oss.go +++ b/pkg/objectstorage/oss.go @@ -20,11 +20,13 @@ import ( "context" "fmt" "io" + "net/http" "strconv" "time" aliyunoss "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/go-http-utils/headers" + "github.com/pkg/errors" ) type oss struct { @@ -86,20 +88,24 @@ func (o *oss) ListBucketMetadatas(ctx context.Context) ([]*BucketMetadata, error } // GetObjectMetadata returns metadata of object. -func (o *oss) GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, error) { +func (o *oss) GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, bool, error) { bucket, err := o.client.Bucket(bucketName) if err != nil { - return nil, err + return nil, false, err } - header, err := bucket.GetObjectMeta(objectKey) + header, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { - return nil, err + if serr, ok := errors.Cause(err).(aliyunoss.ServiceError); ok && serr.StatusCode == http.StatusNotFound { + return nil, false, nil + } + + return nil, false, err } contentLength, err := strconv.ParseInt(header.Get(headers.ContentLength), 10, 64) if err != nil { - return nil, err + return nil, false, err } return &ObjectMetadata{ @@ -111,7 +117,7 @@ func (o *oss) GetObjectMetadata(ctx context.Context, bucketName, objectKey strin ContentType: header.Get(headers.ContentType), Etag: header.Get(headers.ETag), Digest: header.Get(aliyunoss.HTTPHeaderOssMetaPrefix + MetaDigest), - }, nil + }, true, nil } // GetOject returns data of object. diff --git a/pkg/objectstorage/s3.go b/pkg/objectstorage/s3.go index f3c49499747..ecfe38f908a 100644 --- a/pkg/objectstorage/s3.go +++ b/pkg/objectstorage/s3.go @@ -91,13 +91,18 @@ func (s *s3) ListBucketMetadatas(ctx context.Context) ([]*BucketMetadata, error) } // GetObjectMetadata returns metadata of object. -func (s *s3) GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, error) { +func (s *s3) GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, bool, error) { resp, err := s.client.HeadObjectWithContext(ctx, &awss3.HeadObjectInput{ Bucket: aws.String(bucketName), Key: aws.String(objectKey), }) if err != nil { - return nil, err + // S3 is missing this error code. + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { + return nil, false, nil + } + + return nil, false, err } return &ObjectMetadata{ @@ -109,7 +114,7 @@ func (s *s3) GetObjectMetadata(ctx context.Context, bucketName, objectKey string ContentType: aws.StringValue(resp.ContentType), Etag: aws.StringValue(resp.ETag), Digest: aws.StringValue(resp.Metadata[MetaDigest]), - }, nil + }, true, nil } // GetOject returns data of object. @@ -175,15 +180,16 @@ func (s *s3) ListObjectMetadatas(ctx context.Context, bucketName, prefix, marker // IsObjectExist returns whether the object exists. func (s *s3) IsObjectExist(ctx context.Context, bucketName, objectKey string) (bool, error) { - if _, err := s.GetObjectMetadata(ctx, bucketName, objectKey); err != nil { - // S3 is missing this error code. - if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { - return false, nil - } + _, isExist, err := s.GetObjectMetadata(ctx, bucketName, objectKey) + if err != nil { return false, err } + if !isExist { + return false, nil + } + return true, nil }