Skip to content

Commit

Permalink
collect additional metadata for buckets and objects in aws driver (#1399
Browse files Browse the repository at this point in the history 
)

* collect additional metadata for buckets and objects in aws driver

* change names of some metadata fields

* move head object to a separate function, change map/array initialisation method
  • Loading branch information
@devanshjain7
devanshjain7 authored Feb 13, 2023
1 parent 4b427c5 commit 07cac9f
Show file tree
Hide file tree
Showing 7 changed files with 398 additions and 160 deletions.
4 changes: 2 additions & 2 deletions metadata/pkg/constants/Constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,11 @@ const (
OBJECTS_SIZE = "objects.size"
OBJECTS_NAME = "objects.name"
CREATION_DATE = "creationDate"
ACCESS = "access"
BUCKET_ACL = "bucketAcl"
NUMBER_OF_OBJECTS = "numberOfObjects"
NUMBER_OF_FILTERED_OBJECTS = "numberOfFilteredObjects"
BUCKET_NUMBER_OF_FILTERED_OBJECTS = "bucket.numberOfFilteredObjects"
TAGS = "tags"
BUCKET_TAGS = "bucketTags"
)

const (
Expand Down
95 changes: 72 additions & 23 deletions metadata/pkg/drivers/aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
backendpb "github.com/opensds/multi-cloud/backend/proto"
"github.com/opensds/multi-cloud/metadata/pkg/db"
"github.com/opensds/multi-cloud/metadata/pkg/model"
"github.com/opensds/multi-cloud/metadata/pkg/utils"
pb "github.com/opensds/multi-cloud/metadata/proto"
log "github.com/sirupsen/logrus"
)
Expand All @@ -49,6 +50,38 @@ type AwsAdapter struct {
Session *session.Session
}

func GetHeadObject(sess *session.Session, bucketName *string, obj *model.MetaObject) {
svc := s3.New(sess)
meta, err := svc.HeadObject(&s3.HeadObjectInput{Bucket: bucketName, Key: &obj.ObjectName})
if err != nil {
log.Errorf("cannot perform head object on object %v in bucket %v. failed with error: %v", obj.ObjectName, *bucketName, err)
return
}
if meta.ServerSideEncryption != nil {
obj.ServerSideEncryption = *meta.ServerSideEncryption
}
obj.ObjectType = *meta.ContentType
if meta.Expires != nil {
expiresTime, err := time.Parse(time.RFC3339, *meta.Expires)
if err != nil {
log.Errorf("unable to parse given string to time type. error: %v. skipping ExpiresDate field", err)
} else {
obj.ExpiresDate = &expiresTime
}
}
if meta.ReplicationStatus != nil {
obj.ReplicationStatus = *meta.ReplicationStatus
}
if meta.WebsiteRedirectLocation != nil {
obj.RedirectLocation = *meta.WebsiteRedirectLocation
}
metadata := map[string]string{}
for key, val := range meta.Metadata {
metadata[key] = *val
}
obj.Metadata = metadata
}

func ObjectList(sess *session.Session, bucket *model.MetaBucket) error {
svc := s3.New(sess)
output, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket.Name})
Expand All @@ -69,29 +102,33 @@ func ObjectList(sess *session.Session, bucket *model.MetaBucket) error {
totSize += obj.Size
obj.StorageClass = *object.StorageClass

meta, err := svc.HeadObject(&s3.HeadObjectInput{Bucket: &bucket.Name, Key: object.Key})
if err != nil {
log.Errorf("cannot perform head object on object %v in bucket %v. failed with error: %v", *object.Key, bucket.Name, err)
continue
}
if meta.ServerSideEncryption != nil {
obj.ServerSideEncryption = *meta.ServerSideEncryption
}
if meta.VersionId != nil {
obj.VersionId = *meta.VersionId
}
obj.ObjectType = *meta.ContentType
if meta.Expires != nil {
expiresTime, err := time.Parse(time.RFC3339, *meta.Expires)
if err != nil {
log.Errorf("unable to parse given string to time type. error: %v. skipping ExpiresDate field", err)
} else {
obj.ExpiresDate = &expiresTime
tags, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{Bucket: &bucket.Name, Key: &obj.ObjectName})

if err == nil {
tagset := map[string]string{}
for _, tag := range tags.TagSet {
tagset[*tag.Key] = *tag.Value
}
obj.ObjectTags = tagset
if tags.VersionId != nil {
obj.VersionId = *tags.VersionId
}
} else {
log.Errorf("unable to get object tags. failed with error: %v", err)
}
if meta.ReplicationStatus != nil {
obj.ReplicationStatus = *meta.ReplicationStatus

acl, err := svc.GetObjectAcl(&s3.GetObjectAclInput{Bucket: &bucket.Name, Key: &obj.ObjectName})
if err != nil {
log.Errorf("unable to get object Acl. failed with error: %v", err)
} else {
access := []*model.Access{}
for _, grant := range acl.Grants {
access = append(access, utils.AclMapper(grant))
}
obj.ObjectAcl = access
}

GetHeadObject(sess, &bucket.Name, obj)
}
bucket.NumberOfObjects = numObjects
bucket.TotalSize = totSize
Expand Down Expand Up @@ -123,18 +160,30 @@ func GetBucketMeta(buckIdx int, bucket *s3.Bucket, sess *session.Session, bucket
return
}

bucketArray[buckIdx] = buck

tags, err := svc.GetBucketTagging(&s3.GetBucketTaggingInput{Bucket: bucket.Name})

if err == nil {
tagset := make(map[string]string)
tagset := map[string]string{}
for _, tag := range tags.TagSet {
tagset[*tag.Key] = *tag.Value
}
buck.BucketTags = tagset
} else if !strings.Contains(err.Error(), "NoSuchTagSet") {
log.Errorf("unable to get bucket tags. failed with error: %v", err)
}
bucketArray[buckIdx] = buck

acl, err := svc.GetBucketAcl(&s3.GetBucketAclInput{Bucket: bucket.Name})
if err != nil {
log.Errorf("unable to get bucket Acl. failed with error: %v", err)
} else {
access := []*model.Access{}
for _, grant := range acl.Grants {
access = append(access, utils.AclMapper(grant))
}
buck.BucketAcl = access
}
}

func BucketList(sess *session.Session) ([]*model.MetaBucket, error) {
Expand All @@ -153,7 +202,7 @@ func BucketList(sess *session.Session) ([]*model.MetaBucket, error) {
}
wg.Wait()

bucketArrayFiltered := make([]*model.MetaBucket, 0)
bucketArrayFiltered := []*model.MetaBucket{}
for _, buck := range bucketArray {
if buck != nil {
bucketArrayFiltered = append(bucketArrayFiltered, buck)
Expand Down
17 changes: 13 additions & 4 deletions metadata/pkg/model/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ type MetaBucket struct {
Name string `type:"string" json:"name" bson:"name"`
Type string `json:"type,omitempty" bson:"type,omitempty"`
Region string `json:"region,omitempty" bson:"region,omitempty"`
Access string `json:"access,omitempty" bson:"access,omitempty"`
NumberOfObjects int `json:"numberOfObjects" bson:"numberOfObjects,omitempty"`
BucketAcl []*Access `json:"bucketAcl,omitempty" bson:"bucketAcl,omitempty"`
NumberOfObjects int `json:"numberOfObjects" bson:"numberOfObjects"`
NumberOfFilteredObjects int `json:"numberOfFilteredObjects,omitempty" bson:"numberOfFilteredObjects,omitempty"`
Objects []*MetaObject `json:"objects" bson:"objects"`
TotalSize int64 `json:"totalSize" bson:"totalSize"`
FilteredBucketSize int64 `json:"filteredBucketSize,omitempty" bson:"filteredBucketSize"`
BucketTags map[string]string `json:"tags,omitempty" bson:"tags,omitempty"`
BucketTags map[string]string `json:"bucketTags,omitempty" bson:"bucketTags,omitempty"`
}

type MetaObject struct {
Expand All @@ -52,8 +52,17 @@ type MetaObject struct {
RedirectLocation string `json:"redirectLocation,omitempty" bson:"redirectLocation,omitempty"`
ReplicationStatus string `json:"replicationStatus,omitempty" bson:"replicationStatus,omitempty"`
ExpiresDate *time.Time `json:"expiresDate,omitempty" bson:"expiresDate,omitempty"`
GrantControl string `json:"grantControl,omitempty" bson:"grantControl,omitempty"`
ObjectAcl []*Access `json:"objectAcl,omitempty" bson:"objectAcl,omitempty"`
ObjectTags map[string]string `json:"objectTags,omitempty" bson:"objectTags,omitempty"`
Metadata map[string]string `json:"metadata,omitempty" bson:"metadata,omitempty"`
ObjectType string `json:"objectType,omitempty" bson:"objectType,omitempty"`
}

type Access struct {
DisplayName string `json:"displayName,omitempty" bson:"displayName,omitempty"`
EmailAddress string `json:"emailAddress,omitempty" bson:"emailAddress,omitempty"`
ID string `json:"id,omitempty" bson:"id,omitempty"`
Type string `json:"type,omitempty" bson:"type,omitempty"`
URI string `json:"uri,omitempty" bson:"uri,omitempty"`
Permission string `json:"permission,omitempty" bson:"permission,omitempty"`
}
4 changes: 2 additions & 2 deletions metadata/pkg/query-manager/translator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,12 @@ func getBucketsHavingFilteredObjectsQuery(filterConditions bson.A) bson.E {
{constants.NAME, getQualifiedNameForMapBucket(constants.NAME)},
{constants.REGION, getQualifiedNameForMapBucket(constants.REGION)},
{constants.TYPE, getQualifiedNameForMapBucket(constants.TYPE)},
{constants.ACCESS, getQualifiedNameForMapBucket(constants.ACCESS)},
{constants.BUCKET_ACL, getQualifiedNameForMapBucket(constants.BUCKET_ACL)},
{constants.NUMBER_OF_OBJECTS, getQualifiedNameForMapBucket(constants.NUMBER_OF_OBJECTS)},
{constants.NUMBER_OF_FILTERED_OBJECTS, getTotalNumberOfObjects(filterConditions)},
{constants.TOTAL_SIZE, getQualifiedNameForMapBucket(constants.TOTAL_SIZE)},
{constants.FILTERED_BUCKET_SIZE, getTotalSizeForObjects(filterConditions)},
{constants.TAGS, getQualifiedNameForMapBucket(constants.TAGS)},
{constants.BUCKET_TAGS, getQualifiedNameForMapBucket(constants.BUCKET_TAGS)},
//* asking to filter the objects array based on the object level queries given by user
{constants.OBJECTS, getFilteredObjectsQuery(filterConditions)},
},
Expand Down
60 changes: 52 additions & 8 deletions metadata/pkg/utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package utils
import (
"context"

"github.com/aws/aws-sdk-go/service/s3"
backend "github.com/opensds/multi-cloud/backend/proto"
"github.com/opensds/multi-cloud/metadata/pkg/model"
pb "github.com/opensds/multi-cloud/metadata/proto"
Expand All @@ -26,8 +27,7 @@ import (
func GetBackends(unPaginatedResult []*model.MetaBackend) []*pb.BackendMetadata {
var protoBackends []*pb.BackendMetadata
for _, backend := range unPaginatedResult {
var protoBuckets []*pb.BucketMetadata
protoBuckets = GetBuckets(backend, protoBuckets)
protoBuckets := GetBuckets(backend)
protoBackend := &pb.BackendMetadata{
Id: string(backend.Id),
BackendName: backend.BackendName,
Expand All @@ -42,21 +42,23 @@ func GetBackends(unPaginatedResult []*model.MetaBackend) []*pb.BackendMetadata {
return protoBackends
}

func GetBuckets(backend *model.MetaBackend, protoBuckets []*pb.BucketMetadata) []*pb.BucketMetadata {
func GetBuckets(backend *model.MetaBackend) []*pb.BucketMetadata {
var protoBuckets []*pb.BucketMetadata
for _, bucket := range backend.Buckets {

protoAcl := GetAcl(bucket.BucketAcl)
protoObjects := GetObjects(bucket)

protoBucket := &pb.BucketMetadata{
Name: bucket.Name,
Type: bucket.Type,
Region: bucket.Region,
BucketAcl: protoAcl,
BucketSizeInBytes: bucket.TotalSize,
FilteredBucketSize: bucket.FilteredBucketSize,
NumberOfObjects: int32(bucket.NumberOfObjects),
NumberOfFilteredObjects: int32(bucket.NumberOfFilteredObjects),
CreationDate: bucket.CreationDate.String(),
Tags: bucket.BucketTags,
BucketTags: bucket.BucketTags,
Objects: protoObjects,
}
protoBuckets = append(protoBuckets, protoBucket)
Expand All @@ -74,7 +76,7 @@ func GetObjects(bucket *model.MetaBucket) []*pb.ObjectMetadata {
} else {
expiresDateStr = object.ExpiresDate.String()
}

protoAcl := GetAcl(object.ObjectAcl)
protoObject := &pb.ObjectMetadata{
Name: object.ObjectName,
LastModifiedDate: object.LastModifiedDate.String(),
Expand All @@ -83,11 +85,11 @@ func GetObjects(bucket *model.MetaBucket) []*pb.ObjectMetadata {
Type: object.ObjectType,
ServerSideEncryption: object.ServerSideEncryption,
ExpiresDate: expiresDateStr,
GrantControl: object.GrantControl,
ObjectAcl: protoAcl,
VersionId: string(object.VersionId),
RedirectLocation: object.RedirectLocation,
ReplicationStatus: object.ReplicationStatus,
Tags: object.ObjectTags,
ObjectTags: object.ObjectTags,
Metadata: object.Metadata,
StorageClass: object.StorageClass,
}
Expand All @@ -96,6 +98,24 @@ func GetObjects(bucket *model.MetaBucket) []*pb.ObjectMetadata {
return protoObjects
}

func GetAcl(Acl []*model.Access) []*pb.Access {

var protoAcl []*pb.Access

for _, access := range Acl {
protoAccess := &pb.Access{
DisplayName: access.DisplayName,
EmailAddress: access.EmailAddress,
Id: access.ID,
Type: access.Type,
Uri: access.URI,
Permission: access.Permission,
}
protoAcl = append(protoAcl, protoAccess)
}
return protoAcl
}

func GetBackend(ctx context.Context, backedClient backend.BackendService, backendID string) (*backend.BackendDetail,
error) {
backend, err := backedClient.GetBackend(ctx, &backend.GetBackendRequest{
Expand All @@ -117,3 +137,27 @@ func ListBackend(ctx context.Context, backedClient backend.BackendService) ([]*b
}
return backend.Backends, nil
}

func AclMapper(grant *s3.Grant) *model.Access {
access := &model.Access{}
if grant.Grantee.DisplayName != nil {
access.DisplayName = *grant.Grantee.DisplayName
}
if grant.Grantee.EmailAddress != nil {
access.EmailAddress = *grant.Grantee.EmailAddress
}
if grant.Grantee.ID != nil {
access.ID = *grant.Grantee.ID
}
if grant.Grantee.Type != nil {
access.Type = *grant.Grantee.Type
}
if grant.Grantee.URI != nil {
access.URI = *grant.Grantee.URI
}
if grant.Permission != nil {
access.Permission = *grant.Permission
}

return access
}
Loading

0 comments on commit 07cac9f

Please sign in to comment.