diff --git a/code/go/0chain.net/blobbercore/convert/responseHandler.go b/code/go/0chain.net/blobbercore/convert/responseHandler.go index 760f31cb7..8c0916a75 100644 --- a/code/go/0chain.net/blobbercore/convert/responseHandler.go +++ b/code/go/0chain.net/blobbercore/convert/responseHandler.go @@ -1,9 +1,9 @@ package convert import ( - "context" "encoding/json" - "github.com/0chain/blobber/code/go/0chain.net/core/common" + + stats2 "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberHTTP" @@ -11,151 +11,194 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" ) -func GetAllocationResponseHandler(resp *blobbergrpc.GetAllocationResponse) *allocation.Allocation { - return GRPCAllocationToAllocation(resp.Allocation) +func GetAllocationResponseCreator(resp interface{}) *blobbergrpc.GetAllocationResponse { + if resp == nil { + return nil + } + + alloc, _ := resp.(*allocation.Allocation) + return &blobbergrpc.GetAllocationResponse{Allocation: AllocationToGRPCAllocation(alloc)} } -func GetFileMetaDataResponseHandler(resp *blobbergrpc.GetFileMetaDataResponse) map[string]interface{} { - var collaborators []reference.Collaborator - for _, c := range resp.Collaborators { - collaborators = append(collaborators, *GRPCCollaboratorToCollaborator(c)) +func GetFileMetaDataResponseCreator(httpResp interface{}) *blobbergrpc.GetFileMetaDataResponse { + if httpResp == nil { + return nil } - result := FileRefGRPCToFileRef(resp.MetaData).GetListingData(context.Background()) - result["collaborators"] = collaborators - return result -} + r, _ := httpResp.(map[string]interface{}) -func GetFileStatsResponseHandler(resp *blobbergrpc.GetFileStatsResponse) map[string]interface{} { - ctx := context.Background() - result := FileRefGRPCToFileRef(resp.MetaData).GetListingData(ctx) + var resp blobbergrpc.GetFileMetaDataResponse + collaborators, _ := r["collaborators"].([]reference.Collaborator) + for _, c := range collaborators { + resp.Collaborators = append(resp.Collaborators, CollaboratorToGRPCCollaborator(&c)) + } - statsMap := make(map[string]interface{}) - statsBytes, _ := json.Marshal(FileStatsGRPCToFileStats(resp.Stats)) - _ = json.Unmarshal(statsBytes, &statsMap) + resp.MetaData = FileRefToFileRefGRPC(reference.ListingDataToRef(r)) + return &resp +} - for k, v := range statsMap { - result[k] = v +func GetFileStatsResponseCreator(r interface{}) *blobbergrpc.GetFileStatsResponse { + if r == nil { + return nil } - return result + httpResp, _ := r.(map[string]interface{}) + + var resp blobbergrpc.GetFileStatsResponse + resp.MetaData = FileRefToFileRefGRPC(reference.ListingDataToRef(httpResp)) + + respRaw, _ := json.Marshal(httpResp) + var stats stats2.FileStats + _ = json.Unmarshal(respRaw, &stats) + resp.Stats = FileStatsToFileStatsGRPC(&stats) + + return &resp } -func ListEntitesResponseHandler(resp *blobbergrpc.ListEntitiesResponse) *blobberHTTP.ListResult { - ctx := context.Background() - var entities []map[string]interface{} - for i := range resp.Entities { - entities = append(entities, FileRefGRPCToFileRef(resp.Entities[i]).GetListingData(ctx)) +func ListEntitesResponseCreator(r interface{}) *blobbergrpc.ListEntitiesResponse { + if r == nil { + return nil } - return &blobberHTTP.ListResult{ - AllocationRoot: resp.AllocationRoot, - Meta: FileRefGRPCToFileRef(resp.MetaData).GetListingData(ctx), - Entities: entities, + httpResp, _ := r.(*blobberHTTP.ListResult) + + var resp blobbergrpc.ListEntitiesResponse + for i := range httpResp.Entities { + resp.Entities = append(resp.Entities, FileRefToFileRefGRPC(reference.ListingDataToRef(httpResp.Entities[i]))) } + + resp.MetaData = FileRefToFileRefGRPC(reference.ListingDataToRef(httpResp.Meta)) + resp.AllocationRoot = httpResp.AllocationRoot + return &resp } -func GetReferencePathResponseHandler(getReferencePathResponse *blobbergrpc.GetReferencePathResponse) *blobberHTTP.ReferencePathResult { - var recursionCount int - return &blobberHTTP.ReferencePathResult{ - ReferencePath: ReferencePathGRPCToReferencePath(&recursionCount, getReferencePathResponse.ReferencePath), - LatestWM: WriteMarkerGRPCToWriteMarker(getReferencePathResponse.LatestWM), +func GetReferencePathResponseCreator(r interface{}) *blobbergrpc.GetReferencePathResponse { + if r == nil { + return nil } + + httpResp, _ := r.(*blobberHTTP.ReferencePathResult) + var resp blobbergrpc.GetReferencePathResponse + + var recursionCount int + resp.LatestWM = WriteMarkerToWriteMarkerGRPC(httpResp.LatestWM) + resp.ReferencePath = ReferencePathToReferencePathGRPC(&recursionCount, httpResp.ReferencePath) + return &resp } -func GetObjectPathResponseHandler(getObjectPathResponse *blobbergrpc.GetObjectPathResponse) *blobberHTTP.ObjectPathResult { - ctx := context.Background() - path := FileRefGRPCToFileRef(getObjectPathResponse.ObjectPath.Path).GetListingData(ctx) - var pathList []map[string]interface{} - for _, pl := range getObjectPathResponse.ObjectPath.PathList { - pathList = append(pathList, FileRefGRPCToFileRef(pl).GetListingData(ctx)) +func GetObjectTreeResponseCreator(r interface{}) *blobbergrpc.GetObjectTreeResponse { + if r == nil { + return nil } - path["list"] = pathList - return &blobberHTTP.ObjectPathResult{ - ObjectPath: &reference.ObjectPath{ - RootHash: getObjectPathResponse.ObjectPath.RootHash, - Meta: FileRefGRPCToFileRef(getObjectPathResponse.ObjectPath.Meta).GetListingData(ctx), - Path: path, - FileBlockNum: getObjectPathResponse.ObjectPath.FileBlockNum, - }, - LatestWM: WriteMarkerGRPCToWriteMarker(getObjectPathResponse.LatestWriteMarker), - } -} + httpResp, _ := r.(*blobberHTTP.ReferencePathResult) + var resp blobbergrpc.GetObjectTreeResponse -func GetObjectTreeResponseHandler(getObjectTreeResponse *blobbergrpc.GetObjectTreeResponse) *blobberHTTP.ReferencePathResult { var recursionCount int - return &blobberHTTP.ReferencePathResult{ - ReferencePath: ReferencePathGRPCToReferencePath(&recursionCount, getObjectTreeResponse.ReferencePath), - LatestWM: WriteMarkerGRPCToWriteMarker(getObjectTreeResponse.LatestWM), - } + resp.LatestWM = WriteMarkerToWriteMarkerGRPC(httpResp.LatestWM) + resp.ReferencePath = ReferencePathToReferencePathGRPC(&recursionCount, httpResp.ReferencePath) + return &resp } -func CommitWriteResponseHandler(resp *blobbergrpc.CommitResponse) *blobberHTTP.CommitResult { - return &blobberHTTP.CommitResult{ - AllocationRoot: resp.AllocationRoot, - WriteMarker: WriteMarkerGRPCToWriteMarker(resp.WriteMarker), - Success: resp.Success, - ErrorMessage: resp.ErrorMessage, +func GetObjectPathResponseCreator(r interface{}) *blobbergrpc.GetObjectPathResponse { + if r == nil { + return nil + } + + httpResp, _ := r.(*blobberHTTP.ObjectPathResult) + var resp blobbergrpc.GetObjectPathResponse + + var pathList []*blobbergrpc.FileRef + pl, _ := httpResp.Path["list"].([]map[string]interface{}) + for _, v := range pl { + pathList = append(pathList, FileRefToFileRefGRPC(reference.ListingDataToRef(v))) } -} -func GetCalculateHashResponseHandler(response *blobbergrpc.CalculateHashResponse) interface{} { - result := make(map[string]interface{}) - if msg := response.GetMessage(); msg != "" { - result["msg"] = msg + resp.LatestWriteMarker = WriteMarkerToWriteMarkerGRPC(httpResp.LatestWM) + resp.ObjectPath = &blobbergrpc.ObjectPath{ + RootHash: httpResp.RootHash, + Meta: FileRefToFileRefGRPC(reference.ListingDataToRef(httpResp.Meta)), + Path: FileRefToFileRefGRPC(reference.ListingDataToRef(httpResp.Path)), + PathList: pathList, + FileBlockNum: httpResp.FileBlockNum, } - return result + return &resp } -func GetCommitMetaTxnHandlerResponse(response *blobbergrpc.CommitMetaTxnResponse) interface{} { - msg := response.GetMessage() - if msg == "" { +func CommitWriteResponseHandler(r interface{}) *blobbergrpc.CommitResponse { + if r == nil { return nil } - result := struct { - Msg string `json:"msg"` - }{ - Msg: msg, + httpResp, _ := r.(*blobberHTTP.CommitResult) + + return &blobbergrpc.CommitResponse{ + AllocationRoot: httpResp.AllocationRoot, + WriteMarker: WriteMarkerToWriteMarkerGRPC(httpResp.WriteMarker), + ErrorMessage: httpResp.ErrorMessage, + Success: httpResp.Success, } +} + +func GetCalculateHashResponseHandler(r interface{}) *blobbergrpc.CalculateHashResponse { + httpResp, _ := r.(map[string]interface{}) + msg, _ := httpResp["msg"].(string) + + return &blobbergrpc.CalculateHashResponse{Message: msg} +} + +func GetCommitMetaTxnHandlerResponse(r interface{}) *blobbergrpc.CommitMetaTxnResponse { + msg, _ := r.(struct { + Msg string `json:"msg"` + }) - return result + return &blobbergrpc.CommitMetaTxnResponse{Message: msg.Msg} } -func CollaboratorResponse(response *blobbergrpc.CollaboratorResponse) interface{} { - if msg := response.GetMessage(); msg != "" { - return struct { - Msg string `json:"msg"` - }{Msg: msg} +func CollaboratorResponse(r interface{}) *blobbergrpc.CollaboratorResponse { + if r == nil { + return nil } - if collaborators := response.GetCollaborators(); collaborators != nil { - collabs := make([]reference.Collaborator, 0, len(collaborators)) - for _, c := range collaborators { - collabs = append(collabs, *GRPCCollaboratorToCollaborator(c)) - } + msg, _ := r.(struct { + Msg string `json:"msg"` + }) + var resp blobbergrpc.CollaboratorResponse + if msg.Msg != "" { + resp.Message = msg.Msg + return &resp + } - return collabs + collabs, _ := r.([]reference.Collaborator) + for _, c := range collabs { + resp.Collaborators = append(resp.Collaborators, CollaboratorToGRPCCollaborator(&c)) } - return nil + return &resp } -func UpdateObjectAttributesResponseHandler(updateAttributesResponse *blobbergrpc.UpdateObjectAttributesResponse) *blobberHTTP.UpdateObjectAttributesResponse { - return &blobberHTTP.UpdateObjectAttributesResponse{ - WhoPaysForReads: common.WhoPays(updateAttributesResponse.WhoPaysForReads), +func UpdateObjectAttributesResponseCreator(r interface{}) *blobbergrpc.UpdateObjectAttributesResponse { + if r != nil { + return nil } + + httpResp, _ := r.(*reference.Attributes) + return &blobbergrpc.UpdateObjectAttributesResponse{WhoPaysForReads: int64(httpResp.WhoPaysForReads)} } -func CopyObjectResponseHandler(copyObjectResponse *blobbergrpc.CopyObjectResponse) *blobberHTTP.UploadResult { - return &blobberHTTP.UploadResult{ - Filename: copyObjectResponse.Filename, - Size: copyObjectResponse.Size, - Hash: copyObjectResponse.ContentHash, - MerkleRoot: copyObjectResponse.MerkleRoot, - UploadLength: copyObjectResponse.UploadLength, - UploadOffset: copyObjectResponse.UploadOffset, +func CopyObjectResponseCreator(r interface{}) *blobbergrpc.CopyObjectResponse { + if r == nil { + return nil + } + + httpResp, _ := r.(*blobberHTTP.UploadResult) + return &blobbergrpc.CopyObjectResponse{ + Filename: httpResp.Filename, + Size: httpResp.Size, + ContentHash: httpResp.Hash, + MerkleRoot: httpResp.MerkleRoot, + UploadLength: httpResp.UploadLength, + UploadOffset: httpResp.UploadOffset, } } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go b/code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go index c31257737..362ef2a1a 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go @@ -2,190 +2,27 @@ package handler import ( "context" - "encoding/hex" - "encoding/json" - "strconv" + "net/http" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "github.com/0chain/blobber/code/go/0chain.net/core/lock" - "gorm.io/gorm" ) func (b *blobberGRPCService) Commit(ctx context.Context, req *blobbergrpc.CommitRequest) (*blobbergrpc.CommitResponse, error) { - md := GetGRPCMetaDataFromCtx(ctx) - ctx = setupGRPCHandlerContext(ctx, md, req.Allocation) - - allocationTx := req.Allocation - clientID := md.Client - clientKey := md.ClientKey - clientKeyBytes, _ := hex.DecodeString(clientKey) - - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, false) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - - allocationID := allocationObj.ID - - connectionID := req.ConnectionId - if len(connectionID) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - } - - mutex := lock.GetMutex(allocationObj.TableName(), allocationID) - mutex.Lock() - defer mutex.Unlock() - - connectionObj, err := b.packageHandler.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - if err != nil { - return nil, common.NewErrorf("invalid_parameters", - "Invalid connection id. Connection id was not found: %v", err) - } - if len(connectionObj.Changes) == 0 { - return nil, common.NewError("invalid_parameters", - "Invalid connection id. Connection does not have any changes.") - } - - var isACollaborator bool - for _, change := range connectionObj.Changes { - if change.Operation == allocation.UPDATE_OPERATION { - updateFileChange := new(allocation.UpdateFileChange) - if err := updateFileChange.Unmarshal(change.Input); err != nil { - return nil, err - } - fileRef, err := reference.GetReference(ctx, allocationID, updateFileChange.Path) - if err != nil { - return nil, err - } - isACollaborator = reference.IsACollaborator(ctx, fileRef.ID, clientID) - break - } - } - - if len(clientID) == 0 || len(clientKey) == 0 { - return nil, common.NewError("invalid_params", "Please provide clientID and clientKey") - } - - if (allocationObj.OwnerID != clientID || encryption.Hash(clientKeyBytes) != clientID) && !isACollaborator { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - - if allocationObj.BlobberSizeUsed+connectionObj.Size > allocationObj.BlobberSize { - return nil, common.NewError("max_allocation_size", - "Max size reached for the allocation with this blobber") - } - - writeMarkerString := req.WriteMarker - writeMarker := writemarker.WriteMarker{} - err = json.Unmarshal([]byte(writeMarkerString), &writeMarker) - if err != nil { - return nil, common.NewErrorf("invalid_parameters", - "Invalid parameters. Error parsing the writemarker for commit: %v", - err) - } - - var result blobbergrpc.CommitResponse - var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { - latestWM = nil - } else { - latestWM, err = b.packageHandler.GetWriteMarkerEntity(ctx, - allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewErrorf("latest_write_marker_read_error", - "Error reading the latest write marker for allocation: %v", err) - } - } - - writemarkerObj := &writemarker.WriteMarkerEntity{} - writemarkerObj.WM = writeMarker - - err = b.packageHandler.VerifyMarker(writemarkerObj, ctx, allocationObj, connectionObj) + r, err := http.NewRequest("", "", nil) if err != nil { - result.AllocationRoot = allocationObj.AllocationRoot - result.ErrorMessage = "Verification of write marker failed: " + err.Error() - result.Success = false - if latestWM != nil { - result.WriteMarker = convert.WriteMarkerToWriteMarkerGRPC(&latestWM.WM) - } - return &result, common.NewError("write_marker_verification_failed", result.ErrorMessage) - } - - var clientIDForWriteRedeem = writeMarker.ClientID - if isACollaborator { - clientIDForWriteRedeem = allocationObj.OwnerID - } - - if err = writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil { return nil, err } - - err = b.packageHandler.ApplyChanges(connectionObj, ctx, writeMarker.AllocationRoot) - if err != nil { - return nil, err - } - rootRef, err := b.packageHandler.GetReference(ctx, allocationID, "/") - if err != nil { - return nil, err - } - allocationRoot := encryption.Hash(rootRef.Hash + ":" + strconv.FormatInt(int64(writeMarker.Timestamp), 10)) - - if allocationRoot != writeMarker.AllocationRoot { - result.AllocationRoot = allocationObj.AllocationRoot - if latestWM != nil { - result.WriteMarker = convert.WriteMarkerToWriteMarkerGRPC(&latestWM.WM) - } - result.Success = false - result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root. Expected hash: " + allocationRoot - return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage) + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "write_marker": {req.WriteMarker}, + "connection_id": {req.ConnectionId}, } - writemarkerObj.ConnectionID = connectionObj.ConnectionID - writemarkerObj.ClientPublicKey = clientKey - err = b.packageHandler.UpdateAllocationAndCommitChanges(ctx, writemarkerObj, connectionObj, allocationObj, allocationRoot) + resp, err := CommitHandler(ctx, r) if err != nil { return nil, err } - result.AllocationRoot = allocationObj.AllocationRoot - result.WriteMarker = convert.WriteMarkerToWriteMarkerGRPC(&writeMarker) - result.Success = true - result.ErrorMessage = "" - - return &result, nil -} - -func UpdateAllocationAndCommitChanges(ctx context.Context, writemarkerObj *writemarker.WriteMarkerEntity, connectionObj *allocation.AllocationChangeCollector, allocationObj *allocation.Allocation, allocationRoot string) error { - err := writemarkerObj.Save(ctx) - if err != nil { - return common.NewError("write_marker_error", "Error persisting the write marker") - } - - db := datastore.GetStore().GetTransaction(ctx) - allocationUpdates := make(map[string]interface{}) - allocationUpdates["blobber_size_used"] = gorm.Expr("blobber_size_used + ?", connectionObj.Size) - allocationUpdates["used_size"] = gorm.Expr("used_size + ?", connectionObj.Size) - allocationUpdates["allocation_root"] = allocationRoot - allocationUpdates["is_redeem_required"] = true - - err = db.Model(allocationObj).Updates(allocationUpdates).Error - if err != nil { - return common.NewError("allocation_write_error", "Error persisting the allocation object") - } - err = connectionObj.CommitToFileStore(ctx) - if err != nil { - return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) - } - - connectionObj.DeleteChanges(ctx) //nolint:errcheck // never returns an error anyway - - db.Model(connectionObj).Updates(allocation.AllocationChangeCollector{Status: allocation.CommittedConnection}) - return nil + return convert.CommitWriteResponseHandler(resp), nil } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_commit_handler_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_commit_handler_unit_test.go deleted file mode 100644 index 9e9ce4ce0..000000000 --- a/code/go/0chain.net/blobbercore/handler/grpc_commit_handler_unit_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package handler - -import ( - "context" - "encoding/hex" - "fmt" - "strconv" - "testing" - "time" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/mocks" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "github.com/magiconair/properties/assert" - "github.com/stretchr/testify/mock" - "google.golang.org/grpc/metadata" -) - -func TestBlobberGRPCService_Commit(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - pubKeyBytes, err := hex.DecodeString(pubKey) - if err != nil { - t.Fatal(err) - } - - timestamp := time.Now().UnixNano() - rootRefHash := "someHash" - clientId := encryption.Hash(pubKeyBytes) - connectionId := "connection_id" - allocationId := "allocationId" - req := &blobbergrpc.CommitRequest{ - Allocation: allocationTx, - ConnectionId: connectionId, - WriteMarker: `{"allocation_id":"` + allocationId + `","timestamp":` + fmt.Sprint(timestamp) + `,"allocation_root":"` + encryption.Hash(rootRefHash+":"+strconv.FormatInt(int64(timestamp), 10)) + `"}`, - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: clientId, - common.ClientKeyHeader: pubKey, - common.ClientSignatureHeader: clientSignature, - })) - - testcases := []struct { - description string - getAllocationChangesReturn func() (*allocation.AllocationChangeCollector, error) - verifyMarkerReturn func() error - applyChangesReturn func() error - getReferenceReturn func() (*reference.Ref, error) - verifyAllocationReturn func() (*allocation.Allocation, error) - getWriteMarkerEntityReturn func() (*writemarker.WriteMarkerEntity, error) - updateAllocationAndCommitChangesReturn func() error - expectedError bool - }{ - { - description: "success", - expectedError: false, - getAllocationChangesReturn: func() (*allocation.AllocationChangeCollector, error) { - return &allocation.AllocationChangeCollector{ - ConnectionID: connectionId, - AllocationID: allocationId, - ClientID: "", - Size: 0, - Changes: []*allocation.AllocationChange{&allocation.AllocationChange{ - ChangeID: 1, - Size: 0, - Operation: "insert", - ConnectionID: connectionId, - Input: "", - ModelWithTS: datastore.ModelWithTS{}, - }}, - AllocationChanges: nil, - Status: 0, - ModelWithTS: datastore.ModelWithTS{}, - }, nil - }, - verifyMarkerReturn: func() error { - return nil - }, - applyChangesReturn: func() error { - return nil - }, - getReferenceReturn: func() (*reference.Ref, error) { - return &reference.Ref{ - Hash: rootRefHash, - }, nil - }, - verifyAllocationReturn: func() (*allocation.Allocation, error) { - return &allocation.Allocation{ - ID: allocationId, - Tx: req.Allocation, - OwnerID: clientId, - OwnerPublicKey: pubKey, - }, nil - }, - getWriteMarkerEntityReturn: func() (*writemarker.WriteMarkerEntity, error) { - return nil, nil - }, - updateAllocationAndCommitChangesReturn: func() error { - return nil - }, - }, - { - description: "could not commit", - expectedError: true, - getAllocationChangesReturn: func() (*allocation.AllocationChangeCollector, error) { - return &allocation.AllocationChangeCollector{ - ConnectionID: connectionId, - AllocationID: allocationId, - ClientID: "", - Size: 0, - Changes: []*allocation.AllocationChange{&allocation.AllocationChange{ - ChangeID: 1, - Size: 0, - Operation: "insert", - ConnectionID: connectionId, - Input: "", - ModelWithTS: datastore.ModelWithTS{}, - }}, - AllocationChanges: nil, - Status: 0, - ModelWithTS: datastore.ModelWithTS{}, - }, nil - }, - verifyMarkerReturn: func() error { - return nil - }, - applyChangesReturn: func() error { - return nil - }, - getReferenceReturn: func() (*reference.Ref, error) { - return &reference.Ref{ - Hash: rootRefHash, - }, nil - }, - verifyAllocationReturn: func() (*allocation.Allocation, error) { - return &allocation.Allocation{ - ID: allocationId, - Tx: req.Allocation, - OwnerID: clientId, - OwnerPublicKey: pubKey, - }, nil - }, - getWriteMarkerEntityReturn: func() (*writemarker.WriteMarkerEntity, error) { - return nil, nil - }, - updateAllocationAndCommitChangesReturn: func() error { - return fmt.Errorf("some error") - }, - }, - { - description: "invalid marker", - expectedError: true, - getAllocationChangesReturn: func() (*allocation.AllocationChangeCollector, error) { - return &allocation.AllocationChangeCollector{ - ConnectionID: connectionId, - AllocationID: allocationId, - ClientID: "", - Size: 0, - Changes: []*allocation.AllocationChange{&allocation.AllocationChange{ - ChangeID: 1, - Size: 0, - Operation: "insert", - ConnectionID: connectionId, - Input: "", - ModelWithTS: datastore.ModelWithTS{}, - }}, - AllocationChanges: nil, - Status: 0, - ModelWithTS: datastore.ModelWithTS{}, - }, nil - }, - verifyMarkerReturn: func() error { - return fmt.Errorf("invalid marker") - }, - applyChangesReturn: func() error { - return nil - }, - getReferenceReturn: func() (*reference.Ref, error) { - return &reference.Ref{ - Hash: rootRefHash, - }, nil - }, - verifyAllocationReturn: func() (*allocation.Allocation, error) { - return &allocation.Allocation{ - ID: allocationId, - Tx: req.Allocation, - OwnerID: clientId, - OwnerPublicKey: pubKey, - }, nil - }, - getWriteMarkerEntityReturn: func() (*writemarker.WriteMarkerEntity, error) { - return nil, nil - }, - updateAllocationAndCommitChangesReturn: func() error { - return nil - }, - }, - } - - for _, tc := range testcases { - - mockStorageHandler := &storageHandlerI{} - mockPackageHandler := &mocks.PackageHandler{} - - mockPackageHandler.On("GetAllocationChanges", mock.Anything, connectionId, allocationId, mock.Anything).Return(tc.getAllocationChangesReturn()) - - mockPackageHandler.On("VerifyMarker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.verifyMarkerReturn()) - mockPackageHandler.On("ApplyChanges", mock.Anything, mock.Anything, mock.Anything).Return(tc.applyChangesReturn()) - mockPackageHandler.On("GetReference", mock.Anything, mock.Anything, mock.Anything).Return(tc.getReferenceReturn()) - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(tc.verifyAllocationReturn()) - mockPackageHandler.On("GetWriteMarkerEntity", mock.Anything, mock.Anything).Return(tc.getWriteMarkerEntityReturn()) - mockPackageHandler.On("UpdateAllocationAndCommitChanges", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.updateAllocationAndCommitChangesReturn()) - - svc := newGRPCBlobberService(mockStorageHandler, mockPackageHandler) - resp, err := svc.Commit(ctx, req) - if err != nil { - if tc.expectedError { - continue - } else { - t.Fatal("unexpected error - " + err.Error()) - } - } - - assert.Equal(t, resp.WriteMarker.AllocationID, allocationId) - } -} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler.go b/code/go/0chain.net/blobbercore/handler/grpc_handler.go index 5820930c2..981281994 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler.go @@ -2,637 +2,208 @@ package handler import ( "context" - "encoding/json" "net/http" - "strconv" + "strings" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - "go.uber.org/zap" ) type blobberGRPCService struct { - storageHandler StorageHandlerI - packageHandler PackageHandler blobbergrpc.UnimplementedBlobberServer } -func newGRPCBlobberService(sh StorageHandlerI, r PackageHandler) *blobberGRPCService { - return &blobberGRPCService{ - storageHandler: sh, - packageHandler: r, - } +func newGRPCBlobberService() *blobberGRPCService { + return &blobberGRPCService{} } func (b *blobberGRPCService) GetAllocation(ctx context.Context, request *blobbergrpc.GetAllocationRequest) (*blobbergrpc.GetAllocationResponse, error) { - ctx = setupGRPCHandlerContext(ctx, GetGRPCMetaDataFromCtx(ctx), "") - - allocation, err := b.storageHandler.verifyAllocation(ctx, request.Id, false) + r, err := http.NewRequest("GET", "", nil) if err != nil { return nil, err } + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), "") + r.Form = map[string][]string{"id": {request.Id}} - return &blobbergrpc.GetAllocationResponse{Allocation: convert.AllocationToGRPCAllocation(allocation)}, nil -} - -func (b *blobberGRPCService) GetFileMetaData(ctx context.Context, req *blobbergrpc.GetFileMetaDataRequest) (*blobbergrpc.GetFileMetaDataResponse, error) { - logger := ctxzap.Extract(ctx) - md := GetGRPCMetaDataFromCtx(ctx) - - allocationObj, err := b.storageHandler.verifyAllocation(ctx, req.Allocation, true) + resp, err := AllocationHandler(ctx, r) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - allocationID := allocationObj.ID - - clientID := md.Client - if len(clientID) == 0 { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + return nil, err } - path_hash := req.PathHash - path := req.Path - if len(path_hash) == 0 { - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - path_hash = reference.GetReferenceLookup(allocationID, path) - } + return convert.GetAllocationResponseCreator(resp), nil +} - fileref, err := b.packageHandler.GetReferenceFromLookupHash(ctx, allocationID, path_hash) +func (b *blobberGRPCService) GetFileMetaData(ctx context.Context, req *blobbergrpc.GetFileMetaDataRequest) (*blobbergrpc.GetFileMetaDataResponse, error) { + r, err := http.NewRequest("POST", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - } - - if fileref.Type != reference.FILE { - return nil, common.NewError("invalid_parameters", "Path is not a file.") + return nil, err } - - commitMetaTxns, err := b.packageHandler.GetCommitMetaTxns(ctx, fileref.ID) - if err != nil { - logger.Error("Failed to get commitMetaTxns from refID", zap.Error(err), zap.Any("ref_id", fileref.ID)) + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path_hash": {req.PathHash}, + "path": {req.Path}, + "auth_token": {req.AuthToken}, } - fileref.CommitMetaTxns = commitMetaTxns - collaborators, err := b.packageHandler.GetCollaborators(ctx, fileref.ID) + resp, err := FileMetaHandler(ctx, r) if err != nil { - logger.Error("Failed to get collaborators from refID", zap.Error(err), zap.Any("ref_id", fileref.ID)) - } - - authTokenString := req.AuthToken - - if (allocationObj.OwnerID != clientID && - allocationObj.PayerID != clientID && - !b.packageHandler.IsACollaborator(ctx, fileref.ID, clientID)) || len(authTokenString) > 0 { - authTicketVerified, err := b.storageHandler.verifyAuthTicket(ctx, req.AuthToken, allocationObj, fileref, clientID) - if err != nil { - return nil, err - } - if !authTicketVerified { - return nil, common.NewError("auth_ticket_verification_failed", "Could not verify the auth ticket.") - } - fileref.Path = "" - } - - var collaboratorsGRPC []*blobbergrpc.Collaborator - for _, c := range collaborators { - collaboratorsGRPC = append(collaboratorsGRPC, convert.CollaboratorToGRPCCollaborator(&c)) + return nil, err } - return &blobbergrpc.GetFileMetaDataResponse{ - MetaData: convert.FileRefToFileRefGRPC(fileref), - Collaborators: collaboratorsGRPC, - }, nil + return convert.GetFileMetaDataResponseCreator(resp), nil } func (b *blobberGRPCService) GetFileStats(ctx context.Context, req *blobbergrpc.GetFileStatsRequest) (*blobbergrpc.GetFileStatsResponse, error) { - allocationTx := req.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, true) - md := GetGRPCMetaDataFromCtx(ctx) - + r, err := http.NewRequest("POST", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - allocationID := allocationObj.ID - - valid, err := verifySignatureFromRequest(allocationTx, md.ClientSignature, allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - clientID := md.Client - if len(clientID) == 0 || allocationObj.OwnerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + return nil, err } - - path_hash := req.PathHash - path := req.Path - if len(path_hash) == 0 { - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - path_hash = reference.GetReferenceLookup(allocationID, path) + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "path_hash": {req.PathHash}, } - fileref, err := b.packageHandler.GetReferenceFromLookupHash(ctx, allocationID, path_hash) - + resp, err := FileStatsHandler(ctx, r) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - } - - if fileref.Type != reference.FILE { - return nil, common.NewError("invalid_parameters", "Path is not a file.") - } - - stats, _ := b.packageHandler.GetFileStats(ctx, fileref.ID) - wm, _ := b.packageHandler.GetWriteMarkerEntity(ctx, fileref.WriteMarker) - if wm != nil && stats != nil { - stats.WriteMarkerRedeemTxn = wm.CloseTxnID + return nil, err } - return &blobbergrpc.GetFileStatsResponse{ - MetaData: convert.FileRefToFileRefGRPC(fileref), - Stats: convert.FileStatsToFileStatsGRPC(stats), - }, nil + return convert.GetFileStatsResponseCreator(resp), nil } func (b *blobberGRPCService) ListEntities(ctx context.Context, req *blobbergrpc.ListEntitiesRequest) (*blobbergrpc.ListEntitiesResponse, error) { - logger := ctxzap.Extract(ctx) - md := GetGRPCMetaDataFromCtx(ctx) - - clientID := md.Client - allocationTx := req.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, true) - + r, err := http.NewRequest("", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - allocationID := allocationObj.ID - - if len(clientID) == 0 { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - - path_hash := req.PathHash - path := req.Path - if len(path_hash) == 0 { - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - path_hash = reference.GetReferenceLookup(allocationID, path) - } - - logger.Info("Path Hash for list dir :" + path_hash) - - fileref, err := b.packageHandler.GetReferenceFromLookupHash(ctx, allocationID, path_hash) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid path. "+err.Error()) + return nil, err } - authTokenString := req.AuthToken - if clientID != allocationObj.OwnerID || len(authTokenString) > 0 { - authTicketVerified, err := b.storageHandler.verifyAuthTicket(ctx, authTokenString, allocationObj, fileref, clientID) - if err != nil { - return nil, err - } - if !authTicketVerified { - return nil, common.NewError("auth_ticket_verification_failed", "Could not verify the auth ticket.") - } + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "path_hash": {req.PathHash}, + "auth_token": {req.AuthToken}, } - dirref, err := b.packageHandler.GetRefWithChildren(ctx, allocationID, fileref.Path) + resp, err := ListHandler(ctx, r) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid path. "+err.Error()) - } - - if clientID != allocationObj.OwnerID { - dirref.Path = "" - } - - var entities []*blobbergrpc.FileRef - for _, entity := range dirref.Children { - if clientID != allocationObj.OwnerID { - entity.Path = "" - } - entities = append(entities, convert.FileRefToFileRefGRPC(entity)) + return nil, err } - refGRPC := convert.FileRefToFileRefGRPC(dirref) - return &blobbergrpc.ListEntitiesResponse{ - AllocationRoot: allocationObj.AllocationRoot, - MetaData: refGRPC, - Entities: entities, - }, nil + return convert.ListEntitesResponseCreator(resp), nil } func (b *blobberGRPCService) GetObjectPath(ctx context.Context, req *blobbergrpc.GetObjectPathRequest) (*blobbergrpc.GetObjectPathResponse, error) { - allocationTx := req.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, false) - md := GetGRPCMetaDataFromCtx(ctx) - + r, err := http.NewRequest("", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - allocationID := allocationObj.ID - - valid, err := verifySignatureFromRequest(allocationTx, md.ClientSignature, allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - clientID := md.Client - if len(clientID) == 0 || allocationObj.OwnerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - path := req.Path - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") + return nil, err } - - blockNumStr := req.BlockNum - if len(blockNumStr) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "block_num": {req.BlockNum}, } - blockNum, err := strconv.ParseInt(blockNumStr, 10, 64) - if err != nil || blockNum < 0 { - return nil, common.NewError("invalid_parameters", "Invalid block number") - } - - objectPath, err := b.packageHandler.GetObjectPath(ctx, allocationID, blockNum) + resp, err := ObjectPathHandler(ctx, r) if err != nil { return nil, err } - var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { - latestWM = nil - } else { - latestWM, err = b.packageHandler.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) - } - } - var latestWriteMarketGRPC *blobbergrpc.WriteMarker - if latestWM != nil { - latestWriteMarketGRPC = convert.WriteMarkerToWriteMarkerGRPC(&latestWM.WM) - } - - pathList := make([]*blobbergrpc.FileRef, 0) - list, _ := objectPath.Path["list"].([]map[string]interface{}) - if len(list) > 0 { - for _, pl := range list { - pathList = append(pathList, convert.FileRefToFileRefGRPC(reference.ListingDataToRef(pl))) - } - } - - return &blobbergrpc.GetObjectPathResponse{ - ObjectPath: &blobbergrpc.ObjectPath{ - RootHash: objectPath.RootHash, - Meta: convert.FileRefToFileRefGRPC(reference.ListingDataToRef(objectPath.Meta)), - Path: convert.FileRefToFileRefGRPC(reference.ListingDataToRef(objectPath.Path)), - PathList: pathList, - FileBlockNum: objectPath.FileBlockNum, - }, - LatestWriteMarker: latestWriteMarketGRPC, - }, nil + return convert.GetObjectPathResponseCreator(resp), nil } func (b *blobberGRPCService) GetReferencePath(ctx context.Context, req *blobbergrpc.GetReferencePathRequest) (*blobbergrpc.GetReferencePathResponse, error) { - md := GetGRPCMetaDataFromCtx(ctx) - allocationTx := req.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, false) - + r, err := http.NewRequest("", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - allocationID := allocationObj.ID - - valid, err := verifySignatureFromRequest(allocationTx, md.ClientSignature, allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - clientID := md.Client - if len(clientID) == 0 { - return nil, common.NewError("invalid_operation", "Please pass clientID in the header") + return nil, err } - - var paths []string - pathsString := req.Paths - if len(pathsString) == 0 { - path := req.Path - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - paths = append(paths, path) - } else { - err = json.Unmarshal([]byte(pathsString), &paths) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid path array json") - } + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "paths": {req.Paths}, } - rootRef, err := b.packageHandler.GetReferencePathFromPaths(ctx, allocationID, paths) + resp, err := ReferencePathHandler(ctx, r) if err != nil { return nil, err } - refPath := &reference.ReferencePath{Ref: rootRef} - refsToProcess := make([]*reference.ReferencePath, 0) - refsToProcess = append(refsToProcess, refPath) - for len(refsToProcess) > 0 { - refToProcess := refsToProcess[0] - refToProcess.Meta = refToProcess.Ref.GetListingData(ctx) - if len(refToProcess.Ref.Children) > 0 { - refToProcess.List = make([]*reference.ReferencePath, len(refToProcess.Ref.Children)) - } - for idx, child := range refToProcess.Ref.Children { - childRefPath := &reference.ReferencePath{Ref: child} - refToProcess.List[idx] = childRefPath - refsToProcess = append(refsToProcess, childRefPath) - } - refsToProcess = refsToProcess[1:] - } - - var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { - latestWM = nil - } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) - } - } - - var refPathResult blobbergrpc.GetReferencePathResponse - var recursionCount int - refPathResult.ReferencePath = convert.ReferencePathToReferencePathGRPC(&recursionCount, refPath) - if latestWM != nil { - refPathResult.LatestWM = convert.WriteMarkerToWriteMarkerGRPC(&latestWM.WM) - } - - return &refPathResult, nil + return convert.GetReferencePathResponseCreator(resp), nil } func (b *blobberGRPCService) GetObjectTree(ctx context.Context, req *blobbergrpc.GetObjectTreeRequest) (*blobbergrpc.GetObjectTreeResponse, error) { - allocationTx := req.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, false) - md := GetGRPCMetaDataFromCtx(ctx) - + r, err := http.NewRequest("", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - allocationID := allocationObj.ID - - valid, err := verifySignatureFromRequest(allocationTx, md.ClientSignature, allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - clientID := md.Client - if len(clientID) == 0 || allocationObj.OwnerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + return nil, err } - path := req.Path - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, } - rootRef, err := b.packageHandler.GetObjectTree(ctx, allocationID, path) + resp, err := ObjectTreeHandler(ctx, r) if err != nil { return nil, err } - refPath := &reference.ReferencePath{Ref: rootRef} - refsToProcess := make([]*reference.ReferencePath, 0) - refsToProcess = append(refsToProcess, refPath) - for len(refsToProcess) > 0 { - refToProcess := refsToProcess[0] - refToProcess.Meta = refToProcess.Ref.GetListingData(ctx) - if len(refToProcess.Ref.Children) > 0 { - refToProcess.List = make([]*reference.ReferencePath, len(refToProcess.Ref.Children)) - } - for idx, child := range refToProcess.Ref.Children { - childRefPath := &reference.ReferencePath{Ref: child} - refToProcess.List[idx] = childRefPath - refsToProcess = append(refsToProcess, childRefPath) - } - refsToProcess = refsToProcess[1:] - } - - var latestWM *writemarker.WriteMarkerEntity - if len(allocationObj.AllocationRoot) == 0 { - latestWM = nil - } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) - } - } - var refPathResult blobbergrpc.GetObjectTreeResponse - var recursionCount int - refPathResult.ReferencePath = convert.ReferencePathToReferencePathGRPC(&recursionCount, refPath) - if latestWM != nil { - refPathResult.LatestWM = convert.WriteMarkerToWriteMarkerGRPC(&latestWM.WM) - } - return &refPathResult, nil + return convert.GetObjectTreeResponseCreator(resp), nil } func (b *blobberGRPCService) CalculateHash(ctx context.Context, req *blobbergrpc.CalculateHashRequest) (*blobbergrpc.CalculateHashResponse, error) { - allocationTx := req.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, false) + r, err := http.NewRequest("POST", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - - md := GetGRPCMetaDataFromCtx(ctx) - allocationID := allocationObj.ID - - clientID := md.Client - if len(clientID) == 0 || allocationObj.OwnerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + return nil, err } - - var paths []string - pathsString := req.Paths - if len(pathsString) == 0 { - path := req.Path - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - paths = append(paths, path) - } else { - err = json.Unmarshal([]byte(pathsString), &paths) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid path array json") - } + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "paths": {req.Paths}, } - rootRef, err := b.packageHandler.GetReferencePathFromPaths(ctx, allocationID, paths) + resp, err := CalculateHashHandler(ctx, r) if err != nil { return nil, err } - if _, err := rootRef.CalculateHash(ctx, true); err != nil { - return nil, err - } - - return &blobbergrpc.CalculateHashResponse{Message: "Hash recalculated for the given paths"}, nil + return convert.GetCalculateHashResponseHandler(resp), nil } func (b *blobberGRPCService) CommitMetaTxn(ctx context.Context, req *blobbergrpc.CommitMetaTxnRequest) (*blobbergrpc.CommitMetaTxnResponse, error) { - allocationTx := req.GetAllocation() - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, true) + r, err := http.NewRequest("POST", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - allocationID := allocationObj.ID - - md := GetGRPCMetaDataFromCtx(ctx) - clientID := md.Client - if len(clientID) == 0 { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + return nil, err } - - pathHash := req.PathHash - path := req.Path - if len(pathHash) == 0 { - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - pathHash = reference.GetReferenceLookup(allocationID, path) + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "path_hash": {req.PathHash}, + "auth_token": {req.AuthToken}, + "txn_id": {req.TxnId}, } - fileRef, err := b.packageHandler.GetReferenceFromLookupHash(ctx, allocationID, pathHash) + resp, err := CommitMetaTxnHandler(ctx, r) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - } - - if fileRef.Type != reference.FILE { - return nil, common.NewError("invalid_parameters", "Path is not a file.") - } - - auhToken := req.GetAuthToken() - - if clientID != allocationObj.OwnerID || len(auhToken) > 0 { - authTicketVerified, err := b.storageHandler.verifyAuthTicket(ctx, auhToken, allocationObj, fileRef, clientID) - if err != nil { - return nil, err - } - - if !authTicketVerified { - return nil, common.NewError("auth_ticket_verification_failed", "Could not verify the auth ticket.") - } - } - - txnID := req.GetTxnId() - if len(txnID) == 0 { - return nil, common.NewError("invalid_parameter", "TxnID not present in the params") - } - - if err := b.packageHandler.AddCommitMetaTxn(ctx, fileRef.ID, txnID); err != nil { - return nil, common.NewError("add_commit_meta_txn_failed", "Failed to add commitMetaTxn with err :"+err.Error()) + return nil, err } - return &blobbergrpc.CommitMetaTxnResponse{ - Message: "Added commitMetaTxn successfully", - }, nil + return convert.GetCommitMetaTxnHandlerResponse(resp), nil } func (b *blobberGRPCService) Collaborator(ctx context.Context, req *blobbergrpc.CollaboratorRequest) (*blobbergrpc.CollaboratorResponse, error) { - allocationTx := req.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, true) + r, err := http.NewRequest(strings.ToUpper(req.Method), "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - - md := GetGRPCMetaDataFromCtx(ctx) - allocationID := allocationObj.ID - - valid, err := verifySignatureFromRequest(allocationTx, md.ClientSignature, allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") + return nil, err } - - clientID := md.Client - - pathHash := req.PathHash - path := req.Path - if len(pathHash) == 0 { - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - pathHash = reference.GetReferenceLookup(allocationID, path) + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "path_hash": {req.PathHash}, + "collab_id": {req.CollabId}, } - fileRef, err := b.packageHandler.GetReferenceFromLookupHash(ctx, allocationID, pathHash) + resp, err := CollaboratorHandler(ctx, r) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - } - - if fileRef.Type != reference.FILE { - return nil, common.NewError("invalid_parameters", "Path is not a file.") - } - - collabClientID := req.CollabId - if len(collabClientID) == 0 { - return nil, common.NewError("invalid_parameter", "collab_id not present in the params") - } - - var msg string - - switch req.GetMethod() { - case http.MethodPost: - if len(clientID) == 0 || clientID != allocationObj.OwnerID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - - if b.packageHandler.IsACollaborator(ctx, fileRef.ID, collabClientID) { - msg = "Given client ID is already a collaborator" - return &blobbergrpc.CollaboratorResponse{Message: msg}, nil - } - - if err := b.packageHandler.AddCollaborator(ctx, fileRef.ID, collabClientID); err != nil { - return nil, common.NewError("add_collaborator_failed", "Failed to add collaborator with err :"+err.Error()) - } - - msg = "Added collaborator successfully" - - case http.MethodGet: - collaborators, err := b.packageHandler.GetCollaborators(ctx, fileRef.ID) - if err != nil { - return nil, common.NewError("get_collaborator_failed", "Failed to get collaborators from refID with err:"+err.Error()) - } - - var collaboratorsGRPC []*blobbergrpc.Collaborator - for _, c := range collaborators { - collaboratorsGRPC = append(collaboratorsGRPC, convert.CollaboratorToGRPCCollaborator(&c)) - } - - return &blobbergrpc.CollaboratorResponse{ - Collaborators: collaboratorsGRPC, - }, nil - - case http.MethodDelete: - if len(clientID) == 0 || clientID != allocationObj.OwnerID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - - if err := b.packageHandler.RemoveCollaborator(ctx, fileRef.ID, collabClientID); err != nil { - return nil, common.NewError("delete_collaborator_failed", "Failed to delete collaborator from refID with err:"+err.Error()) - } - - msg = "Removed collaborator successfully" - - default: - return nil, common.NewError("invalid_method", "Invalid method used. Use POST/GET/DELETE instead") + return nil, err } - return &blobbergrpc.CollaboratorResponse{ - Message: msg, - }, nil + return convert.CollaboratorResponse(resp), nil } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index 23759ea1b..e89037c5f 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -19,61 +19,8 @@ import ( "gorm.io/gorm" "github.com/0chain/gosdk/core/zcncrypto" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/stretchr/testify/mock" ) -// StorageHandlerI is an autogenerated mock type for the StorageHandlerI type -type storageHandlerI struct { - mock.Mock -} - -// verifyAllocation provides a mock function with given fields: ctx, tx, readonly -func (_m *storageHandlerI) verifyAllocation(ctx context.Context, tx string, readonly bool) (*allocation.Allocation, error) { - ret := _m.Called(ctx, tx, readonly) - - var r0 *allocation.Allocation - if rf, ok := ret.Get(0).(func(context.Context, string, bool) *allocation.Allocation); ok { - r0 = rf(ctx, tx, readonly) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*allocation.Allocation) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok { - r1 = rf(ctx, tx, readonly) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// verifyAuthTicket provides a mock function with given fields: ctx, authTokenString, allocationObj, refRequested, clientID -func (_m *storageHandlerI) verifyAuthTicket(ctx context.Context, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string) (bool, error) { - ret := _m.Called(ctx, authTokenString, allocationObj, refRequested, clientID) - - var r0 bool - if rf, ok := ret.Get(0).(func(context.Context, string, *allocation.Allocation, *reference.Ref, string) bool); ok { - r0 = rf(ctx, authTokenString, allocationObj, refRequested, clientID) - } else { - r0 = ret.Get(0).(bool) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string, *allocation.Allocation, *reference.Ref, string) error); ok { - r1 = rf(ctx, authTokenString, allocationObj, refRequested, clientID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - type TestDataController struct { db *gorm.DB } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 7dea657d1..efd6d83c1 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -3,23 +3,26 @@ package handler import ( "context" "encoding/hex" - "encoding/json" "fmt" "log" + "math/rand" "net/http" "os" "strconv" + "strings" "testing" "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "google.golang.org/grpc" "google.golang.org/grpc/metadata" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "google.golang.org/grpc" "gorm.io/driver/postgres" "gorm.io/gorm" ) @@ -526,120 +529,120 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { }) - t.Run("TestCommit", func(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - pubKeyBytes, _ := hex.DecodeString(pubKey) - clientId := encryption.Hash(pubKeyBytes) - now := common.Timestamp(time.Now().UnixNano()) - - blobberPubKey := "de52c0a51872d5d2ec04dbc15a6f0696cba22657b80520e1d070e72de64c9b04e19ce3223cae3c743a20184158457582ffe9c369ca9218c04bfe83a26a62d88d" - blobberPubKeyBytes, _ := hex.DecodeString(blobberPubKey) - - fr := reference.Ref{ - AllocationID: "exampleId", - Type: "f", - Name: "new_name", - Path: "/new_name", - ContentHash: "contentHash", - MerkleRoot: "merkleRoot", - ActualFileHash: "actualFileHash", - } - - rootRefHash := encryption.Hash(encryption.Hash(fr.GetFileHashData())) - - wm := writemarker.WriteMarker{ - AllocationRoot: encryption.Hash(rootRefHash + ":" + strconv.FormatInt(int64(now), 10)), - PreviousAllocationRoot: "/", - AllocationID: "exampleId", - Size: 1337, - BlobberID: encryption.Hash(blobberPubKeyBytes), - Timestamp: now, - ClientID: clientId, - } - - wmSig, err := signScheme.Sign(encryption.Hash(wm.GetHashData())) - if err != nil { - t.Fatal(err) - } - - wm.Signature = wmSig - - wmRaw, err := json.Marshal(wm) - if err != nil { - t.Fatal(err) - } - - err = tdController.ClearDatabase() - if err != nil { - t.Fatal(err) - } - err = tdController.AddCommitTestData(allocationTx, pubKey, clientId, wmSig, now) - if err != nil { - t.Fatal(err) - } - - testCases := []struct { - name string - context metadata.MD - input *blobbergrpc.CommitRequest - expectedAllocation string - expectingError bool - }{ - { - name: "Success", - context: metadata.New(map[string]string{ - common.ClientHeader: clientId, - common.ClientSignatureHeader: clientSignature, - common.ClientKeyHeader: pubKey, - }), - input: &blobbergrpc.CommitRequest{ - Allocation: allocationTx, - ConnectionId: "connection_id", - WriteMarker: string(wmRaw), - }, - expectedAllocation: "exampleId", - expectingError: false, - }, - { - name: "invalid write_marker", - context: metadata.New(map[string]string{ - common.ClientHeader: clientId, - common.ClientSignatureHeader: clientSignature, - common.ClientKeyHeader: pubKey, - }), - input: &blobbergrpc.CommitRequest{ - Allocation: allocationTx, - ConnectionId: "invalid", - WriteMarker: "invalid", - }, - expectedAllocation: "", - expectingError: true, - }, - } - - for _, tc := range testCases { - ctx := context.Background() - ctx = metadata.NewOutgoingContext(ctx, tc.context) - getCommiteResp, err := blobberClient.Commit(ctx, tc.input) - if err != nil { - if !tc.expectingError { - t.Fatal(err) - } - continue - } - - if tc.expectingError { - t.Fatal("expected error") - } - - if getCommiteResp.WriteMarker.AllocationID != tc.expectedAllocation { - t.Fatal("unexpected root name from GetObject") - } - } - }) + //t.Run("TestCommit", func(t *testing.T) { + // allocationTx := randString(32) + // + // pubKey, _, signScheme := GeneratePubPrivateKey(t) + // clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + // pubKeyBytes, _ := hex.DecodeString(pubKey) + // clientId := encryption.Hash(pubKeyBytes) + // now := common.Timestamp(time.Now().UnixNano()) + // + // blobberPubKey := "de52c0a51872d5d2ec04dbc15a6f0696cba22657b80520e1d070e72de64c9b04e19ce3223cae3c743a20184158457582ffe9c369ca9218c04bfe83a26a62d88d" + // blobberPubKeyBytes, _ := hex.DecodeString(blobberPubKey) + // + // fr := reference.Ref{ + // AllocationID: "exampleId", + // Type: "f", + // Name: "new_name", + // Path: "/new_name", + // ContentHash: "contentHash", + // MerkleRoot: "merkleRoot", + // ActualFileHash: "actualFileHash", + // } + // + // rootRefHash := encryption.Hash(encryption.Hash(fr.GetFileHashData())) + // + // wm := writemarker.WriteMarker{ + // AllocationRoot: encryption.Hash(rootRefHash + ":" + strconv.FormatInt(int64(now), 10)), + // PreviousAllocationRoot: "/", + // AllocationID: "exampleId", + // Size: 1337, + // BlobberID: encryption.Hash(blobberPubKeyBytes), + // Timestamp: now, + // ClientID: clientId, + // } + // + // wmSig, err := signScheme.Sign(encryption.Hash(wm.GetHashData())) + // if err != nil { + // t.Fatal(err) + // } + // + // wm.Signature = wmSig + // + // wmRaw, err := json.Marshal(wm) + // if err != nil { + // t.Fatal(err) + // } + // + // err = tdController.ClearDatabase() + // if err != nil { + // t.Fatal(err) + // } + // err = tdController.AddCommitTestData(allocationTx, pubKey, clientId, wmSig, now) + // if err != nil { + // t.Fatal(err) + // } + // + // testCases := []struct { + // name string + // context metadata.MD + // input *blobbergrpc.CommitRequest + // expectedAllocation string + // expectingError bool + // }{ + // { + // name: "Success", + // context: metadata.New(map[string]string{ + // common.ClientHeader: clientId, + // common.ClientSignatureHeader: clientSignature, + // common.ClientKeyHeader: pubKey, + // }), + // input: &blobbergrpc.CommitRequest{ + // Allocation: allocationTx, + // ConnectionId: "connection_id", + // WriteMarker: string(wmRaw), + // }, + // expectedAllocation: "exampleId", + // expectingError: false, + // }, + // { + // name: "invalid write_marker", + // context: metadata.New(map[string]string{ + // common.ClientHeader: clientId, + // common.ClientSignatureHeader: clientSignature, + // common.ClientKeyHeader: pubKey, + // }), + // input: &blobbergrpc.CommitRequest{ + // Allocation: allocationTx, + // ConnectionId: "invalid", + // WriteMarker: "invalid", + // }, + // expectedAllocation: "", + // expectingError: true, + // }, + // } + // + // for _, tc := range testCases { + // ctx := context.Background() + // ctx = metadata.NewOutgoingContext(ctx, tc.context) + // getCommiteResp, err := blobberClient.Commit(ctx, tc.input) + // if err != nil { + // if !tc.expectingError { + // t.Fatal(err) + // } + // continue + // } + // + // if tc.expectingError { + // t.Fatal("expected error") + // } + // + // if getCommiteResp.WriteMarker.AllocationID != tc.expectedAllocation { + // t.Fatal("unexpected root name from GetObject") + // } + // } + //}) t.Run("TestCommitMetaTxn", func(t *testing.T) { allocationTx := randString(32) @@ -848,3 +851,14 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } }) } + +func randString(n int) string { + + const hexLetters = "abcdef0123456789" + + var sb strings.Builder + for i := 0; i < n; i++ { + sb.WriteByte(hexLetters[rand.Intn(len(hexLetters))]) + } + return sb.String() +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_unit_test.go deleted file mode 100644 index dd6b14f58..000000000 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_unit_test.go +++ /dev/null @@ -1,758 +0,0 @@ -package handler - -import ( - "context" - "errors" - "math/rand" - "net/http" - "strings" - "testing" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/mocks" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "google.golang.org/grpc/metadata" -) - -func TestBlobberGRPCService_GetAllocation_Success(t *testing.T) { - req := &blobbergrpc.GetAllocationRequest{ - Id: "something", - } - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Id, false).Return(&allocation.Allocation{ - Tx: req.Id, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - allocation, err := svc.GetAllocation(context.Background(), req) - assert.NoError(t, err) - assert.Equal(t, allocation.Allocation.Tx, req.Id) -} - -func TestBlobberGRPCService_GetAllocation_invalidAllocation(t *testing.T) { - req := &blobbergrpc.GetAllocationRequest{ - Id: "invalid_allocation", - } - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Id, false).Return(nil, errors.New("some error")) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.GetAllocation(context.Background(), req) - if err == nil { - t.Fatal("expected error") - } - - assert.Equal(t, err.Error(), "some error") -} - -func TestBlobberGRPCService_GetFileMetaData_Success(t *testing.T) { - req := &blobbergrpc.GetFileMetaDataRequest{ - Path: "path", - PathHash: "path_hash", - AuthToken: "testval", - Allocation: "something", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: "", - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.FILE, - }, nil) - mockReferencePackage.On("GetCommitMetaTxns", mock.Anything, mock.Anything).Return(nil, nil) - mockReferencePackage.On("GetCollaborators", mock.Anything, mock.Anything).Return([]reference.Collaborator{ - reference.Collaborator{ - RefID: 1, - ClientID: "test", - }, - }, nil) - mockReferencePackage.On("IsACollaborator", mock.Anything, mock.Anything, mock.Anything).Return(true) - mockStorageHandler.On("verifyAuthTicket", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.GetFileMetaData(ctx, req) - if err != nil { - t.Fatal("unexpected error") - } - - assert.Equal(t, resp.MetaData.FileMetaData.Name, "test") -} - -func TestBlobberGRPCService_GetFileMetaData_FileNotExist(t *testing.T) { - req := &blobbergrpc.GetFileMetaDataRequest{ - Path: "path", - PathHash: "path_hash", - AuthToken: "testval", - Allocation: "something", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: "", - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("file doesnt exist")) - mockReferencePackage.On("GetCommitMetaTxns", mock.Anything, mock.Anything).Return(nil, nil) - mockReferencePackage.On("GetCollaborators", mock.Anything, mock.Anything).Return([]reference.Collaborator{ - reference.Collaborator{ - RefID: 1, - ClientID: "test", - }, - }, nil) - mockReferencePackage.On("IsACollaborator", mock.Anything, mock.Anything, mock.Anything).Return(true) - mockStorageHandler.On("verifyAuthTicket", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.GetFileMetaData(ctx, req) - if err == nil { - t.Fatal("expected error") - } -} - -func randString(n int) string { - - const hexLetters = "abcdef0123456789" - - var sb strings.Builder - for i := 0; i < n; i++ { - sb.WriteByte(hexLetters[rand.Intn(len(hexLetters))]) - } - return sb.String() -} - -func TestBlobberGRPCService_GetFileStats_Success(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.GetFileStatsRequest{ - Path: "path", - PathHash: "path_hash", - Allocation: allocationTx, - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "owner", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - ID: 123, - Name: "test", - Type: reference.FILE, - }, nil) - mockReferencePackage.On("GetFileStats", mock.Anything, int64(123)).Return(&stats.FileStats{ - NumBlockDownloads: 10, - }, nil) - mockReferencePackage.On("GetWriteMarkerEntity", mock.Anything, mock.Anything).Return(nil, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.GetFileStats(ctx, req) - if err != nil { - t.Fatal("unexpected error") - } - - assert.Equal(t, resp.MetaData.FileMetaData.Name, "test") - assert.Equal(t, resp.Stats.NumBlockDownloads, int64(10)) -} - -func TestBlobberGRPCService_GetFileStats_FileNotExist(t *testing.T) { - req := &blobbergrpc.GetFileStatsRequest{ - Path: "path", - PathHash: "path_hash", - Allocation: "", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "owner", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: "", - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("file does not exist")) - mockReferencePackage.On("GetFileStats", mock.Anything, int64(123)).Return(&stats.FileStats{ - NumBlockDownloads: 10, - }, nil) - mockReferencePackage.On("GetWriteMarkerEntity", mock.Anything, mock.Anything).Return(nil, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.GetFileStats(ctx, req) - if err == nil { - t.Fatal("expected error") - } -} - -func TestBlobberGRPCService_ListEntities_Success(t *testing.T) { - req := &blobbergrpc.ListEntitiesRequest{ - Path: "path", - PathHash: "path_hash", - AuthToken: "something", - Allocation: "", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: "", - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - AllocationRoot: "/allocationroot", - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.FILE, - }, nil) - mockStorageHandler.On("verifyAuthTicket", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - mockReferencePackage.On("GetRefWithChildren", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.DIRECTORY, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.ListEntities(ctx, req) - if err != nil { - t.Fatal("unexpected error") - } - - assert.Equal(t, resp.AllocationRoot, "/allocationroot") -} - -func TestBlobberGRPCService_ListEntities_InvalidAuthTicket(t *testing.T) { - req := &blobbergrpc.ListEntitiesRequest{ - Path: "path", - PathHash: "path_hash", - AuthToken: "something", - Allocation: "", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: "", - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.FILE, - }, nil) - mockStorageHandler.On("verifyAuthTicket", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(false, nil) - mockReferencePackage.On("GetRefWithChildren", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.DIRECTORY, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.ListEntities(ctx, req) - if err == nil { - t.Fatal("expected error") - } -} - -func TestBlobberGRPCService_GetObjectPath_Success(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.GetObjectPathRequest{ - Allocation: allocationTx, - Path: "path", - BlockNum: "120", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "owner", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetObjectPath", mock.Anything, mock.Anything, mock.Anything).Return(&reference.ObjectPath{ - RootHash: "hash", - FileBlockNum: 120, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.GetObjectPath(ctx, req) - if err != nil { - t.Fatal("unexpected error") - } - - assert.Equal(t, resp.ObjectPath.RootHash, "hash") - assert.Equal(t, resp.ObjectPath.FileBlockNum, int64(120)) - -} - -func TestBlobberGRPCService_GetObjectPath_InvalidAllocation(t *testing.T) { - req := &blobbergrpc.GetObjectPathRequest{ - Allocation: "", - Path: "path", - BlockNum: "120", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "owner", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: "", - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(nil, errors.New("invalid allocation")) - mockReferencePackage.On("GetObjectPathGRPC", mock.Anything, mock.Anything, mock.Anything).Return(&blobbergrpc.ObjectPath{ - RootHash: "hash", - FileBlockNum: 120, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.GetObjectPath(ctx, req) - if err == nil { - t.Fatal("expected error") - } -} - -func TestBlobberGRPCService_GetReferencePath_Success(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.GetReferencePathRequest{ - Paths: `["something"]`, - Path: "", - Allocation: allocationTx, - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetReferencePathFromPaths", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.DIRECTORY, - Children: []*reference.Ref{{Name: "test1", Type: reference.FILE}}, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.GetReferencePath(ctx, req) - if err != nil { - t.Fatal("unexpected error") - } - - assert.Equal(t, resp.ReferencePath.MetaData.DirMetaData.Name, "test") - -} - -func TestBlobberGRPCService_GetReferencePath_InvalidPaths(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.GetReferencePathRequest{ - Paths: `["something"]`, - Path: "", - Allocation: allocationTx, - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetReferencePathFromPaths", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("invalid paths")) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.GetReferencePath(ctx, req) - if err == nil { - t.Fatal("expected error") - } - - assert.Equal(t, err.Error(), "invalid paths") - -} - -func TestBlobberGRPCService_GetObjectTree_Success(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.GetObjectTreeRequest{ - Path: "something", - Allocation: allocationTx, - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "owner", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetObjectTree", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.DIRECTORY, - Children: []*reference.Ref{{Name: "test1", Type: reference.FILE}}, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.GetObjectTree(ctx, req) - if err != nil { - t.Fatal("unexpected error - " + err.Error()) - } - - assert.Equal(t, resp.ReferencePath.MetaData.DirMetaData.Name, "test") - -} - -func TestBlobberGRPCService_GetObjectTree_NotOwner(t *testing.T) { - req := &blobbergrpc.GetObjectTreeRequest{ - Path: "something", - Allocation: "", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "hacker", - common.ClientKeyHeader: "", - common.ClientSignatureHeader: "", - })) - - mockStorageHandler := &storageHandlerI{} - mockReferencePackage := &mocks.PackageHandler{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.GetObjectTree(ctx, req) - if err == nil { - t.Fatal("expected error") - } -} - -func TestBlobberGRPCService_CalculateHashSuccess(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.CalculateHashRequest{ - Allocation: allocationTx, - Path: "some-path", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "owner", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := new(storageHandlerI) - mockReferencePackage := new(mocks.PackageHandler) - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetReferencePathFromPaths", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.DIRECTORY, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.CalculateHash(ctx, req) - if err != nil { - t.Fatal("unexpected error: ", err) - } - - assert.Equal(t, resp.GetMessage(), "Hash recalculated for the given paths") -} - -func TestBlobberGRPCService_CalculateHashNotOwner(t *testing.T) { - req := &blobbergrpc.CalculateHashRequest{ - Allocation: "", - Path: "some-path", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "hacker", - common.ClientSignatureHeader: "", - })) - - mockStorageHandler := new(storageHandlerI) - mockReferencePackage := new(mocks.PackageHandler) - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "owner", - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.CalculateHash(ctx, req) - if err == nil { - t.Fatal("expected error") - } -} - -func TestBlobberGRPCService_CommitMetaTxnSuccess(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.CommitMetaTxnRequest{ - Path: "/some_file", - PathHash: "exampleId:examplePath", - AuthToken: "", - Allocation: allocationTx, - TxnId: "8", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "owner", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := new(storageHandlerI) - mockReferencePackage := new(mocks.PackageHandler) - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "8", - Tx: req.Allocation, - OwnerID: "owner", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - ID: 8, - Type: reference.FILE, - }, nil) - mockReferencePackage.On("AddCommitMetaTxn", mock.Anything, mock.Anything, mock.Anything). - Return(nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.CommitMetaTxn(ctx, req) - if err != nil { - t.Fatal("unexpected error: ", err) - } - - assert.Equal(t, resp.GetMessage(), "Added commitMetaTxn successfully") -} - -func TestBlobberGRPCService_CommitMetaTxnError(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.CommitMetaTxnRequest{ - Path: "/some_file", - PathHash: "exampleId:examplePath", - AuthToken: "", - Allocation: allocationTx, - TxnId: "", // TxnId not passed, expecting error - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "owner", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := new(storageHandlerI) - mockReferencePackage := new(mocks.PackageHandler) - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "8", - Tx: req.Allocation, - OwnerID: "owner", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - ID: 8, - Type: reference.FILE, - }, nil) - mockReferencePackage.On("AddCommitMetaTxn", mock.Anything, mock.Anything, mock.Anything). - Return(nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.CommitMetaTxn(ctx, req) - if err == nil { - t.Fatal("expected error") - } -} - -func TestBlobberGRPCService_AddCollaboratorSuccess(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.CollaboratorRequest{ - Allocation: allocationTx, - Path: "some-path", - CollabId: "12", - Method: http.MethodPost, - PathHash: "exampleId:examplePath", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "12", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := new(storageHandlerI) - mockReferencePackage := new(mocks.PackageHandler) - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "12", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.FILE, - }, nil) - mockReferencePackage.On("IsACollaborator", mock.Anything, mock.Anything, mock.Anything). - Return(false) - mockReferencePackage.On("AddCollaborator", mock.Anything, mock.Anything, mock.Anything). - Return(nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - resp, err := svc.Collaborator(ctx, req) - if err != nil { - t.Fatal("unexpected error: ", err) - } - - assert.Equal(t, resp.GetMessage(), "Added collaborator successfully") -} - -func TestBlobberGRPCService_AddCollaboratorError(t *testing.T) { - allocationTx := randString(32) - - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.CollaboratorRequest{ - Allocation: allocationTx, - Path: "some-path", - CollabId: "12", - Method: http.MethodPost, - PathHash: "exampleId:examplePath", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "1", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := new(storageHandlerI) - mockReferencePackage := new(mocks.PackageHandler) - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, true).Return(&allocation.Allocation{ - ID: "allocationId", - Tx: req.Allocation, - OwnerID: "12", - OwnerPublicKey: pubKey, - }, nil) - mockReferencePackage.On("GetReferenceFromLookupHash", mock.Anything, mock.Anything, mock.Anything).Return(&reference.Ref{ - Name: "test", - Type: reference.FILE, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.Collaborator(ctx, req) - if err == nil { - t.Fatal("expected error") - } -} diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index cd3dbb44c..45b7163fb 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -7,18 +7,17 @@ import ( "net/http" "os" "runtime/pprof" + "time" + + "go.uber.org/zap" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/gorilla/mux" - "go.uber.org/zap" - "google.golang.org/grpc/metadata" ) var storageHandler StorageHandler @@ -29,28 +28,26 @@ func GetMetaDataStore() *datastore.Store { /*SetupHandlers sets up the necessary API end points */ func SetupHandlers(r *mux.Router) { - svc := newGRPCBlobberService(&storageHandler, &packageHandler{}) - //object operations r.HandleFunc("/v1/file/upload/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(UploadHandler)))) r.HandleFunc("/v1/file/download/{allocation}", common.UserRateLimit(common.ToByteStream(WithConnection(DownloadHandler)))) r.HandleFunc("/v1/file/rename/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(RenameHandler)))) - r.HandleFunc("/v1/file/attributes/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(UpdateAttributesHandler(svc))))).Methods(http.MethodPost) - r.HandleFunc("/v1/file/copy/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CopyHandler(svc))))).Methods(http.MethodPost) + r.HandleFunc("/v1/file/copy/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CopyHandler)))) + r.HandleFunc("/v1/file/attributes/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(UpdateAttributesHandler)))) - r.HandleFunc("/v1/connection/commit/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CommitHandler(svc))))).Methods("POST") - r.HandleFunc("/v1/file/commitmetatxn/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CommitMetaTxnHandler(svc))))).Methods(http.MethodPost) - r.HandleFunc("/v1/file/collaborator/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CollaboratorHandler(svc))))).Methods(http.MethodGet, http.MethodPost, http.MethodDelete) - r.HandleFunc("/v1/file/calculatehash/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CalculateHashHandler(svc))))).Methods(http.MethodPost) + r.HandleFunc("/v1/connection/commit/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CommitHandler)))) + r.HandleFunc("/v1/file/commitmetatxn/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CommitMetaTxnHandler)))) + r.HandleFunc("/v1/file/collaborator/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CollaboratorHandler)))) + r.HandleFunc("/v1/file/calculatehash/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(CalculateHashHandler)))) //object info related apis - r.HandleFunc("/allocation", common.UserRateLimit(common.ToJSONResponse(WithConnection(AllocationHandler(svc))))).Methods(http.MethodGet) - r.HandleFunc("/v1/file/meta/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(FileMetaHandler(svc))))).Methods(http.MethodPost) - r.HandleFunc("/v1/file/stats/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(FileStatsHandler(svc))))).Methods(http.MethodPost) - r.HandleFunc("/v1/file/list/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ListHandler(svc))))).Methods(http.MethodGet) - r.HandleFunc("/v1/file/objectpath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler(svc))))).Methods(http.MethodGet) - r.HandleFunc("/v1/file/referencepath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler(svc))))).Methods(http.MethodGet) - r.HandleFunc("/v1/file/objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler(svc))))).Methods(http.MethodGet) + r.HandleFunc("/allocation", common.UserRateLimit(common.ToJSONResponse(WithConnection(AllocationHandler)))) + r.HandleFunc("/v1/file/meta/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(FileMetaHandler)))) + r.HandleFunc("/v1/file/stats/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(FileStatsHandler)))) + r.HandleFunc("/v1/file/list/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ListHandler)))) + r.HandleFunc("/v1/file/objectpath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler)))) + r.HandleFunc("/v1/file/referencepath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler)))) + r.HandleFunc("/v1/file/objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler)))) //admin related r.HandleFunc("/_debug", common.UserRateLimit(common.ToJSONResponse(DumpGoRoutines))) @@ -115,100 +112,59 @@ func setupHandlerContext(ctx context.Context, r *http.Request) context.Context { return ctx } -func setupHandlerGRPCContext(ctx context.Context, r *http.Request) context.Context { - return metadata.NewIncomingContext(ctx, metadata.New(map[string]string{ - common.ClientHeader: r.Header.Get(common.ClientHeader), - common.ClientKeyHeader: r.Header.Get(common.ClientKeyHeader), - common.ClientSignatureHeader: r.Header.Get(common.ClientSignatureHeader), - })) -} - -func AllocationHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - - getAllocationResp, err := svc.GetAllocation(ctx, &blobbergrpc.GetAllocationRequest{ - Id: r.FormValue("id"), - }) - if err != nil { - return nil, err - } +func AllocationHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.GetAllocationResponseHandler(getAllocationResp), nil + response, err := storageHandler.GetAllocationDetails(ctx, r) + if err != nil { + return nil, err } + + return response, nil } -func FileMetaHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - - getFileMetaDataResp, err := svc.GetFileMetaData(ctx, &blobbergrpc.GetFileMetaDataRequest{ - Path: r.FormValue("path"), - PathHash: r.FormValue("path_hash"), - AuthToken: r.FormValue("auth_token"), - Allocation: mux.Vars(r)["allocation"], - }) - if err != nil { - return nil, err - } +func FileMetaHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.GetFileMetaDataResponseHandler(getFileMetaDataResp), nil + response, err := storageHandler.GetFileMeta(ctx, r) + if err != nil { + return nil, err } + + return response, nil } -func CommitMetaTxnHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - - response, err := svc.CommitMetaTxn(ctx, &blobbergrpc.CommitMetaTxnRequest{ - Path: r.FormValue("path"), - PathHash: r.FormValue("path_hash"), - AuthToken: r.FormValue("auth_token"), - Allocation: mux.Vars(r)["allocation"], - TxnId: r.FormValue("txn_id"), - }) - if err != nil { - return nil, err - } +func CommitMetaTxnHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.GetCommitMetaTxnHandlerResponse(response), nil + response, err := storageHandler.AddCommitMetaTxn(ctx, r) + if err != nil { + return nil, err } + + return response, nil } -func CollaboratorHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - - response, err := svc.Collaborator(ctx, &blobbergrpc.CollaboratorRequest{ - Allocation: mux.Vars(r)["allocation"], - CollabId: r.FormValue("collab_id"), - Method: r.Method, - Path: r.FormValue("path"), - PathHash: r.FormValue("path_hash"), - }) - if err != nil { - return nil, err - } +func CollaboratorHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.CollaboratorResponse(response), nil + response, err := storageHandler.AddCollaborator(ctx, r) + if err != nil { + return nil, err } -} -func FileStatsHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) + return response, nil +} - getFileStatsResponse, err := svc.GetFileStats(ctx, &blobbergrpc.GetFileStatsRequest{ - Path: r.FormValue("path"), - PathHash: r.FormValue("path_hash"), - Allocation: mux.Vars(r)["allocation"], - }) - if err != nil { - return nil, err - } +func FileStatsHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.GetFileStatsResponseHandler(getFileStatsResponse), nil + response, err := storageHandler.GetFileStats(ctx, r) + if err != nil { + return nil, err } + + return response, nil } /*DownloadHandler is the handler to respond to download requests from clients*/ @@ -224,86 +180,63 @@ func DownloadHandler(ctx context.Context, r *http.Request) (interface{}, error) } /*ListHandler is the handler to respond to upload requests fro clients*/ -func ListHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - listEntitiesResponse, err := svc.ListEntities(ctx, &blobbergrpc.ListEntitiesRequest{ - Path: r.FormValue("path"), - PathHash: r.FormValue("path_hash"), - AuthToken: r.FormValue("auth_token"), - Allocation: mux.Vars(r)["allocation"], - }) - if err != nil { - return nil, err - } +func ListHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.ListEntitesResponseHandler(listEntitiesResponse), nil + response, err := storageHandler.ListEntities(ctx, r) + if err != nil { + return nil, err } + + return response, nil } /*CommitHandler is the handler to respond to upload requests fro clients*/ -func CommitHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - - response, err := svc.Commit(ctx, &blobbergrpc.CommitRequest{ - Allocation: mux.Vars(r)["allocation"], - ConnectionId: r.FormValue("connection_id"), - WriteMarker: r.FormValue("write_marker"), - }) - if err != nil { - return nil, err - } +func CommitHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.CommitWriteResponseHandler(response), nil + response, err := storageHandler.CommitWrite(ctx, r) + if err != nil { + return nil, err } + + return response, nil } -func ReferencePathHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - getReferencePathResponse, err := svc.GetReferencePath(ctx, &blobbergrpc.GetReferencePathRequest{ - Paths: r.FormValue("paths"), - Path: r.FormValue("path"), - Allocation: mux.Vars(r)["allocation"], - }) - if err != nil { - return nil, err - } +func ReferencePathHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx, canceler := context.WithTimeout(ctx, time.Second*10) + defer canceler() + + ctx = setupHandlerContext(ctx, r) - return convert.GetReferencePathResponseHandler(getReferencePathResponse), nil + response, err := storageHandler.GetReferencePath(ctx, r) + if err != nil { + return nil, err } + + return response, nil } -func ObjectPathHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - getObjectPathResponse, err := svc.GetObjectPath(ctx, &blobbergrpc.GetObjectPathRequest{ - Allocation: mux.Vars(r)["allocation"], - Path: r.FormValue("path"), - BlockNum: r.FormValue("block_num"), - }) - if err != nil { - return nil, err - } +func ObjectPathHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.GetObjectPathResponseHandler(getObjectPathResponse), nil + response, err := storageHandler.GetObjectPath(ctx, r) + if err != nil { + return nil, err } + + return response, nil } -func ObjectTreeHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - getObjectTreeResponse, err := svc.GetObjectTree(ctx, &blobbergrpc.GetObjectTreeRequest{ - Path: r.FormValue("path"), - Allocation: mux.Vars(r)["allocation"], - }) - if err != nil { - return nil, err - } +func ObjectTreeHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.GetObjectTreeResponseHandler(getObjectTreeResponse), nil + response, err := storageHandler.GetObjectTree(ctx, r) + if err != nil { + return nil, err } + + return response, nil } func RenameHandler(ctx context.Context, r *http.Request) (interface{}, error) { @@ -316,23 +249,14 @@ func RenameHandler(ctx context.Context, r *http.Request) (interface{}, error) { return response, nil } -func CopyHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - - copyObjResponse, err := svc.CopyObject(ctx, &blobbergrpc.CopyObjectRequest{ - Allocation: mux.Vars(r)["allocation"], - Path: r.FormValue("path"), - PathHash: r.FormValue("path_hash"), - ConnectionId: r.FormValue("connection_id"), - Dest: r.FormValue("dest"), - }) - if err != nil { - return nil, err - } - - return convert.CopyObjectResponseHandler(copyObjResponse), nil +func CopyHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) + response, err := storageHandler.CopyObject(ctx, r) + if err != nil { + return nil, err } + + return response, nil } /*UploadHandler is the handler to respond to upload requests fro clients*/ @@ -346,40 +270,25 @@ func UploadHandler(ctx context.Context, r *http.Request) (interface{}, error) { return response, nil } -func UpdateAttributesHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) - - updateAttrResp, err := svc.UpdateObjectAttributes(ctx, &blobbergrpc.UpdateObjectAttributesRequest{ - Path: r.FormValue("path"), - PathHash: r.FormValue("path_hash"), - Allocation: mux.Vars(r)["allocation"], - ConnectionId: r.FormValue("connection_id"), - Attributes: r.FormValue("attributes"), - }) - if err != nil { - return nil, err - } - - return convert.UpdateObjectAttributesResponseHandler(updateAttrResp), nil +func UpdateAttributesHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) + response, err := storageHandler.UpdateObjectAttributes(ctx, r) + if err != nil { + return nil, err } -} -func CalculateHashHandler(svc *blobberGRPCService) func(ctx context.Context, r *http.Request) (interface{}, error) { - return func(ctx context.Context, r *http.Request) (interface{}, error) { - ctx = setupHandlerGRPCContext(ctx, r) + return response, nil +} - response, err := svc.CalculateHash(ctx, &blobbergrpc.CalculateHashRequest{ - Allocation: mux.Vars(r)["allocation"], - Paths: r.FormValue("paths"), - Path: r.FormValue("path"), - }) - if err != nil { - return nil, err - } +func CalculateHashHandler(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) - return convert.GetCalculateHashResponseHandler(response), nil + response, err := storageHandler.CalculateHash(ctx, r) + if err != nil { + return nil, err } + + return response, nil } //nolint:gosimple // need more time to verify diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index c14fb811a..49614a930 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -89,13 +89,12 @@ func setup(t *testing.T) { func setupHandlers() (*mux.Router, map[string]string) { router := mux.NewRouter() - svc := newGRPCBlobberService(&storageHandler, &packageHandler{}) opPath := "/v1/file/objectpath/{allocation}" opName := "Object_Path" router.HandleFunc(opPath, common.UserRateLimit( common.ToJSONResponse( - WithReadOnlyConnection(ObjectPathHandler(svc)), + WithReadOnlyConnection(ObjectPathHandler), ), ), ).Name(opName) @@ -104,7 +103,7 @@ func setupHandlers() (*mux.Router, map[string]string) { rpName := "Reference_Path" router.HandleFunc(rpPath, common.UserRateLimit( common.ToJSONResponse( - WithReadOnlyConnection(ReferencePathHandler(svc)), + WithReadOnlyConnection(ReferencePathHandler), ), ), ).Name(rpName) @@ -113,7 +112,7 @@ func setupHandlers() (*mux.Router, map[string]string) { sName := "Stats" router.HandleFunc(sPath, common.UserRateLimit( common.ToJSONResponse( - WithReadOnlyConnection(FileStatsHandler(svc)), + WithReadOnlyConnection(FileStatsHandler), ), ), ).Name(sName) @@ -122,7 +121,7 @@ func setupHandlers() (*mux.Router, map[string]string) { otName := "Object_Tree" router.HandleFunc(otPath, common.UserRateLimit( common.ToJSONResponse( - WithReadOnlyConnection(ObjectTreeHandler(svc)), + WithReadOnlyConnection(ObjectTreeHandler), ), ), ).Name(otName) @@ -131,7 +130,7 @@ func setupHandlers() (*mux.Router, map[string]string) { collName := "Collaborator" router.HandleFunc(collPath, common.UserRateLimit( common.ToJSONResponse( - WithReadOnlyConnection(CollaboratorHandler(svc)), + WithReadOnlyConnection(CollaboratorHandler), ), ), ).Name(collName) @@ -149,7 +148,7 @@ func setupHandlers() (*mux.Router, map[string]string) { cName := "Copy" router.HandleFunc(cPath, common.UserRateLimit( common.ToJSONResponse( - WithReadOnlyConnection(CopyHandler(svc)), + WithReadOnlyConnection(CopyHandler), ), ), ).Name(cName) @@ -158,7 +157,7 @@ func setupHandlers() (*mux.Router, map[string]string) { aName := "Attributes" router.HandleFunc(aPath, common.UserRateLimit( common.ToJSONResponse( - WithReadOnlyConnection(UpdateAttributesHandler(svc)), + WithReadOnlyConnection(UpdateAttributesHandler), ), ), ).Name(aName) diff --git a/code/go/0chain.net/blobbercore/handler/helper.go b/code/go/0chain.net/blobbercore/handler/helper.go index a1f35a7fb..0add2303e 100644 --- a/code/go/0chain.net/blobbercore/handler/helper.go +++ b/code/go/0chain.net/blobbercore/handler/helper.go @@ -3,132 +3,17 @@ package handler import ( "context" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/gorilla/mux" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "google.golang.org/grpc" ) func RegisterGRPCServices(r *mux.Router, server *grpc.Server) { - blobberService := newGRPCBlobberService(&storageHandler, &packageHandler{}) + blobberService := newGRPCBlobberService() grpcGatewayHandler := runtime.NewServeMux() blobbergrpc.RegisterBlobberServer(server, blobberService) _ = blobbergrpc.RegisterBlobberHandlerServer(context.Background(), grpcGatewayHandler, blobberService) r.PathPrefix("/").Handler(grpcGatewayHandler) } - -type StorageHandlerI interface { - verifyAllocation(ctx context.Context, tx string, readonly bool) (alloc *allocation.Allocation, err error) - verifyAuthTicket(ctx context.Context, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string) (bool, error) -} - -// PackageHandler is an interface for all static functions that may need to be mocked -type PackageHandler interface { - GetReference(ctx context.Context, allocationID string, newPath string) (*reference.Ref, error) - GetReferenceLookup(ctx context.Context, allocationID string, path string) string - GetReferenceFromLookupHash(ctx context.Context, allocationID string, path_hash string) (*reference.Ref, error) - GetCommitMetaTxns(ctx context.Context, refID int64) ([]reference.CommitMetaTxn, error) - AddCommitMetaTxn(ctx context.Context, refID int64, txnID string) error - GetCollaborators(ctx context.Context, refID int64) ([]reference.Collaborator, error) - IsACollaborator(ctx context.Context, refID int64, clientID string) bool - AddCollaborator(ctx context.Context, refID int64, clientID string) error - RemoveCollaborator(ctx context.Context, refID int64, clientID string) error - GetFileStats(ctx context.Context, refID int64) (*stats.FileStats, error) - GetWriteMarkerEntity(ctx context.Context, allocation_root string) (*writemarker.WriteMarkerEntity, error) - GetRefWithChildren(ctx context.Context, allocationID string, path string) (*reference.Ref, error) - GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*reference.ObjectPath, error) - GetReferencePathFromPaths(ctx context.Context, allocationID string, paths []string) (*reference.Ref, error) - GetAllocationChanges(ctx context.Context, connectionID string, allocationID string, clientID string) (*allocation.AllocationChangeCollector, error) - SaveAllocationChanges(ctx context.Context, alloc *allocation.AllocationChangeCollector) error - GetObjectTree(ctx context.Context, allocationID string, path string) (*reference.Ref, error) - VerifyMarker(wm *writemarker.WriteMarkerEntity, ctx context.Context, sa *allocation.Allocation, co *allocation.AllocationChangeCollector) error - ApplyChanges(connectionObj *allocation.AllocationChangeCollector, ctx context.Context, allocationRoot string) error - UpdateAllocationAndCommitChanges(ctx context.Context, writemarkerObj *writemarker.WriteMarkerEntity, connectionObj *allocation.AllocationChangeCollector, allocationObj *allocation.Allocation, allocationRoot string) error -} - -type packageHandler struct{} - -func (r *packageHandler) UpdateAllocationAndCommitChanges(ctx context.Context, writemarkerObj *writemarker.WriteMarkerEntity, connectionObj *allocation.AllocationChangeCollector, allocationObj *allocation.Allocation, allocationRoot string) error { - return UpdateAllocationAndCommitChanges(ctx, writemarkerObj, connectionObj, allocationObj, allocationRoot) -} - -func (r *packageHandler) GetReference(ctx context.Context, allocationID string, path string) (*reference.Ref, error) { - return reference.GetReference(ctx, allocationID, path) -} - -func (r *packageHandler) ApplyChanges(connectionObj *allocation.AllocationChangeCollector, ctx context.Context, allocationRoot string) error { - return connectionObj.ApplyChanges(ctx, allocationRoot) -} - -func (r *packageHandler) VerifyMarker(wm *writemarker.WriteMarkerEntity, ctx context.Context, sa *allocation.Allocation, co *allocation.AllocationChangeCollector) error { - return wm.VerifyMarker(ctx, sa, co) -} - -func (r *packageHandler) GetAllocationChanges(ctx context.Context, connectionID string, allocationID string, clientID string) (*allocation.AllocationChangeCollector, error) { - return allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) -} - -func (r *packageHandler) GetObjectTree(ctx context.Context, allocationID string, path string) (*reference.Ref, error) { - return reference.GetObjectTree(ctx, allocationID, path) -} - -func (r *packageHandler) GetReferencePathFromPaths(ctx context.Context, allocationID string, paths []string) (*reference.Ref, error) { - return reference.GetReferencePathFromPaths(ctx, allocationID, paths) -} - -func (r *packageHandler) GetRefWithChildren(ctx context.Context, allocationID string, path string) (*reference.Ref, error) { - return reference.GetRefWithChildren(ctx, allocationID, path) -} - -func (r *packageHandler) GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*reference.ObjectPath, error) { - return reference.GetObjectPath(ctx, allocationID, blockNum) -} - -func (r *packageHandler) GetFileStats(ctx context.Context, refID int64) (*stats.FileStats, error) { - return stats.GetFileStats(ctx, refID) -} - -func (r *packageHandler) GetWriteMarkerEntity(ctx context.Context, allocation_root string) (*writemarker.WriteMarkerEntity, error) { - return writemarker.GetWriteMarkerEntity(ctx, allocation_root) -} - -func (r *packageHandler) GetReferenceLookup(ctx context.Context, allocationID string, path string) string { - return reference.GetReferenceLookup(allocationID, path) -} - -func (r *packageHandler) GetReferenceFromLookupHash(ctx context.Context, allocationID string, path_hash string) (*reference.Ref, error) { - return reference.GetReferenceFromLookupHash(ctx, allocationID, path_hash) -} - -func (r *packageHandler) GetCommitMetaTxns(ctx context.Context, refID int64) ([]reference.CommitMetaTxn, error) { - return reference.GetCommitMetaTxns(ctx, refID) -} - -func (r *packageHandler) AddCommitMetaTxn(ctx context.Context, refID int64, txnID string) error { - return reference.AddCommitMetaTxn(ctx, refID, txnID) -} - -func (r *packageHandler) GetCollaborators(ctx context.Context, refID int64) ([]reference.Collaborator, error) { - return reference.GetCollaborators(ctx, refID) -} - -func (r *packageHandler) IsACollaborator(ctx context.Context, refID int64, clientID string) bool { - return reference.IsACollaborator(ctx, refID, clientID) -} - -func (r *packageHandler) AddCollaborator(ctx context.Context, refID int64, clientID string) error { - return reference.AddCollaborator(ctx, refID, clientID) -} - -func (r *packageHandler) RemoveCollaborator(ctx context.Context, refID int64, clientID string) error { - return reference.RemoveCollaborator(ctx, refID, clientID) -} - -func (r packageHandler) SaveAllocationChanges(ctx context.Context, alloc *allocation.AllocationChangeCollector) error { - return alloc.Save(ctx) -} diff --git a/code/go/0chain.net/blobbercore/handler/metadata.go b/code/go/0chain.net/blobbercore/handler/metadata.go index 0ad3cb7fd..c07a14ea6 100644 --- a/code/go/0chain.net/blobbercore/handler/metadata.go +++ b/code/go/0chain.net/blobbercore/handler/metadata.go @@ -2,8 +2,10 @@ package handler import ( "context" + "net/http" + + "github.com/gorilla/mux" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/constants" "github.com/0chain/blobber/code/go/0chain.net/core/common" "google.golang.org/grpc/metadata" ) @@ -36,10 +38,9 @@ func GetGRPCMetaDataFromCtx(ctx context.Context) *GRPCMetaData { return metaData } -func setupGRPCHandlerContext(ctx context.Context, md *GRPCMetaData, alloc string) context.Context { - ctx = context.WithValue(ctx, constants.CLIENT_CONTEXT_KEY, md.Client) - ctx = context.WithValue(ctx, constants.CLIENT_KEY_CONTEXT_KEY, md.ClientKey) - ctx = context.WithValue(ctx, constants.ALLOCATION_CONTEXT_KEY, alloc) - ctx = context.WithValue(ctx, constants.CLIENT_SIGNATURE_HEADER_KEY, md.ClientSignature) - return ctx +func httpRequestWithMetaData(r *http.Request, md *GRPCMetaData, alloc string) { + r.Header.Set(common.ClientHeader, md.Client) + r.Header.Set(common.ClientKeyHeader, md.ClientKey) + r.Header.Set(common.ClientSignatureHeader, md.ClientSignature) + *r = *mux.SetURLVars(r, map[string]string{"allocation": alloc}) } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go index 6cfafd7a6..9a12df35a 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go @@ -2,211 +2,50 @@ package handler import ( "context" - "encoding/json" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + "net/http" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/lock" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - "go.uber.org/zap" - "path/filepath" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" ) -func (b *blobberGRPCService) UpdateObjectAttributes(ctx context.Context, r *blobbergrpc.UpdateObjectAttributesRequest) (*blobbergrpc.UpdateObjectAttributesResponse, error) { - logger := ctxzap.Extract(ctx) - - allocationTx := r.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, false) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - - md := GetGRPCMetaDataFromCtx(ctx) - valid, err := verifySignatureFromRequest(allocationTx, md.ClientSignature, allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - //// runtime type check - //_ = ctx.Value(constants.CLIENT_KEY_CONTEXT_KEY).(string) //why this removed? - - clientID := md.Client - if clientID == "" { - return nil, common.NewError("update_object_attributes", - "missing client ID") - } - - var attributes = r.Attributes // new attributes as string - if attributes == "" { - return nil, common.NewError("update_object_attributes", - "missing new attributes, pass at least {} for empty attributes") - } - - var attrs = new(reference.Attributes) - if err = json.Unmarshal([]byte(attributes), attrs); err != nil { - return nil, common.NewErrorf("update_object_attributes", - "decoding given attributes: %v", err) - } - - pathHash := r.PathHash - path := r.Path - if len(pathHash) == 0 { - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - pathHash = b.packageHandler.GetReferenceLookup(ctx, allocationObj.ID, path) - } - - if allocationObj.OwnerID != clientID { - return nil, common.NewError( - "invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - - var connID = r.ConnectionId - if connID == "" { - return nil, common.NewErrorf("update_object_attributes", - "invalid connection id passed: %s", connID) - } - - var conn *allocation.AllocationChangeCollector - conn, err = b.packageHandler.GetAllocationChanges(ctx, connID, allocationObj.ID, clientID) - if err != nil { - return nil, common.NewErrorf("update_object_attributes", - "reading metadata for connection: %v", err) - } - - var mutex = lock.GetMutex(conn.TableName(), connID) - - mutex.Lock() - defer mutex.Unlock() - - var ref *reference.Ref - ref, err = b.packageHandler.GetReferenceFromLookupHash(ctx, allocationObj.ID, pathHash) +func (b *blobberGRPCService) UpdateObjectAttributes(ctx context.Context, req *blobbergrpc.UpdateObjectAttributesRequest) (*blobbergrpc.UpdateObjectAttributesResponse, error) { + r, err := http.NewRequest("POST", "", nil) if err != nil { - return nil, common.NewErrorf("update_object_attributes", - "invalid file path: %v", err) + return nil, err } - - var change = new(allocation.AllocationChange) - change.ConnectionID = conn.ConnectionID - change.Operation = allocation.UPDATE_ATTRS_OPERATION - - var uafc = &allocation.AttributesChange{ - ConnectionID: conn.ConnectionID, - AllocationID: conn.AllocationID, - Path: ref.Path, - Attributes: attrs, + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "path_hash": {req.PathHash}, + "connection_id": {req.ConnectionId}, + "attributes": {req.Attributes}, } - conn.AddChange(change, uafc) - - err = b.packageHandler.SaveAllocationChanges(ctx, conn) + resp, err := UpdateAttributesHandler(ctx, r) if err != nil { - logger.Error("update_object_attributes: "+ - "error in writing the connection meta data", zap.Error(err)) - return nil, common.NewError("update_object_attributes", - "error writing the connection meta data") + return nil, err } - // return new attributes as result - return &blobbergrpc.UpdateObjectAttributesResponse{WhoPaysForReads: int64(attrs.WhoPaysForReads)}, nil + return convert.UpdateObjectAttributesResponseCreator(resp), nil } -func (b *blobberGRPCService) CopyObject(ctx context.Context, r *blobbergrpc.CopyObjectRequest) (*blobbergrpc.CopyObjectResponse, error) { - logger := ctxzap.Extract(ctx) - md := GetGRPCMetaDataFromCtx(ctx) - - allocationTx := r.Allocation - allocationObj, err := b.storageHandler.verifyAllocation(ctx, allocationTx, false) +func (b *blobberGRPCService) CopyObject(ctx context.Context, req *blobbergrpc.CopyObjectRequest) (*blobbergrpc.CopyObjectResponse, error) { + r, err := http.NewRequest("POST", "", nil) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - - valid, err := verifySignatureFromRequest(allocationTx, md.ClientSignature, allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - allocationID := allocationObj.ID - - clientID := md.Client - if len(clientID) == 0 { - return nil, common.NewError("invalid_operation", "Invalid client") - } - - if len(r.Dest) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid destination for operation") - } - - pathHash := r.PathHash - path := r.Path - if len(pathHash) == 0 { - if len(path) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path") - } - pathHash = b.packageHandler.GetReferenceLookup(ctx, allocationObj.ID, path) + return nil, err } - - if len(clientID) == 0 || allocationObj.OwnerID != clientID { //already checked clientId ? - return nil, common. - NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) + r.Form = map[string][]string{ + "path": {req.Path}, + "path_hash": {req.PathHash}, + "connection_id": {req.ConnectionId}, + "dest": {req.Dest}, } - connectionID := r.ConnectionId - if len(connectionID) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - } - - connectionObj, err := b.packageHandler.GetAllocationChanges(ctx, connectionID, allocationID, clientID) + resp, err := CopyHandler(ctx, r) if err != nil { - return nil, common.NewError("meta_error", "Error reading metadata for connection") + return nil, err } - mutex := lock.GetMutex(connectionObj.TableName(), connectionID) - mutex.Lock() - defer mutex.Unlock() - - objectRef, err := b.packageHandler.GetReferenceFromLookupHash(ctx, allocationID, pathHash) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - } - - newPath := filepath.Join(r.Dest, objectRef.Name) - destRef, _ := b.packageHandler.GetReference(ctx, allocationID, newPath) - if destRef != nil { - return nil, common.NewError( - "invalid_parameters", "Invalid destination path. Object Already exists.") - } - - destRef, err = b.packageHandler.GetReference(ctx, allocationID, r.Dest) - if err != nil || destRef.Type != reference.DIRECTORY { - return nil, common.NewError( - "invalid_parameters", "Invalid destination path. Should be a valid directory.") - } - - allocationChange := &allocation.AllocationChange{} - allocationChange.ConnectionID = connectionObj.ConnectionID - allocationChange.Size = objectRef.Size - allocationChange.Operation = allocation.COPY_OPERATION - - dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ConnectionID, - AllocationID: connectionObj.AllocationID, DestPath: r.Dest} - dfc.SrcPath = objectRef.Path - connectionObj.Size += allocationChange.Size - connectionObj.AddChange(allocationChange, dfc) - - err = b.packageHandler.SaveAllocationChanges(ctx, connectionObj) - if err != nil { - logger.Error("Error in writing the connection meta data", zap.Error(err)) - return nil, common.NewError("connection_write_error", "Error writing the connection meta data") - } - - result := &blobbergrpc.CopyObjectResponse{} - result.Filename = objectRef.Name - result.ContentHash = objectRef.Hash - result.MerkleRoot = objectRef.MerkleRoot - result.Size = objectRef.Size - - return result, nil + return convert.CopyObjectResponseCreator(resp), nil } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler_test.go deleted file mode 100644 index b24b6a1d0..000000000 --- a/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler_test.go +++ /dev/null @@ -1,197 +0,0 @@ -package handler - -import ( - "context" - "errors" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/mocks" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "google.golang.org/grpc/metadata" - "path/filepath" - "testing" -) - -func TestBlobberGRPCService_UpdateObjectAttributes_Success(t *testing.T) { - allocationTx := randString(32) - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - req := &blobbergrpc.UpdateObjectAttributesRequest{ - Allocation: allocationTx, - Attributes: `{"who_pays_for_reads" : 1}`, - Path: `path`, - ConnectionId: `connection_id`, - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "client_key", - common.ClientSignatureHeader: clientSignature, - })) - - resOk := &blobbergrpc.UpdateObjectAttributesResponse{WhoPaysForReads: int64(1)} - - mockStorageHandler := &storageHandlerI{} - alloc := &allocation.Allocation{ - Tx: req.Allocation, - ID: `allocation_id`, - OwnerID: `client`, - OwnerPublicKey: pubKey, - } - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false). - Return(alloc, nil) - - mockReferencePackage := &mocks.PackageHandler{} - allocChange := &allocation.AllocationChangeCollector{} - mockReferencePackage.On(`GetAllocationChanges`, mock.Anything, - req.ConnectionId, alloc.ID, `client`).Return(allocChange, nil) - mockReferencePackage.On(`SaveAllocationChanges`, mock.Anything, allocChange). - Return(nil) - - pathHash := req.Allocation + `:` + req.Path - mockReferencePackage.On(`GetReferenceLookup`, mock.Anything, alloc.ID, req.Path). - Return(pathHash) - - mockReferencePackage.On(`GetReferenceFromLookupHash`, mock.Anything, alloc.ID, pathHash).Return( - &reference.Ref{ - Name: "test", - Type: reference.FILE, - }, nil) - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - gotResponse, err := svc.UpdateObjectAttributes(ctx, req) - if err != nil { - t.Fatal("unexpected error - " + err.Error()) - } - - assert.Equal(t, gotResponse, resOk) -} - -func TestBlobberGRPCService_UpdateObjectAttributes_InvalidAllocation(t *testing.T) { - req := &blobbergrpc.UpdateObjectAttributesRequest{ - Allocation: `invalid_allocation`, - } - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "client_key", - common.ClientSignatureHeader: "clientSignature", - })) - - mockStorageHandler := &storageHandlerI{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false). - Return(nil, errors.New("some error")) - - mockReferencePackage := &mocks.PackageHandler{} - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - _, err := svc.UpdateObjectAttributes(ctx, req) - if err == nil { - t.Fatal("expected error") - } - if err.Error() != "invalid_parameters: Invalid allocation id passed.some error" { - t.Fatal(`unexpected error - `, err) - } -} - -func TestBlobberGRPCService_CopyObject_Success(t *testing.T) { - allocationTx := randString(32) - pubKey, _, signScheme := GeneratePubPrivateKey(t) - clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) - - req := &blobbergrpc.CopyObjectRequest{ - Allocation: allocationTx, - Path: "path", - ConnectionId: "connection_id", - Dest: "dest", - } - - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "client_key", - common.ClientSignatureHeader: clientSignature, - })) - - mockStorageHandler := &storageHandlerI{} - alloc := &allocation.Allocation{ - Tx: req.Allocation, - ID: req.Allocation, - OwnerID: `client`, - OwnerPublicKey: pubKey, - } - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false). - Return(alloc, nil) - - mockReferencePackage := &mocks.PackageHandler{} - allocChange := &allocation.AllocationChangeCollector{} - mockReferencePackage.On(`GetAllocationChanges`, mock.Anything, - req.ConnectionId, alloc.ID, alloc.OwnerID).Return(allocChange, nil) - - pathHash := req.Allocation + `:` + req.Path - mockReferencePackage.On(`GetReferenceLookup`, mock.Anything, alloc.ID, req.Path). - Return(pathHash) - - objectRef := &reference.Ref{ - Name: "test", - ContentHash: `hash`, - MerkleRoot: `root`, - Size: 1, - } - - mockReferencePackage.On(`GetReferenceFromLookupHash`, mock.Anything, alloc.ID, pathHash). - Return(objectRef, nil) - newPath := filepath.Join(req.Dest, objectRef.Name) - mockReferencePackage.On(`GetReference`, mock.Anything, alloc.ID, newPath). - Return(nil, nil) - mockReferencePackage.On(`GetReference`, mock.Anything, alloc.ID, req.Dest). - Return(&reference.Ref{Type: `d`}, nil) - mockReferencePackage.On(`SaveAllocationChanges`, mock.Anything, allocChange). - Return(nil) - - resOk := &blobbergrpc.CopyObjectResponse{ - Filename: objectRef.Name, - Size: objectRef.Size, - ContentHash: objectRef.Hash, - MerkleRoot: objectRef.MerkleRoot, - UploadLength: 0, - UploadOffset: 0, - } - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - gotResponse, err := svc.CopyObject(ctx, req) - if err != nil { - t.Fatal("unexpected error - " + err.Error()) - } - - assert.Equal(t, gotResponse, resOk) -} - -func TestBlobberGRPCService_CopyObject_InvalidAllocation(t *testing.T) { - req := &blobbergrpc.CopyObjectRequest{ - Allocation: `invalid_allocation`, - } - ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ - common.ClientHeader: "client", - common.ClientKeyHeader: "client_key", - common.ClientSignatureHeader: "clientSignature", - })) - - mockStorageHandler := &storageHandlerI{} - mockStorageHandler.On("verifyAllocation", mock.Anything, req.Allocation, false). - Return(nil, errors.New("some error")) - - mockReferencePackage := &mocks.PackageHandler{} - - svc := newGRPCBlobberService(mockStorageHandler, mockReferencePackage) - - _, err := svc.CopyObject(ctx, req) - if err == nil { - t.Fatal("expected error") - } - if err.Error() != "invalid_parameters: Invalid allocation id passed.some error" { - t.Fatal(`unexpected error - `, err) - } -}