From 3c600d3f80157c15d9727c679f3d05e30c0641fa Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Mon, 28 Jun 2021 21:58:10 +0530 Subject: [PATCH 01/18] upload attributes handler integrated tests implemented --- .../blobbercore/convert/responseHandler.go | 9 +- .../handler/grpc_handler_helper_unit_test.go | 60 +++++++++++++ .../handler/grpc_handler_integration_test.go | 86 +++++++++++++++++++ .../handler/object_operation_handler.go | 3 + 4 files changed, 157 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/convert/responseHandler.go b/code/go/0chain.net/blobbercore/convert/responseHandler.go index 34c99bbc3..20242218d 100644 --- a/code/go/0chain.net/blobbercore/convert/responseHandler.go +++ b/code/go/0chain.net/blobbercore/convert/responseHandler.go @@ -3,6 +3,7 @@ package convert import ( "context" "encoding/json" + "fmt" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -182,11 +183,17 @@ func CollaboratorResponseCreator(r interface{}) *blobbergrpc.CollaboratorRespons } func UpdateObjectAttributesResponseCreator(r interface{}) *blobbergrpc.UpdateObjectAttributesResponse { + + fmt.Println(`response - `, r) + if r != nil { return nil } - httpResp, _ := r.(*reference.Attributes) + httpResp, ok := r.(*reference.Attributes) + if !ok { + fmt.Println(`response not okay - `, r) + } return &blobbergrpc.UpdateObjectAttributesResponse{WhoPaysForReads: int64(httpResp.WhoPaysForReads)} } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index e89037c5f..305385728 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -544,3 +544,63 @@ VALUES return nil } + +func (c *TestDataController) AddAttributesTestData(allocationTx, pubkey, clientId string) error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id, blobber_size, allocation_root) +VALUES ('exampleId' ,'` + allocationTx + `','` + clientId + `','` + pubkey + `',` + fmt.Sprint(expTime) + `,'examplePayerId', 99999999, '/'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO allocation_connections (connection_id, allocation_id, client_id, size, status) +VALUES ('connection_id' ,'exampleId','` + clientId + `', 1337, 1); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash, parent_path) +VALUES +(1234,'exampleId','exampleId:examplePath','exampleId:examplePath','d','root','/','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'), +(123,'exampleId','exampleId:examplePath','exampleId:examplePath','f','some_file','/some_file','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 8361e34dc..0d8de03b0 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -905,6 +905,92 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } } }) + + t.Run("TestUpdateAttributes", func(t *testing.T) { + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + pubKeyBytes, _ := hex.DecodeString(pubKey) + clientId := encryption.Hash(pubKeyBytes) + + if err := tdController.ClearDatabase(); err != nil { + t.Fatal(err) + } + if err := tdController.AddAttributesTestData(allocationTx, pubKey, clientId); err != nil { + t.Fatal(err) + } + + attr := &reference.Attributes{WhoPaysForReads: common.WhoPaysOwner} + attrBytes, err := json.Marshal(attr) + if err != nil { + t.Fatal(err) + } + + testCases := []struct { + name string + context metadata.MD + input *blobbergrpc.UpdateObjectAttributesRequest + expectedMessage int + expectingError bool + }{ + { + name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: clientId, + common.ClientSignatureHeader: clientSignature, + common.ClientKeyHeader: pubKey, + }), + input: &blobbergrpc.UpdateObjectAttributesRequest{ + Allocation: allocationTx, + Path: "/some_file", + PathHash: "exampleId:examplePath", + ConnectionId: "connection_id", + Attributes: string(attrBytes), + }, + expectedMessage: int(attr.WhoPaysForReads), + expectingError: false, + }, + { + name: "Fail", + context: metadata.New(map[string]string{ + common.ClientHeader: clientId, + common.ClientSignatureHeader: clientSignature, + common.ClientKeyHeader: pubKey, + }), + input: &blobbergrpc.UpdateObjectAttributesRequest{ + Allocation: "", + Path: "", + PathHash: "", + ConnectionId: "", + Attributes: "", + }, + expectedMessage: 0, + expectingError: true, + }, + } + + for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) + response, err := blobberClient.UpdateObjectAttributes(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + if response.GetWhoPaysForReads() != int64(tc.expectedMessage) { + t.Fatal("failed!") + } + } + }) } func randString(n int) string { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 7ccc89fe3..ed220050e 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberHTTP" @@ -704,6 +705,8 @@ func (fsh *StorageHandler) UpdateObjectAttributes(ctx context.Context, "missing new attributes, pass at least {} for empty attributes") } + fmt.Println(`attributes --- `, attributes) + var attrs = new(reference.Attributes) if err = json.Unmarshal([]byte(attributes), attrs); err != nil { return nil, common.NewErrorf("update_object_attributes", From ad10f776832ffed5d2f6dd527c5cc310671257f8 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Mon, 28 Jun 2021 22:20:39 +0530 Subject: [PATCH 02/18] update attributes handler integrated tests implemented --- .../blobbercore/convert/responseHandler.go | 13 +++---------- .../handler/grpc_handler_integration_test.go | 2 +- .../blobbercore/handler/object_operation_handler.go | 4 ---- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/code/go/0chain.net/blobbercore/convert/responseHandler.go b/code/go/0chain.net/blobbercore/convert/responseHandler.go index 20242218d..a2f72e208 100644 --- a/code/go/0chain.net/blobbercore/convert/responseHandler.go +++ b/code/go/0chain.net/blobbercore/convert/responseHandler.go @@ -3,8 +3,6 @@ package convert import ( "context" "encoding/json" - "fmt" - "github.com/0chain/blobber/code/go/0chain.net/core/common" stats2 "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" @@ -183,17 +181,12 @@ func CollaboratorResponseCreator(r interface{}) *blobbergrpc.CollaboratorRespons } func UpdateObjectAttributesResponseCreator(r interface{}) *blobbergrpc.UpdateObjectAttributesResponse { - - fmt.Println(`response - `, r) - - if r != nil { + if r == nil { return nil } - httpResp, ok := r.(*reference.Attributes) - if !ok { - fmt.Println(`response not okay - `, r) - } + httpResp, _ := r.(*reference.Attributes) + return &blobbergrpc.UpdateObjectAttributesResponse{WhoPaysForReads: int64(httpResp.WhoPaysForReads)} } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 0d8de03b0..f0158cb5b 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -921,7 +921,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { t.Fatal(err) } - attr := &reference.Attributes{WhoPaysForReads: common.WhoPaysOwner} + attr := &reference.Attributes{WhoPaysForReads: common.WhoPays3rdParty} attrBytes, err := json.Marshal(attr) if err != nil { t.Fatal(err) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index ed220050e..ec91938a0 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -5,8 +5,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "fmt" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberHTTP" "net/http" @@ -705,8 +703,6 @@ func (fsh *StorageHandler) UpdateObjectAttributes(ctx context.Context, "missing new attributes, pass at least {} for empty attributes") } - fmt.Println(`attributes --- `, attributes) - var attrs = new(reference.Attributes) if err = json.Unmarshal([]byte(attributes), attrs); err != nil { return nil, common.NewErrorf("update_object_attributes", From f217228fd42f2f7e79e556e3b56df3e2ce88e29a Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Mon, 28 Jun 2021 22:47:06 +0530 Subject: [PATCH 03/18] copy object handler integrated tests implemented --- .../handler/grpc_handler_helper_unit_test.go | 60 ++++++++++++++ .../handler/grpc_handler_integration_test.go | 80 +++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index 305385728..da189506d 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -604,3 +604,63 @@ VALUES return nil } + +func (c *TestDataController) AddCopyObjectData(allocationTx, pubkey, clientId string) error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id, blobber_size, allocation_root) +VALUES ('exampleId' ,'` + allocationTx + `','` + clientId + `','` + pubkey + `',` + fmt.Sprint(expTime) + `,'examplePayerId', 99999999, '/'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO allocation_connections (connection_id, allocation_id, client_id, size, status) +VALUES ('connection_id' ,'exampleId','` + clientId + `', 1337, 1); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash, parent_path) +VALUES +(1234,'exampleId','exampleId:examplePath','exampleId:examplePath','d','root','/copy','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'), +(123,'exampleId','exampleId:examplePath','exampleId:examplePath','f','some_file','/some_file','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index f0158cb5b..b84cc13a3 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -991,6 +991,86 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } } }) + + t.Run("TestCopyObject", func(t *testing.T) { + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + pubKeyBytes, _ := hex.DecodeString(pubKey) + clientId := encryption.Hash(pubKeyBytes) + + if err := tdController.ClearDatabase(); err != nil { + t.Fatal(err) + } + if err := tdController.AddCopyObjectData(allocationTx, pubKey, clientId); err != nil { + t.Fatal(err) + } + + testCases := []struct { + name string + context metadata.MD + input *blobbergrpc.CopyObjectRequest + expectedMessage string + expectingError bool + }{ + { + name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: clientId, + common.ClientSignatureHeader: clientSignature, + common.ClientKeyHeader: pubKey, + }), + input: &blobbergrpc.CopyObjectRequest{ + Allocation: allocationTx, + Path: "/some_file", + PathHash: "exampleId:examplePath", + ConnectionId: "connection_id", + Dest: "/copy", + }, + expectedMessage: "some_file", + expectingError: false, + }, + { + name: "Fail", + context: metadata.New(map[string]string{ + common.ClientHeader: clientId, + common.ClientSignatureHeader: clientSignature, + common.ClientKeyHeader: pubKey, + }), + input: &blobbergrpc.CopyObjectRequest{ + Allocation: "", + Path: "", + PathHash: "", + ConnectionId: "", + Dest: "", + }, + expectedMessage: "", + expectingError: true, + }, + } + + for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) + response, err := blobberClient.CopyObject(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + if response.GetFilename() != tc.expectedMessage { + t.Fatal("failed!") + } + } + }) } func randString(n int) string { From 77f6db76bcfd769eb70ddcb1fe0c86d60d475752 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Mon, 28 Jun 2021 23:03:08 +0530 Subject: [PATCH 04/18] rename object handler integrated tests implemented --- .../handler/grpc_handler_helper_unit_test.go | 60 ++++++++++++++ .../handler/grpc_handler_integration_test.go | 80 +++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index da189506d..4a0f9cac3 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -664,3 +664,63 @@ VALUES return nil } + +func (c *TestDataController) AddRenameTestData(allocationTx, pubkey, clientId string) error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id, blobber_size, allocation_root) +VALUES ('exampleId' ,'` + allocationTx + `','` + clientId + `','` + pubkey + `',` + fmt.Sprint(expTime) + `,'examplePayerId', 99999999, '/'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO allocation_connections (connection_id, allocation_id, client_id, size, status) +VALUES ('connection_id' ,'exampleId','` + clientId + `', 1337, 1); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash, parent_path) +VALUES +(1234,'exampleId','exampleId:examplePath','exampleId:examplePath','d','root','/','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'), +(123,'exampleId','exampleId:examplePath','exampleId:examplePath','f','some_file','/some_file','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index b84cc13a3..bb4f97ebe 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -1071,6 +1071,86 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } } }) + + t.Run("TestRenameObject", func(t *testing.T) { + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + pubKeyBytes, _ := hex.DecodeString(pubKey) + clientId := encryption.Hash(pubKeyBytes) + + if err := tdController.ClearDatabase(); err != nil { + t.Fatal(err) + } + if err := tdController.AddRenameTestData(allocationTx, pubKey, clientId); err != nil { + t.Fatal(err) + } + + testCases := []struct { + name string + context metadata.MD + input *blobbergrpc.RenameObjectRequest + expectedMessage string + expectingError bool + }{ + { + name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: clientId, + common.ClientSignatureHeader: clientSignature, + common.ClientKeyHeader: pubKey, + }), + input: &blobbergrpc.RenameObjectRequest{ + Allocation: allocationTx, + Path: "/some_file", + PathHash: "exampleId:examplePath", + ConnectionId: "connection_id", + NewName: "some_new_file", + }, + expectedMessage: "some_new_file", + expectingError: false, + }, + { + name: "Fail", + context: metadata.New(map[string]string{ + common.ClientHeader: clientId, + common.ClientSignatureHeader: clientSignature, + common.ClientKeyHeader: pubKey, + }), + input: &blobbergrpc.RenameObjectRequest{ + Allocation: "", + Path: "", + PathHash: "", + ConnectionId: "", + NewName: "", + }, + expectedMessage: "", + expectingError: true, + }, + } + + for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) + response, err := blobberClient.RenameObject(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + if response.GetFilename() != tc.expectedMessage { + t.Fatal("failed!") + } + } + }) } func randString(n int) string { From 25229dd9cb0e33d2adeed9492151c754318dd2e3 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Tue, 29 Jun 2021 00:14:59 +0530 Subject: [PATCH 05/18] download handler request multipart implemented --- .../0chain.net/blobbercore/convert/convert.go | 58 ++++++++++ .../handler/grpc_handler_integration_test.go | 106 ++++++++++++++++++ .../handler/object_operation_grpc_handler.go | 13 +-- 3 files changed, 165 insertions(+), 12 deletions(-) diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go index 308ef6c87..09b39f04b 100644 --- a/code/go/0chain.net/blobbercore/convert/convert.go +++ b/code/go/0chain.net/blobbercore/convert/convert.go @@ -1,8 +1,12 @@ package convert import ( + "bytes" "context" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "mime/multipart" + "net/http" + "strings" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" @@ -419,3 +423,57 @@ func convertDirMetaDataGRPCToDirRef(dirref *blobbergrpc.DirMetaData) *reference. UpdatedAt: time.Unix(0, dirref.UpdatedAt), } } + +func DownloadFileGRPCToHTTP(req *blobbergrpc.DownloadFileRequest) (*http.Request, error) { + body := bytes.NewBuffer([]byte{}) + writer := multipart.NewWriter(body) + + err := writer.WriteField("path", req.Path) + if err != nil { + return nil, err + } + + err = writer.WriteField("path_hash", req.PathHash) + if err != nil { + return nil, err + } + + err = writer.WriteField("rx_pay", req.RxPay) + if err != nil { + return nil, err + } + + err = writer.WriteField("block_num", req.BlockNum) + if err != nil { + return nil, err + } + + err = writer.WriteField("num_blocks", req.NumBlocks) + if err != nil { + return nil, err + } + + err = writer.WriteField("read_marker", req.ReadMarker) + if err != nil { + return nil, err + } + + err = writer.WriteField("auth_token", req.AuthToken) + if err != nil { + return nil, err + } + + err = writer.WriteField("content", req.Content) + if err != nil { + return nil, err + } + + r, err := http.NewRequest("POST", "", strings.NewReader(body.String())) + if err != nil { + return nil, err + } + + r.Header.Set("Content-Type", writer.FormDataContentType()) + + return r, nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index bb4f97ebe..9c68f2cab 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" "log" "math/rand" "net/http" @@ -1151,6 +1152,111 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } } }) + + t.Run("TestDownload", func(t *testing.T) { + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + pubKeyBytes, _ := hex.DecodeString(pubKey) + clientId := encryption.Hash(pubKeyBytes) + + if err := tdController.ClearDatabase(); err != nil { + t.Fatal(err) + } + if err := tdController.AddRenameTestData(allocationTx, pubKey, clientId); err != nil { + t.Fatal(err) + } + + blobberPubKey := "de52c0a51872d5d2ec04dbc15a6f0696cba22657b80520e1d070e72de64c9b04e19ce3223cae3c743a20184158457582ffe9c369ca9218c04bfe83a26a62d88d" + blobberPubKeyBytes, _ := hex.DecodeString(blobberPubKey) + + rm := readmarker.ReadMarker{ + ClientID: "", + ClientPublicKey: "", + BlobberID: encryption.Hash(blobberPubKeyBytes), + AllocationID: "", + OwnerID: "", + Timestamp: 0, + ReadCounter: 0, + Signature: "", + Suspend: 0, + PayerID: "", + AuthTicket: nil, + } + + rmString, err := json.Marshal(rm) + if err != nil { + t.Fatal(err) + } + + testCases := []struct { + name string + context metadata.MD + input *blobbergrpc.DownloadFileRequest + expectedMessage string + expectingError bool + }{ + { + name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: clientId, + common.ClientSignatureHeader: clientSignature, + common.ClientKeyHeader: pubKey, + }), + input: &blobbergrpc.DownloadFileRequest{ + Allocation: allocationTx, + Path: "/some_file", + PathHash: "exampleId:examplePath", + ReadMarker: string(rmString), + }, + expectedMessage: "some_new_file", + expectingError: false, + }, + { + name: "Fail", + context: metadata.New(map[string]string{ + common.ClientHeader: clientId, + common.ClientSignatureHeader: clientSignature, + common.ClientKeyHeader: pubKey, + }), + input: &blobbergrpc.DownloadFileRequest{ + Allocation: "", + Path: "", + PathHash: "", + RxPay: "", + BlockNum: "", + NumBlocks: "", + ReadMarker: "", + AuthToken: "", + Content: "", + }, + expectedMessage: "", + expectingError: true, + }, + } + + for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) + response, err := blobberClient.DownloadFile(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + if response.GetPath() != tc.expectedMessage { + t.Fatal("failed!") + } + } + }) } func randString(n int) string { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go index c9d1f9cd5..83b748ac6 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_grpc_handler.go @@ -71,23 +71,12 @@ func (b *blobberGRPCService) RenameObject(ctx context.Context, req *blobbergrpc. } func (b *blobberGRPCService) DownloadFile(ctx context.Context, req *blobbergrpc.DownloadFileRequest) (*blobbergrpc.DownloadFileResponse, error) { - - r, err := http.NewRequest("POST", "", nil) + r, err := convert.DownloadFileGRPCToHTTP(req) if err != nil { return nil, err } httpRequestWithMetaData(r, GetGRPCMetaDataFromCtx(ctx), req.Allocation) - r.Form = map[string][]string{ - "path": {req.Path}, - "path_hash": {req.PathHash}, - "rx_pay": {req.RxPay}, - "block_num": {req.BlockNum}, - "num_blocks": {req.NumBlocks}, - "read_marker": {req.ReadMarker}, - "auth_token": {req.AuthToken}, - "content": {req.AuthToken}, - } resp, err := DownloadHandler(ctx, r) if err != nil { From 5852f507c2a072d54fdc1f837099df0e5428539c Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Tue, 29 Jun 2021 00:25:39 +0530 Subject: [PATCH 06/18] download handler request multipart writer closed --- code/go/0chain.net/blobbercore/convert/convert.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go index 09b39f04b..45122609e 100644 --- a/code/go/0chain.net/blobbercore/convert/convert.go +++ b/code/go/0chain.net/blobbercore/convert/convert.go @@ -468,6 +468,8 @@ func DownloadFileGRPCToHTTP(req *blobbergrpc.DownloadFileRequest) (*http.Request return nil, err } + writer.Close() + r, err := http.NewRequest("POST", "", strings.NewReader(body.String())) if err != nil { return nil, err From 8f1028a1066819ddd405e3202215aa653f488073 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Tue, 29 Jun 2021 02:46:47 +0530 Subject: [PATCH 07/18] download handler integration tests WIP --- .../handler/grpc_handler_helper_unit_test.go | 76 +++++++++++++++++++ .../handler/grpc_handler_integration_test.go | 38 ++++++---- 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index 4a0f9cac3..d7842c8e8 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -724,3 +724,79 @@ VALUES return nil } + +func (c *TestDataController) AddDownloadTestData(allocationTx, pubkey, clientId, wmSig string, now common.Timestamp) error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id, blobber_size, allocation_root) +VALUES ('exampleId' ,'` + allocationTx + `','` + clientId + `','` + pubkey + `',` + fmt.Sprint(expTime) + `,'examplePayerId', 99999999, '/'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO allocation_connections (connection_id, allocation_id, client_id, size, status) +VALUES ('connection_id' ,'exampleId','` + clientId + `', 1337, 1); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO allocation_changes (id, connection_id, operation, size, input) +VALUES (1 ,'connection_id','rename', 1200, '{"allocation_id":"exampleId","path":"/some_file","new_name":"new_name"}'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO write_markers(prev_allocation_root, allocation_root, status, allocation_id, size, client_id, signature, blobber_id, timestamp, connection_id, client_key) +VALUES ('/', '/', 2,'exampleId', 1337, '` + clientId + `','` + wmSig + `','blobber_id', ` + fmt.Sprint(now) + `, 'connection_id', '` + pubkey + `'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash, parent_path) +VALUES +(1234,'exampleId','exampleId:examplePath','exampleId:examplePath','d','root','/','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'), +(123,'exampleId','exampleId:examplePath','exampleId:examplePath','f','some_file','/some_file','someHash','customMeta','sonMonWenMyFile','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 9c68f2cab..9d9e95143 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -1156,40 +1156,51 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { t.Run("TestDownload", func(t *testing.T) { allocationTx := randString(32) + _, err := os.Create(`example`) + if err != nil { + t.Fatal(err) + } + defer os.Remove(`example`) + pubKey, _, signScheme := GeneratePubPrivateKey(t) clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) pubKeyBytes, _ := hex.DecodeString(pubKey) clientId := encryption.Hash(pubKeyBytes) + now := common.Timestamp(time.Now().Unix()) + allocationId := `exampleId` if err := tdController.ClearDatabase(); err != nil { t.Fatal(err) } - if err := tdController.AddRenameTestData(allocationTx, pubKey, clientId); err != nil { - t.Fatal(err) - } blobberPubKey := "de52c0a51872d5d2ec04dbc15a6f0696cba22657b80520e1d070e72de64c9b04e19ce3223cae3c743a20184158457582ffe9c369ca9218c04bfe83a26a62d88d" blobberPubKeyBytes, _ := hex.DecodeString(blobberPubKey) rm := readmarker.ReadMarker{ - ClientID: "", - ClientPublicKey: "", BlobberID: encryption.Hash(blobberPubKeyBytes), - AllocationID: "", - OwnerID: "", - Timestamp: 0, - ReadCounter: 0, - Signature: "", - Suspend: 0, - PayerID: "", - AuthTicket: nil, + AllocationID: allocationId, + ClientPublicKey: pubKey, + ClientID: clientId, + OwnerID: clientId, + Timestamp: now, + ReadCounter: 1337, } + rmSig, err := signScheme.Sign(encryption.Hash(rm.GetHashData())) + if err != nil { + t.Fatal(err) + } + rm.Signature = rmSig + rmString, err := json.Marshal(rm) if err != nil { t.Fatal(err) } + if err := tdController.AddDownloadTestData(allocationTx, pubKey, clientId, rmSig, now); err != nil { + t.Fatal(err) + } + testCases := []struct { name string context metadata.MD @@ -1209,6 +1220,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { Path: "/some_file", PathHash: "exampleId:examplePath", ReadMarker: string(rmString), + BlockNum: "10", }, expectedMessage: "some_new_file", expectingError: false, From 1d29d5a02f1da63be7f13807477a542378b891d3 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Tue, 29 Jun 2021 04:14:38 +0530 Subject: [PATCH 08/18] download handler integration tests WIP --- .../blobbercore/convert/responseHandler.go | 22 +++++--- .../handler/grpc_handler_helper_unit_test.go | 10 +--- .../handler/grpc_handler_integration_test.go | 56 ++++++++++++++++--- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/code/go/0chain.net/blobbercore/convert/responseHandler.go b/code/go/0chain.net/blobbercore/convert/responseHandler.go index a2f72e208..3d2cd4a6b 100644 --- a/code/go/0chain.net/blobbercore/convert/responseHandler.go +++ b/code/go/0chain.net/blobbercore/convert/responseHandler.go @@ -376,12 +376,20 @@ func DownloadFileResponseCreator(r interface{}) *blobbergrpc.DownloadFileRespons return nil } - httpResp, _ := r.(*blobberHTTP.DownloadResponse) - return &blobbergrpc.DownloadFileResponse{ - Success: httpResp.Success, - Data: httpResp.Data, - AllocationId: httpResp.AllocationID, - Path: httpResp.Path, - LatestRm: ReadMarkerToReadMarkerGRPC(httpResp.LatestRM), + switch httpResp := r.(type) { + case []byte: + return &blobbergrpc.DownloadFileResponse{ + Data: httpResp, + } + case *blobberHTTP.DownloadResponse: + return &blobbergrpc.DownloadFileResponse{ + Success: httpResp.Success, + Data: httpResp.Data, + AllocationId: httpResp.AllocationID, + Path: httpResp.Path, + LatestRm: ReadMarkerToReadMarkerGRPC(httpResp.LatestRM), + } } + + return nil } diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index d7842c8e8..72a20d28a 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -775,19 +775,11 @@ VALUES (1 ,'connection_id','rename', 1200, '{"allocation_id":"exampleId","path": return err } - _, err = tx.Exec(` -INSERT INTO write_markers(prev_allocation_root, allocation_root, status, allocation_id, size, client_id, signature, blobber_id, timestamp, connection_id, client_key) -VALUES ('/', '/', 2,'exampleId', 1337, '` + clientId + `','` + wmSig + `','blobber_id', ` + fmt.Sprint(now) + `, 'connection_id', '` + pubkey + `'); -`) - if err != nil { - return err - } - _, err = tx.Exec(` INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash, parent_path) VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','d','root','/','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'), -(123,'exampleId','exampleId:examplePath','exampleId:examplePath','f','some_file','/some_file','someHash','customMeta','sonMonWenMyFile','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'); +(123,'exampleId','exampleId:examplePath','exampleId:examplePath','f','some_file','/some_file','someHash','customMeta','tmpMonWenMyFile','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash','/'); `) if err != nil { return err diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 9d9e95143..b312da48b 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "io" "log" "math/rand" "net/http" @@ -1156,11 +1157,52 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { t.Run("TestDownload", func(t *testing.T) { allocationTx := randString(32) - _, err := os.Create(`example`) + root, _ := os.Getwd() + path := strings.Split(root, `code`) + err := os.Mkdir(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon`, 0700) if err != nil { t.Fatal(err) } - defer os.Remove(`example`) + err = os.Mkdir(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, 0700) + if err != nil { + t.Fatal(err) + } + f, err := os.Create(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) + if err != nil { + t.Fatal(err) + } + file, err := os.Open(root + "/grpc_handler_integration_test.go") + if err != nil { + t.Fatal(err) + } + stat, err := file.Stat() + if err != nil { + t.Fatal(err) + } + fileB := make([]byte, stat.Size()) + if _, err := io.ReadFull(file, fileB); err != nil { + t.Fatal(err) + } + _, err = f.Write(fileB) + if err != nil { + t.Fatal(err) + } + _, err = f.Write(fileB) + if err != nil { + t.Fatal(err) + } + _, err = f.Write(fileB) + if err != nil { + t.Fatal(err) + } + fmt.Println(`file size`, stat.Size()) + file.Close() + f.Close() + defer func() { + os.Remove(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) + os.Remove(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`) + os.Remove(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon`) + }() pubKey, _, signScheme := GeneratePubPrivateKey(t) clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) @@ -1183,7 +1225,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { ClientID: clientId, OwnerID: clientId, Timestamp: now, - ReadCounter: 1337, + //ReadCounter: 1337, } rmSig, err := signScheme.Sign(encryption.Hash(rm.GetHashData())) @@ -1220,7 +1262,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { Path: "/some_file", PathHash: "exampleId:examplePath", ReadMarker: string(rmString), - BlockNum: "10", + BlockNum: "2", }, expectedMessage: "some_new_file", expectingError: false, @@ -1251,7 +1293,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { for _, tc := range testCases { ctx := context.Background() ctx = metadata.NewOutgoingContext(ctx, tc.context) - response, err := blobberClient.DownloadFile(ctx, tc.input) + _, err := blobberClient.DownloadFile(ctx, tc.input) if err != nil { if !tc.expectingError { t.Fatal(err) @@ -1263,10 +1305,6 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { if tc.expectingError { t.Fatal("expected error") } - - if response.GetPath() != tc.expectedMessage { - t.Fatal("failed!") - } } }) } From 0cadb698fb239d7887c28d51a95399631ff94702 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Tue, 29 Jun 2021 04:35:55 +0530 Subject: [PATCH 09/18] download handler integration tests edit makefile --- Makefile | 4 +++- .../handler/grpc_handler_integration_test.go | 14 +------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index dc1cdf24d..1110730da 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,6 @@ lint: golangci-lint run --timeout 2m0s; integration-tests: - go test ./... -args integration; \ No newline at end of file + mkdir -p docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen + go test ./... -args integration; + rm -r docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon \ No newline at end of file diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index b312da48b..14b2e5386 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -1159,14 +1159,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { root, _ := os.Getwd() path := strings.Split(root, `code`) - err := os.Mkdir(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon`, 0700) - if err != nil { - t.Fatal(err) - } - err = os.Mkdir(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, 0700) - if err != nil { - t.Fatal(err) - } + f, err := os.Create(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) if err != nil { t.Fatal(err) @@ -1198,11 +1191,6 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { fmt.Println(`file size`, stat.Size()) file.Close() f.Close() - defer func() { - os.Remove(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) - os.Remove(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`) - os.Remove(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon`) - }() pubKey, _, signScheme := GeneratePubPrivateKey(t) clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) From 2c81950f3e893e61fe35e3c9e9839829caf12955 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 19:50:51 +0530 Subject: [PATCH 10/18] make file mkdir removed and added to test file --- Makefile | 2 - .../handler/grpc_handler_integration_test.go | 56 +++++++++++-------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 1110730da..642ae15ed 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,4 @@ lint: golangci-lint run --timeout 2m0s; integration-tests: - mkdir -p docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen go test ./... -args integration; - rm -r docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon \ No newline at end of file diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 61b9ed42e..fae61a532 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -6,8 +6,8 @@ import ( "encoding/json" "fmt" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "io" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "io" "log" "math/rand" "net/http" @@ -1161,37 +1161,49 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { root, _ := os.Getwd() path := strings.Split(root, `code`) + err := os.MkdirAll(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, os.ModePerm) + f, err := os.Create(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) if err != nil { t.Fatal(err) } + defer f.Close() + defer func() { + err := os.RemoveAll(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon`) + if err != nil { + t.Fatal(err) + } + }() file, err := os.Open(root + "/grpc_handler_integration_test.go") if err != nil { t.Fatal(err) } - stat, err := file.Stat() - if err != nil { - t.Fatal(err) - } - fileB := make([]byte, stat.Size()) - if _, err := io.ReadFull(file, fileB); err != nil { - t.Fatal(err) - } - _, err = f.Write(fileB) - if err != nil { - t.Fatal(err) - } - _, err = f.Write(fileB) - if err != nil { - t.Fatal(err) - } - _, err = f.Write(fileB) + defer file.Close() + + _, err = io.Copy(f, file) if err != nil { t.Fatal(err) } - fmt.Println(`file size`, stat.Size()) - file.Close() - f.Close() + //stat, err := file.Stat() + //if err != nil { + // t.Fatal(err) + //} + //fileB := make([]byte, stat.Size()) + //if _, err := io.ReadFull(file, fileB); err != nil { + // t.Fatal(err) + //} + //_, err = f.Write(fileB) + //if err != nil { + // t.Fatal(err) + //} + //_, err = f.Write(fileB) + //if err != nil { + // t.Fatal(err) + //} + //_, err = f.Write(fileB) + //if err != nil { + // t.Fatal(err) + //} pubKey, _, signScheme := GeneratePubPrivateKey(t) clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) @@ -1251,7 +1263,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { Path: "/some_file", PathHash: "exampleId:examplePath", ReadMarker: string(rmString), - BlockNum: "2", + BlockNum: "1", }, expectedMessage: "some_new_file", expectingError: false, From 5ec54818f8737195c128c24238db25fc8d130cc8 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 19:59:27 +0530 Subject: [PATCH 11/18] lint fixed --- .../handler/grpc_handler_integration_test.go | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index fae61a532..f7e999c3b 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -1161,7 +1161,10 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { root, _ := os.Getwd() path := strings.Split(root, `code`) - err := os.MkdirAll(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, os.ModePerm) + err = os.MkdirAll(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, os.ModePerm) + if err != nil { + t.Fatal(err) + } f, err := os.Create(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) if err != nil { @@ -1184,26 +1187,6 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { if err != nil { t.Fatal(err) } - //stat, err := file.Stat() - //if err != nil { - // t.Fatal(err) - //} - //fileB := make([]byte, stat.Size()) - //if _, err := io.ReadFull(file, fileB); err != nil { - // t.Fatal(err) - //} - //_, err = f.Write(fileB) - //if err != nil { - // t.Fatal(err) - //} - //_, err = f.Write(fileB) - //if err != nil { - // t.Fatal(err) - //} - //_, err = f.Write(fileB) - //if err != nil { - // t.Fatal(err) - //} pubKey, _, signScheme := GeneratePubPrivateKey(t) clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) From 7381c2df1ded63016545619845bae8f26d85355e Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 20:12:39 +0530 Subject: [PATCH 12/18] permission changes added --- .../handler/grpc_handler_integration_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index f7e999c3b..c8a9ffbb3 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -1161,22 +1161,28 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { root, _ := os.Getwd() path := strings.Split(root, `code`) - err = os.MkdirAll(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, os.ModePerm) + err = os.Chmod(path[0]+`docker.local/blobber1/files/files`, 0777) if err != nil { t.Fatal(err) } - f, err := os.Create(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) + err = os.MkdirAll(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, os.ModePerm) if err != nil { t.Fatal(err) } - defer f.Close() defer func() { err := os.RemoveAll(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon`) if err != nil { t.Fatal(err) } }() + + f, err := os.Create(path[0] + `docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen/MyFile`) + if err != nil { + t.Fatal(err) + } + defer f.Close() + file, err := os.Open(root + "/grpc_handler_integration_test.go") if err != nil { t.Fatal(err) From 4f4d560574bd6310eb79bcc0d8abd3d7f16e92da Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 20:34:43 +0530 Subject: [PATCH 13/18] add permission to test workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a60c0d996..e0e1be791 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: cd docker.local/blobber1 ../bin/blobber.start_bls.sh /dev/null & cd ../.. - make integration-tests + sudo make integration-tests lint: runs-on: ubuntu-20.04 steps: From 33903b07da40bc459e9513d25a5b2def596d69d0 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 20:51:23 +0530 Subject: [PATCH 14/18] remove permission to test workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0e1be791..a60c0d996 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: cd docker.local/blobber1 ../bin/blobber.start_bls.sh /dev/null & cd ../.. - sudo make integration-tests + make integration-tests lint: runs-on: ubuntu-20.04 steps: From 8e2f479f26e0d3e61e80bed5d141dca2a209ec26 Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 20:57:24 +0530 Subject: [PATCH 15/18] add permission to test workflow makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 642ae15ed..ba3dd4b74 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,4 @@ lint: golangci-lint run --timeout 2m0s; integration-tests: - go test ./... -args integration; + sudo go test ./... -args integration; From 718247d4bce14781e3f5f4e0c1770cf4dd203e1a Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 22:12:51 +0530 Subject: [PATCH 16/18] add permission to test --- Makefile | 2 +- .../blobbercore/handler/grpc_handler_integration_test.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ba3dd4b74..642ae15ed 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,4 @@ lint: golangci-lint run --timeout 2m0s; integration-tests: - sudo go test ./... -args integration; + go test ./... -args integration; diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index c8a9ffbb3..afcdf8934 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -1161,6 +1161,11 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { root, _ := os.Getwd() path := strings.Split(root, `code`) + err = os.Chown(path[0]+`docker.local/blobber1/files/files`, os.Getuid(), os.Getgid()) + if err != nil { + t.Fatal(err) + } + err = os.Chmod(path[0]+`docker.local/blobber1/files/files`, 0777) if err != nil { t.Fatal(err) From ac85f3e6b0ab580418d542ef57ad5607b411e26c Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 22:37:58 +0530 Subject: [PATCH 17/18] add sudo permission to test in makefile --- Makefile | 2 +- .../blobbercore/handler/grpc_handler_integration_test.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 642ae15ed..ba3dd4b74 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,4 @@ lint: golangci-lint run --timeout 2m0s; integration-tests: - go test ./... -args integration; + sudo go test ./... -args integration; diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index afcdf8934..c8a9ffbb3 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -1161,11 +1161,6 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { root, _ := os.Getwd() path := strings.Split(root, `code`) - err = os.Chown(path[0]+`docker.local/blobber1/files/files`, os.Getuid(), os.Getgid()) - if err != nil { - t.Fatal(err) - } - err = os.Chmod(path[0]+`docker.local/blobber1/files/files`, 0777) if err != nil { t.Fatal(err) From fa1999e0b6e18e598ea169279e7ab28e940c5dbf Mon Sep 17 00:00:00 2001 From: nipunapathirana Date: Wed, 30 Jun 2021 22:48:44 +0530 Subject: [PATCH 18/18] remove chmod in test file --- .../blobbercore/handler/grpc_handler_integration_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index c8a9ffbb3..401f2717b 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -1161,11 +1161,6 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { root, _ := os.Getwd() path := strings.Split(root, `code`) - err = os.Chmod(path[0]+`docker.local/blobber1/files/files`, 0777) - if err != nil { - t.Fatal(err) - } - err = os.MkdirAll(path[0]+`docker.local/blobber1/files/files/exa/mpl/eId/objects/tmp/Mon/Wen`, os.ModePerm) if err != nil { t.Fatal(err)