From 4edd89da476d7365f15b9e788d80e7b1ceb520d8 Mon Sep 17 00:00:00 2001 From: Ebrahim Gomaa Date: Mon, 6 Nov 2023 17:23:54 +0200 Subject: [PATCH 1/4] Node recovery conductor tests (#1259) * notify on validator tickert generatd * fix race condition --- code/go/0chain.net/conductor/conductrpc/client.go | 5 +++++ code/go/0chain.net/conductor/conductrpc/entity.go | 6 ++++++ code/go/0chain.net/conductor/conductrpc/server.go | 4 ++++ code/go/0chain.net/conductor/conductrpc/state.go | 1 + .../storage/challenge_handler_integration_tests.go | 6 ++++++ 5 files changed, 22 insertions(+) diff --git a/code/go/0chain.net/conductor/conductrpc/client.go b/code/go/0chain.net/conductor/conductrpc/client.go index f21410310..5ef0afff2 100644 --- a/code/go/0chain.net/conductor/conductrpc/client.go +++ b/code/go/0chain.net/conductor/conductrpc/client.go @@ -53,6 +53,11 @@ func (c *client) blobberCommitted(blobberID string) (err error) { return } +func (c *client) validationTicketGenerated(ticket ValidtorTicket) (err error) { + err = c.client.Call("Server.ValidatorTicket", &ticket, nil) + return +} + func (c *client) sendFileMetaRoot(m map[string]string) (err error) { err = c.client.Call("Server.GetFileMetaRoot", m, nil) return diff --git a/code/go/0chain.net/conductor/conductrpc/entity.go b/code/go/0chain.net/conductor/conductrpc/entity.go index cc1d6096e..c52db9bf3 100644 --- a/code/go/0chain.net/conductor/conductrpc/entity.go +++ b/code/go/0chain.net/conductor/conductrpc/entity.go @@ -112,6 +112,12 @@ func (e *Entity) BlobberCommitted(blobberID string) { } } +func (e *Entity) ValidatorTicket(ticket ValidtorTicket) { + err := e.client.validationTicketGenerated(ticket) + if err != nil { + log.Println(err) + } +} func (e *Entity) SendFileMetaRoot(blobberID, fileMetaRoot string, ctxCncl context.CancelFunc) { m := map[string]string{ diff --git a/code/go/0chain.net/conductor/conductrpc/server.go b/code/go/0chain.net/conductor/conductrpc/server.go index 01979c52a..00c23fc14 100644 --- a/code/go/0chain.net/conductor/conductrpc/server.go +++ b/code/go/0chain.net/conductor/conductrpc/server.go @@ -16,3 +16,7 @@ type ( RoundName = config.RoundName Number = config.Number ) + +type ValidtorTicket struct { + ValidatorId string +} diff --git a/code/go/0chain.net/conductor/conductrpc/state.go b/code/go/0chain.net/conductor/conductrpc/state.go index f503156a5..3c9f57dfb 100644 --- a/code/go/0chain.net/conductor/conductrpc/state.go +++ b/code/go/0chain.net/conductor/conductrpc/state.go @@ -62,6 +62,7 @@ type State struct { BlobberUpload BlobberUpload BlobberDelete BlobberDelete AdversarialValidator AdversarialValidator + NotifyOnValidationTicketGeneration bool StopWMCommit *bool FailRenameCommit []string GetFileMetaRoot bool diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go b/code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go index 26e8705fc..04e782ca8 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go @@ -49,5 +49,11 @@ func ChallengeHandler(ctx context.Context, r *http.Request) (interface{}, error, res, err := challengeHandler(ctx, r) + if state.NotifyOnValidationTicketGeneration { + conductrpc.Client().ValidatorTicket(conductrpc.ValidtorTicket{ + ValidatorId: node.Self.ID, + }) + } + return res, err, true } From 259c0cd6016eddb09910cf71250d18799f11ae75 Mon Sep 17 00:00:00 2001 From: Jayash Satolia <73050737+Jayashsatolia403@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:25:30 +0530 Subject: [PATCH 2/4] Added changes for fail upload commit (#1318) --- .../allocation/file_changer_upload.go | 2 +- .../file_changer_upload_integration.go | 28 +++++++++++++++++++ .../allocation/file_changer_upload_main.go | 15 ++++++++++ .../0chain.net/conductor/conductrpc/state.go | 1 + 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go create mode 100644 code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index dbe7bb3c3..c92a74248 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -21,7 +21,7 @@ type UploadFileChanger struct { } // ApplyChange update references, and create a new FileRef -func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, +func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { totalRefs, err := reference.CountRefs(ctx, nf.AllocationID) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go new file mode 100644 index 000000000..c9e1fa580 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go @@ -0,0 +1,28 @@ +//go:build integration_tests +// +build integration_tests + +package allocation + +import ( + "context" + "errors" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/node" +) + +func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, + allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { + + state := conductrpc.Client().State() + if state.FailUploadCommit != nil { + for _, nodeId := range state.FailUploadCommit { + if nodeId == node.Self.ID { + return nil, errors.New("error directed by conductor") + } + } + } + return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) +} diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go new file mode 100644 index 000000000..5a41a40aa --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go @@ -0,0 +1,15 @@ +//go:build !integration_tests +// +build !integration_tests + +package allocation + +import ( + "context" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" +) + +func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, + allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { + return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) +} diff --git a/code/go/0chain.net/conductor/conductrpc/state.go b/code/go/0chain.net/conductor/conductrpc/state.go index 3c9f57dfb..ccf1053f1 100644 --- a/code/go/0chain.net/conductor/conductrpc/state.go +++ b/code/go/0chain.net/conductor/conductrpc/state.go @@ -65,6 +65,7 @@ type State struct { NotifyOnValidationTicketGeneration bool StopWMCommit *bool FailRenameCommit []string + FailUploadCommit []string GetFileMetaRoot bool } From d8c650d77c73c0ee5125c9813a90325094e52ad9 Mon Sep 17 00:00:00 2001 From: Ebrahim Gomaa Date: Thu, 9 Nov 2023 16:56:45 +0200 Subject: [PATCH 3/4] Feat/download verify ct (#1317) * notify on validator tickert generatd * fix race condition * miss up download for CT * debug logs --- .../blobbercore/filestore/storage.go | 1 + .../blobbercore/filestore/tree_validation.go | 2 +- .../tree_validation_integration_tests.go | 59 +++++++++++++++++++ .../filestore/tree_validation_main.go | 11 ++++ .../handler/object_operation_handler.go | 3 + .../0chain.net/conductor/conductrpc/state.go | 1 + .../conductor-config/0chain_blobber.yaml | 2 +- .../conductor-config/0chain_validator.yaml | 2 +- 8 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go create mode 100644 code/go/0chain.net/blobbercore/filestore/tree_validation_main.go diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index e142fb686..e12f2a44a 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -528,6 +528,7 @@ func (fs *FileStore) GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadRes dataSize: readBlockIn.FileSize, } + logging.Logger.Debug("calling GetMerkleProofOfMultipleIndexes", zap.Any("readBlockIn", readBlockIn), zap.Any("vmp", vmp)) nodes, indexes, err := vp.GetMerkleProofOfMultipleIndexes(file, nodesSize, startBlock, endBlock) if err != nil { return nil, common.NewError("get_merkle_proof_error", err.Error()) diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index b44145707..d0e8fba5e 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -288,7 +288,7 @@ type validationTreeProof struct { // If endInd - startInd is whole file then no proof is required at all. // startInd and endInd is taken as closed interval. So to get proof for data at index 0 both startInd // and endInd would be 0. -func (v *validationTreeProof) GetMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( +func (v *validationTreeProof) getMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( [][][]byte, [][]int, error) { if startInd < 0 || endInd < 0 { diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go new file mode 100644 index 000000000..f5a51fc82 --- /dev/null +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go @@ -0,0 +1,59 @@ +//go:build integration_tests +// +build integration_tests + +package filestore + +import ( + "io" + + crpc "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" +) + +func (v *validationTreeProof) GetMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( + [][][]byte, [][]int, error) { + nodes, indexes, err := v.getMerkleProofOfMultipleIndexes(r, nodesSize, startInd, endInd) + if err != nil { + return nodes, indexes, err + } + + state := crpc.Client().State() + if state == nil { + return nodes, indexes, err + } + + logging.Logger.Debug("[conductor] just after getMerkleProofOfMultipleIndexes", + zap.Bool("miss_up_download", state.MissUpDownload), + zap.Int("start_ind", startInd), + zap.Int("end_ind", endInd), + zap.Int64("nodes_size", nodesSize), + zap.Int("output_nodes_size", len(nodes)), + zap.Int("output_indexes_size", len(indexes)), + zap.Any("nodes", nodes), + zap.Any("indexes", indexes), + ) + + if state.MissUpDownload { + logging.Logger.Debug("miss up/download", + zap.Bool("miss_up_download", state.MissUpDownload), + zap.Int("start_ind", startInd), + zap.Int("end_ind", endInd), + zap.Int64("nodes_size", nodesSize), + zap.Int("output_nodes_size", len(nodes)), + zap.Int("output_indexes_size", len(indexes)), + zap.Any("nodes", nodes), + zap.Any("indexes", indexes), + ) + + for i := range nodes { + nodes[i][0] = []byte("wrong data") + } + + for i := range indexes { + indexes[i][0] = 0 + } + } + + return nodes, indexes, err +} \ No newline at end of file diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_main.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_main.go new file mode 100644 index 000000000..d0dc7a5a2 --- /dev/null +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_main.go @@ -0,0 +1,11 @@ +//go:build !integration_tests +// +build !integration_tests + +package filestore + +import "io" + +func (v *validationTreeProof) GetMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( + [][][]byte, [][]int, error) { + return v.getMerkleProofOfMultipleIndexes(r, nodesSize, startInd, endInd) +} diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index c177c21ea..99dae6ade 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -25,6 +25,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "go.uber.org/zap" @@ -379,6 +380,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i IsPrecommit: fromPreCommit, } + logging.Logger.Info("calling GetFileBlock for thumb", zap.Any("rbi", rbi)) fileDownloadResponse, err = filestore.GetFileStore().GetFileBlock(rbi) if err != nil { return nil, common.NewErrorf("download_file", "couldn't get thumbnail block: %v", err) @@ -398,6 +400,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i VerifyDownload: dr.VerifyDownload, IsPrecommit: fromPreCommit, } + logging.Logger.Info("calling GetFileBlock", zap.Any("rbi", rbi)) fileDownloadResponse, err = filestore.GetFileStore().GetFileBlock(rbi) if err != nil { return nil, common.NewErrorf("download_file", "couldn't get file block: %v", err) diff --git a/code/go/0chain.net/conductor/conductrpc/state.go b/code/go/0chain.net/conductor/conductrpc/state.go index ccf1053f1..5d5da96a6 100644 --- a/code/go/0chain.net/conductor/conductrpc/state.go +++ b/code/go/0chain.net/conductor/conductrpc/state.go @@ -67,6 +67,7 @@ type State struct { FailRenameCommit []string FailUploadCommit []string GetFileMetaRoot bool + MissUpDownload bool } // Name returns NodeName by given NodeID. diff --git a/docker.local/conductor-config/0chain_blobber.yaml b/docker.local/conductor-config/0chain_blobber.yaml index 60f56ded6..85fc52664 100644 --- a/docker.local/conductor-config/0chain_blobber.yaml +++ b/docker.local/conductor-config/0chain_blobber.yaml @@ -1,7 +1,7 @@ version: "1.0" logging: - level: "info" + level: "debug" console: true # printing log to console is only supported in development mode info: diff --git a/docker.local/conductor-config/0chain_validator.yaml b/docker.local/conductor-config/0chain_validator.yaml index abc25d78b..35883065e 100644 --- a/docker.local/conductor-config/0chain_validator.yaml +++ b/docker.local/conductor-config/0chain_validator.yaml @@ -17,7 +17,7 @@ rate_limiters: proxy: true logging: - level: "error" + level: "debug" console: true # printing log to console is only supported in development mode healthcheck: From 577752cd743e7833ddf166676af31b46228278c3 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Thu, 9 Nov 2023 20:55:08 +0530 Subject: [PATCH 4/4] Improve download performance (#1315) * improve download file * write data * rmv content length * fix unit test --- .../blobbercore/handler/handler_download_test.go | 13 ++++++------- .../blobbercore/handler/object_operation_handler.go | 3 +++ code/go/0chain.net/core/common/handler.go | 7 ++++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler_download_test.go b/code/go/0chain.net/blobbercore/handler/handler_download_test.go index fd9d0d340..fca5304dd 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_download_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_download_test.go @@ -301,6 +301,7 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, alloc.OwnerID) r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) @@ -478,7 +479,7 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", pathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set("X-Auth-Token", authTicket) @@ -559,7 +560,7 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", pathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set("X-Auth-Token", authTicket) @@ -673,7 +674,7 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set("X-Auth-Token", authTicket) @@ -793,7 +794,7 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set("X-Auth-Token", authTicket) @@ -912,7 +913,7 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set("X-Auth-Token", authTicket) @@ -1005,9 +1006,7 @@ func TestHandlers_Download(t *testing.T) { m := make(map[string]interface{}) err = json.Unmarshal(data, &m) require.NoError(t, err) - if test.wantCode != http.StatusOK || test.wantBody != "" { - fmt.Println("fprint", test.args.w.Body.String()) var body string if m["Data"] != nil { body = m["Data"].(string) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 99dae6ade..37049c0b1 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -436,6 +436,9 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i addDailyBlocks(clientID, dr.NumBlocks) AddDownloadedData(clientID, dr.NumBlocks) }() + if !dr.VerifyDownload { + return fileDownloadResponse.Data, nil + } return fileDownloadResponse, nil } diff --git a/code/go/0chain.net/core/common/handler.go b/code/go/0chain.net/core/common/handler.go index 27ca8585c..d4de9afb8 100644 --- a/code/go/0chain.net/core/common/handler.go +++ b/code/go/0chain.net/core/common/handler.go @@ -82,7 +82,12 @@ func ToByteStream(handler JSONResponderF) ReqRespHandlerf { w.Write(rawdata) //nolint:errcheck } else { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(data) //nolint:errcheck + byteData, err := json.Marshal(data) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + w.Write(byteData) //nolint:errcheck } } }