From fc7adee5d18954669e945e57ec6493ae083e1c24 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Thu, 21 Dec 2023 23:01:58 +0530 Subject: [PATCH 1/4] wait for repair and increase numBlocks (#1338) --- wasmsdk/allocation.go | 4 ++++ zboxcore/sdk/allocation.go | 2 -- zboxcore/sdk/sdk.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/wasmsdk/allocation.go b/wasmsdk/allocation.go index 3812cf199..44b9500c0 100644 --- a/wasmsdk/allocation.go +++ b/wasmsdk/allocation.go @@ -194,6 +194,10 @@ func updateAllocationWithRepair(allocationID string, hash, err := allocationObj.UpdateWithRepair(size, extend, uint64(lock), addBlobberId, removeBlobberId, false, &sdk.FileOptionsParameters{}, statusBar) if err == nil { clearAllocation(allocationID) + wg.Wait() + if statusBar.err != nil { + return hash, statusBar.err + } } return hash, err diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 922350615..ea68b4304 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -388,7 +388,6 @@ func (a *Allocation) RepairFile(file sys.File, remotepath string, if ref.EncryptedKey != "" { opts = []ChunkedUploadOption{ WithMask(mask), - WithChunkNumber(10), WithStatusCallback(status), WithEncrypt(true), WithEncryptedPoint(ref.EncryptedKeyPoint), @@ -396,7 +395,6 @@ func (a *Allocation) RepairFile(file sys.File, remotepath string, } else { opts = []ChunkedUploadOption{ WithMask(mask), - WithChunkNumber(10), WithStatusCallback(status), } } diff --git a/zboxcore/sdk/sdk.go b/zboxcore/sdk/sdk.go index 04601f146..414a95e5e 100644 --- a/zboxcore/sdk/sdk.go +++ b/zboxcore/sdk/sdk.go @@ -53,7 +53,7 @@ type StatusCallback interface { RepairCompleted(filesRepaired int) } -var numBlockDownloads = 10 +var numBlockDownloads = 100 var sdkInitialized = false var networkWorkerTimerInHours = 1 var shouldVerifyHash = true From 5697527646f720827d76bc3a179c52d30639a903 Mon Sep 17 00:00:00 2001 From: Ebrahim Gomaa Date: Fri, 22 Dec 2023 07:55:16 +0200 Subject: [PATCH 2/4] reorder wait group done (#1340) --- core/transaction/entity.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/transaction/entity.go b/core/transaction/entity.go index 6c06356b0..70782c439 100644 --- a/core/transaction/entity.go +++ b/core/transaction/entity.go @@ -267,6 +267,7 @@ func SendTransactionSync(txn *Transaction, miners []string) error { if err != nil { fails <- err } + wg.Done() }() //nolint } wg.Wait() @@ -297,9 +298,6 @@ func SendTransactionSync(txn *Transaction, miners []string) error { } func sendTransactionToURL(url string, txn *Transaction, wg *sync.WaitGroup) ([]byte, error) { - if wg != nil { - defer wg.Done() - } postReq, err := util.NewHTTPPostRequest(url, txn) if err != nil { //Logger.Error("Error in serializing the transaction", txn, err.Error()) From c4073cde471331cebc6f45c52fe9244ed93fd661 Mon Sep 17 00:00:00 2001 From: Ebrahim Gomaa Date: Mon, 25 Dec 2023 22:03:43 +0200 Subject: [PATCH 3/4] hotfix / remove hard coded prefix handling of encrypted upload (#1344) * remove hard coded prefix handling of encrypted upload * fix for other parts of the file --- wasmsdk/blobber.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index cd9d59338..53fcab0b3 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -589,9 +589,7 @@ func multiUpload(jsonBulkUploadOptions string) (MultiUploadResult, error) { wg.Add(1) encrypt := option.Encrypt remotePath := option.RemotePath - if strings.HasPrefix(remotePath, "/Encrypted") { - encrypt = true - } + fileReader := jsbridge.NewFileReader(option.ReadChunkFuncName, option.FileSize) mimeType, err := zboxutil.GetFileContentType(fileReader) if err != nil { @@ -678,9 +676,7 @@ func uploadWithJsFuncs(allocationID, remotePath string, readChunkFuncName string } } wg.Add(1) - if strings.HasPrefix(remotePath, "/Encrypted") { - encrypt = true - } + fileReader := jsbridge.NewFileReader(readChunkFuncName, fileSize) @@ -760,9 +756,6 @@ func upload(allocationID, remotePath string, fileBytes, thumbnailBytes []byte, w wg := &sync.WaitGroup{} statusBar := &StatusBar{wg: wg} wg.Add(1) - if strings.HasPrefix(remotePath, "/Encrypted") { - encrypt = true - } fileReader := bytes.NewReader(fileBytes) From ea5f217dd0eeeda83fa4b584f1b616278cb79541 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Sun, 31 Dec 2023 21:37:44 +0530 Subject: [PATCH 4/4] increase batch size (#1349) --- zboxcore/sdk/allocation.go | 3 +-- zboxcore/sdk/chunked_upload.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index ea68b4304..1e833b8e1 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -42,7 +42,7 @@ var ( noBLOBBERS = errors.New("", "No Blobbers set in this allocation") notInitialized = errors.New("sdk_not_initialized", "Please call InitStorageSDK Init and use GetAllocation to get the allocation object") IsWasm = false - MultiOpBatchSize = 10 + MultiOpBatchSize = 50 ) const ( @@ -242,7 +242,6 @@ func GetWritePriceRange() (PriceRange, error) { func SetWasm() { IsWasm = true BatchSize = 5 - MultiOpBatchSize = 7 } func getPriceRange(name string) (PriceRange, error) { diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index e6846cc75..223522def 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -752,7 +752,7 @@ func (su *ChunkedUpload) uploadProcessor() { su.consensus.Reset() var pos uint64 var errCount int32 - swg := sizedwaitgroup.New(BatchSize) + swg := sizedwaitgroup.New(BatchSize * 2) for i := su.uploadMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { pos = uint64(i.TrailingZeros()) swg.Add()