From 7407aaa439137b0d3ff09080b91225066816967b Mon Sep 17 00:00:00 2001 From: Pradip Parmar Date: Thu, 17 Feb 2022 09:22:52 +0530 Subject: [PATCH 1/8] delete file query update. Signed-off-by: Pradip Parmar --- code/go/0chain.net/blobbercore/allocation/deletefilechange.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index d878ffd21..f22039731 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -58,7 +58,7 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) for contenthash := range nf.ContentHash { var count int64 - err := db.Table((&reference.Ref{}).TableName()).Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash}).Count(&count).Error + err := db.Table((&reference.Ref{}).TableName()).Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash}).Where("deleted_at IS null").Count(&count).Error if err == nil && count == 0 { Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { From 05dbc8bd6c88517da59a6017e01ea631f137b13b Mon Sep 17 00:00:00 2001 From: Pradip Parmar Date: Fri, 18 Feb 2022 06:44:01 +0530 Subject: [PATCH 2/8] thumbnail actual file delete handling. Signed-off-by: Pradip Parmar --- code/go/0chain.net/blobbercore/reference/object.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 28575d38d..b0f265f4a 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -71,7 +71,7 @@ func DeleteObject(ctx context.Context, allocationID, path string) (*Ref, map[str GetTransaction(ctx) var deletedObjects []*Ref - txDelete := db.Clauses(clause.Returning{Columns: []clause.Column{{Name: "content_hash"}, {Name: "type"}}}) + txDelete := db.Clauses(clause.Returning{Columns: []clause.Column{{Name: "content_hash"}, {Name: "thumbnail_hash"}, {Name: "type"}}}) path = filepath.Join("/", path) txDelete = txDelete.Where("allocation_id = ? and deleted_at IS NULL and (path LIKE ? or path = ?) and path != ? ", allocationID, (path + "%"), path, "/") @@ -85,6 +85,9 @@ func DeleteObject(ctx context.Context, allocationID, path string) (*Ref, map[str for _, it := range deletedObjects { if it.Type == FILE { deletedFiles[it.ContentHash] = true + if it.ThumbnailHash != "" { + deletedFiles[it.ThumbnailHash] = true + } } } From b69db781fa49e8c3ab81a81d8194c0b69e0f41e5 Mon Sep 17 00:00:00 2001 From: Pradip Parmar Date: Sat, 19 Feb 2022 11:10:08 +0530 Subject: [PATCH 3/8] handling for nil minio Signed-off-by: Pradip Parmar --- code/go/0chain.net/blobbercore/filestore/fs_store.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index fd1e7c9f9..18cb3e8dc 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -603,8 +603,11 @@ func (fs *FileFSStore) DownloadFromCloud(fileHash, filePath string) error { } func (fs *FileFSStore) RemoveFromCloud(fileHash string) error { - if _, err := fs.Minio.StatObject(MinioConfig.BucketName, fileHash, minio.StatObjectOptions{}); err == nil { - return fs.Minio.RemoveObject(MinioConfig.BucketName, fileHash) + if fs != nil && fs.Minio != nil { + _, err := fs.Minio.StatObject(MinioConfig.BucketName, fileHash, minio.StatObjectOptions{}) + if err == nil { + return fs.Minio.RemoveObject(MinioConfig.BucketName, fileHash) + } } return nil } From e25a6a25db3a3122c2b0b03541ffb17d82947c53 Mon Sep 17 00:00:00 2001 From: Pradip Parmar Date: Sun, 20 Feb 2022 00:43:37 +0530 Subject: [PATCH 4/8] updated where condition for thumbnail delete. Signed-off-by: Pradip Parmar --- code/go/0chain.net/blobbercore/allocation/deletefilechange.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index f22039731..66aa6ef00 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -58,7 +58,7 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) for contenthash := range nf.ContentHash { var count int64 - err := db.Table((&reference.Ref{}).TableName()).Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash}).Where("deleted_at IS null").Count(&count).Error + err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Count(&count).Error if err == nil && count == 0 { Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { From d60cca7ca383344d623f09e861cac99112701c69 Mon Sep 17 00:00:00 2001 From: Pradip Parmar Date: Sun, 20 Feb 2022 02:08:37 +0530 Subject: [PATCH 5/8] added handling to delete old files after created new for updated. Signed-off-by: Pradip Parmar --- .../allocation/deletefilechange.go | 2 +- .../allocation/file_changer_update.go | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index 66aa6ef00..618791755 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -58,7 +58,7 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) for contenthash := range nf.ContentHash { var count int64 - err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Count(&count).Error + err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error if err == nil && count == 0 { Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 53e7f7fcd..7d464dd33 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -5,6 +5,8 @@ import ( "encoding/json" "path/filepath" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" @@ -15,6 +17,7 @@ import ( ) type UpdateFileChanger struct { + deleteHash map[string]bool BaseFileChanger } @@ -59,6 +62,21 @@ func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *Allocati return nil, common.NewError("file_not_found", "File to update not found in blobber") } existingRef := dirRef.Children[idx] + // remove changed thumbnail and files + nf.deleteHash = make(map[string]bool) + if existingRef.ThumbnailHash != "" && existingRef.ThumbnailHash != nf.ThumbnailHash { + nf.deleteHash[existingRef.ThumbnailHash] = true + } + if existingRef.ContentHash != "" && existingRef.ContentHash != nf.Hash { + nf.deleteHash[existingRef.ThumbnailHash] = true + } + + err = nf.removeOldFiles(ctx) + if err != nil { + Logger.Error("error in old file remove", zap.Any("change", nf)) + return nil, common.NewError("olc_file_not_removed", "Not able to remove old file") + } + existingRef.ActualFileHash = nf.ActualHash existingRef.ActualFileSize = nf.ActualSize existingRef.MimeType = nf.MimeType @@ -84,6 +102,21 @@ func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *Allocati return rootRef, err } +func (nf *UpdateFileChanger) removeOldFiles(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + for contenthash := range nf.deleteHash { + var count int64 + err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error + if err == nil && count == 1 { + Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) + if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { + Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err)) + } + } + } + return nil +} + func (nf *UpdateFileChanger) Marshal() (string, error) { ret, err := json.Marshal(nf) if err != nil { From 3784251964303d07297b28db34d18a6e04596b34 Mon Sep 17 00:00:00 2001 From: Pradip Parmar Date: Mon, 21 Feb 2022 09:12:59 +0530 Subject: [PATCH 6/8] fix for file update. code restructure. Signed-off-by: Pradip Parmar --- .../blobbercore/allocation/file_changer_update.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 7d464dd33..1af88e223 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -68,13 +68,7 @@ func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *Allocati nf.deleteHash[existingRef.ThumbnailHash] = true } if existingRef.ContentHash != "" && existingRef.ContentHash != nf.Hash { - nf.deleteHash[existingRef.ThumbnailHash] = true - } - - err = nf.removeOldFiles(ctx) - if err != nil { - Logger.Error("error in old file remove", zap.Any("change", nf)) - return nil, common.NewError("olc_file_not_removed", "Not able to remove old file") + nf.deleteHash[existingRef.ContentHash] = true } existingRef.ActualFileHash = nf.ActualHash @@ -102,19 +96,19 @@ func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *Allocati return rootRef, err } -func (nf *UpdateFileChanger) removeOldFiles(ctx context.Context) error { +func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) for contenthash := range nf.deleteHash { var count int64 err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error - if err == nil && count == 1 { + if err == nil && count == 0 { Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err)) } } } - return nil + return nf.BaseFileChanger.CommitToFileStore(ctx) } func (nf *UpdateFileChanger) Marshal() (string, error) { From 371ebeeb345a8646ad3e954249394112d9e30947 Mon Sep 17 00:00:00 2001 From: Pradip Parmar Date: Tue, 22 Feb 2022 09:54:25 +0530 Subject: [PATCH 7/8] unit test for update file with store check. Signed-off-by: Pradip Parmar --- .../allocation/updatefilechange_test.go | 329 ++++++++++++++++++ .../blobbercore/filestore/mock_store.go | 24 +- .../object_operation_handler_bench_test.go | 2 +- 3 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go new file mode 100644 index 000000000..6d3e6d4ea --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -0,0 +1,329 @@ +package allocation + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/chain" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/config" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk/zboxcore/client" + mocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "google.golang.org/grpc/metadata" +) + +func init() { + resetMockFileBlock() + common.ConfigRateLimits() + chain.SetServerChain(&chain.Chain{}) + config.Configuration.SignatureScheme = "bls0chain" + logging.Logger = zap.NewNop() + + dir, _ := os.Getwd() + if _, err := filestore.SetupFSStoreI(dir+"/tmp", MockFileBlockGetter{}); err != nil { + panic(err) + } +} + +func TestBlobberCore_UpdateFile(t *testing.T) { + setup(t) + setupEncryptionScheme() + + sch := zcncrypto.NewSignatureScheme("bls0chain") + mnemonic := "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" + _, err := sch.RecoverKeys(mnemonic) + if err != nil { + t.Fatal(err) + } + ts := time.Now().Add(time.Hour) + alloc := makeTestAllocation(common.Timestamp(ts.Unix())) + alloc.OwnerPublicKey = sch.GetPublicKey() + alloc.OwnerID = client.GetClientID() + + testCases := []struct { + name string + context metadata.MD + allocChange *AllocationChange + path string + filename string + allocRoot string + thumbnailHash string + hash string + allocationID string + expectedMessage string + expectingError bool + setupDbMock func() + initDir, expectedDir map[string]map[string]bool + }{ + { + name: "Update thumbnail hash", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "thumbnail_hash", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash_old", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash_old": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + }, + { + name: "Update content hash", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "thumbnail_hash", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash_old", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash_old": true, + "thumbnail_hash": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + }, + { + name: "Remove thumbnail", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + }, + }, + }, + } + + for _, tc := range testCases { + datastore.MocketTheStore(t, true) + filestore.UseMock(tc.initDir) + tc.setupDbMock() + + ctx := context.TODO() + db := datastore.GetStore().GetDB().Begin() + ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + + change := &UpdateFileChanger{ + BaseFileChanger: BaseFileChanger{ + Path: tc.path, + Filename: tc.filename, + ActualSize: 2310, + ActualThumbnailSize: 92, + ActualThumbnailHash: tc.thumbnailHash, + Attributes: reference.Attributes{WhoPaysForReads: common.WhoPaysOwner}, + AllocationID: tc.allocationID, + Hash: tc.hash, + Size: 2310, + ThumbnailHash: tc.thumbnailHash, + ThumbnailSize: 92, + ChunkSize: 65536, + IsFinal: true, + }, + } + + _, err := func() (*reference.Ref, error) { + resp, err := change.ProcessChange(ctx, tc.allocChange, tc.allocRoot) + if err != nil { + return nil, err + } + + err = change.CommitToFileStore(ctx) + return resp, err + }() + + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + + if tc.expectingError && strings.Contains(tc.expectedMessage, err.Error()) { + t.Fatal("expected error " + tc.expectedMessage) + break + } + + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + require.EqualValues(t, tc.expectedDir, tc.initDir) + } +} diff --git a/code/go/0chain.net/blobbercore/filestore/mock_store.go b/code/go/0chain.net/blobbercore/filestore/mock_store.go index 3eb356f76..4ff6c312e 100644 --- a/code/go/0chain.net/blobbercore/filestore/mock_store.go +++ b/code/go/0chain.net/blobbercore/filestore/mock_store.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "io" "mime/multipart" @@ -12,15 +13,17 @@ import ( ) type MockStore struct { + d map[string]map[string]bool } var mockStore *MockStore -func UseMock() { +func UseMock(initData map[string]map[string]bool) { if mockStore == nil { - mockStore = &MockStore{} + mockStore = &MockStore{d: initData} } + mockStore.d = initData fileStore = mockStore } @@ -67,6 +70,7 @@ func (ms *MockStore) GetFileBlock(allocationID string, fileData *FileInputData, } func (ms *MockStore) CommitWrite(allocationID string, fileData *FileInputData, connectionID string) (bool, error) { + ms.addFileInDataObj(allocationID, fileData.Hash) return true, nil } @@ -74,8 +78,13 @@ func (ms *MockStore) GetFileBlockForChallenge(allocationID string, fileData *Fil return nil, nil, constants.ErrNotImplemented } func (ms *MockStore) DeleteFile(allocationID, contentHash string) error { + if ms.d == nil || ms.d[allocationID] == nil || !ms.d[allocationID][contentHash] { + return errors.New("file not available related to content") + } + delete(ms.d[allocationID], contentHash) return nil } + func (ms *MockStore) GetTotalDiskSizeUsed() (int64, error) { return 0, constants.ErrNotImplemented } @@ -97,3 +106,14 @@ func (ms *MockStore) DownloadFromCloud(fileHash, filePath string) error { func (ms *MockStore) SetupAllocation(allocationID string, skipCreate bool) (*StoreAllocation, error) { return nil, constants.ErrNotImplemented } + +func (ms *MockStore) addFileInDataObj(allocationID, contentHash string) { + if contentHash == "" { + return + } + if ms.d == nil { + ms.d = make(map[string]map[string]bool, 0) + } + dataObj := ms.d[allocationID] + dataObj[contentHash] = true +} diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go index 4763c426d..526addeeb 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go @@ -107,7 +107,7 @@ func BenchmarkUploadFileWithNoDisk(b *testing.B) { //GB := 1024 * MB datastore.UseMocket(false) - filestore.UseMock() + filestore.UseMock(nil) blobber := mock.NewBlobberClient() allocationID := "benchmark_uploadfile" From 47fa8eb71f76f07ca7afba77f0de6a12b4d79503 Mon Sep 17 00:00:00 2001 From: Pradip Parmar Date: Tue, 22 Feb 2022 19:37:02 +0530 Subject: [PATCH 8/8] removed unwanted code from updatefilechange test Signed-off-by: Pradip Parmar --- .../allocation/updatefilechange_test.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go index 6d3e6d4ea..cf7164bf9 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -2,7 +2,6 @@ package allocation import ( "context" - "os" "strings" "testing" "time" @@ -10,9 +9,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/config" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/core/zcncrypto" "github.com/0chain/gosdk/zboxcore/client" @@ -23,21 +20,10 @@ import ( ) func init() { - resetMockFileBlock() - common.ConfigRateLimits() - chain.SetServerChain(&chain.Chain{}) - config.Configuration.SignatureScheme = "bls0chain" logging.Logger = zap.NewNop() - - dir, _ := os.Getwd() - if _, err := filestore.SetupFSStoreI(dir+"/tmp", MockFileBlockGetter{}); err != nil { - panic(err) - } } func TestBlobberCore_UpdateFile(t *testing.T) { - setup(t) - setupEncryptionScheme() sch := zcncrypto.NewSignatureScheme("bls0chain") mnemonic := "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe"