diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index d878ffd21..618791755 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -58,7 +58,7 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) for contenthash := range nf.ContentHash { var count int64 - err := db.Table((&reference.Ref{}).TableName()).Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash}).Count(&count).Error + err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error if err == nil && count == 0 { Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 53e7f7fcd..1af88e223 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -5,6 +5,8 @@ import ( "encoding/json" "path/filepath" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" @@ -15,6 +17,7 @@ import ( ) type UpdateFileChanger struct { + deleteHash map[string]bool BaseFileChanger } @@ -59,6 +62,15 @@ func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *Allocati return nil, common.NewError("file_not_found", "File to update not found in blobber") } existingRef := dirRef.Children[idx] + // remove changed thumbnail and files + nf.deleteHash = make(map[string]bool) + if existingRef.ThumbnailHash != "" && existingRef.ThumbnailHash != nf.ThumbnailHash { + nf.deleteHash[existingRef.ThumbnailHash] = true + } + if existingRef.ContentHash != "" && existingRef.ContentHash != nf.Hash { + nf.deleteHash[existingRef.ContentHash] = true + } + existingRef.ActualFileHash = nf.ActualHash existingRef.ActualFileSize = nf.ActualSize existingRef.MimeType = nf.MimeType @@ -84,6 +96,21 @@ func (nf *UpdateFileChanger) ProcessChange(ctx context.Context, change *Allocati return rootRef, err } +func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + for contenthash := range nf.deleteHash { + var count int64 + err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error + if err == nil && count == 0 { + Logger.Info("Deleting content file", zap.String("content_hash", contenthash)) + if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil { + Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err)) + } + } + } + return nf.BaseFileChanger.CommitToFileStore(ctx) +} + func (nf *UpdateFileChanger) Marshal() (string, error) { ret, err := json.Marshal(nf) if err != nil { diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go new file mode 100644 index 000000000..cf7164bf9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -0,0 +1,315 @@ +package allocation + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk/zboxcore/client" + mocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "google.golang.org/grpc/metadata" +) + +func init() { + logging.Logger = zap.NewNop() +} + +func TestBlobberCore_UpdateFile(t *testing.T) { + + sch := zcncrypto.NewSignatureScheme("bls0chain") + mnemonic := "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" + _, err := sch.RecoverKeys(mnemonic) + if err != nil { + t.Fatal(err) + } + ts := time.Now().Add(time.Hour) + alloc := makeTestAllocation(common.Timestamp(ts.Unix())) + alloc.OwnerPublicKey = sch.GetPublicKey() + alloc.OwnerID = client.GetClientID() + + testCases := []struct { + name string + context metadata.MD + allocChange *AllocationChange + path string + filename string + allocRoot string + thumbnailHash string + hash string + allocationID string + expectedMessage string + expectingError bool + setupDbMock func() + initDir, expectedDir map[string]map[string]bool + }{ + { + name: "Update thumbnail hash", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "thumbnail_hash", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash_old", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash_old": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + }, + { + name: "Update content hash", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "thumbnail_hash", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash_old", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash_old": true, + "thumbnail_hash": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + }, + { + name: "Remove thumbnail", + allocChange: &AllocationChange{}, + allocRoot: "/", + path: "/test_file", + filename: "test_file", + hash: "content_hash", + thumbnailHash: "", + allocationID: alloc.ID, + expectingError: false, + setupDbMock: func() { + mocket.Catcher.Reset() + + query := `SELECT * FROM "reference_objects" WHERE` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "id": 1, + "level": 0, + "lookup_hash": "lookup_hash_root", + "path": "/", + "name": "/", + "allocation_id": alloc.ID, + "parent_path": "", + "content_hash": "", + "thumbnail_size": 00, + "thumbnail_hash": "", + "type": reference.DIRECTORY, + }, + { + "id": 2, + "level": 1, + "lookup_hash": "lookup_hash", + "path": "/test_file", + "name": "test_file", + "content_hash": "content_hash", + "thumbnail_size": 300, + "thumbnail_hash": "thumbnail_hash", + "allocation_id": alloc.ID, + "parent_path": "/", + "type": reference.FILE, + }, + }, + ) + + query = `UPDATE "reference_objects" SET` + mocket.Catcher.NewMock().WithQuery(query).WithReply( + []map[string]interface{}{ + { + "rows_affected": 1, + }, + }, + ) + + }, + initDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + "thumbnail_hash": true, + }, + }, + expectedDir: map[string]map[string]bool{ + alloc.ID: { + "content_hash": true, + }, + }, + }, + } + + for _, tc := range testCases { + datastore.MocketTheStore(t, true) + filestore.UseMock(tc.initDir) + tc.setupDbMock() + + ctx := context.TODO() + db := datastore.GetStore().GetDB().Begin() + ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + + change := &UpdateFileChanger{ + BaseFileChanger: BaseFileChanger{ + Path: tc.path, + Filename: tc.filename, + ActualSize: 2310, + ActualThumbnailSize: 92, + ActualThumbnailHash: tc.thumbnailHash, + Attributes: reference.Attributes{WhoPaysForReads: common.WhoPaysOwner}, + AllocationID: tc.allocationID, + Hash: tc.hash, + Size: 2310, + ThumbnailHash: tc.thumbnailHash, + ThumbnailSize: 92, + ChunkSize: 65536, + IsFinal: true, + }, + } + + _, err := func() (*reference.Ref, error) { + resp, err := change.ProcessChange(ctx, tc.allocChange, tc.allocRoot) + if err != nil { + return nil, err + } + + err = change.CommitToFileStore(ctx) + return resp, err + }() + + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + + if tc.expectingError && strings.Contains(tc.expectedMessage, err.Error()) { + t.Fatal("expected error " + tc.expectedMessage) + break + } + + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + require.EqualValues(t, tc.expectedDir, tc.initDir) + } +} diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index fd1e7c9f9..18cb3e8dc 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -603,8 +603,11 @@ func (fs *FileFSStore) DownloadFromCloud(fileHash, filePath string) error { } func (fs *FileFSStore) RemoveFromCloud(fileHash string) error { - if _, err := fs.Minio.StatObject(MinioConfig.BucketName, fileHash, minio.StatObjectOptions{}); err == nil { - return fs.Minio.RemoveObject(MinioConfig.BucketName, fileHash) + if fs != nil && fs.Minio != nil { + _, err := fs.Minio.StatObject(MinioConfig.BucketName, fileHash, minio.StatObjectOptions{}) + if err == nil { + return fs.Minio.RemoveObject(MinioConfig.BucketName, fileHash) + } } return nil } diff --git a/code/go/0chain.net/blobbercore/filestore/mock_store.go b/code/go/0chain.net/blobbercore/filestore/mock_store.go index 3eb356f76..4ff6c312e 100644 --- a/code/go/0chain.net/blobbercore/filestore/mock_store.go +++ b/code/go/0chain.net/blobbercore/filestore/mock_store.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "io" "mime/multipart" @@ -12,15 +13,17 @@ import ( ) type MockStore struct { + d map[string]map[string]bool } var mockStore *MockStore -func UseMock() { +func UseMock(initData map[string]map[string]bool) { if mockStore == nil { - mockStore = &MockStore{} + mockStore = &MockStore{d: initData} } + mockStore.d = initData fileStore = mockStore } @@ -67,6 +70,7 @@ func (ms *MockStore) GetFileBlock(allocationID string, fileData *FileInputData, } func (ms *MockStore) CommitWrite(allocationID string, fileData *FileInputData, connectionID string) (bool, error) { + ms.addFileInDataObj(allocationID, fileData.Hash) return true, nil } @@ -74,8 +78,13 @@ func (ms *MockStore) GetFileBlockForChallenge(allocationID string, fileData *Fil return nil, nil, constants.ErrNotImplemented } func (ms *MockStore) DeleteFile(allocationID, contentHash string) error { + if ms.d == nil || ms.d[allocationID] == nil || !ms.d[allocationID][contentHash] { + return errors.New("file not available related to content") + } + delete(ms.d[allocationID], contentHash) return nil } + func (ms *MockStore) GetTotalDiskSizeUsed() (int64, error) { return 0, constants.ErrNotImplemented } @@ -97,3 +106,14 @@ func (ms *MockStore) DownloadFromCloud(fileHash, filePath string) error { func (ms *MockStore) SetupAllocation(allocationID string, skipCreate bool) (*StoreAllocation, error) { return nil, constants.ErrNotImplemented } + +func (ms *MockStore) addFileInDataObj(allocationID, contentHash string) { + if contentHash == "" { + return + } + if ms.d == nil { + ms.d = make(map[string]map[string]bool, 0) + } + dataObj := ms.d[allocationID] + dataObj[contentHash] = true +} diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go index 4763c426d..526addeeb 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go @@ -107,7 +107,7 @@ func BenchmarkUploadFileWithNoDisk(b *testing.B) { //GB := 1024 * MB datastore.UseMocket(false) - filestore.UseMock() + filestore.UseMock(nil) blobber := mock.NewBlobberClient() allocationID := "benchmark_uploadfile" diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 28575d38d..b0f265f4a 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -71,7 +71,7 @@ func DeleteObject(ctx context.Context, allocationID, path string) (*Ref, map[str GetTransaction(ctx) var deletedObjects []*Ref - txDelete := db.Clauses(clause.Returning{Columns: []clause.Column{{Name: "content_hash"}, {Name: "type"}}}) + txDelete := db.Clauses(clause.Returning{Columns: []clause.Column{{Name: "content_hash"}, {Name: "thumbnail_hash"}, {Name: "type"}}}) path = filepath.Join("/", path) txDelete = txDelete.Where("allocation_id = ? and deleted_at IS NULL and (path LIKE ? or path = ?) and path != ? ", allocationID, (path + "%"), path, "/") @@ -85,6 +85,9 @@ func DeleteObject(ctx context.Context, allocationID, path string) (*Ref, map[str for _, it := range deletedObjects { if it.Type == FILE { deletedFiles[it.ContentHash] = true + if it.ThumbnailHash != "" { + deletedFiles[it.ThumbnailHash] = true + } } }