From 03245871a3cfc81310624e940ba0b80a4c1ea5a2 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 26 Feb 2022 16:20:44 +0800 Subject: [PATCH 01/18] feat(writemarker): updated WriteMarker mutex --- .../blobbercore/datastore/postgres_schema.go | 2 +- .../blobbercore/writemarker/mutex.go | 36 +++++-------------- .../blobbercore/writemarker/mutext_test.go | 28 +++++++-------- 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go index 0ad6c3287..d0c1bc2d4 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go @@ -9,7 +9,7 @@ const ( // WriteLock WriteMarker lock type WriteLock struct { AllocationID string `gorm:"primaryKey, column:allocation_id"` - SessionID string `gorm:"column:session_id"` + ConnectionID string `gorm:"column:connection_id"` CreatedAt time.Time `gorm:"column:created_at"` } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index 120163930..c5b42a7c9 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -7,7 +7,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/errors" "github.com/0chain/gosdk/constants" @@ -24,9 +23,8 @@ const ( ) type LockResult struct { - Status LockStatus `json:"status,omitempty"` - CreatedAt time.Time `json:"created_at,omitempty"` - RootNode *reference.HashNode `json:"root_node,omitempty"` + Status LockStatus `json:"status,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` } // Mutex WriteMarker mutex @@ -35,7 +33,7 @@ type Mutex struct { } // Lock -func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, requestTime *time.Time) (*LockResult, error) { +func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, requestTime *time.Time) (*LockResult, error) { m.Mutex.Lock() defer m.Mutex.Unlock() @@ -43,8 +41,8 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.Throw(constants.ErrInvalidParameter, "allocationID") } - if len(sessionID) == 0 { - return nil, errors.Throw(constants.ErrInvalidParameter, "sessionID") + if len(connectionID) == 0 { + return nil, errors.Throw(constants.ErrInvalidParameter, "connectionID") } if requestTime == nil { @@ -65,7 +63,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques if errors.Is(err, gorm.ErrRecordNotFound) { lock = datastore.WriteLock{ AllocationID: allocationID, - SessionID: sessionID, + ConnectionID: connectionID, CreatedAt: *requestTime, } @@ -74,15 +72,9 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt, - RootNode: rootNode, }, nil } @@ -97,7 +89,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques // locked, but it is timeout if now.After(timeout) { - lock.SessionID = sessionID + lock.ConnectionID = connectionID lock.CreatedAt = *requestTime err = db.Save(&lock).Error @@ -105,30 +97,18 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt, - RootNode: rootNode, }, nil } //try lock by same session, return old lock directly - if lock.SessionID == sessionID && lock.CreatedAt.Equal(*requestTime) { - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - + if lock.ConnectionID == connectionID && lock.CreatedAt.Equal(*requestTime) { return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt, - RootNode: rootNode, }, nil } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index 0c09c4f47..e36f0fe15 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -23,7 +23,7 @@ func TestMutext_LockShouldWork(t *testing.T) { tests := []struct { name string allocationID string - sessionID string + connectionID string requestTime time.Time mock func() assert func(*testing.T, *LockResult, error) @@ -31,7 +31,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "Lock should work", allocationID: "lock_allocation_id", - sessionID: "lock_session_id", + connectionID: "lock_connection_id", requestTime: now, mock: func() { @@ -44,7 +44,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "retry lock by same request should work if it is not timeout", allocationID: "lock_same_allocation_id", - sessionID: "lock_same_session_id", + connectionID: "lock_same_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -53,7 +53,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_allocation_id", - "session_id": "lock_same_session_id", + "connection_id": "lock_same_connection_id", "created_at": now, }, }) @@ -66,7 +66,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should be pending if it already is locked by other session ", allocationID: "lock_allocation_id", - sessionID: "lock_pending_session_id", + connectionID: "lock_pending_connection_id", requestTime: time.Now(), mock: func() { gomocket.Catcher.NewMock(). @@ -75,7 +75,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_allocation_id", - "session_id": "lock_session_id", + "connection_id": "lock_connection_id", "created_at": time.Now().Add(-5 * time.Second), }, }) @@ -88,7 +88,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should ok if it is timeout", allocationID: "lock_timeout_allocation_id", - sessionID: "lock_timeout_2nd_session_id", + connectionID: "lock_timeout_2nd_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -97,7 +97,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_timeout_allocation_id", - "session_id": "lock_timeout_1st_session_id", + "connection_id": "lock_timeout_1st_connection_id", "created_at": time.Now().Add(31 * time.Second), }, }) @@ -116,7 +116,7 @@ func TestMutext_LockShouldWork(t *testing.T) { if it.mock != nil { it.mock() } - r, err := m.Lock(context.TODO(), it.allocationID, it.sessionID, &it.requestTime) + r, err := m.Lock(context.TODO(), it.allocationID, it.connectionID, &it.requestTime) it.assert(test, r, err) @@ -139,7 +139,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { tests := []struct { name string allocationID string - sessionID string + connectionID string requestTime time.Time mock func() assert func(*testing.T, *LockResult, error) @@ -147,7 +147,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "Lock should not work if request_time is timeout", allocationID: "lock_allocation_id", - sessionID: "lock_session_id", + connectionID: "lock_connection_id", requestTime: time.Now().Add(31 * time.Second), mock: func() { config.Configuration.WriteMarkerLockTimeout = 30 * time.Second @@ -160,7 +160,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "retry lock by same request should not work if it is timeout", allocationID: "lock_same_timeout_allocation_id", - sessionID: "lock_same_timeout_session_id", + connectionID: "lock_same_timeout_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -169,7 +169,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_timeout_allocation_id", - "session_id": "lock_same_timeout_session_id", + "connection_id": "lock_same_timeout_connection_id", "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), }, }) @@ -188,7 +188,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { if it.mock != nil { it.mock() } - r, err := m.Lock(context.TODO(), it.allocationID, it.sessionID, &it.requestTime) + r, err := m.Lock(context.TODO(), it.allocationID, it.connectionID, &it.requestTime) it.assert(test, r, err) From 322ac3e4cb3c1aeb4d1835519994ba56c8b76082 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 26 Feb 2022 17:05:52 +0800 Subject: [PATCH 02/18] feat(writemarker): updated CreateAt with int64 --- code/go/0chain.net/blobbercore/writemarker/mutex.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index c5b42a7c9..daeaddab0 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -24,7 +24,7 @@ const ( type LockResult struct { Status LockStatus `json:"status,omitempty"` - CreatedAt time.Time `json:"created_at,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` } // Mutex WriteMarker mutex @@ -74,7 +74,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } @@ -99,7 +99,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } @@ -108,14 +108,14 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req if lock.ConnectionID == connectionID && lock.CreatedAt.Equal(*requestTime) { return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } // pending return &LockResult{ Status: LockStatusPending, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } From 9ba857a6d6f6c8680f422528915b178d8186272b Mon Sep 17 00:00:00 2001 From: Lz Date: Mon, 28 Feb 2022 22:34:47 +0800 Subject: [PATCH 03/18] fix(ref): load root hash node --- .../0chain.net/blobbercore/handler/handler.go | 6 +- .../blobbercore/handler/handler_ref.go | 16 ++ .../blobbercore/reference/{dao.go => hash.go} | 55 ++++--- .../blobbercore/reference/hash_test.go | 154 ++++++++++++++++++ .../blobbercore/writemarker/mutex.go | 2 +- code/go/0chain.net/core/common/constants.go | 6 + 6 files changed, 212 insertions(+), 27 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/handler/handler_ref.go rename code/go/0chain.net/blobbercore/reference/{dao.go => hash.go} (58%) create mode 100644 code/go/0chain.net/blobbercore/reference/hash_test.go diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index bc2eba128..51ea0cbec 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -78,8 +78,10 @@ func SetupHandlers(r *mux.Router) { // lightweight http handler without heavy postgres transaction to improve performance - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost) - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost, http.MethodOptions) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete, http.MethodOptions) + + r.HandleFunc("/v1/refs/root/{allocation}", WithHandler(LoadRootNode)).Methods(http.MethodGet, http.MethodOptions) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/blobbercore/handler/handler_ref.go b/code/go/0chain.net/blobbercore/handler/handler_ref.go new file mode 100644 index 000000000..686a5c8d4 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_ref.go @@ -0,0 +1,16 @@ +//go:build !integration_tests +// +build !integration_tests + +package handler + +import "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + +// LoadRootNode load root node with its descendant nodes +func LoadRootNode(ctx *Context) (interface{}, error) { + + root, err := reference.LoadRootNode(ctx, ctx.AllocationTx) + if err != nil { + return nil, err + } + return root, nil +} diff --git a/code/go/0chain.net/blobbercore/reference/dao.go b/code/go/0chain.net/blobbercore/reference/hash.go similarity index 58% rename from code/go/0chain.net/blobbercore/reference/dao.go rename to code/go/0chain.net/blobbercore/reference/hash.go index 363ce4825..1250bed22 100644 --- a/code/go/0chain.net/blobbercore/reference/dao.go +++ b/code/go/0chain.net/blobbercore/reference/hash.go @@ -14,47 +14,54 @@ func LoadRootNode(ctx context.Context, allocationID string) (*HashNode, error) { db := datastore.GetStore().GetDB() - db = db.Where("allocation_id = ? and deleted_at IS NULL", allocationID) + db = db.Raw(` +SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects +WHERE allocation_id = ? and deleted_at IS NULL +ORDER BY level desc, path`, allocationID) - db = db.Order("level desc, path") + rows, err := db.Rows() + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } - dict := make(map[string][]*HashNode) + defer rows.Close() - var nodes []*HashNode - // it is better to load them in batched if there are a lot of objects in db - err := db.FindInBatches(&nodes, 100, func(tx *gorm.DB, batch int) error { - // batch processing found records - for _, object := range nodes { - dict[object.ParentPath] = append(dict[object.ParentPath], object) + nodes := make(map[string]*HashNode) + for rows.Next() { - for _, child := range dict[object.Path] { - object.AddChild(child) - } + node := &HashNode{} + err = db.ScanRows(rows, node) + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } - return nil - }).Error + _, ok := nodes[node.Path] + if ok { + return nil, common.ErrDuplicatedNode + } + + nodes[node.Path] = node + + parent, ok := nodes[node.ParentPath] + if ok { + parent.AddChild(node) + } - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } // create empty dir if root is missing - if len(dict) == 0 { + if len(nodes) == 0 { return &HashNode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil } - rootNodes, ok := dict[""] + root, ok := nodes["/"] if ok { - if len(rootNodes) == 1 { - return rootNodes[0], nil - } - - return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") + return root, nil } - return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") + return nil, common.ErrMissingRootNode } const ( diff --git a/code/go/0chain.net/blobbercore/reference/hash_test.go b/code/go/0chain.net/blobbercore/reference/hash_test.go new file mode 100644 index 000000000..153ec278b --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/hash_test.go @@ -0,0 +1,154 @@ +package reference + +import ( + "context" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHash_Should_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *HashNode, error) + }{ + { + name: "No any node should work", + allocationID: "allocation_none", + mock: func() { + gomocket.Catcher.Reset().NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_none"). + WithReply(nil) + }, + assert: func(test *testing.T, allocationID string, r *HashNode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 0) + + require.Equal(test, allocationID, r.AllocationID) + require.Equal(test, DIRECTORY, r.Type) + require.Equal(test, "/", r.Name) + require.Equal(test, "/", r.Path) + require.Equal(test, "", r.ContentHash) + require.Equal(test, "", r.MerkleRoot) + require.Equal(test, "", r.ActualFileHash) + require.EqualValues(test, 0, r.ChunkSize) + require.EqualValues(test, 0, r.Size) + require.EqualValues(test, 0, r.ActualFileSize) + + buf, e := r.Attributes.MarshalJSON() //nolint + require.Nil(test, e) + + require.Equal(test, "null", string(buf)) + + require.Equal(test, "", r.ParentPath) + + }, + }, + { + name: "Nested node should work", + allocationID: "allocation_nested", + mock: func() { + gomocket.Catcher.Reset().NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_nested"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *HashNode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 2) + + require.Equal(test, r.Children[0].Name, "sub1") + require.Len(test, r.Children[0].Children, 1) + require.Equal(test, r.Children[0].Children[0].Name, "file1") + require.Equal(test, r.Children[1].Name, "sub2") + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRootNode(context.TODO(), it.allocationID) + require.Nil(test, err) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index daeaddab0..fa8fe342d 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -132,7 +132,7 @@ func (*Mutex) Unlock(ctx context.Context, allocationID string, sessionID string) db := datastore.GetStore().GetDB() - err := db.Where("allocation_id = ? and session_id =? ", allocationID, sessionID).Delete(&datastore.WriteLock{}).Error + err := db.Where("allocation_id = ? and session_id = ? ", allocationID, sessionID).Delete(&datastore.WriteLock{}).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil diff --git a/code/go/0chain.net/core/common/constants.go b/code/go/0chain.net/core/common/constants.go index 2345aebe1..de9dc6ae2 100644 --- a/code/go/0chain.net/core/common/constants.go +++ b/code/go/0chain.net/core/common/constants.go @@ -32,4 +32,10 @@ var ( // ErrEntityNotFound entity can't found in db ErrEntityNotFound = errors.New("entity not found") + + // ErrMissingRootNode root node is missing + ErrMissingRootNode = errors.New("root node is missing") + + // ErrDuplicatedNode duplicated nodes + ErrDuplicatedNode = errors.New("duplicated nodes") ) From 9470a783cee3fb2c0d8ea4f33b5222c68b853a3a Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 09:52:21 +0800 Subject: [PATCH 04/18] fix(ref): added unit tests for hashnode --- .../0chain.net/blobbercore/handler/handler.go | 2 +- .../{handler_ref.go => handler_hashnode.go} | 6 +- .../handler/handler_hashnode_test.go | 113 +++++ .../blobbercore/reference/entity.go | 40 +- .../blobbercore/reference/hash_test.go | 154 ------- .../reference/{hash.go => hashnode.go} | 26 +- .../blobbercore/reference/hashnode_test.go | 390 ++++++++++++++++++ 7 files changed, 519 insertions(+), 212 deletions(-) rename code/go/0chain.net/blobbercore/handler/{handler_ref.go => handler_hashnode.go} (53%) create mode 100644 code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go delete mode 100644 code/go/0chain.net/blobbercore/reference/hash_test.go rename code/go/0chain.net/blobbercore/reference/{hash.go => hashnode.go} (59%) create mode 100644 code/go/0chain.net/blobbercore/reference/hashnode_test.go diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 51ea0cbec..a85570168 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -81,7 +81,7 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost, http.MethodOptions) r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete, http.MethodOptions) - r.HandleFunc("/v1/refs/root/{allocation}", WithHandler(LoadRootNode)).Methods(http.MethodGet, http.MethodOptions) + r.HandleFunc("/v1/hashnode/root/{allocation}", WithHandler(LoadRootHashnode)).Methods(http.MethodGet, http.MethodOptions) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/blobbercore/handler/handler_ref.go b/code/go/0chain.net/blobbercore/handler/handler_hashnode.go similarity index 53% rename from code/go/0chain.net/blobbercore/handler/handler_ref.go rename to code/go/0chain.net/blobbercore/handler/handler_hashnode.go index 686a5c8d4..2a916353a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_ref.go +++ b/code/go/0chain.net/blobbercore/handler/handler_hashnode.go @@ -5,10 +5,10 @@ package handler import "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" -// LoadRootNode load root node with its descendant nodes -func LoadRootNode(ctx *Context) (interface{}, error) { +// LoadRootHashnode load root node with its descendant nodes +func LoadRootHashnode(ctx *Context) (interface{}, error) { - root, err := reference.LoadRootNode(ctx, ctx.AllocationTx) + root, err := reference.LoadRootHashnode(ctx, ctx.AllocationTx) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go b/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go new file mode 100644 index 000000000..1515b9878 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go @@ -0,0 +1,113 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/gorilla/mux" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHashnodeHanders_LoadRootHashnode(t *testing.T) { + + datastore.UseMocket(true) + + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_handler_load_root"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + r := mux.NewRouter() + SetupHandlers(r) + + req, err := http.NewRequest(http.MethodGet, "/v1/refs/root/{allocation}", nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(WithHandler(func(ctx *Context) (interface{}, error) { + ctx.AllocationTx = "allocation_handler_load_root" + return LoadRootHashnode(ctx) + })) + + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + + var root reference.Hashnode + + err = json.Unmarshal(rr.Body.Bytes(), &root) + require.Nil(t, err) + + require.NotNil(t, root) + require.Len(t, root.Children, 2) + + require.Equal(t, root.Children[0].Name, "sub1") + require.Len(t, root.Children[0].Children, 1) + require.Equal(t, root.Children[0].Children[0].Name, "file1") + require.Equal(t, root.Children[1].Name, "sub2") +} diff --git a/code/go/0chain.net/blobbercore/reference/entity.go b/code/go/0chain.net/blobbercore/reference/entity.go index 2fae95c28..d0e2ee523 100644 --- a/code/go/0chain.net/blobbercore/reference/entity.go +++ b/code/go/0chain.net/blobbercore/reference/entity.go @@ -1,14 +1,11 @@ package reference import ( - "strconv" - "strings" - "gorm.io/datatypes" ) -// HashNode ref node in hash tree -type HashNode struct { +// Hashnode ref node in hash tree +type Hashnode struct { // hash data AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"` Type string `gorm:"column:type" json:"type,omitempty"` @@ -24,46 +21,23 @@ type HashNode struct { // other data ParentPath string `gorm:"parent_path" json:"-"` - Children []*HashNode `gorm:"-" json:"children,omitempty"` + Children []*Hashnode `gorm:"-" json:"children,omitempty"` } // TableName get table name of Ref -func (HashNode) TableName() string { +func (Hashnode) TableName() string { return TableNameReferenceObjects } -func (n *HashNode) AddChild(c *HashNode) { +func (n *Hashnode) AddChild(c *Hashnode) { if n.Children == nil { - n.Children = make([]*HashNode, 0, 10) + n.Children = make([]*Hashnode, 0, 10) } n.Children = append(n.Children, c) } // GetLookupHash get lookuphash -func (n *HashNode) GetLookupHash() string { +func (n *Hashnode) GetLookupHash() string { return GetReferenceLookup(n.AllocationID, n.Path) } - -// GetHashCode get hash code -func (n *HashNode) GetHashCode() string { - - if len(n.Attributes) == 0 { - n.Attributes = datatypes.JSON("{}") - } - hashArray := []string{ - n.AllocationID, - n.Type, - n.Name, - n.Path, - strconv.FormatInt(n.Size, 10), - n.ContentHash, - n.MerkleRoot, - strconv.FormatInt(n.ActualFileSize, 10), - n.ActualFileHash, - string(n.Attributes), - strconv.FormatInt(n.ChunkSize, 10), - } - - return strings.Join(hashArray, ":") -} diff --git a/code/go/0chain.net/blobbercore/reference/hash_test.go b/code/go/0chain.net/blobbercore/reference/hash_test.go deleted file mode 100644 index 153ec278b..000000000 --- a/code/go/0chain.net/blobbercore/reference/hash_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package reference - -import ( - "context" - "testing" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - gomocket "github.com/selvatico/go-mocket" - "github.com/stretchr/testify/require" -) - -func TestHash_Should_Work(t *testing.T) { - - datastore.UseMocket(true) - - tests := []struct { - name string - allocationID string - mock func() - assert func(*testing.T, string, *HashNode, error) - }{ - { - name: "No any node should work", - allocationID: "allocation_none", - mock: func() { - gomocket.Catcher.Reset().NewMock(). - WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path -FROM reference_objects`).WithArgs("allocation_none"). - WithReply(nil) - }, - assert: func(test *testing.T, allocationID string, r *HashNode, err error) { - require.NotNil(test, r) - require.Len(test, r.Children, 0) - - require.Equal(test, allocationID, r.AllocationID) - require.Equal(test, DIRECTORY, r.Type) - require.Equal(test, "/", r.Name) - require.Equal(test, "/", r.Path) - require.Equal(test, "", r.ContentHash) - require.Equal(test, "", r.MerkleRoot) - require.Equal(test, "", r.ActualFileHash) - require.EqualValues(test, 0, r.ChunkSize) - require.EqualValues(test, 0, r.Size) - require.EqualValues(test, 0, r.ActualFileSize) - - buf, e := r.Attributes.MarshalJSON() //nolint - require.Nil(test, e) - - require.Equal(test, "null", string(buf)) - - require.Equal(test, "", r.ParentPath) - - }, - }, - { - name: "Nested node should work", - allocationID: "allocation_nested", - mock: func() { - gomocket.Catcher.Reset().NewMock(). - WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path -FROM reference_objects`). - WithArgs("allocation_nested"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "allocation_nested", - "type": "D", - "name": "/", - "path": "/", - "content_hash": "", - "merkle_root": "", - "actual_file_hash": "", - "attributes": []byte("null"), - "chunk_size": 0, - "size": 0, - "actual_file_size": 0, - "parent_path": "", - }, - { - "allocation_id": "allocation_nested", - "type": "D", - "name": "sub1", - "path": "/sub1", - "content_hash": "", - "merkle_root": "", - "actual_file_hash": "", - "attributes": []byte("null"), - "chunk_size": 0, - "size": 0, - "actual_file_size": 0, - "parent_path": "/", - }, - { - "allocation_id": "allocation_nested", - "type": "D", - "name": "sub2", - "path": "/sub2", - "content_hash": "", - "merkle_root": "", - "actual_file_hash": "", - "attributes": []byte("null"), - "chunk_size": 0, - "size": 0, - "actual_file_size": 0, - "parent_path": "/", - }, - { - "allocation_id": "allocation_nested", - "type": "D", - "name": "file1", - "path": "/sub1/file1", - "content_hash": "", - "merkle_root": "", - "actual_file_hash": "", - "attributes": []byte("null"), - "chunk_size": 0, - "size": 0, - "actual_file_size": 0, - "parent_path": "/sub1", - }, - }) - - }, - assert: func(test *testing.T, allocationID string, r *HashNode, err error) { - require.NotNil(test, r) - require.Len(test, r.Children, 2) - - require.Equal(test, r.Children[0].Name, "sub1") - require.Len(test, r.Children[0].Children, 1) - require.Equal(test, r.Children[0].Children[0].Name, "file1") - require.Equal(test, r.Children[1].Name, "sub2") - - }, - }, - } - - for _, it := range tests { - - t.Run(it.name, - func(test *testing.T) { - if it.mock != nil { - it.mock() - } - - r, err := LoadRootNode(context.TODO(), it.allocationID) - require.Nil(test, err) - - it.assert(test, it.allocationID, r, err) - - }, - ) - - } - -} diff --git a/code/go/0chain.net/blobbercore/reference/hash.go b/code/go/0chain.net/blobbercore/reference/hashnode.go similarity index 59% rename from code/go/0chain.net/blobbercore/reference/hash.go rename to code/go/0chain.net/blobbercore/reference/hashnode.go index 1250bed22..a2393e6bd 100644 --- a/code/go/0chain.net/blobbercore/reference/hash.go +++ b/code/go/0chain.net/blobbercore/reference/hashnode.go @@ -6,11 +6,10 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/errors" - "gorm.io/gorm" ) -// LoadRootNode load root node with its descendant nodes -func LoadRootNode(ctx context.Context, allocationID string) (*HashNode, error) { +// LoadRootHashnode load root node with its descendant nodes +func LoadRootHashnode(ctx context.Context, allocationID string) (*Hashnode, error) { db := datastore.GetStore().GetDB() @@ -27,10 +26,10 @@ ORDER BY level desc, path`, allocationID) defer rows.Close() - nodes := make(map[string]*HashNode) + nodes := make(map[string]*Hashnode) for rows.Next() { - node := &HashNode{} + node := &Hashnode{} err = db.ScanRows(rows, node) if err != nil { return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) @@ -52,7 +51,7 @@ ORDER BY level desc, path`, allocationID) // create empty dir if root is missing if len(nodes) == 0 { - return &HashNode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil + return &Hashnode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil } root, ok := nodes["/"] @@ -63,18 +62,3 @@ ORDER BY level desc, path`, allocationID) return nil, common.ErrMissingRootNode } - -const ( - SQLWhereGetByAllocationTxAndPath = "reference_objects.allocation_id = ? and reference_objects.path = ? and deleted_at is NULL" -) - -// DryRun Creates a prepared statement when executing any SQL and caches them to speed up future calls -// https://gorm.io/docs/performance.html#Caches-Prepared-Statement -func DryRun(db *gorm.DB) { - - // https://gorm.io/docs/session.html#DryRun - // Session mode - //tx := db.Session(&gorm.Session{PrepareStmt: true, DryRun: true}) - - // use Table instead of Model to reduce reflect times -} diff --git a/code/go/0chain.net/blobbercore/reference/hashnode_test.go b/code/go/0chain.net/blobbercore/reference/hashnode_test.go new file mode 100644 index 000000000..d14a565c8 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/hashnode_test.go @@ -0,0 +1,390 @@ +package reference + +import ( + "context" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHashabelNode_Should_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *Hashnode, error) + }{ + { + name: "No any node should work", + allocationID: "allocation_none", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_none"). + WithReply(nil) + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 0) + + require.Equal(test, allocationID, r.AllocationID) + require.Equal(test, DIRECTORY, r.Type) + require.Equal(test, "/", r.Name) + require.Equal(test, "/", r.Path) + require.Equal(test, "", r.ContentHash) + require.Equal(test, "", r.MerkleRoot) + require.Equal(test, "", r.ActualFileHash) + require.EqualValues(test, 0, r.ChunkSize) + require.EqualValues(test, 0, r.Size) + require.EqualValues(test, 0, r.ActualFileSize) + + buf, e := r.Attributes.MarshalJSON() //nolint + require.Nil(test, e) + + require.Equal(test, "null", string(buf)) + + require.Equal(test, "", r.ParentPath) + + }, + }, + { + name: "Nested node should work", + allocationID: "allocation_nested", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_nested"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 2) + + require.Equal(test, r.Children[0].Name, "sub1") + require.Len(test, r.Children[0].Children, 1) + require.Equal(test, r.Children[0].Children[0].Name, "file1") + require.Equal(test, r.Children[1].Name, "sub2") + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRoot(context.TODO(), it.allocationID) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} + +func TestHashabelNode_Should_Not_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *Hashnode, error) + }{ + { + name: "Missing root node should not work", + allocationID: "allocation_missing", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_missing"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_missing", + "type": "D", + "name": "sub", + "path": "/sub", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }}) + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrMissingRootNode, err) + + }, + }, + { + name: "Duplicated root node should not work", + allocationID: "allocation_duplicated_root", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_duplicated_root"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrDuplicatedNode, err) + + }, + }, + { + name: "Duplicated node should not work", + allocationID: "allocation_duplicated_node", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_duplicated_node"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrDuplicatedNode, err) + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRoot(context.TODO(), it.allocationID) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} From 77be1373253be7cbe54b6374cb15a03005457aea Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 09:59:37 +0800 Subject: [PATCH 05/18] fix(ref): added unit tests for hashnode --- code/go/0chain.net/blobbercore/reference/hashnode_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/hashnode_test.go b/code/go/0chain.net/blobbercore/reference/hashnode_test.go index d14a565c8..47e8da6fd 100644 --- a/code/go/0chain.net/blobbercore/reference/hashnode_test.go +++ b/code/go/0chain.net/blobbercore/reference/hashnode_test.go @@ -142,7 +142,7 @@ FROM reference_objects`). it.mock() } - r, err := LoadRoot(context.TODO(), it.allocationID) + r, err := LoadRootHashnode(context.TODO(), it.allocationID) it.assert(test, it.allocationID, r, err) @@ -378,7 +378,7 @@ FROM reference_objects`). it.mock() } - r, err := LoadRoot(context.TODO(), it.allocationID) + r, err := LoadRootHashnode(context.TODO(), it.allocationID) it.assert(test, it.allocationID, r, err) From 1aa34df4e1e041dadb4da522752bcf52551eee70 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 26 Feb 2022 16:20:44 +0800 Subject: [PATCH 06/18] feat(writemarker): updated WriteMarker mutex --- .../blobbercore/datastore/postgres_schema.go | 2 +- .../blobbercore/writemarker/mutex.go | 36 +++++-------------- .../blobbercore/writemarker/mutext_test.go | 28 +++++++-------- 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go index 0ad6c3287..d0c1bc2d4 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go @@ -9,7 +9,7 @@ const ( // WriteLock WriteMarker lock type WriteLock struct { AllocationID string `gorm:"primaryKey, column:allocation_id"` - SessionID string `gorm:"column:session_id"` + ConnectionID string `gorm:"column:connection_id"` CreatedAt time.Time `gorm:"column:created_at"` } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index 120163930..c5b42a7c9 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -7,7 +7,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/errors" "github.com/0chain/gosdk/constants" @@ -24,9 +23,8 @@ const ( ) type LockResult struct { - Status LockStatus `json:"status,omitempty"` - CreatedAt time.Time `json:"created_at,omitempty"` - RootNode *reference.HashNode `json:"root_node,omitempty"` + Status LockStatus `json:"status,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` } // Mutex WriteMarker mutex @@ -35,7 +33,7 @@ type Mutex struct { } // Lock -func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, requestTime *time.Time) (*LockResult, error) { +func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, requestTime *time.Time) (*LockResult, error) { m.Mutex.Lock() defer m.Mutex.Unlock() @@ -43,8 +41,8 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.Throw(constants.ErrInvalidParameter, "allocationID") } - if len(sessionID) == 0 { - return nil, errors.Throw(constants.ErrInvalidParameter, "sessionID") + if len(connectionID) == 0 { + return nil, errors.Throw(constants.ErrInvalidParameter, "connectionID") } if requestTime == nil { @@ -65,7 +63,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques if errors.Is(err, gorm.ErrRecordNotFound) { lock = datastore.WriteLock{ AllocationID: allocationID, - SessionID: sessionID, + ConnectionID: connectionID, CreatedAt: *requestTime, } @@ -74,15 +72,9 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt, - RootNode: rootNode, }, nil } @@ -97,7 +89,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques // locked, but it is timeout if now.After(timeout) { - lock.SessionID = sessionID + lock.ConnectionID = connectionID lock.CreatedAt = *requestTime err = db.Save(&lock).Error @@ -105,30 +97,18 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, sessionID string, reques return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt, - RootNode: rootNode, }, nil } //try lock by same session, return old lock directly - if lock.SessionID == sessionID && lock.CreatedAt.Equal(*requestTime) { - rootNode, err := reference.LoadRootNode(ctx, allocationID) - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - + if lock.ConnectionID == connectionID && lock.CreatedAt.Equal(*requestTime) { return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt, - RootNode: rootNode, }, nil } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index 0c09c4f47..e36f0fe15 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -23,7 +23,7 @@ func TestMutext_LockShouldWork(t *testing.T) { tests := []struct { name string allocationID string - sessionID string + connectionID string requestTime time.Time mock func() assert func(*testing.T, *LockResult, error) @@ -31,7 +31,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "Lock should work", allocationID: "lock_allocation_id", - sessionID: "lock_session_id", + connectionID: "lock_connection_id", requestTime: now, mock: func() { @@ -44,7 +44,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "retry lock by same request should work if it is not timeout", allocationID: "lock_same_allocation_id", - sessionID: "lock_same_session_id", + connectionID: "lock_same_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -53,7 +53,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_allocation_id", - "session_id": "lock_same_session_id", + "connection_id": "lock_same_connection_id", "created_at": now, }, }) @@ -66,7 +66,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should be pending if it already is locked by other session ", allocationID: "lock_allocation_id", - sessionID: "lock_pending_session_id", + connectionID: "lock_pending_connection_id", requestTime: time.Now(), mock: func() { gomocket.Catcher.NewMock(). @@ -75,7 +75,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_allocation_id", - "session_id": "lock_session_id", + "connection_id": "lock_connection_id", "created_at": time.Now().Add(-5 * time.Second), }, }) @@ -88,7 +88,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should ok if it is timeout", allocationID: "lock_timeout_allocation_id", - sessionID: "lock_timeout_2nd_session_id", + connectionID: "lock_timeout_2nd_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -97,7 +97,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_timeout_allocation_id", - "session_id": "lock_timeout_1st_session_id", + "connection_id": "lock_timeout_1st_connection_id", "created_at": time.Now().Add(31 * time.Second), }, }) @@ -116,7 +116,7 @@ func TestMutext_LockShouldWork(t *testing.T) { if it.mock != nil { it.mock() } - r, err := m.Lock(context.TODO(), it.allocationID, it.sessionID, &it.requestTime) + r, err := m.Lock(context.TODO(), it.allocationID, it.connectionID, &it.requestTime) it.assert(test, r, err) @@ -139,7 +139,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { tests := []struct { name string allocationID string - sessionID string + connectionID string requestTime time.Time mock func() assert func(*testing.T, *LockResult, error) @@ -147,7 +147,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "Lock should not work if request_time is timeout", allocationID: "lock_allocation_id", - sessionID: "lock_session_id", + connectionID: "lock_connection_id", requestTime: time.Now().Add(31 * time.Second), mock: func() { config.Configuration.WriteMarkerLockTimeout = 30 * time.Second @@ -160,7 +160,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "retry lock by same request should not work if it is timeout", allocationID: "lock_same_timeout_allocation_id", - sessionID: "lock_same_timeout_session_id", + connectionID: "lock_same_timeout_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -169,7 +169,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_timeout_allocation_id", - "session_id": "lock_same_timeout_session_id", + "connection_id": "lock_same_timeout_connection_id", "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), }, }) @@ -188,7 +188,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { if it.mock != nil { it.mock() } - r, err := m.Lock(context.TODO(), it.allocationID, it.sessionID, &it.requestTime) + r, err := m.Lock(context.TODO(), it.allocationID, it.connectionID, &it.requestTime) it.assert(test, r, err) From 37d7449449b09470eed48cd2f64591bf79634533 Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 26 Feb 2022 17:05:52 +0800 Subject: [PATCH 07/18] feat(writemarker): updated CreateAt with int64 --- code/go/0chain.net/blobbercore/writemarker/mutex.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index c5b42a7c9..daeaddab0 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -24,7 +24,7 @@ const ( type LockResult struct { Status LockStatus `json:"status,omitempty"` - CreatedAt time.Time `json:"created_at,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` } // Mutex WriteMarker mutex @@ -74,7 +74,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } @@ -99,7 +99,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } @@ -108,14 +108,14 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req if lock.ConnectionID == connectionID && lock.CreatedAt.Equal(*requestTime) { return &LockResult{ Status: LockStatusOK, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } // pending return &LockResult{ Status: LockStatusPending, - CreatedAt: lock.CreatedAt, + CreatedAt: lock.CreatedAt.Unix(), }, nil } From a6e5c37c74ae3bdd45c54ddd26515f5d681aeab7 Mon Sep 17 00:00:00 2001 From: Lz Date: Mon, 28 Feb 2022 22:34:47 +0800 Subject: [PATCH 08/18] fix(ref): load root hash node --- .../0chain.net/blobbercore/handler/handler.go | 6 +- .../blobbercore/handler/handler_ref.go | 16 ++ .../blobbercore/reference/{dao.go => hash.go} | 55 ++++--- .../blobbercore/reference/hash_test.go | 154 ++++++++++++++++++ .../blobbercore/writemarker/mutex.go | 2 +- code/go/0chain.net/core/common/constants.go | 6 + 6 files changed, 212 insertions(+), 27 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/handler/handler_ref.go rename code/go/0chain.net/blobbercore/reference/{dao.go => hash.go} (58%) create mode 100644 code/go/0chain.net/blobbercore/reference/hash_test.go diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index bc2eba128..51ea0cbec 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -78,8 +78,10 @@ func SetupHandlers(r *mux.Router) { // lightweight http handler without heavy postgres transaction to improve performance - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost) - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost, http.MethodOptions) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete, http.MethodOptions) + + r.HandleFunc("/v1/refs/root/{allocation}", WithHandler(LoadRootNode)).Methods(http.MethodGet, http.MethodOptions) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/blobbercore/handler/handler_ref.go b/code/go/0chain.net/blobbercore/handler/handler_ref.go new file mode 100644 index 000000000..686a5c8d4 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_ref.go @@ -0,0 +1,16 @@ +//go:build !integration_tests +// +build !integration_tests + +package handler + +import "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + +// LoadRootNode load root node with its descendant nodes +func LoadRootNode(ctx *Context) (interface{}, error) { + + root, err := reference.LoadRootNode(ctx, ctx.AllocationTx) + if err != nil { + return nil, err + } + return root, nil +} diff --git a/code/go/0chain.net/blobbercore/reference/dao.go b/code/go/0chain.net/blobbercore/reference/hash.go similarity index 58% rename from code/go/0chain.net/blobbercore/reference/dao.go rename to code/go/0chain.net/blobbercore/reference/hash.go index 363ce4825..1250bed22 100644 --- a/code/go/0chain.net/blobbercore/reference/dao.go +++ b/code/go/0chain.net/blobbercore/reference/hash.go @@ -14,47 +14,54 @@ func LoadRootNode(ctx context.Context, allocationID string) (*HashNode, error) { db := datastore.GetStore().GetDB() - db = db.Where("allocation_id = ? and deleted_at IS NULL", allocationID) + db = db.Raw(` +SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects +WHERE allocation_id = ? and deleted_at IS NULL +ORDER BY level desc, path`, allocationID) - db = db.Order("level desc, path") + rows, err := db.Rows() + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } - dict := make(map[string][]*HashNode) + defer rows.Close() - var nodes []*HashNode - // it is better to load them in batched if there are a lot of objects in db - err := db.FindInBatches(&nodes, 100, func(tx *gorm.DB, batch int) error { - // batch processing found records - for _, object := range nodes { - dict[object.ParentPath] = append(dict[object.ParentPath], object) + nodes := make(map[string]*HashNode) + for rows.Next() { - for _, child := range dict[object.Path] { - object.AddChild(child) - } + node := &HashNode{} + err = db.ScanRows(rows, node) + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } - return nil - }).Error + _, ok := nodes[node.Path] + if ok { + return nil, common.ErrDuplicatedNode + } + + nodes[node.Path] = node + + parent, ok := nodes[node.ParentPath] + if ok { + parent.AddChild(node) + } - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } // create empty dir if root is missing - if len(dict) == 0 { + if len(nodes) == 0 { return &HashNode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil } - rootNodes, ok := dict[""] + root, ok := nodes["/"] if ok { - if len(rootNodes) == 1 { - return rootNodes[0], nil - } - - return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") + return root, nil } - return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") + return nil, common.ErrMissingRootNode } const ( diff --git a/code/go/0chain.net/blobbercore/reference/hash_test.go b/code/go/0chain.net/blobbercore/reference/hash_test.go new file mode 100644 index 000000000..153ec278b --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/hash_test.go @@ -0,0 +1,154 @@ +package reference + +import ( + "context" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHash_Should_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *HashNode, error) + }{ + { + name: "No any node should work", + allocationID: "allocation_none", + mock: func() { + gomocket.Catcher.Reset().NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_none"). + WithReply(nil) + }, + assert: func(test *testing.T, allocationID string, r *HashNode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 0) + + require.Equal(test, allocationID, r.AllocationID) + require.Equal(test, DIRECTORY, r.Type) + require.Equal(test, "/", r.Name) + require.Equal(test, "/", r.Path) + require.Equal(test, "", r.ContentHash) + require.Equal(test, "", r.MerkleRoot) + require.Equal(test, "", r.ActualFileHash) + require.EqualValues(test, 0, r.ChunkSize) + require.EqualValues(test, 0, r.Size) + require.EqualValues(test, 0, r.ActualFileSize) + + buf, e := r.Attributes.MarshalJSON() //nolint + require.Nil(test, e) + + require.Equal(test, "null", string(buf)) + + require.Equal(test, "", r.ParentPath) + + }, + }, + { + name: "Nested node should work", + allocationID: "allocation_nested", + mock: func() { + gomocket.Catcher.Reset().NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_nested"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *HashNode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 2) + + require.Equal(test, r.Children[0].Name, "sub1") + require.Len(test, r.Children[0].Children, 1) + require.Equal(test, r.Children[0].Children[0].Name, "file1") + require.Equal(test, r.Children[1].Name, "sub2") + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRootNode(context.TODO(), it.allocationID) + require.Nil(test, err) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index daeaddab0..fa8fe342d 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -132,7 +132,7 @@ func (*Mutex) Unlock(ctx context.Context, allocationID string, sessionID string) db := datastore.GetStore().GetDB() - err := db.Where("allocation_id = ? and session_id =? ", allocationID, sessionID).Delete(&datastore.WriteLock{}).Error + err := db.Where("allocation_id = ? and session_id = ? ", allocationID, sessionID).Delete(&datastore.WriteLock{}).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil diff --git a/code/go/0chain.net/core/common/constants.go b/code/go/0chain.net/core/common/constants.go index 2345aebe1..de9dc6ae2 100644 --- a/code/go/0chain.net/core/common/constants.go +++ b/code/go/0chain.net/core/common/constants.go @@ -32,4 +32,10 @@ var ( // ErrEntityNotFound entity can't found in db ErrEntityNotFound = errors.New("entity not found") + + // ErrMissingRootNode root node is missing + ErrMissingRootNode = errors.New("root node is missing") + + // ErrDuplicatedNode duplicated nodes + ErrDuplicatedNode = errors.New("duplicated nodes") ) From cf6720b2392d240b5bb5ad390b27233b6060ef0b Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 09:52:21 +0800 Subject: [PATCH 09/18] fix(ref): added unit tests for hashnode --- .../0chain.net/blobbercore/handler/handler.go | 2 +- .../{handler_ref.go => handler_hashnode.go} | 6 +- .../handler/handler_hashnode_test.go | 113 +++++ .../blobbercore/reference/entity.go | 40 +- .../blobbercore/reference/hash_test.go | 154 ------- .../reference/{hash.go => hashnode.go} | 26 +- .../blobbercore/reference/hashnode_test.go | 390 ++++++++++++++++++ 7 files changed, 519 insertions(+), 212 deletions(-) rename code/go/0chain.net/blobbercore/handler/{handler_ref.go => handler_hashnode.go} (53%) create mode 100644 code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go delete mode 100644 code/go/0chain.net/blobbercore/reference/hash_test.go rename code/go/0chain.net/blobbercore/reference/{hash.go => hashnode.go} (59%) create mode 100644 code/go/0chain.net/blobbercore/reference/hashnode_test.go diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 51ea0cbec..a85570168 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -81,7 +81,7 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost, http.MethodOptions) r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete, http.MethodOptions) - r.HandleFunc("/v1/refs/root/{allocation}", WithHandler(LoadRootNode)).Methods(http.MethodGet, http.MethodOptions) + r.HandleFunc("/v1/hashnode/root/{allocation}", WithHandler(LoadRootHashnode)).Methods(http.MethodGet, http.MethodOptions) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/blobbercore/handler/handler_ref.go b/code/go/0chain.net/blobbercore/handler/handler_hashnode.go similarity index 53% rename from code/go/0chain.net/blobbercore/handler/handler_ref.go rename to code/go/0chain.net/blobbercore/handler/handler_hashnode.go index 686a5c8d4..2a916353a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_ref.go +++ b/code/go/0chain.net/blobbercore/handler/handler_hashnode.go @@ -5,10 +5,10 @@ package handler import "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" -// LoadRootNode load root node with its descendant nodes -func LoadRootNode(ctx *Context) (interface{}, error) { +// LoadRootHashnode load root node with its descendant nodes +func LoadRootHashnode(ctx *Context) (interface{}, error) { - root, err := reference.LoadRootNode(ctx, ctx.AllocationTx) + root, err := reference.LoadRootHashnode(ctx, ctx.AllocationTx) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go b/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go new file mode 100644 index 000000000..1515b9878 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go @@ -0,0 +1,113 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/gorilla/mux" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHashnodeHanders_LoadRootHashnode(t *testing.T) { + + datastore.UseMocket(true) + + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_handler_load_root"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + r := mux.NewRouter() + SetupHandlers(r) + + req, err := http.NewRequest(http.MethodGet, "/v1/refs/root/{allocation}", nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(WithHandler(func(ctx *Context) (interface{}, error) { + ctx.AllocationTx = "allocation_handler_load_root" + return LoadRootHashnode(ctx) + })) + + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + + var root reference.Hashnode + + err = json.Unmarshal(rr.Body.Bytes(), &root) + require.Nil(t, err) + + require.NotNil(t, root) + require.Len(t, root.Children, 2) + + require.Equal(t, root.Children[0].Name, "sub1") + require.Len(t, root.Children[0].Children, 1) + require.Equal(t, root.Children[0].Children[0].Name, "file1") + require.Equal(t, root.Children[1].Name, "sub2") +} diff --git a/code/go/0chain.net/blobbercore/reference/entity.go b/code/go/0chain.net/blobbercore/reference/entity.go index 2fae95c28..d0e2ee523 100644 --- a/code/go/0chain.net/blobbercore/reference/entity.go +++ b/code/go/0chain.net/blobbercore/reference/entity.go @@ -1,14 +1,11 @@ package reference import ( - "strconv" - "strings" - "gorm.io/datatypes" ) -// HashNode ref node in hash tree -type HashNode struct { +// Hashnode ref node in hash tree +type Hashnode struct { // hash data AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"` Type string `gorm:"column:type" json:"type,omitempty"` @@ -24,46 +21,23 @@ type HashNode struct { // other data ParentPath string `gorm:"parent_path" json:"-"` - Children []*HashNode `gorm:"-" json:"children,omitempty"` + Children []*Hashnode `gorm:"-" json:"children,omitempty"` } // TableName get table name of Ref -func (HashNode) TableName() string { +func (Hashnode) TableName() string { return TableNameReferenceObjects } -func (n *HashNode) AddChild(c *HashNode) { +func (n *Hashnode) AddChild(c *Hashnode) { if n.Children == nil { - n.Children = make([]*HashNode, 0, 10) + n.Children = make([]*Hashnode, 0, 10) } n.Children = append(n.Children, c) } // GetLookupHash get lookuphash -func (n *HashNode) GetLookupHash() string { +func (n *Hashnode) GetLookupHash() string { return GetReferenceLookup(n.AllocationID, n.Path) } - -// GetHashCode get hash code -func (n *HashNode) GetHashCode() string { - - if len(n.Attributes) == 0 { - n.Attributes = datatypes.JSON("{}") - } - hashArray := []string{ - n.AllocationID, - n.Type, - n.Name, - n.Path, - strconv.FormatInt(n.Size, 10), - n.ContentHash, - n.MerkleRoot, - strconv.FormatInt(n.ActualFileSize, 10), - n.ActualFileHash, - string(n.Attributes), - strconv.FormatInt(n.ChunkSize, 10), - } - - return strings.Join(hashArray, ":") -} diff --git a/code/go/0chain.net/blobbercore/reference/hash_test.go b/code/go/0chain.net/blobbercore/reference/hash_test.go deleted file mode 100644 index 153ec278b..000000000 --- a/code/go/0chain.net/blobbercore/reference/hash_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package reference - -import ( - "context" - "testing" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - gomocket "github.com/selvatico/go-mocket" - "github.com/stretchr/testify/require" -) - -func TestHash_Should_Work(t *testing.T) { - - datastore.UseMocket(true) - - tests := []struct { - name string - allocationID string - mock func() - assert func(*testing.T, string, *HashNode, error) - }{ - { - name: "No any node should work", - allocationID: "allocation_none", - mock: func() { - gomocket.Catcher.Reset().NewMock(). - WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path -FROM reference_objects`).WithArgs("allocation_none"). - WithReply(nil) - }, - assert: func(test *testing.T, allocationID string, r *HashNode, err error) { - require.NotNil(test, r) - require.Len(test, r.Children, 0) - - require.Equal(test, allocationID, r.AllocationID) - require.Equal(test, DIRECTORY, r.Type) - require.Equal(test, "/", r.Name) - require.Equal(test, "/", r.Path) - require.Equal(test, "", r.ContentHash) - require.Equal(test, "", r.MerkleRoot) - require.Equal(test, "", r.ActualFileHash) - require.EqualValues(test, 0, r.ChunkSize) - require.EqualValues(test, 0, r.Size) - require.EqualValues(test, 0, r.ActualFileSize) - - buf, e := r.Attributes.MarshalJSON() //nolint - require.Nil(test, e) - - require.Equal(test, "null", string(buf)) - - require.Equal(test, "", r.ParentPath) - - }, - }, - { - name: "Nested node should work", - allocationID: "allocation_nested", - mock: func() { - gomocket.Catcher.Reset().NewMock(). - WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path -FROM reference_objects`). - WithArgs("allocation_nested"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "allocation_nested", - "type": "D", - "name": "/", - "path": "/", - "content_hash": "", - "merkle_root": "", - "actual_file_hash": "", - "attributes": []byte("null"), - "chunk_size": 0, - "size": 0, - "actual_file_size": 0, - "parent_path": "", - }, - { - "allocation_id": "allocation_nested", - "type": "D", - "name": "sub1", - "path": "/sub1", - "content_hash": "", - "merkle_root": "", - "actual_file_hash": "", - "attributes": []byte("null"), - "chunk_size": 0, - "size": 0, - "actual_file_size": 0, - "parent_path": "/", - }, - { - "allocation_id": "allocation_nested", - "type": "D", - "name": "sub2", - "path": "/sub2", - "content_hash": "", - "merkle_root": "", - "actual_file_hash": "", - "attributes": []byte("null"), - "chunk_size": 0, - "size": 0, - "actual_file_size": 0, - "parent_path": "/", - }, - { - "allocation_id": "allocation_nested", - "type": "D", - "name": "file1", - "path": "/sub1/file1", - "content_hash": "", - "merkle_root": "", - "actual_file_hash": "", - "attributes": []byte("null"), - "chunk_size": 0, - "size": 0, - "actual_file_size": 0, - "parent_path": "/sub1", - }, - }) - - }, - assert: func(test *testing.T, allocationID string, r *HashNode, err error) { - require.NotNil(test, r) - require.Len(test, r.Children, 2) - - require.Equal(test, r.Children[0].Name, "sub1") - require.Len(test, r.Children[0].Children, 1) - require.Equal(test, r.Children[0].Children[0].Name, "file1") - require.Equal(test, r.Children[1].Name, "sub2") - - }, - }, - } - - for _, it := range tests { - - t.Run(it.name, - func(test *testing.T) { - if it.mock != nil { - it.mock() - } - - r, err := LoadRootNode(context.TODO(), it.allocationID) - require.Nil(test, err) - - it.assert(test, it.allocationID, r, err) - - }, - ) - - } - -} diff --git a/code/go/0chain.net/blobbercore/reference/hash.go b/code/go/0chain.net/blobbercore/reference/hashnode.go similarity index 59% rename from code/go/0chain.net/blobbercore/reference/hash.go rename to code/go/0chain.net/blobbercore/reference/hashnode.go index 1250bed22..a2393e6bd 100644 --- a/code/go/0chain.net/blobbercore/reference/hash.go +++ b/code/go/0chain.net/blobbercore/reference/hashnode.go @@ -6,11 +6,10 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/errors" - "gorm.io/gorm" ) -// LoadRootNode load root node with its descendant nodes -func LoadRootNode(ctx context.Context, allocationID string) (*HashNode, error) { +// LoadRootHashnode load root node with its descendant nodes +func LoadRootHashnode(ctx context.Context, allocationID string) (*Hashnode, error) { db := datastore.GetStore().GetDB() @@ -27,10 +26,10 @@ ORDER BY level desc, path`, allocationID) defer rows.Close() - nodes := make(map[string]*HashNode) + nodes := make(map[string]*Hashnode) for rows.Next() { - node := &HashNode{} + node := &Hashnode{} err = db.ScanRows(rows, node) if err != nil { return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) @@ -52,7 +51,7 @@ ORDER BY level desc, path`, allocationID) // create empty dir if root is missing if len(nodes) == 0 { - return &HashNode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil + return &Hashnode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil } root, ok := nodes["/"] @@ -63,18 +62,3 @@ ORDER BY level desc, path`, allocationID) return nil, common.ErrMissingRootNode } - -const ( - SQLWhereGetByAllocationTxAndPath = "reference_objects.allocation_id = ? and reference_objects.path = ? and deleted_at is NULL" -) - -// DryRun Creates a prepared statement when executing any SQL and caches them to speed up future calls -// https://gorm.io/docs/performance.html#Caches-Prepared-Statement -func DryRun(db *gorm.DB) { - - // https://gorm.io/docs/session.html#DryRun - // Session mode - //tx := db.Session(&gorm.Session{PrepareStmt: true, DryRun: true}) - - // use Table instead of Model to reduce reflect times -} diff --git a/code/go/0chain.net/blobbercore/reference/hashnode_test.go b/code/go/0chain.net/blobbercore/reference/hashnode_test.go new file mode 100644 index 000000000..d14a565c8 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/hashnode_test.go @@ -0,0 +1,390 @@ +package reference + +import ( + "context" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHashabelNode_Should_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *Hashnode, error) + }{ + { + name: "No any node should work", + allocationID: "allocation_none", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_none"). + WithReply(nil) + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 0) + + require.Equal(test, allocationID, r.AllocationID) + require.Equal(test, DIRECTORY, r.Type) + require.Equal(test, "/", r.Name) + require.Equal(test, "/", r.Path) + require.Equal(test, "", r.ContentHash) + require.Equal(test, "", r.MerkleRoot) + require.Equal(test, "", r.ActualFileHash) + require.EqualValues(test, 0, r.ChunkSize) + require.EqualValues(test, 0, r.Size) + require.EqualValues(test, 0, r.ActualFileSize) + + buf, e := r.Attributes.MarshalJSON() //nolint + require.Nil(test, e) + + require.Equal(test, "null", string(buf)) + + require.Equal(test, "", r.ParentPath) + + }, + }, + { + name: "Nested node should work", + allocationID: "allocation_nested", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_nested"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 2) + + require.Equal(test, r.Children[0].Name, "sub1") + require.Len(test, r.Children[0].Children, 1) + require.Equal(test, r.Children[0].Children[0].Name, "file1") + require.Equal(test, r.Children[1].Name, "sub2") + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRoot(context.TODO(), it.allocationID) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} + +func TestHashabelNode_Should_Not_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *Hashnode, error) + }{ + { + name: "Missing root node should not work", + allocationID: "allocation_missing", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_missing"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_missing", + "type": "D", + "name": "sub", + "path": "/sub", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }}) + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrMissingRootNode, err) + + }, + }, + { + name: "Duplicated root node should not work", + allocationID: "allocation_duplicated_root", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_duplicated_root"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrDuplicatedNode, err) + + }, + }, + { + name: "Duplicated node should not work", + allocationID: "allocation_duplicated_node", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_duplicated_node"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrDuplicatedNode, err) + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRoot(context.TODO(), it.allocationID) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} From 2d6b112358bc723fdba885604cfd3cb2ba8a68e9 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 09:59:37 +0800 Subject: [PATCH 10/18] fix(ref): added unit tests for hashnode --- code/go/0chain.net/blobbercore/reference/hashnode_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/hashnode_test.go b/code/go/0chain.net/blobbercore/reference/hashnode_test.go index d14a565c8..47e8da6fd 100644 --- a/code/go/0chain.net/blobbercore/reference/hashnode_test.go +++ b/code/go/0chain.net/blobbercore/reference/hashnode_test.go @@ -142,7 +142,7 @@ FROM reference_objects`). it.mock() } - r, err := LoadRoot(context.TODO(), it.allocationID) + r, err := LoadRootHashnode(context.TODO(), it.allocationID) it.assert(test, it.allocationID, r, err) @@ -378,7 +378,7 @@ FROM reference_objects`). it.mock() } - r, err := LoadRoot(context.TODO(), it.allocationID) + r, err := LoadRootHashnode(context.TODO(), it.allocationID) it.assert(test, it.allocationID, r, err) From c43b9dbc8de66b616ffd18741e93e448b46690e8 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 14:51:07 +0800 Subject: [PATCH 11/18] fix(ref):fixed typo --- code/go/0chain.net/blobbercore/reference/hashnode_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/hashnode_test.go b/code/go/0chain.net/blobbercore/reference/hashnode_test.go index 47e8da6fd..c6a8d5ae5 100644 --- a/code/go/0chain.net/blobbercore/reference/hashnode_test.go +++ b/code/go/0chain.net/blobbercore/reference/hashnode_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestHashabelNode_Should_Work(t *testing.T) { +func TestHashnode_Should_Work(t *testing.T) { datastore.UseMocket(true) @@ -153,7 +153,7 @@ FROM reference_objects`). } -func TestHashabelNode_Should_Not_Work(t *testing.T) { +func TestHashnode_Should_Not_Work(t *testing.T) { datastore.UseMocket(true) From 54452f991229eb44d4c154c80631fb1b157a4dc9 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 14:52:44 +0800 Subject: [PATCH 12/18] fix(ref):fixed typo --- code/go/0chain.net/blobbercore/reference/hashnode_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/hashnode_test.go b/code/go/0chain.net/blobbercore/reference/hashnode_test.go index 358712382..c6a8d5ae5 100644 --- a/code/go/0chain.net/blobbercore/reference/hashnode_test.go +++ b/code/go/0chain.net/blobbercore/reference/hashnode_test.go @@ -10,11 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -<<<<<<< HEAD func TestHashnode_Should_Work(t *testing.T) { -======= -func TestHashabelNode_Should_Work(t *testing.T) { ->>>>>>> 2d6b112358bc723fdba885604cfd3cb2ba8a68e9 datastore.UseMocket(true) @@ -157,11 +153,7 @@ FROM reference_objects`). } -<<<<<<< HEAD func TestHashnode_Should_Not_Work(t *testing.T) { -======= -func TestHashabelNode_Should_Not_Work(t *testing.T) { ->>>>>>> 2d6b112358bc723fdba885604cfd3cb2ba8a68e9 datastore.UseMocket(true) From 38141c145d8c5e7648e537c793ca7d59c33f5c7a Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 19:22:02 +0800 Subject: [PATCH 13/18] fix(writelock):renamed connectionID with sessionID --- code/go/0chain.net/blobbercore/datastore/postgres_schema.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go index d0c1bc2d4..0ad6c3287 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go @@ -9,7 +9,7 @@ const ( // WriteLock WriteMarker lock type WriteLock struct { AllocationID string `gorm:"primaryKey, column:allocation_id"` - ConnectionID string `gorm:"column:connection_id"` + SessionID string `gorm:"column:session_id"` CreatedAt time.Time `gorm:"column:created_at"` } From 534c8847684b0ab2469528f77298d58a54618bd9 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 19:23:14 +0800 Subject: [PATCH 14/18] fix(writelock):renamed connectionID with sessionID --- .../blobbercore/writemarker/mutex.go | 6 +++--- .../blobbercore/writemarker/mutext_test.go | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index fa8fe342d..2474cd5b4 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -63,7 +63,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req if errors.Is(err, gorm.ErrRecordNotFound) { lock = datastore.WriteLock{ AllocationID: allocationID, - ConnectionID: connectionID, + SessionID: connectionID, CreatedAt: *requestTime, } @@ -89,7 +89,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req // locked, but it is timeout if now.After(timeout) { - lock.ConnectionID = connectionID + lock.SessionID = connectionID lock.CreatedAt = *requestTime err = db.Save(&lock).Error @@ -105,7 +105,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req } //try lock by same session, return old lock directly - if lock.ConnectionID == connectionID && lock.CreatedAt.Equal(*requestTime) { + if lock.SessionID == connectionID && lock.CreatedAt.Equal(*requestTime) { return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt.Unix(), diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index e36f0fe15..6c3978a08 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -31,7 +31,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "Lock should work", allocationID: "lock_allocation_id", - connectionID: "lock_connection_id", + connectionID: "lock_session_id", requestTime: now, mock: func() { @@ -44,7 +44,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "retry lock by same request should work if it is not timeout", allocationID: "lock_same_allocation_id", - connectionID: "lock_same_connection_id", + connectionID: "lock_same_session_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -53,7 +53,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_allocation_id", - "connection_id": "lock_same_connection_id", + "session_id": "lock_same_session_id", "created_at": now, }, }) @@ -66,7 +66,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should be pending if it already is locked by other session ", allocationID: "lock_allocation_id", - connectionID: "lock_pending_connection_id", + connectionID: "lock_pending_session_id", requestTime: time.Now(), mock: func() { gomocket.Catcher.NewMock(). @@ -75,7 +75,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_allocation_id", - "connection_id": "lock_connection_id", + "session_id": "lock_session_id", "created_at": time.Now().Add(-5 * time.Second), }, }) @@ -88,7 +88,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should ok if it is timeout", allocationID: "lock_timeout_allocation_id", - connectionID: "lock_timeout_2nd_connection_id", + connectionID: "lock_timeout_2nd_session_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -97,7 +97,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_timeout_allocation_id", - "connection_id": "lock_timeout_1st_connection_id", + "session_id": "lock_timeout_1st_session_id", "created_at": time.Now().Add(31 * time.Second), }, }) @@ -147,7 +147,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "Lock should not work if request_time is timeout", allocationID: "lock_allocation_id", - connectionID: "lock_connection_id", + connectionID: "lock_session_id", requestTime: time.Now().Add(31 * time.Second), mock: func() { config.Configuration.WriteMarkerLockTimeout = 30 * time.Second @@ -160,7 +160,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "retry lock by same request should not work if it is timeout", allocationID: "lock_same_timeout_allocation_id", - connectionID: "lock_same_timeout_connection_id", + connectionID: "lock_same_timeout_session_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -169,7 +169,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_timeout_allocation_id", - "connection_id": "lock_same_timeout_connection_id", + "session_id": "lock_same_timeout_session_id", "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), }, }) From d68f246a8be0c62c0b1d586f3c37422ff7bd8b65 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 1 Mar 2022 19:28:08 +0800 Subject: [PATCH 15/18] fix(writelock):revered sessionID to connectionID --- .../blobbercore/datastore/postgres_schema.go | 2 +- .../handler/handler_writemarker.go | 6 +++--- .../handler/handler_writemarker_test.go | 4 ++-- .../blobbercore/writemarker/mutex.go | 12 +++++------ .../blobbercore/writemarker/mutext_test.go | 20 +++++++++---------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go index 0ad6c3287..d0c1bc2d4 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres_schema.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres_schema.go @@ -9,7 +9,7 @@ const ( // WriteLock WriteMarker lock type WriteLock struct { AllocationID string `gorm:"primaryKey, column:allocation_id"` - SessionID string `gorm:"column:session_id"` + ConnectionID string `gorm:"column:connection_id"` CreatedAt time.Time `gorm:"column:created_at"` } diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go index 02196fd5f..2338dde80 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go @@ -11,10 +11,10 @@ var WriteMarkerMutext = &writemarker.Mutex{} // LockWriteMarker try to lock writemarker for specified allocation id, and return latest RefTree func LockWriteMarker(ctx *Context) (interface{}, error) { - sessionID := ctx.FormValue("session_id") + connectionID := ctx.FormValue("connection_id") requestTime := ctx.FormTime("request_time") - result, err := WriteMarkerMutext.Lock(ctx, ctx.AllocationTx, sessionID, requestTime) + result, err := WriteMarkerMutext.Lock(ctx, ctx.AllocationTx, connectionID, requestTime) if err != nil { return nil, err } @@ -24,7 +24,7 @@ func LockWriteMarker(ctx *Context) (interface{}, error) { // UnlockWriteMarker release WriteMarkerMutex func UnlockWriteMarker(ctx *Context) (interface{}, error) { - sessionID := ctx.FormValue("session_id") + sessionID := ctx.FormValue("connection_id") err := WriteMarkerMutext.Unlock(ctx, ctx.AllocationTx, sessionID) if err != nil { diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go index 48779982f..0e18064df 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go @@ -28,7 +28,7 @@ func TestWriteMarkerHandlers_Lock(t *testing.T) { now := time.Now() - formWriter.WriteField("session_id", "session_id") //nolint: errcheck + formWriter.WriteField("connection_id", "connection_id") //nolint: errcheck formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck formWriter.Close() @@ -68,7 +68,7 @@ func TestWriteMarkerHandlers_Unlock(t *testing.T) { now := time.Now() - formWriter.WriteField("session_id", "session_id") //nolint: errcheck + formWriter.WriteField("connection_id", "connection_id") //nolint: errcheck formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck formWriter.Close() diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index 2474cd5b4..6a3764485 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -63,7 +63,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req if errors.Is(err, gorm.ErrRecordNotFound) { lock = datastore.WriteLock{ AllocationID: allocationID, - SessionID: connectionID, + ConnectionID: connectionID, CreatedAt: *requestTime, } @@ -89,7 +89,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req // locked, but it is timeout if now.After(timeout) { - lock.SessionID = connectionID + lock.ConnectionID = connectionID lock.CreatedAt = *requestTime err = db.Save(&lock).Error @@ -105,7 +105,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req } //try lock by same session, return old lock directly - if lock.SessionID == connectionID && lock.CreatedAt.Equal(*requestTime) { + if lock.ConnectionID == connectionID && lock.CreatedAt.Equal(*requestTime) { return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt.Unix(), @@ -120,19 +120,19 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req } -func (*Mutex) Unlock(ctx context.Context, allocationID string, sessionID string) error { +func (*Mutex) Unlock(ctx context.Context, allocationID string, connectionID string) error { if len(allocationID) == 0 { return nil } - if len(sessionID) == 0 { + if len(connectionID) == 0 { return nil } db := datastore.GetStore().GetDB() - err := db.Where("allocation_id = ? and session_id = ? ", allocationID, sessionID).Delete(&datastore.WriteLock{}).Error + err := db.Where("allocation_id = ? and connection_id = ? ", allocationID, connectionID).Delete(&datastore.WriteLock{}).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index 6c3978a08..e36f0fe15 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -31,7 +31,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "Lock should work", allocationID: "lock_allocation_id", - connectionID: "lock_session_id", + connectionID: "lock_connection_id", requestTime: now, mock: func() { @@ -44,7 +44,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "retry lock by same request should work if it is not timeout", allocationID: "lock_same_allocation_id", - connectionID: "lock_same_session_id", + connectionID: "lock_same_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -53,7 +53,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_allocation_id", - "session_id": "lock_same_session_id", + "connection_id": "lock_same_connection_id", "created_at": now, }, }) @@ -66,7 +66,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should be pending if it already is locked by other session ", allocationID: "lock_allocation_id", - connectionID: "lock_pending_session_id", + connectionID: "lock_pending_connection_id", requestTime: time.Now(), mock: func() { gomocket.Catcher.NewMock(). @@ -75,7 +75,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_allocation_id", - "session_id": "lock_session_id", + "connection_id": "lock_connection_id", "created_at": time.Now().Add(-5 * time.Second), }, }) @@ -88,7 +88,7 @@ func TestMutext_LockShouldWork(t *testing.T) { { name: "lock should ok if it is timeout", allocationID: "lock_timeout_allocation_id", - connectionID: "lock_timeout_2nd_session_id", + connectionID: "lock_timeout_2nd_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -97,7 +97,7 @@ func TestMutext_LockShouldWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_timeout_allocation_id", - "session_id": "lock_timeout_1st_session_id", + "connection_id": "lock_timeout_1st_connection_id", "created_at": time.Now().Add(31 * time.Second), }, }) @@ -147,7 +147,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "Lock should not work if request_time is timeout", allocationID: "lock_allocation_id", - connectionID: "lock_session_id", + connectionID: "lock_connection_id", requestTime: time.Now().Add(31 * time.Second), mock: func() { config.Configuration.WriteMarkerLockTimeout = 30 * time.Second @@ -160,7 +160,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { { name: "retry lock by same request should not work if it is timeout", allocationID: "lock_same_timeout_allocation_id", - connectionID: "lock_same_timeout_session_id", + connectionID: "lock_same_timeout_connection_id", requestTime: now, mock: func() { gomocket.Catcher.NewMock(). @@ -169,7 +169,7 @@ func TestMutext_LockShouldNotWork(t *testing.T) { WithReply([]map[string]interface{}{ { "allocation_id": "lock_same_timeout_allocation_id", - "session_id": "lock_same_timeout_session_id", + "connection_id": "lock_same_timeout_connection_id", "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), }, }) From 52987f99dc8ba9a7d08d3ffa83ebe2f170e4c942 Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 2 Mar 2022 07:47:18 +0800 Subject: [PATCH 16/18] fix(gomod): upgraded gorm --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index d9e844bef..741c36ada 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d gorm.io/driver/postgres v1.3.1 - gorm.io/gorm v1.23.1 + gorm.io/gorm v1.23.2 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index a154c6548..24b4559e4 100644 --- a/go.sum +++ b/go.sum @@ -1638,8 +1638,9 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.23.1 h1:aj5IlhDzEPsoIyOPtTRVI+SyaN1u6k613sbt4pwbxG0= gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.2 h1:xmq9QRMWL8HTJyhAUBXy8FqIIQCYESeKfJL4DoGKiWQ= +gorm.io/gorm v1.23.2/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From e39d682dbe2a896b930b385335d065bafa45a06b Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 2 Mar 2022 16:04:15 +0800 Subject: [PATCH 17/18] feat(writemarker): fixed sql issue on postgres --- .../0chain.net/blobbercore/handler/context.go | 6 +-- .../blobbercore/writemarker/mutex.go | 4 +- .../blobbercore/writemarker/mutext_test.go | 48 +++++++++---------- go.mod | 2 +- go.sum | 4 +- 5 files changed, 31 insertions(+), 33 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/context.go b/code/go/0chain.net/blobbercore/handler/context.go index 0f27146a5..7286d2779 100644 --- a/code/go/0chain.net/blobbercore/handler/context.go +++ b/code/go/0chain.net/blobbercore/handler/context.go @@ -84,8 +84,7 @@ func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.Re statusCode = http.StatusInternalServerError } - buf, _ := json.Marshal(err) - http.Error(w, string(buf), statusCode) + http.Error(w, err.Error(), statusCode) return } @@ -97,8 +96,7 @@ func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.Re statusCode = http.StatusInternalServerError } - buf, _ := json.Marshal(err) - http.Error(w, string(buf), statusCode) + http.Error(w, err.Error(), statusCode) return } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index 6a3764485..8c2349f92 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -67,7 +67,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req CreatedAt: *requestTime, } - err = db.Create(&lock).Error + err = db.Table(datastore.TableNameWriteLock).Create(&lock).Error if err != nil { return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } @@ -92,7 +92,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req lock.ConnectionID = connectionID lock.CreatedAt = *requestTime - err = db.Save(&lock).Error + err = db.Table(datastore.TableNameWriteLock).Where("allocation_id=?", allocationID).Save(&lock).Error if err != nil { return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index e36f0fe15..b7fb23810 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -61,6 +61,7 @@ func TestMutext_LockShouldWork(t *testing.T) { assert: func(test *testing.T, r *LockResult, err error) { require.Nil(test, err) require.Equal(test, LockStatusOK, r.Status) + require.EqualValues(test, now.Unix(), r.CreatedAt) }, }, { @@ -86,7 +87,7 @@ func TestMutext_LockShouldWork(t *testing.T) { }, }, { - name: "lock should ok if it is timeout", + name: "lock should be ok if it is timeout", allocationID: "lock_timeout_allocation_id", connectionID: "lock_timeout_2nd_connection_id", requestTime: now, @@ -107,6 +108,28 @@ func TestMutext_LockShouldWork(t *testing.T) { require.Equal(test, LockStatusPending, r.Status) }, }, + { + name: "retry lock by same request should work if it is timeout", + allocationID: "lock_same_timeout_allocation_id", + connectionID: "lock_same_timeout_connection_id", + requestTime: now, + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). + WithArgs("lock_same_timeout_allocation_id"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "lock_same_timeout_allocation_id", + "connection_id": "lock_same_timeout_connection_id", + "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), + }, + }) + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.Nil(test, err) + require.NotNil(test, r) + }, + }, } for _, it := range tests { @@ -134,7 +157,6 @@ func TestMutext_LockShouldNotWork(t *testing.T) { config.Configuration.WriteMarkerLockTimeout = 30 * time.Second m := &Mutex{} - now := time.Now() tests := []struct { name string @@ -157,28 +179,6 @@ func TestMutext_LockShouldNotWork(t *testing.T) { require.NotNil(test, err) }, }, - { - name: "retry lock by same request should not work if it is timeout", - allocationID: "lock_same_timeout_allocation_id", - connectionID: "lock_same_timeout_connection_id", - requestTime: now, - mock: func() { - gomocket.Catcher.NewMock(). - WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). - WithArgs("lock_same_timeout_allocation_id"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "lock_same_timeout_allocation_id", - "connection_id": "lock_same_timeout_connection_id", - "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), - }, - }) - }, - assert: func(test *testing.T, r *LockResult, err error) { - require.NotNil(test, err) - require.Nil(test, r) - }, - }, } for _, it := range tests { diff --git a/go.mod b/go.mod index 741c36ada..bb7973944 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15 + github.com/0chain/gosdk v1.7.2 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-ini/ini v1.55.0 // indirect diff --git a/go.sum b/go.sum index 24b4559e4..c9a7595e4 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15 h1:cOqg66kR646dc1NNgduE0Ridlg0bDDvhxOPC9rhrNWk= -github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= +github.com/0chain/gosdk v1.7.2 h1:iWEc36gy0Puzzj9X/OshQ2B4I1XP6EeoAo2vZ2S5A/8= +github.com/0chain/gosdk v1.7.2/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= From c5d068f35de6af0b49cc22c285c937659a300f49 Mon Sep 17 00:00:00 2001 From: Lz Date: Wed, 2 Mar 2022 16:10:27 +0800 Subject: [PATCH 18/18] feat(gomod): updated gosdk --- code/go/0chain.net/blobbercore/mock/init.go | 8 ++++++-- code/go/0chain.net/core/transaction/http.go | 8 ++++---- go.mod | 2 +- go.sum | 5 ++--- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/code/go/0chain.net/blobbercore/mock/init.go b/code/go/0chain.net/blobbercore/mock/init.go index 2564584ad..e1c2fa978 100644 --- a/code/go/0chain.net/blobbercore/mock/init.go +++ b/code/go/0chain.net/blobbercore/mock/init.go @@ -1,10 +1,12 @@ package mock import ( + "encoding/json" "io" "net/http" "net/http/httptest" + "github.com/0chain/gosdk/core/zcncrypto" "github.com/0chain/gosdk/sdks" "github.com/0chain/gosdk/sdks/blobber" ) @@ -18,11 +20,13 @@ const ( // ) func NewBlobberClient() *blobber.Blobber { - z := sdks.New("9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85", "40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a", "bls0chain") - err := z.InitWallet(zboxWallet) + wallet := &zcncrypto.Wallet{} + err := json.Unmarshal([]byte(zboxWallet), wallet) //nolint: errcheck if err != nil { panic("mock: z.InitWallet " + err.Error()) } + z := sdks.New("9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85", "40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a", "bls0chain", wallet) + z.NewRequest = func(method, url string, body io.Reader) (*http.Request, error) { return httptest.NewRequest(method, url, body), nil } diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index 57ba421de..dee99d909 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -1,6 +1,7 @@ package transaction import ( + "bytes" "context" "encoding/hex" "hash/fnv" @@ -107,7 +108,7 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] //leave first item for ErrTooLessConfirmation var msgList = make([]string, 1, numSharders) - r := resty.New(transport, func(req *http.Request, resp *http.Response, cancelFunc context.CancelFunc, err error) error { + r := resty.New(transport, func(req *http.Request, resp *http.Response, respBody []byte, cancelFunc context.CancelFunc, err error) error { if err != nil { //network issue msgList = append(msgList, err.Error()) return err @@ -116,7 +117,6 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] url := req.URL.String() if resp.StatusCode != http.StatusOK { - resp.Body.Close() errorMsg := "[sharder]" + resp.Status + ": " + url msgList = append(msgList, errorMsg) @@ -124,9 +124,9 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] } hash := fnv.New32() //use fnv for better performance - teeReader := io.TeeReader(resp.Body, hash) + + teeReader := io.TeeReader(bytes.NewReader(respBody), hash) resBody, err := io.ReadAll(teeReader) - resp.Body.Close() if err != nil { errorMsg := "[sharder]body: " + url + " " + err.Error() diff --git a/go.mod b/go.mod index bb7973944..54279efc6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.7.2 + github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-ini/ini v1.55.0 // indirect diff --git a/go.sum b/go.sum index c9a7595e4..3c8684a81 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.7.2 h1:iWEc36gy0Puzzj9X/OshQ2B4I1XP6EeoAo2vZ2S5A/8= -github.com/0chain/gosdk v1.7.2/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= +github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da h1:Mno3gifncwxsLEfkGUnsKqTvCDUbUioJXkYYH8Thw+8= +github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -878,7 +878,6 @@ github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so=