diff --git a/code/go/0chain.net/blobbercore/handler/context.go b/code/go/0chain.net/blobbercore/handler/context.go index 0f27146a5..7286d2779 100644 --- a/code/go/0chain.net/blobbercore/handler/context.go +++ b/code/go/0chain.net/blobbercore/handler/context.go @@ -84,8 +84,7 @@ func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.Re statusCode = http.StatusInternalServerError } - buf, _ := json.Marshal(err) - http.Error(w, string(buf), statusCode) + http.Error(w, err.Error(), statusCode) return } @@ -97,8 +96,7 @@ func WithHandler(handler func(ctx *Context) (interface{}, error)) func(w http.Re statusCode = http.StatusInternalServerError } - buf, _ := json.Marshal(err) - http.Error(w, string(buf), statusCode) + http.Error(w, err.Error(), statusCode) return } diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index cb7a2710d..29f0069b8 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -78,8 +78,10 @@ func SetupHandlers(r *mux.Router) { // lightweight http handler without heavy postgres transaction to improve performance - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost) - r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(LockWriteMarker)).Methods(http.MethodPost, http.MethodOptions) + r.HandleFunc("/v1/writemarker/lock/{allocation}", WithHandler(UnlockWriteMarker)).Methods(http.MethodDelete, http.MethodOptions) + + r.HandleFunc("/v1/hashnode/root/{allocation}", WithHandler(LoadRootHashnode)).Methods(http.MethodGet, http.MethodOptions) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/blobbercore/handler/handler_hashnode.go b/code/go/0chain.net/blobbercore/handler/handler_hashnode.go new file mode 100644 index 000000000..2a916353a --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_hashnode.go @@ -0,0 +1,16 @@ +//go:build !integration_tests +// +build !integration_tests + +package handler + +import "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + +// LoadRootHashnode load root node with its descendant nodes +func LoadRootHashnode(ctx *Context) (interface{}, error) { + + root, err := reference.LoadRootHashnode(ctx, ctx.AllocationTx) + if err != nil { + return nil, err + } + return root, nil +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go b/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go new file mode 100644 index 000000000..1515b9878 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/handler_hashnode_test.go @@ -0,0 +1,113 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/gorilla/mux" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHashnodeHanders_LoadRootHashnode(t *testing.T) { + + datastore.UseMocket(true) + + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_handler_load_root"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_handler_load_root", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + r := mux.NewRouter() + SetupHandlers(r) + + req, err := http.NewRequest(http.MethodGet, "/v1/refs/root/{allocation}", nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(WithHandler(func(ctx *Context) (interface{}, error) { + ctx.AllocationTx = "allocation_handler_load_root" + return LoadRootHashnode(ctx) + })) + + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + + var root reference.Hashnode + + err = json.Unmarshal(rr.Body.Bytes(), &root) + require.Nil(t, err) + + require.NotNil(t, root) + require.Len(t, root.Children, 2) + + require.Equal(t, root.Children[0].Name, "sub1") + require.Len(t, root.Children[0].Children, 1) + require.Equal(t, root.Children[0].Children[0].Name, "file1") + require.Equal(t, root.Children[1].Name, "sub2") +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go index 02196fd5f..2338dde80 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go @@ -11,10 +11,10 @@ var WriteMarkerMutext = &writemarker.Mutex{} // LockWriteMarker try to lock writemarker for specified allocation id, and return latest RefTree func LockWriteMarker(ctx *Context) (interface{}, error) { - sessionID := ctx.FormValue("session_id") + connectionID := ctx.FormValue("connection_id") requestTime := ctx.FormTime("request_time") - result, err := WriteMarkerMutext.Lock(ctx, ctx.AllocationTx, sessionID, requestTime) + result, err := WriteMarkerMutext.Lock(ctx, ctx.AllocationTx, connectionID, requestTime) if err != nil { return nil, err } @@ -24,7 +24,7 @@ func LockWriteMarker(ctx *Context) (interface{}, error) { // UnlockWriteMarker release WriteMarkerMutex func UnlockWriteMarker(ctx *Context) (interface{}, error) { - sessionID := ctx.FormValue("session_id") + sessionID := ctx.FormValue("connection_id") err := WriteMarkerMutext.Unlock(ctx, ctx.AllocationTx, sessionID) if err != nil { diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go index 48779982f..0e18064df 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker_test.go @@ -28,7 +28,7 @@ func TestWriteMarkerHandlers_Lock(t *testing.T) { now := time.Now() - formWriter.WriteField("session_id", "session_id") //nolint: errcheck + formWriter.WriteField("connection_id", "connection_id") //nolint: errcheck formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck formWriter.Close() @@ -68,7 +68,7 @@ func TestWriteMarkerHandlers_Unlock(t *testing.T) { now := time.Now() - formWriter.WriteField("session_id", "session_id") //nolint: errcheck + formWriter.WriteField("connection_id", "connection_id") //nolint: errcheck formWriter.WriteField("request_time", strconv.FormatInt(now.Unix(), 10)) //nolint: errcheck formWriter.Close() diff --git a/code/go/0chain.net/blobbercore/mock/init.go b/code/go/0chain.net/blobbercore/mock/init.go index 2564584ad..e1c2fa978 100644 --- a/code/go/0chain.net/blobbercore/mock/init.go +++ b/code/go/0chain.net/blobbercore/mock/init.go @@ -1,10 +1,12 @@ package mock import ( + "encoding/json" "io" "net/http" "net/http/httptest" + "github.com/0chain/gosdk/core/zcncrypto" "github.com/0chain/gosdk/sdks" "github.com/0chain/gosdk/sdks/blobber" ) @@ -18,11 +20,13 @@ const ( // ) func NewBlobberClient() *blobber.Blobber { - z := sdks.New("9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85", "40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a", "bls0chain") - err := z.InitWallet(zboxWallet) + wallet := &zcncrypto.Wallet{} + err := json.Unmarshal([]byte(zboxWallet), wallet) //nolint: errcheck if err != nil { panic("mock: z.InitWallet " + err.Error()) } + z := sdks.New("9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85", "40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a", "bls0chain", wallet) + z.NewRequest = func(method, url string, body io.Reader) (*http.Request, error) { return httptest.NewRequest(method, url, body), nil } diff --git a/code/go/0chain.net/blobbercore/reference/dao.go b/code/go/0chain.net/blobbercore/reference/dao.go deleted file mode 100644 index 363ce4825..000000000 --- a/code/go/0chain.net/blobbercore/reference/dao.go +++ /dev/null @@ -1,73 +0,0 @@ -package reference - -import ( - "context" - - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/errors" - "gorm.io/gorm" -) - -// LoadRootNode load root node with its descendant nodes -func LoadRootNode(ctx context.Context, allocationID string) (*HashNode, error) { - - db := datastore.GetStore().GetDB() - - db = db.Where("allocation_id = ? and deleted_at IS NULL", allocationID) - - db = db.Order("level desc, path") - - dict := make(map[string][]*HashNode) - - var nodes []*HashNode - // it is better to load them in batched if there are a lot of objects in db - err := db.FindInBatches(&nodes, 100, func(tx *gorm.DB, batch int) error { - // batch processing found records - for _, object := range nodes { - dict[object.ParentPath] = append(dict[object.ParentPath], object) - - for _, child := range dict[object.Path] { - object.AddChild(child) - } - } - - return nil - }).Error - - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - - // create empty dir if root is missing - if len(dict) == 0 { - return &HashNode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil - } - - rootNodes, ok := dict[""] - - if ok { - if len(rootNodes) == 1 { - return rootNodes[0], nil - } - - return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") - } - - return nil, errors.Throw(common.ErrInternal, "invalid_ref_tree: / is missing or invalid") -} - -const ( - SQLWhereGetByAllocationTxAndPath = "reference_objects.allocation_id = ? and reference_objects.path = ? and deleted_at is NULL" -) - -// DryRun Creates a prepared statement when executing any SQL and caches them to speed up future calls -// https://gorm.io/docs/performance.html#Caches-Prepared-Statement -func DryRun(db *gorm.DB) { - - // https://gorm.io/docs/session.html#DryRun - // Session mode - //tx := db.Session(&gorm.Session{PrepareStmt: true, DryRun: true}) - - // use Table instead of Model to reduce reflect times -} diff --git a/code/go/0chain.net/blobbercore/reference/entity.go b/code/go/0chain.net/blobbercore/reference/entity.go index 2fae95c28..d0e2ee523 100644 --- a/code/go/0chain.net/blobbercore/reference/entity.go +++ b/code/go/0chain.net/blobbercore/reference/entity.go @@ -1,14 +1,11 @@ package reference import ( - "strconv" - "strings" - "gorm.io/datatypes" ) -// HashNode ref node in hash tree -type HashNode struct { +// Hashnode ref node in hash tree +type Hashnode struct { // hash data AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"` Type string `gorm:"column:type" json:"type,omitempty"` @@ -24,46 +21,23 @@ type HashNode struct { // other data ParentPath string `gorm:"parent_path" json:"-"` - Children []*HashNode `gorm:"-" json:"children,omitempty"` + Children []*Hashnode `gorm:"-" json:"children,omitempty"` } // TableName get table name of Ref -func (HashNode) TableName() string { +func (Hashnode) TableName() string { return TableNameReferenceObjects } -func (n *HashNode) AddChild(c *HashNode) { +func (n *Hashnode) AddChild(c *Hashnode) { if n.Children == nil { - n.Children = make([]*HashNode, 0, 10) + n.Children = make([]*Hashnode, 0, 10) } n.Children = append(n.Children, c) } // GetLookupHash get lookuphash -func (n *HashNode) GetLookupHash() string { +func (n *Hashnode) GetLookupHash() string { return GetReferenceLookup(n.AllocationID, n.Path) } - -// GetHashCode get hash code -func (n *HashNode) GetHashCode() string { - - if len(n.Attributes) == 0 { - n.Attributes = datatypes.JSON("{}") - } - hashArray := []string{ - n.AllocationID, - n.Type, - n.Name, - n.Path, - strconv.FormatInt(n.Size, 10), - n.ContentHash, - n.MerkleRoot, - strconv.FormatInt(n.ActualFileSize, 10), - n.ActualFileHash, - string(n.Attributes), - strconv.FormatInt(n.ChunkSize, 10), - } - - return strings.Join(hashArray, ":") -} diff --git a/code/go/0chain.net/blobbercore/reference/hashnode.go b/code/go/0chain.net/blobbercore/reference/hashnode.go new file mode 100644 index 000000000..a2393e6bd --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/hashnode.go @@ -0,0 +1,64 @@ +package reference + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/errors" +) + +// LoadRootHashnode load root node with its descendant nodes +func LoadRootHashnode(ctx context.Context, allocationID string) (*Hashnode, error) { + + db := datastore.GetStore().GetDB() + + db = db.Raw(` +SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects +WHERE allocation_id = ? and deleted_at IS NULL +ORDER BY level desc, path`, allocationID) + + rows, err := db.Rows() + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + defer rows.Close() + + nodes := make(map[string]*Hashnode) + for rows.Next() { + + node := &Hashnode{} + err = db.ScanRows(rows, node) + if err != nil { + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + } + + _, ok := nodes[node.Path] + if ok { + return nil, common.ErrDuplicatedNode + } + + nodes[node.Path] = node + + parent, ok := nodes[node.ParentPath] + if ok { + parent.AddChild(node) + } + + } + + // create empty dir if root is missing + if len(nodes) == 0 { + return &Hashnode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil + } + + root, ok := nodes["/"] + + if ok { + return root, nil + } + + return nil, common.ErrMissingRootNode +} diff --git a/code/go/0chain.net/blobbercore/reference/hashnode_test.go b/code/go/0chain.net/blobbercore/reference/hashnode_test.go new file mode 100644 index 000000000..c6a8d5ae5 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/hashnode_test.go @@ -0,0 +1,390 @@ +package reference + +import ( + "context" + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + gomocket "github.com/selvatico/go-mocket" + "github.com/stretchr/testify/require" +) + +func TestHashnode_Should_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *Hashnode, error) + }{ + { + name: "No any node should work", + allocationID: "allocation_none", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_none"). + WithReply(nil) + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 0) + + require.Equal(test, allocationID, r.AllocationID) + require.Equal(test, DIRECTORY, r.Type) + require.Equal(test, "/", r.Name) + require.Equal(test, "/", r.Path) + require.Equal(test, "", r.ContentHash) + require.Equal(test, "", r.MerkleRoot) + require.Equal(test, "", r.ActualFileHash) + require.EqualValues(test, 0, r.ChunkSize) + require.EqualValues(test, 0, r.Size) + require.EqualValues(test, 0, r.ActualFileSize) + + buf, e := r.Attributes.MarshalJSON() //nolint + require.Nil(test, e) + + require.Equal(test, "null", string(buf)) + + require.Equal(test, "", r.ParentPath) + + }, + }, + { + name: "Nested node should work", + allocationID: "allocation_nested", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_nested"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_nested", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.NotNil(test, r) + require.Len(test, r.Children, 2) + + require.Equal(test, r.Children[0].Name, "sub1") + require.Len(test, r.Children[0].Children, 1) + require.Equal(test, r.Children[0].Children[0].Name, "file1") + require.Equal(test, r.Children[1].Name, "sub2") + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRootHashnode(context.TODO(), it.allocationID) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} + +func TestHashnode_Should_Not_Work(t *testing.T) { + + datastore.UseMocket(true) + + tests := []struct { + name string + allocationID string + mock func() + assert func(*testing.T, string, *Hashnode, error) + }{ + { + name: "Missing root node should not work", + allocationID: "allocation_missing", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`).WithArgs("allocation_missing"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_missing", + "type": "D", + "name": "sub", + "path": "/sub", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }}) + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrMissingRootNode, err) + + }, + }, + { + name: "Duplicated root node should not work", + allocationID: "allocation_duplicated_root", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_duplicated_root"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_root", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrDuplicatedNode, err) + + }, + }, + { + name: "Duplicated node should not work", + allocationID: "allocation_duplicated_node", + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT allocation_id, type, name, path, content_hash, merkle_root, actual_file_hash, attributes, chunk_size,size,actual_file_size, parent_path +FROM reference_objects`). + WithArgs("allocation_duplicated_node"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "/", + "path": "/", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "sub1", + "path": "/sub1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "sub2", + "path": "/sub2", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + { + "allocation_id": "allocation_duplicated_node", + "type": "D", + "name": "file1", + "path": "/sub1/file1", + "content_hash": "", + "merkle_root": "", + "actual_file_hash": "", + "attributes": []byte("null"), + "chunk_size": 0, + "size": 0, + "actual_file_size": 0, + "parent_path": "/sub1", + }, + }) + + }, + assert: func(test *testing.T, allocationID string, r *Hashnode, err error) { + require.Nil(test, r) + require.ErrorIs(test, common.ErrDuplicatedNode, err) + + }, + }, + } + + for _, it := range tests { + + t.Run(it.name, + func(test *testing.T) { + if it.mock != nil { + it.mock() + } + + r, err := LoadRootHashnode(context.TODO(), it.allocationID) + + it.assert(test, it.allocationID, r, err) + + }, + ) + + } + +} diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index daeaddab0..8c2349f92 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -67,7 +67,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req CreatedAt: *requestTime, } - err = db.Create(&lock).Error + err = db.Table(datastore.TableNameWriteLock).Create(&lock).Error if err != nil { return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } @@ -92,7 +92,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req lock.ConnectionID = connectionID lock.CreatedAt = *requestTime - err = db.Save(&lock).Error + err = db.Table(datastore.TableNameWriteLock).Where("allocation_id=?", allocationID).Save(&lock).Error if err != nil { return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } @@ -120,19 +120,19 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string, req } -func (*Mutex) Unlock(ctx context.Context, allocationID string, sessionID string) error { +func (*Mutex) Unlock(ctx context.Context, allocationID string, connectionID string) error { if len(allocationID) == 0 { return nil } - if len(sessionID) == 0 { + if len(connectionID) == 0 { return nil } db := datastore.GetStore().GetDB() - err := db.Where("allocation_id = ? and session_id =? ", allocationID, sessionID).Delete(&datastore.WriteLock{}).Error + err := db.Where("allocation_id = ? and connection_id = ? ", allocationID, connectionID).Delete(&datastore.WriteLock{}).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index e36f0fe15..b7fb23810 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -61,6 +61,7 @@ func TestMutext_LockShouldWork(t *testing.T) { assert: func(test *testing.T, r *LockResult, err error) { require.Nil(test, err) require.Equal(test, LockStatusOK, r.Status) + require.EqualValues(test, now.Unix(), r.CreatedAt) }, }, { @@ -86,7 +87,7 @@ func TestMutext_LockShouldWork(t *testing.T) { }, }, { - name: "lock should ok if it is timeout", + name: "lock should be ok if it is timeout", allocationID: "lock_timeout_allocation_id", connectionID: "lock_timeout_2nd_connection_id", requestTime: now, @@ -107,6 +108,28 @@ func TestMutext_LockShouldWork(t *testing.T) { require.Equal(test, LockStatusPending, r.Status) }, }, + { + name: "retry lock by same request should work if it is timeout", + allocationID: "lock_same_timeout_allocation_id", + connectionID: "lock_same_timeout_connection_id", + requestTime: now, + mock: func() { + gomocket.Catcher.NewMock(). + WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). + WithArgs("lock_same_timeout_allocation_id"). + WithReply([]map[string]interface{}{ + { + "allocation_id": "lock_same_timeout_allocation_id", + "connection_id": "lock_same_timeout_connection_id", + "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), + }, + }) + }, + assert: func(test *testing.T, r *LockResult, err error) { + require.Nil(test, err) + require.NotNil(test, r) + }, + }, } for _, it := range tests { @@ -134,7 +157,6 @@ func TestMutext_LockShouldNotWork(t *testing.T) { config.Configuration.WriteMarkerLockTimeout = 30 * time.Second m := &Mutex{} - now := time.Now() tests := []struct { name string @@ -157,28 +179,6 @@ func TestMutext_LockShouldNotWork(t *testing.T) { require.NotNil(test, err) }, }, - { - name: "retry lock by same request should not work if it is timeout", - allocationID: "lock_same_timeout_allocation_id", - connectionID: "lock_same_timeout_connection_id", - requestTime: now, - mock: func() { - gomocket.Catcher.NewMock(). - WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). - WithArgs("lock_same_timeout_allocation_id"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "lock_same_timeout_allocation_id", - "connection_id": "lock_same_timeout_connection_id", - "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), - }, - }) - }, - assert: func(test *testing.T, r *LockResult, err error) { - require.NotNil(test, err) - require.Nil(test, r) - }, - }, } for _, it := range tests { diff --git a/code/go/0chain.net/core/common/constants.go b/code/go/0chain.net/core/common/constants.go index 4314f134b..8ed461d5c 100644 --- a/code/go/0chain.net/core/common/constants.go +++ b/code/go/0chain.net/core/common/constants.go @@ -33,6 +33,11 @@ var ( // ErrEntityNotFound entity can't found in db ErrEntityNotFound = errors.New("entity not found") + // ErrMissingRootNode root node is missing + ErrMissingRootNode = errors.New("root node is missing") + + // ErrDuplicatedNode duplicated nodes + ErrDuplicatedNode = errors.New("duplicated nodes") // ErrFileWasDeleted file already was deleted ErrFileWasDeleted = errors.New("file was deleted") ) diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index 57ba421de..dee99d909 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -1,6 +1,7 @@ package transaction import ( + "bytes" "context" "encoding/hex" "hash/fnv" @@ -107,7 +108,7 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] //leave first item for ErrTooLessConfirmation var msgList = make([]string, 1, numSharders) - r := resty.New(transport, func(req *http.Request, resp *http.Response, cancelFunc context.CancelFunc, err error) error { + r := resty.New(transport, func(req *http.Request, resp *http.Response, respBody []byte, cancelFunc context.CancelFunc, err error) error { if err != nil { //network issue msgList = append(msgList, err.Error()) return err @@ -116,7 +117,6 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] url := req.URL.String() if resp.StatusCode != http.StatusOK { - resp.Body.Close() errorMsg := "[sharder]" + resp.Status + ": " + url msgList = append(msgList, errorMsg) @@ -124,9 +124,9 @@ func makeSCRestAPICall(scAddress string, relativePath string, params map[string] } hash := fnv.New32() //use fnv for better performance - teeReader := io.TeeReader(resp.Body, hash) + + teeReader := io.TeeReader(bytes.NewReader(respBody), hash) resBody, err := io.ReadAll(teeReader) - resp.Body.Close() if err != nil { errorMsg := "[sharder]body: " + url + " " + err.Error() diff --git a/go.mod b/go.mod index d9e844bef..54279efc6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15 + github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-ini/ini v1.55.0 // indirect @@ -38,7 +38,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gorm.io/datatypes v0.0.0-20200806042100-bc394008dd0d gorm.io/driver/postgres v1.3.1 - gorm.io/gorm v1.23.1 + gorm.io/gorm v1.23.2 nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index a154c6548..3c8684a81 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15 h1:cOqg66kR646dc1NNgduE0Ridlg0bDDvhxOPC9rhrNWk= -github.com/0chain/gosdk v1.7.1-0.20220219170933-3eac488a6f15/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= +github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da h1:Mno3gifncwxsLEfkGUnsKqTvCDUbUioJXkYYH8Thw+8= +github.com/0chain/gosdk v1.7.3-0.20220302075118-819a9fe8d1da/go.mod h1:G/JUrqvT2WStxFbSpJKnU1Wt37GyatimoqPJfEE10bs= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -878,7 +878,6 @@ github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= @@ -1638,8 +1637,9 @@ gorm.io/driver/sqlserver v0.2.5/go.mod h1:TcPfkdce5b8qlCMgyUeUdm7HQa1ZzWUuxzI+od gorm.io/gorm v0.2.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v0.2.27/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.23.1 h1:aj5IlhDzEPsoIyOPtTRVI+SyaN1u6k613sbt4pwbxG0= gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.2 h1:xmq9QRMWL8HTJyhAUBXy8FqIIQCYESeKfJL4DoGKiWQ= +gorm.io/gorm v1.23.2/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=