Skip to content

Commit

Permalink
Sanity check. Remove mod go-eth (#103)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dpunish3r
dpunish3r committed Jan 3, 2024
1 parent 3a715fb commit 96cb9e5
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 31 deletions.
61 changes: 49 additions & 12 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,12 @@ const (
)

var (
sanityEntry uint64 = 0
sanityBlock uint64 = 0
sanityBookmark uint64 = 0
initSanityEntry bool = false
initSanityBlock bool = false
initSanityBookmark bool = false
sanityEntry uint64 = 0
sanityBlock uint64 = 0
sanityBookmark uint64 = 0
)

// main runs a datastream server or client
Expand Down Expand Up @@ -465,20 +468,34 @@ func printEntryNum(e *datastreamer.FileEntry, c *datastreamer.StreamClient, s *d

// checkEntryBlockSanity checks entry, bookmark, and block sequence consistency
func checkEntryBlockSanity(e *datastreamer.FileEntry, c *datastreamer.StreamClient, s *datastreamer.StreamServer) error {
// Sanity check initialization
if !initSanityEntry {
initSanityEntry = true
if c.FromEntry > 0 {
sanityEntry = c.FromEntry
} else {
sanityEntry = 0
}
}

// Log work in progress
if e.Number%100000 == 0 {
log.Infof("Sanity check entry #%d...", e.Number)
log.Infof("Checking entry #%d...", e.Number)
}

// Sanity check for entry sequence
if sanityEntry > 0 {
if e.Number != sanityEntry {
log.Infof("SANITY CHECK failed: Entry received[%d] | Entry expected[%d]", e.Number, sanityEntry)
if e.Number < sanityEntry {
log.Warnf("(X) SANITY CHECK failed: REPEATED entries? Received[%d] | Entry expected[%d]", e.Number, sanityEntry)
} else {
log.Warnf("(X) SANITY CHECK failed: GAP entries? Received[%d] | Entry expected[%d]", e.Number, sanityEntry)
}
return errors.New("sanity check failed for entry sequence")
}
} else {
if e.Number != 0 {
log.Infof("SANITY CHECK failed: Entry received[%d] | Entry expected[0]", e.Number)
log.Warnf("(X) SANITY CHECK failed: Entry received[%d] | Entry expected[0]", e.Number)
return errors.New("sanity check failed for entry sequence")
}
}
Expand All @@ -489,13 +506,23 @@ func checkEntryBlockSanity(e *datastreamer.FileEntry, c *datastreamer.StreamClie
blockNum := binary.LittleEndian.Uint64(e.Data[8:16])
if sanityBlock > 0 {
if blockNum != sanityBlock {
log.Infof("SANITY CHECK failed: Block received[%d] | Block expected[%d]", blockNum, sanityBlock)
if blockNum < sanityBlock {
log.Infof("(X) SANITY CHECK failed (%d): REPEATED blocks? Received[%d] | Block expected[%d]", e.Number, blockNum, sanityBlock)
} else {
log.Infof("(X) SANITY CHECK failed (%d): GAP blocks? Received[%d] | Block expected[%d]", e.Number, blockNum, sanityBlock)
}
sanityBlock = blockNum
}
} else {
if blockNum != 0 {
log.Infof("SANITY CHECK failed: Block received[%d] | Block expected[0]", blockNum)
sanityBlock = 0
if initSanityBlock {
log.Infof("(X) SANITY CHECK failed (%d): Block received[%d] | Block expected[0]", e.Number, blockNum)
sanityBlock = 0
} else {
log.Infof("SANITY CHECK note (%d): First Block received[%d]", e.Number, blockNum)
sanityBlock = blockNum
}
initSanityBlock = true
}
}
sanityBlock++
Expand All @@ -506,13 +533,23 @@ func checkEntryBlockSanity(e *datastreamer.FileEntry, c *datastreamer.StreamClie
bookmarkNum := binary.LittleEndian.Uint64(e.Data[1:9])
if sanityBookmark > 0 {
if bookmarkNum != sanityBookmark {
log.Infof("SANITY CHECK failed: Bookmark received[%d] | Bookmark expected[%d]", bookmarkNum, sanityBookmark)
if bookmarkNum < sanityBookmark {
log.Infof("(X) SANITY CHECK failed (%d): REPEATED bookmarks? Received[%d] | Bookmark expected[%d]", e.Number, bookmarkNum, sanityBookmark)
} else {
log.Infof("(X) SANITY CHECK failed (%d): GAP bookmarks? Received[%d] | Bookmark expected[%d]", e.Number, bookmarkNum, sanityBookmark)
}
sanityBookmark = bookmarkNum
}
} else {
if bookmarkNum != 0 {
log.Infof("SANITY CHECK failed: Bookmark received[%d] | Bookmark expected[0]", bookmarkNum)
sanityBookmark = 0
if initSanityBookmark {
log.Infof("(X) SANITY CHECK failed (%d): Bookmark received[%d] | Bookmark expected[0]", e.Number, bookmarkNum)
sanityBookmark = 0
} else {
log.Infof("SANITY CHECK note (%d): First Bookmark received[%d]", e.Number, bookmarkNum)
sanityBookmark = bookmarkNum
}
initSanityBookmark = true
}
}
sanityBookmark++
Expand Down
66 changes: 54 additions & 12 deletions datastreamer/datastreamer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package datastreamer_test

import (
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"os"
Expand All @@ -10,14 +11,55 @@ import (

"github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer"
"github.com/0xPolygonHermez/zkevm-data-streamer/log"
"github.com/ethereum/go-ethereum/common"
"github.com/stretchr/testify/require"
)

// AUX ------------------------------------------------------------------------
const hashLength = 32

type hash [hashLength]byte

func (h *hash) setBytes(b []byte) {
if len(b) > len(h) {
b = b[len(b)-hashLength:]
}

copy(h[hashLength-len(b):], b)
}

func bytesToHash(b []byte) hash {
var h hash
h.setBytes(b)
return h
}

func has0xPrefix(str string) bool {
return len(str) >= 2 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')
}

func hex2Bytes(str string) []byte {
h, _ := hex.DecodeString(str)
return h
}

func fromHex(s string) []byte {
if has0xPrefix(s) {
s = s[2:]
}
if len(s)%2 == 1 {
s = "0" + s
}
return hex2Bytes(s)
}

func hexToHash(s string) hash { return bytesToHash(fromHex(s)) }

// ----------------------------------------------------------------------------

type TestEntry struct {
FieldA uint64 // 8 bytes
FieldB common.Hash // 32 bytes
FieldC []byte // n bytes
FieldA uint64 // 8 bytes
FieldB hash // 32 bytes
FieldC []byte // n bytes
}

type TestBookmark struct {
Expand All @@ -41,7 +83,7 @@ func (t TestEntry) Encode() []byte {

func (t TestEntry) Decode(bytes []byte) TestEntry {
t.FieldA = binary.LittleEndian.Uint64(bytes[:8])
t.FieldB = common.BytesToHash(bytes[8:40])
t.FieldB = bytesToHash(bytes[8:40])
t.FieldC = bytes[40:]
return t
}
Expand Down Expand Up @@ -69,27 +111,27 @@ var (
testEntries = []TestEntry{
{
FieldA: 0,
FieldB: common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldB: hexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldC: []byte("test entry 0"),
},
{
FieldA: 1,
FieldB: common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldB: hexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldC: []byte("test entry 1"),
},
{
FieldA: 2,
FieldB: common.HexToHash("0x2234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldB: hexToHash("0x2234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldC: []byte("test entry 2"),
},
{
FieldA: 3,
FieldB: common.HexToHash("0x3234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldB: hexToHash("0x3234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldC: []byte("test entry 3"),
},
{
FieldA: 4,
FieldB: common.HexToHash("0x3234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldB: hexToHash("0x3234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
FieldC: []byte("large test entry 4 large test entry 4 large test entry 4 large test entry 4" +
"large test entry 4 large test entry 4 large test entry 4 large test entry 4" +
"large test entry 4 large test entry 4 large test entry 4 large test entry 4" +
Expand All @@ -105,13 +147,13 @@ var (

badUpdateEntry = TestEntry{
FieldA: 10,
FieldB: common.HexToHash("0xa1cdef7890abcdef1234567890abcdef1234567890abcdef1234567890123456"),
FieldB: hexToHash("0xa1cdef7890abcdef1234567890abcdef1234567890abcdef1234567890123456"),
FieldC: []byte("test entry not updated"),
}

okUpdateEntry = TestEntry{
FieldA: 11,
FieldB: common.HexToHash("0xa2cdef7890abcdef1234567890abcdef1234567890abcdef1234567890123456"),
FieldB: hexToHash("0xa2cdef7890abcdef1234567890abcdef1234567890abcdef1234567890123456"),
FieldC: []byte("update entry"),
}

Expand Down
2 changes: 0 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ module github.com/0xPolygonHermez/zkevm-data-streamer
go 1.19

require (
github.com/ethereum/go-ethereum v1.13.2
github.com/hermeznetwork/tracerr v0.3.2
github.com/mitchellh/mapstructure v1.5.0
github.com/spf13/viper v1.17.0
Expand Down Expand Up @@ -33,7 +32,6 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect
Expand Down
6 changes: 1 addition & 5 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw=
github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
Expand Down Expand Up @@ -225,8 +223,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand Down Expand Up @@ -526,8 +522,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down

0 comments on commit 96cb9e5

Please sign in to comment.