Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/feature/snapshot' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
@jubeless
jubeless committed Jul 27, 2020
2 parents f68c757 + 5eeeb38 commit cce276e
Show file tree
Hide file tree
Showing 8 changed files with 182 additions and 60 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
@@ -1,2 +1,3 @@
.envrc
.idea
.DS_Store
2 changes: 1 addition & 1 deletion snapshot/reader.go
View file
Expand Up @@ -92,7 +92,7 @@ func (r *Reader) Next() (*Section, error) {
r.nextOffset = uint64(beginOffset) + sectionSize + 8 // well well, sectionSize includes the rowCount I guess?

return &Section{
Name: strings.TrimRight(str, string([]byte{0x00})),
Name: stringToSectionName(strings.TrimRight(str, string([]byte{0x00}))),
Offset: uint64(beginOffset),
Size: sectionSize,
RowCount: rowCount,
Expand Down
116 changes: 93 additions & 23 deletions snapshot/section.go
View file
Expand Up @@ -2,8 +2,80 @@ package snapshot

import "io"

type SectionName string

const (
SectionNameChainSnapshotHeader SectionName = "eosio::chain::chain_snapshot_header"
SectionNameBlockState SectionName = "eosio::chain::block_state"
SectionNameAccountObject SectionName = "eosio::chain::account_object"
SectionNameAccountMetadataObject SectionName = "eosio::chain::account_metadata_object"
SectionNameAccountRamCorrectionObject SectionName = "eosio::chain::account_ram_correction_object"
SectionNameGlobalPropertyObject SectionName = "eosio::chain::global_property_object"
SectionNameProtocolStateObject SectionName = "eosio::chain::protocol_state_object"
SectionNameDynamicGlobalPropertyObject SectionName = "eosio::chain::dynamic_global_property_object"
SectionNameBlockSummaryObject SectionName = "eosio::chain::block_summary_object"
SectionNameTransactionObject SectionName = "eosio::chain::transaction_object"
SectionNameGeneratedTransactionObject SectionName = "eosio::chain::generated_transaction_object"
SectionNameCodeObject SectionName = "eosio::chain::code_object"
SectionNameContractTables SectionName = "contract_tables"
SectionNamePermissionObject SectionName = "eosio::chain::permission_object"
SectionNamePermissionLinkObject SectionName = "eosio::chain::permission_link_object"
SectionNameResourceLimitsObject SectionName = "eosio::chain::resource_limits::resource_limits_object"
SectionNameResourceUsageObject SectionName = "eosio::chain::resource_limits::resource_usage_object"
SectionNameResourceLimitsStateObject SectionName = "eosio::chain::resource_limits::resource_limits_state_object"
SectionNameResourceLimitsConfigObject SectionName = "eosio::chain::resource_limits::resource_limits_config_object"
SectionNameGenesisState SectionName = "eosio::chain::genesis_state"
)

func stringToSectionName(name string) SectionName {
switch name {
case "eosio::chain::chain_snapshot_header":
return SectionNameChainSnapshotHeader
case "eosio::chain::block_state":
return SectionNameBlockState
case "eosio::chain::account_object":
return SectionNameAccountObject
case "eosio::chain::account_metadata_object":
return SectionNameAccountMetadataObject
case "eosio::chain::account_ram_correction_object":
return SectionNameAccountRamCorrectionObject
case "eosio::chain::global_property_object":
return SectionNameGlobalPropertyObject
case "eosio::chain::protocol_state_object":
return SectionNameProtocolStateObject
case "eosio::chain::dynamic_global_property_object":
return SectionNameDynamicGlobalPropertyObject
case "eosio::chain::block_summary_object":
return SectionNameBlockSummaryObject
case "eosio::chain::transaction_object":
return SectionNameTransactionObject
case "eosio::chain::generated_transaction_object":
return SectionNameGeneratedTransactionObject
case "eosio::chain::code_object":
return SectionNameCodeObject
case "contract_tables":
return SectionNameContractTables
case "eosio::chain::permission_object":
return SectionNamePermissionObject
case "eosio::chain::permission_link_object":
return SectionNamePermissionLinkObject
case "eosio::chain::resource_limits::resource_limits_object":
return SectionNameResourceLimitsObject
case "eosio::chain::resource_limits::resource_usage_object":
return SectionNameResourceUsageObject
case "eosio::chain::resource_limits::resource_limits_state_object":
return SectionNameResourceLimitsStateObject
case "eosio::chain::resource_limits::resource_limits_config_object":
return SectionNameResourceLimitsConfigObject
case "eosio::chain::genesis_state":
return SectionNameGenesisState
default:
panic("unsupported section name: " + name)
}
}

type Section struct {
Name string
Name SectionName
Offset uint64
Size uint64 // This includes the section name and row count
BufferSize uint64 // This represents the bytes that are following the section header
Expand All @@ -20,48 +92,46 @@ type callbackFunc func(obj interface{}) error

func (s *Section) Process(f callbackFunc) error {
switch s.Name {
case "eosio::chain::chain_snapshot_header":
case SectionNameChainSnapshotHeader:
return s.readChainSnapshotHeader(f)
case "eosio::chain::block_state":
case SectionNameBlockState:
return s.readBlockState(f)
case "eosio::chain::account_object":
case SectionNameAccountObject:
return s.readAccountObjects(f)
case "eosio::chain::account_metadata_object":
case SectionNameAccountMetadataObject:
return s.readAccountMetadataObjects(f)
case "eosio::chain::account_ram_correction_object":
case SectionNameAccountRamCorrectionObject:
return s.readAccountRAMCorrectionObject(f)
case "eosio::chain::global_property_object":
case SectionNameGlobalPropertyObject:
return s.readGlobalPropertyObject(f)
case "eosio::chain::protocol_state_object":
case SectionNameProtocolStateObject:
return s.readProtocolStateObject(f)
case "eosio::chain::dynamic_global_property_object":
case SectionNameDynamicGlobalPropertyObject:
return s.readDynamicGlobalPropertyObject(f)
case "eosio::chain::block_summary_object":
case SectionNameBlockSummaryObject:
return s.readBlockSummary(f)
case "eosio::chain::transaction_object":
case SectionNameTransactionObject:
return s.readTransactionObject(f)
case "eosio::chain::generated_transaction_object":
case SectionNameGeneratedTransactionObject:
return s.readGeneratedTransactionObject(f)
case "eosio::chain::code_object":
case SectionNameCodeObject:
return s.readCodeObject(f)
case "contract_tables":
case SectionNameContractTables:
return s.readContractTables(f)
case "eosio::chain::permission_object":
case SectionNamePermissionObject:
return s.readPermissionObject(f)
case "eosio::chain::permission_link_object":
case SectionNamePermissionLinkObject:
return s.readPermissionLinkObject(f)
case "eosio::chain::resource_limits::resource_limits_object":
case SectionNameResourceLimitsObject:
return s.readResourceLimitsObject(f)
case "eosio::chain::resource_limits::resource_usage_object":
case SectionNameResourceUsageObject:
return s.readResourceUsageObject(f)
case "eosio::chain::resource_limits::resource_limits_state_object":
case SectionNameResourceLimitsStateObject:
return s.readResourceLimitsStateObject(f)
case "eosio::chain::resource_limits::resource_limits_config_object":
case SectionNameResourceLimitsConfigObject:
return s.readResourceLimitsConfigObject(f)

case "eosio::chain::genesis_state":
case SectionNameGenesisState:
return s.readGenesisState(f)

default:
panic("unsupported section: " + s.Name)
}
Expand Down
118 changes: 86 additions & 32 deletions snapshot/snapshot_test.go
View file
Expand Up @@ -4,50 +4,100 @@ import (
"fmt"
"io"
"os"
"strings"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)

func TestSnapshotRead(t *testing.T) {
// filename := "/tmp/0125111385-07750c59b24ed52d2dbf2048b67b58e9c9bd53ff5cc4550277718c1d5d800f73-snapshot.bin" // mainnet
// filename := "/tmp/0003212331-0031042b02b2cf711fee6e1e24da94101fa6c1ea9ece568d5f13232473429db1-snapshot.bin" // kylin
filename := os.Getenv("READ_SNAPSHOT_FILE")
if filename == "" || !fileExists(filename) {
t.Skipf("Environment varaible 'READ_SNAPSHOT_FILE' not set or value %q is not an exisiting file", filename)
return

//if os.Getenv("READ_SNAPSHOT_FILE") != "true" {
// t.Skipf("Environment varaible 'READ_SNAPSHOT_FILE' not set to true")
// return
//}

logger, _ := zap.NewDevelopment()
tests := []struct {
name string
testFile string
}{
{name: "eos-local dev", testFile: "eos-jdev_0000000638.bin"},
{name: "eos-dev1", testFile: "eos-dev1_0004841949.bin"},
{name: "Battlefield - b8d703ed1", testFile: "battlefield-snapshot.bin"},
}
r, err := NewReader(filename)
fmt.Println("Filename", filename)
defer r.Close()

assert.NoError(t, err)
assert.Equal(t, r.Header.Version, uint32(1))
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testFile := testData(test.testFile)

for {
section, err := r.Next()
if err == io.EOF {
break
}
assert.NoError(t, err)
fmt.Println("Section", section.Name, "rows", section.RowCount, "bytes", section.BufferSize, "offset", section.Offset)
if !fileExists(testFile) {
logger.Error("test file not found", zap.String("testfile", testFile))
return
}

if strings.Contains(section.Name, "contract") {
require.NoError(t, section.Process(func(o interface{}) error {
switch obj := o.(type) {
case *TableIDObject:
fmt.Println("Table ID", obj.Code, obj.Scope, obj.TableName)
case *KeyValueObject:
fmt.Println("KV", obj.PrimKey, obj.Value)
default:
fmt.Printf("Ignoring row %T\n", obj)
}
return nil
r, err := NewReader(testFile)
require.NoError(t, err)
defer r.Close()

assert.NoError(t, err)
assert.Equal(t, r.Header.Version, uint32(1))

}))
}
for {
section, err := r.Next()
if err == io.EOF {
break
}
assert.NoError(t, err)
logger.Info("new section",
zap.String("section_name", string(section.Name)),
zap.Uint64("row_count", section.RowCount),
zap.Uint64("bytes_count", section.BufferSize),
zap.Uint64("bytes_count", section.Offset),
)
switch section.Name {
case SectionNameAccountObject:
require.NoError(t, section.Process(func(o interface{}) error {
acc, ok := o.(AccountObject)
if !ok {
return fmt.Errorf("process account object: unexpected object type: %T", o)
}
logger.Info("new account object",
zap.String("name", string(acc.Name)),
)
return nil
}))
case SectionNameContractTables:
var currentTable *TableIDObject
require.NoError(t, section.Process(func(o interface{}) error {
switch obj := o.(type) {
case *TableIDObject:
logger.Info("Table ID", zap.Reflect("table_id", obj))
currentTable = obj
case *KeyValueObject:
if currentTable.Count < 20 {
logger.Info("Key Value Object", zap.Reflect("kv", obj))
}
case *Index64Object:
logger.Info("Index64Object", zap.Reflect("index_64_object", obj))
case *Index128Object:
logger.Info("Index128Object", zap.Reflect("index_128_object", obj))
case *Index256Object:
logger.Info("Index256Object", zap.Reflect("index_256_object", obj))
case *IndexDoubleObject:
logger.Info("IndexDoubleObject", zap.Reflect("index_double_object", obj))
case *IndexLongDoubleObject:
logger.Info("IndexLongDoubleObject", zap.Reflect("index_long_object", obj))
default:
fmt.Printf("Ignoring row %T\n", obj)
}
return nil
}))
}
}
})
}
}

Expand All @@ -63,3 +113,7 @@ func fileExists(path string) bool {

return !info.IsDir()
}

func testData(filename string) string {
return filepath.Join("test-data", filename)
}
Binary file added snapshot/test-data/battlefield-snapshot.bin
View file
Binary file not shown.
Binary file added snapshot/test-data/eos-dev1_0004841949.bin
View file
Binary file not shown.
Binary file added snapshot/test-data/eos-jdev_0000000638.bin
View file
Binary file not shown.
5 changes: 1 addition & 4 deletions snapshot/typesv3.go
View file
Expand Up @@ -14,12 +14,10 @@ type TableIDObject struct {
Scope string
TableName string
Payer string
Count uint32
Count uint32 // represents the number of rows & indices for a given table
}

type ContractRow struct {
TableID *TableIDObject

PrimKey string
Payer string
}
Expand Down Expand Up @@ -131,7 +129,6 @@ func (section *Section) readContractTables(f callbackFunc) error {
}

contractRow := ContractRow{
TableID: t,
PrimKey: eos.NameToString(binary.LittleEndian.Uint64(head[0:8])),
Payer: eos.NameToString(binary.LittleEndian.Uint64(head[8:16])),
}
Expand Down

0 comments on commit cce276e

Please sign in to comment.