diff --git a/.gitignore b/.gitignore index f1d97b07..c494c3b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .envrc .idea +.DS_Store diff --git a/snapshot/reader.go b/snapshot/reader.go index d32996a4..e9c2e419 100644 --- a/snapshot/reader.go +++ b/snapshot/reader.go @@ -92,7 +92,7 @@ func (r *Reader) Next() (*Section, error) { r.nextOffset = uint64(beginOffset) + sectionSize + 8 // well well, sectionSize includes the rowCount I guess? return &Section{ - Name: strings.TrimRight(str, string([]byte{0x00})), + Name: stringToSectionName(strings.TrimRight(str, string([]byte{0x00}))), Offset: uint64(beginOffset), Size: sectionSize, RowCount: rowCount, diff --git a/snapshot/section.go b/snapshot/section.go index cffe5562..4336db42 100644 --- a/snapshot/section.go +++ b/snapshot/section.go @@ -2,8 +2,80 @@ package snapshot import "io" +type SectionName string + +const ( + SectionNameChainSnapshotHeader SectionName = "eosio::chain::chain_snapshot_header" + SectionNameBlockState SectionName = "eosio::chain::block_state" + SectionNameAccountObject SectionName = "eosio::chain::account_object" + SectionNameAccountMetadataObject SectionName = "eosio::chain::account_metadata_object" + SectionNameAccountRamCorrectionObject SectionName = "eosio::chain::account_ram_correction_object" + SectionNameGlobalPropertyObject SectionName = "eosio::chain::global_property_object" + SectionNameProtocolStateObject SectionName = "eosio::chain::protocol_state_object" + SectionNameDynamicGlobalPropertyObject SectionName = "eosio::chain::dynamic_global_property_object" + SectionNameBlockSummaryObject SectionName = "eosio::chain::block_summary_object" + SectionNameTransactionObject SectionName = "eosio::chain::transaction_object" + SectionNameGeneratedTransactionObject SectionName = "eosio::chain::generated_transaction_object" + SectionNameCodeObject SectionName = "eosio::chain::code_object" + SectionNameContractTables SectionName = "contract_tables" + SectionNamePermissionObject SectionName = "eosio::chain::permission_object" + SectionNamePermissionLinkObject SectionName = "eosio::chain::permission_link_object" + SectionNameResourceLimitsObject SectionName = "eosio::chain::resource_limits::resource_limits_object" + SectionNameResourceUsageObject SectionName = "eosio::chain::resource_limits::resource_usage_object" + SectionNameResourceLimitsStateObject SectionName = "eosio::chain::resource_limits::resource_limits_state_object" + SectionNameResourceLimitsConfigObject SectionName = "eosio::chain::resource_limits::resource_limits_config_object" + SectionNameGenesisState SectionName = "eosio::chain::genesis_state" +) + +func stringToSectionName(name string) SectionName { + switch name { + case "eosio::chain::chain_snapshot_header": + return SectionNameChainSnapshotHeader + case "eosio::chain::block_state": + return SectionNameBlockState + case "eosio::chain::account_object": + return SectionNameAccountObject + case "eosio::chain::account_metadata_object": + return SectionNameAccountMetadataObject + case "eosio::chain::account_ram_correction_object": + return SectionNameAccountRamCorrectionObject + case "eosio::chain::global_property_object": + return SectionNameGlobalPropertyObject + case "eosio::chain::protocol_state_object": + return SectionNameProtocolStateObject + case "eosio::chain::dynamic_global_property_object": + return SectionNameDynamicGlobalPropertyObject + case "eosio::chain::block_summary_object": + return SectionNameBlockSummaryObject + case "eosio::chain::transaction_object": + return SectionNameTransactionObject + case "eosio::chain::generated_transaction_object": + return SectionNameGeneratedTransactionObject + case "eosio::chain::code_object": + return SectionNameCodeObject + case "contract_tables": + return SectionNameContractTables + case "eosio::chain::permission_object": + return SectionNamePermissionObject + case "eosio::chain::permission_link_object": + return SectionNamePermissionLinkObject + case "eosio::chain::resource_limits::resource_limits_object": + return SectionNameResourceLimitsObject + case "eosio::chain::resource_limits::resource_usage_object": + return SectionNameResourceUsageObject + case "eosio::chain::resource_limits::resource_limits_state_object": + return SectionNameResourceLimitsStateObject + case "eosio::chain::resource_limits::resource_limits_config_object": + return SectionNameResourceLimitsConfigObject + case "eosio::chain::genesis_state": + return SectionNameGenesisState + default: + panic("unsupported section name: " + name) + } +} + type Section struct { - Name string + Name SectionName Offset uint64 Size uint64 // This includes the section name and row count BufferSize uint64 // This represents the bytes that are following the section header @@ -20,48 +92,46 @@ type callbackFunc func(obj interface{}) error func (s *Section) Process(f callbackFunc) error { switch s.Name { - case "eosio::chain::chain_snapshot_header": + case SectionNameChainSnapshotHeader: return s.readChainSnapshotHeader(f) - case "eosio::chain::block_state": + case SectionNameBlockState: return s.readBlockState(f) - case "eosio::chain::account_object": + case SectionNameAccountObject: return s.readAccountObjects(f) - case "eosio::chain::account_metadata_object": + case SectionNameAccountMetadataObject: return s.readAccountMetadataObjects(f) - case "eosio::chain::account_ram_correction_object": + case SectionNameAccountRamCorrectionObject: return s.readAccountRAMCorrectionObject(f) - case "eosio::chain::global_property_object": + case SectionNameGlobalPropertyObject: return s.readGlobalPropertyObject(f) - case "eosio::chain::protocol_state_object": + case SectionNameProtocolStateObject: return s.readProtocolStateObject(f) - case "eosio::chain::dynamic_global_property_object": + case SectionNameDynamicGlobalPropertyObject: return s.readDynamicGlobalPropertyObject(f) - case "eosio::chain::block_summary_object": + case SectionNameBlockSummaryObject: return s.readBlockSummary(f) - case "eosio::chain::transaction_object": + case SectionNameTransactionObject: return s.readTransactionObject(f) - case "eosio::chain::generated_transaction_object": + case SectionNameGeneratedTransactionObject: return s.readGeneratedTransactionObject(f) - case "eosio::chain::code_object": + case SectionNameCodeObject: return s.readCodeObject(f) - case "contract_tables": + case SectionNameContractTables: return s.readContractTables(f) - case "eosio::chain::permission_object": + case SectionNamePermissionObject: return s.readPermissionObject(f) - case "eosio::chain::permission_link_object": + case SectionNamePermissionLinkObject: return s.readPermissionLinkObject(f) - case "eosio::chain::resource_limits::resource_limits_object": + case SectionNameResourceLimitsObject: return s.readResourceLimitsObject(f) - case "eosio::chain::resource_limits::resource_usage_object": + case SectionNameResourceUsageObject: return s.readResourceUsageObject(f) - case "eosio::chain::resource_limits::resource_limits_state_object": + case SectionNameResourceLimitsStateObject: return s.readResourceLimitsStateObject(f) - case "eosio::chain::resource_limits::resource_limits_config_object": + case SectionNameResourceLimitsConfigObject: return s.readResourceLimitsConfigObject(f) - - case "eosio::chain::genesis_state": + case SectionNameGenesisState: return s.readGenesisState(f) - default: panic("unsupported section: " + s.Name) } diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index 6e4a8a8a..1b4e111d 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -4,50 +4,100 @@ import ( "fmt" "io" "os" - "strings" + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestSnapshotRead(t *testing.T) { - // filename := "/tmp/0125111385-07750c59b24ed52d2dbf2048b67b58e9c9bd53ff5cc4550277718c1d5d800f73-snapshot.bin" // mainnet - // filename := "/tmp/0003212331-0031042b02b2cf711fee6e1e24da94101fa6c1ea9ece568d5f13232473429db1-snapshot.bin" // kylin - filename := os.Getenv("READ_SNAPSHOT_FILE") - if filename == "" || !fileExists(filename) { - t.Skipf("Environment varaible 'READ_SNAPSHOT_FILE' not set or value %q is not an exisiting file", filename) - return + + //if os.Getenv("READ_SNAPSHOT_FILE") != "true" { + // t.Skipf("Environment varaible 'READ_SNAPSHOT_FILE' not set to true") + // return + //} + + logger, _ := zap.NewDevelopment() + tests := []struct { + name string + testFile string + }{ + {name: "eos-local dev", testFile: "eos-jdev_0000000638.bin"}, + {name: "eos-dev1", testFile: "eos-dev1_0004841949.bin"}, + {name: "Battlefield - b8d703ed1", testFile: "battlefield-snapshot.bin"}, } - r, err := NewReader(filename) - fmt.Println("Filename", filename) - defer r.Close() - assert.NoError(t, err) - assert.Equal(t, r.Header.Version, uint32(1)) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testFile := testData(test.testFile) - for { - section, err := r.Next() - if err == io.EOF { - break - } - assert.NoError(t, err) - fmt.Println("Section", section.Name, "rows", section.RowCount, "bytes", section.BufferSize, "offset", section.Offset) + if !fileExists(testFile) { + logger.Error("test file not found", zap.String("testfile", testFile)) + return + } - if strings.Contains(section.Name, "contract") { - require.NoError(t, section.Process(func(o interface{}) error { - switch obj := o.(type) { - case *TableIDObject: - fmt.Println("Table ID", obj.Code, obj.Scope, obj.TableName) - case *KeyValueObject: - fmt.Println("KV", obj.PrimKey, obj.Value) - default: - fmt.Printf("Ignoring row %T\n", obj) - } - return nil + r, err := NewReader(testFile) + require.NoError(t, err) + defer r.Close() + + assert.NoError(t, err) + assert.Equal(t, r.Header.Version, uint32(1)) - })) - } + for { + section, err := r.Next() + if err == io.EOF { + break + } + assert.NoError(t, err) + logger.Info("new section", + zap.String("section_name", string(section.Name)), + zap.Uint64("row_count", section.RowCount), + zap.Uint64("bytes_count", section.BufferSize), + zap.Uint64("bytes_count", section.Offset), + ) + switch section.Name { + case SectionNameAccountObject: + require.NoError(t, section.Process(func(o interface{}) error { + acc, ok := o.(AccountObject) + if !ok { + return fmt.Errorf("process account object: unexpected object type: %T", o) + } + logger.Info("new account object", + zap.String("name", string(acc.Name)), + ) + return nil + })) + case SectionNameContractTables: + var currentTable *TableIDObject + require.NoError(t, section.Process(func(o interface{}) error { + switch obj := o.(type) { + case *TableIDObject: + logger.Info("Table ID", zap.Reflect("table_id", obj)) + currentTable = obj + case *KeyValueObject: + if currentTable.Count < 20 { + logger.Info("Key Value Object", zap.Reflect("kv", obj)) + } + case *Index64Object: + logger.Info("Index64Object", zap.Reflect("index_64_object", obj)) + case *Index128Object: + logger.Info("Index128Object", zap.Reflect("index_128_object", obj)) + case *Index256Object: + logger.Info("Index256Object", zap.Reflect("index_256_object", obj)) + case *IndexDoubleObject: + logger.Info("IndexDoubleObject", zap.Reflect("index_double_object", obj)) + case *IndexLongDoubleObject: + logger.Info("IndexLongDoubleObject", zap.Reflect("index_long_object", obj)) + default: + fmt.Printf("Ignoring row %T\n", obj) + } + return nil + })) + } + } + }) } } @@ -63,3 +113,7 @@ func fileExists(path string) bool { return !info.IsDir() } + +func testData(filename string) string { + return filepath.Join("test-data", filename) +} diff --git a/snapshot/test-data/battlefield-snapshot.bin b/snapshot/test-data/battlefield-snapshot.bin new file mode 100644 index 00000000..724fb03b Binary files /dev/null and b/snapshot/test-data/battlefield-snapshot.bin differ diff --git a/snapshot/test-data/eos-dev1_0004841949.bin b/snapshot/test-data/eos-dev1_0004841949.bin new file mode 100644 index 00000000..ee7f2cca Binary files /dev/null and b/snapshot/test-data/eos-dev1_0004841949.bin differ diff --git a/snapshot/test-data/eos-jdev_0000000638.bin b/snapshot/test-data/eos-jdev_0000000638.bin new file mode 100644 index 00000000..32681434 Binary files /dev/null and b/snapshot/test-data/eos-jdev_0000000638.bin differ diff --git a/snapshot/typesv3.go b/snapshot/typesv3.go index d1d78e49..f63d5794 100644 --- a/snapshot/typesv3.go +++ b/snapshot/typesv3.go @@ -14,12 +14,10 @@ type TableIDObject struct { Scope string TableName string Payer string - Count uint32 + Count uint32 // represents the number of rows & indices for a given table } type ContractRow struct { - TableID *TableIDObject - PrimKey string Payer string } @@ -131,7 +129,6 @@ func (section *Section) readContractTables(f callbackFunc) error { } contractRow := ContractRow{ - TableID: t, PrimKey: eos.NameToString(binary.LittleEndian.Uint64(head[0:8])), Payer: eos.NameToString(binary.LittleEndian.Uint64(head[8:16])), }