From 0ac53e83c84905820013cecde71d5bd61759c5cb Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 19 Apr 2018 01:07:39 +0800 Subject: [PATCH 01/37] sharding/client: Add utils.go for blob serialization(#92) --- sharding/client/utils.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 sharding/client/utils.go diff --git a/sharding/client/utils.go b/sharding/client/utils.go new file mode 100644 index 000000000000..a4c577ec8138 --- /dev/null +++ b/sharding/client/utils.go @@ -0,0 +1,39 @@ +package client + +import ( + "fmt" +) + +type collationbody []byte + +var ( + collationsizelimit = int64(2 * *20) + chunkSize = int64(32) + indicatorSize = int64(1) + chunkDataSize = chunkSize - indicatorSize +) + +func (c *collationbody) validateBody() bool { + return len(c) < 2^20 && len(c) > 0 +} + +func createChunks(c collationbody) { + validCollation, err := c.validateBody + if err != nil { + fmt.Errorf("Error %v", err) + } + if !validCollation { + fmt.Errorf("Error %v", err) + } + index := int64(len(c)) / chunkDataSize + + /* + for i = 0; i < index; i++ { + serialisedblob[i*chunksize] = 0 + for f = 0; f Date: Sun, 22 Apr 2018 17:57:12 +0800 Subject: [PATCH 02/37] sharding/client: Changing Validate Body(#92) --- sharding/client/utils.go | 49 ++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index a4c577ec8138..cecff0b95683 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -7,33 +7,64 @@ import ( type collationbody []byte var ( - collationsizelimit = int64(2 * *20) + collationsizelimit = int64(2 ^ 20) chunkSize = int64(32) indicatorSize = int64(1) chunkDataSize = chunkSize - indicatorSize ) -func (c *collationbody) validateBody() bool { - return len(c) < 2^20 && len(c) > 0 +/* Validate that the collation body is within its bounds and if +the size of the body is below the limit it is simply appended +till it reaches the required limit */ + +func (cb collationbody) validateBody() error { + + length := int64(len(cb)) + + if length == 0 { + return fmt.Errorf("Collation Body has to be a non-zero value") + } + + if length > collationsizelimit { + return fmt.Errorf("Collation Body is over the size limit") + } + + if length < collationsizelimit { + x := make([]byte, (collationsizelimit - length)) + cb = append(cb, x...) + fmt.Printf("%b", x) + + } + + return nil +} + +func main() { + + x := []byte{'h', 'e', 'g', 'g'} + t := x.validateBody() + fmt.Printf("%b %s", x, t) + } -func createChunks(c collationbody) { - validCollation, err := c.validateBody +/* +func createChunks(cb collationbody) { + validCollation := cb.validateBody if err != nil { fmt.Errorf("Error %v", err) } if !validCollation { fmt.Errorf("Error %v", err) } - index := int64(len(c)) / chunkDataSize + index := int64(len(cb)) / chunkDataSize + - /* for i = 0; i < index; i++ { serialisedblob[i*chunksize] = 0 for f = 0; f Date: Sun, 22 Apr 2018 23:09:22 +0800 Subject: [PATCH 03/37] sharding/client: Adding Parse Blob(#92) --- sharding/client/utils.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index cecff0b95683..29f12c18796c 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -10,27 +10,32 @@ var ( collationsizelimit = int64(2 ^ 20) chunkSize = int64(32) indicatorSize = int64(1) + numberOfChunks = collationsizelimit / chunkSize chunkDataSize = chunkSize - indicatorSize + totalDatasize = numberOfChunks * chunkDataSize ) +func (cb collationbody) length() int64 { + + return int64(len(cb)) +} + /* Validate that the collation body is within its bounds and if the size of the body is below the limit it is simply appended till it reaches the required limit */ func (cb collationbody) validateBody() error { - length := int64(len(cb)) - - if length == 0 { + if cb.length() == 0 { return fmt.Errorf("Collation Body has to be a non-zero value") } - if length > collationsizelimit { + if cb.length() > totalDatasize { return fmt.Errorf("Collation Body is over the size limit") } - if length < collationsizelimit { - x := make([]byte, (collationsizelimit - length)) + if cb.length() < totalDatasize { + x := make([]byte, (totalDatasize - cb.length())) cb = append(cb, x...) fmt.Printf("%b", x) @@ -39,12 +44,17 @@ func (cb collationbody) validateBody() error { return nil } -func main() { +/* + add +*/ + +func (cb collationbody) ParseBlob() { + terminalLength := cb.length() % chunkDataSize + chunksNumber := cb.length() / chunkDataSize - x := []byte{'h', 'e', 'g', 'g'} - t := x.validateBody() - fmt.Printf("%b %s", x, t) + if terminalLength != 0 { + } } /* From c36155c08a35735644a7c4cc41b66bdc40f2afd8 Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 23 Apr 2018 00:39:41 +0800 Subject: [PATCH 04/37] sharding/client: Add conditions for terminal chunks(#92) --- sharding/client/utils.go | 49 ++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 29f12c18796c..c7972f619703 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -34,47 +34,36 @@ func (cb collationbody) validateBody() error { return fmt.Errorf("Collation Body is over the size limit") } - if cb.length() < totalDatasize { - x := make([]byte, (totalDatasize - cb.length())) - cb = append(cb, x...) - fmt.Printf("%b", x) - - } - return nil } -/* - add -*/ +// Parse Collation body and modify it accordingly func (cb collationbody) ParseBlob() { + terminalLength := cb.length() % chunkDataSize chunksNumber := cb.length() / chunkDataSize + indicatorByte := make([]byte, 1) + indicatorByte[0] = 0 + var tempbody collationbody - if terminalLength != 0 { + // Appends empty indicator bytes to non terminal-chunks + for i := int64(1); i <= chunksNumber; i++ { + tempbody = append(tempbody, append(indicatorByte, cb[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } -} + // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk + if terminalLength != 0 { + indicatorByte[0] = byte(terminalLength) + tempbody = append(tempbody, append(indicatorByte, cb[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) -/* -func createChunks(cb collationbody) { - validCollation := cb.validateBody - if err != nil { - fmt.Errorf("Error %v", err) - } - if !validCollation { - fmt.Errorf("Error %v", err) } - index := int64(len(cb)) / chunkDataSize - + cb = tempbody - for i = 0; i < index; i++ { - serialisedblob[i*chunksize] = 0 - for f = 0; f Date: Wed, 2 May 2018 00:36:16 +0800 Subject: [PATCH 05/37] sharding/client: Change to serialize blobs(#92) --- sharding/client/utils.go | 56 ++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index c7972f619703..0950dbdb4a22 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -2,12 +2,11 @@ package client import ( "fmt" + "math" ) -type collationbody []byte - var ( - collationsizelimit = int64(2 ^ 20) + collationsizelimit = int64(math.Pow(float64(2), float64(20))) chunkSize = int64(32) indicatorSize = int64(1) numberOfChunks = collationsizelimit / chunkSize @@ -15,6 +14,14 @@ var ( totalDatasize = numberOfChunks * chunkDataSize ) +type collationbody []byte + +type body interface { + length() int64 + validateblob() error + ParseBlob() +} + func (cb collationbody) length() int64 { return int64(len(cb)) @@ -39,13 +46,36 @@ func (cb collationbody) validateBody() error { // Parse Collation body and modify it accordingly -func (cb collationbody) ParseBlob() { +func (cb collationbody) serializeBlob() []byte { terminalLength := cb.length() % chunkDataSize chunksNumber := cb.length() / chunkDataSize indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - var tempbody collationbody + var tempbody []byte + + // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right + + if chunksNumber == 0 { + paddedbytes := make([]byte, cb.length()-terminalLength) + indicatorByte[0] = byte(terminalLength) + tempbody = append(indicatorByte, append(cb, paddedbytes...)...) + return tempbody + } + + //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 + + if terminalLength == 0 { + + for i := int64(1); i < chunksNumber; i++ { + tempbody = append(tempbody, append(indicatorByte, cb[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + } + indicatorByte[0] = byte(chunkDataSize) + tempbody = append(tempbody, append(indicatorByte, cb[(chunksNumber-1)*chunkDataSize:chunksNumber*chunkDataSize]...)...) + return tempbody + + } // Appends empty indicator bytes to non terminal-chunks for i := int64(1); i <= chunksNumber; i++ { @@ -53,17 +83,11 @@ func (cb collationbody) ParseBlob() { } // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk - if terminalLength != 0 { - indicatorByte[0] = byte(terminalLength) - tempbody = append(tempbody, append(indicatorByte, cb[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + indicatorByte[0] = byte(terminalLength) + tempbody = append(tempbody, append(indicatorByte, cb[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + emptyBytes := make([]byte, (chunkDataSize - cb.length())) + cb = append(cb, emptyBytes...) - } - cb = tempbody + return tempbody - // Pad the collation body with empty bytes until it is equal to 1 Mib - if cb.length() < collationsizelimit { - emptyBytes := make([]byte, (collationsizelimit - cb.length())) - cb = append(cb, emptyBytes...) - - } } From ec2408c1a46b425ae682f6f2c7db388bdcd79ffa Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 2 May 2018 13:00:37 +0800 Subject: [PATCH 06/37] sharding/client: Adding deserialize blobs(#92) --- sharding/client/utils.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 0950dbdb4a22..30c284570238 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -3,6 +3,7 @@ package client import ( "fmt" "math" + "reflect" ) var ( @@ -18,8 +19,8 @@ type collationbody []byte type body interface { length() int64 - validateblob() error - ParseBlob() + validateBody() error + serializeBlob() []byte } func (cb collationbody) length() int64 { @@ -44,6 +45,24 @@ func (cb collationbody) validateBody() error { return nil } +func deserializeBlob(blob body) []byte { + deserializedblob := blob.(collationbody) + length := deserializedblob.length() + chunksNumber := chunkSize / length + indicatorByte := make([]byte, 1) + indicatorByte[0] = 0 + tempbody := []byte{0} + + for i := int64(1); i <= chunksNumber; i++ { + + if reflect.TypeOf(deserializedblob[:(i-1)*chunksNumber]) == reflect.TypeOf(indicatorByte) { + + } + + } + +} + // Parse Collation body and modify it accordingly func (cb collationbody) serializeBlob() []byte { @@ -52,7 +71,7 @@ func (cb collationbody) serializeBlob() []byte { chunksNumber := cb.length() / chunkDataSize indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - var tempbody []byte + tempbody := []byte{} // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right From bf04ddb21c717d08b60f9ece6af81c66ba682f82 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 2 May 2018 15:46:46 +0800 Subject: [PATCH 07/37] sharding/client: Fixing deserializeblobs(#92) --- sharding/client/utils.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 30c284570238..554425e8cb13 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -15,7 +15,7 @@ var ( totalDatasize = numberOfChunks * chunkDataSize ) -type collationbody []byte +type txblob []byte type body interface { length() int64 @@ -23,7 +23,7 @@ type body interface { serializeBlob() []byte } -func (cb collationbody) length() int64 { +func (cb txblob) length() int64 { return int64(len(cb)) } @@ -32,7 +32,7 @@ func (cb collationbody) length() int64 { the size of the body is below the limit it is simply appended till it reaches the required limit */ -func (cb collationbody) validateBody() error { +func (cb txblob) validateBody() error { if cb.length() == 0 { return fmt.Errorf("Collation Body has to be a non-zero value") @@ -45,27 +45,36 @@ func (cb collationbody) validateBody() error { return nil } -func deserializeBlob(blob body) []byte { - deserializedblob := blob.(collationbody) - length := deserializedblob.length() +func deserializebody(collationbody []byte) []body { + length := int64(len(collationbody)) chunksNumber := chunkSize / length indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - tempbody := []byte{0} + txblobs := []body{} + var tempbody txblob for i := int64(1); i <= chunksNumber; i++ { - if reflect.TypeOf(deserializedblob[:(i-1)*chunksNumber]) == reflect.TypeOf(indicatorByte) { + if reflect.TypeOf(collationbody[(i-1)*chunkSize]) == reflect.TypeOf(indicatorByte) { + tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) + + } else { + terminalIndex := int64(collationbody[(i-1)*chunkSize]) + tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) + txblobs = append(txblobs, tempbody) + tempbody = txblob{} } } + return txblobs + } // Parse Collation body and modify it accordingly -func (cb collationbody) serializeBlob() []byte { +func (cb txblob) serializeBlob() []byte { terminalLength := cb.length() % chunkDataSize chunksNumber := cb.length() / chunkDataSize From c1ff07039304dd5d0b3f22113caac4ec130c84b2 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 2 May 2018 16:16:57 +0800 Subject: [PATCH 08/37] sharding/client: Adding Main serialize function(#92) --- sharding/client/utils.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 554425e8cb13..c0d2c13fdaf3 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -17,7 +17,7 @@ var ( type txblob []byte -type body interface { +type blob interface { length() int64 validateBody() error serializeBlob() []byte @@ -45,12 +45,12 @@ func (cb txblob) validateBody() error { return nil } -func deserializebody(collationbody []byte) []body { +func deserializebody(collationbody []byte) []blob { length := int64(len(collationbody)) chunksNumber := chunkSize / length indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - txblobs := []body{} + txblobs := []blob{} var tempbody txblob for i := int64(1); i <= chunksNumber; i++ { @@ -72,7 +72,21 @@ func deserializebody(collationbody []byte) []body { } -// Parse Collation body and modify it accordingly +func serialize(rawtx []blob) []byte { + length := int64(len(rawtx)) + serialisedData := []byte{} + + for i := int64(1); i < length; i++ { + data := rawtx[length].(txblob) + refinedData := data.serializeBlob() + serialisedData = append(serialisedData, refinedData...) + txblob(serialisedData).validateBody() + + } + return serialisedData +} + +// Parse blob and modify it accordingly func (cb txblob) serializeBlob() []byte { From bc174f32b99f8aa24949e21e8c3b5554f6efa57a Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 2 May 2018 16:57:59 +0800 Subject: [PATCH 09/37] sharding/client: Cleaning Up and adding errors(#92) --- sharding/client/utils.go | 117 +++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index c0d2c13fdaf3..ec5efe9ee375 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -4,6 +4,8 @@ import ( "fmt" "math" "reflect" + + "github.com/ethereum/go-ethereum/log" ) var ( @@ -19,7 +21,6 @@ type txblob []byte type blob interface { length() int64 - validateBody() error serializeBlob() []byte } @@ -28,64 +29,6 @@ func (cb txblob) length() int64 { return int64(len(cb)) } -/* Validate that the collation body is within its bounds and if -the size of the body is below the limit it is simply appended -till it reaches the required limit */ - -func (cb txblob) validateBody() error { - - if cb.length() == 0 { - return fmt.Errorf("Collation Body has to be a non-zero value") - } - - if cb.length() > totalDatasize { - return fmt.Errorf("Collation Body is over the size limit") - } - - return nil -} - -func deserializebody(collationbody []byte) []blob { - length := int64(len(collationbody)) - chunksNumber := chunkSize / length - indicatorByte := make([]byte, 1) - indicatorByte[0] = 0 - txblobs := []blob{} - var tempbody txblob - - for i := int64(1); i <= chunksNumber; i++ { - - if reflect.TypeOf(collationbody[(i-1)*chunkSize]) == reflect.TypeOf(indicatorByte) { - tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) - - } else { - terminalIndex := int64(collationbody[(i-1)*chunkSize]) - tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) - txblobs = append(txblobs, tempbody) - tempbody = txblob{} - - } - - } - - return txblobs - -} - -func serialize(rawtx []blob) []byte { - length := int64(len(rawtx)) - serialisedData := []byte{} - - for i := int64(1); i < length; i++ { - data := rawtx[length].(txblob) - refinedData := data.serializeBlob() - serialisedData = append(serialisedData, refinedData...) - txblob(serialisedData).validateBody() - - } - return serialisedData -} - // Parse blob and modify it accordingly func (cb txblob) serializeBlob() []byte { @@ -133,3 +76,59 @@ func (cb txblob) serializeBlob() []byte { return tempbody } + +func Serialize(rawtx []blob) ([]byte, error) { + length := int64(len(rawtx)) + + if length == 0 { + return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") + } + serialisedData := []byte{} + + for i := int64(0); i < length; i++ { + + blobLength := txblob(serialisedData).length() + data := rawtx[i].(txblob) + refinedData := data.serializeBlob() + serialisedData = append(serialisedData, refinedData...) + + if txblob(serialisedData).length() > collationsizelimit { + log.Info(fmt.Sprintf("The total number of interfaces added to the collation body are: %d", i)) + serialisedData = serialisedData[:blobLength] + return serialisedData, nil + + } + + } + return serialisedData, nil +} + +// Collation body deserialised and separated into its respective interfaces + +func Deserializebody(collationbody []byte) []blob { + + length := int64(len(collationbody)) + chunksNumber := chunkSize / length + indicatorByte := make([]byte, 1) + indicatorByte[0] = 0 + txblobs := []blob{} + var tempbody txblob + + for i := int64(1); i <= chunksNumber; i++ { + + if reflect.TypeOf(collationbody[(i-1)*chunkSize]) == reflect.TypeOf(indicatorByte) { + tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) + + } else { + terminalIndex := int64(collationbody[(i-1)*chunkSize]) + tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) + txblobs = append(txblobs, tempbody) + tempbody = txblob{} + + } + + } + + return txblobs + +} From 19aa0ebe3c41992248089e0ea21058b276e112c8 Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 3 May 2018 20:37:22 +0800 Subject: [PATCH 10/37] sharding/client: Make serialization similar to JSON marshalling(#92) --- sharding/client/utils.go | 56 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index ec5efe9ee375..1eeb064bfaca 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -1,6 +1,7 @@ package client import ( + "errors" "fmt" "math" "reflect" @@ -17,24 +18,27 @@ var ( totalDatasize = numberOfChunks * chunkDataSize ) -type txblob []byte +func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) { + val := reflect.ValueOf(arg) + if val.Kind() == kind { + return val, nil -type blob interface { - length() int64 - serializeBlob() []byte -} - -func (cb txblob) length() int64 { - - return int64(len(cb)) + } + err := errors.New("Interface Conversion a failure") + return val, err } // Parse blob and modify it accordingly -func (cb txblob) serializeBlob() []byte { +func serializeBlob(cb interface{}) ([]byte, error) { - terminalLength := cb.length() % chunkDataSize - chunksNumber := cb.length() / chunkDataSize + blob, err := convertInterface(cb, reflect.Slice) + if err != nil { + return nil, fmt.Errorf("Error: %v", err) + } + length := int64(blob.Len()) + terminalLength := length % chunkDataSize + chunksNumber := length / chunkDataSize indicatorByte := make([]byte, 1) indicatorByte[0] = 0 tempbody := []byte{} @@ -42,10 +46,10 @@ func (cb txblob) serializeBlob() []byte { // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right if chunksNumber == 0 { - paddedbytes := make([]byte, cb.length()-terminalLength) + paddedbytes := make([]byte, length-terminalLength) indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(cb, paddedbytes...)...) - return tempbody + tempbody = append(indicatorByte, append(cb.([]byte), paddedbytes...)...) + return tempbody, nil } //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 @@ -53,41 +57,41 @@ func (cb txblob) serializeBlob() []byte { if terminalLength == 0 { for i := int64(1); i < chunksNumber; i++ { - tempbody = append(tempbody, append(indicatorByte, cb[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } indicatorByte[0] = byte(chunkDataSize) - tempbody = append(tempbody, append(indicatorByte, cb[(chunksNumber-1)*chunkDataSize:chunksNumber*chunkDataSize]...)...) - return tempbody + tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(chunksNumber-1)*chunkDataSize:chunksNumber*chunkDataSize]...)...) + return tempbody, nil } // Appends empty indicator bytes to non terminal-chunks for i := int64(1); i <= chunksNumber; i++ { - tempbody = append(tempbody, append(indicatorByte, cb[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk indicatorByte[0] = byte(terminalLength) - tempbody = append(tempbody, append(indicatorByte, cb[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) - emptyBytes := make([]byte, (chunkDataSize - cb.length())) - cb = append(cb, emptyBytes...) + tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + emptyBytes := make([]byte, (chunkDataSize - terminalLength)) + tempbody = append(tempbody, emptyBytes...) - return tempbody + return tempbody, nil } -func Serialize(rawtx []blob) ([]byte, error) { +func Serialize(rawtx []interface{}) ([]byte, error) { length := int64(len(rawtx)) if length == 0 { return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") } - serialisedData := []byte{} + serialisedData := []byte{0, 2} for i := int64(0); i < length; i++ { - blobLength := txblob(serialisedData).length() + blobLength := length(serialisedData) data := rawtx[i].(txblob) refinedData := data.serializeBlob() serialisedData = append(serialisedData, refinedData...) From f38af88d85952d4bb329357f81af7e6d26036ddb Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 3 May 2018 21:28:44 +0800 Subject: [PATCH 11/37] sharding/client: Finish changing Deserializebody(#92) --- sharding/client/utils.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 1eeb064bfaca..a3d6113bacac 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -81,22 +81,26 @@ func serializeBlob(cb interface{}) ([]byte, error) { } +// Serialize takes a set of transactions and converts them to a single byte array func Serialize(rawtx []interface{}) ([]byte, error) { length := int64(len(rawtx)) if length == 0 { return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") } - serialisedData := []byte{0, 2} + serialisedData := []byte{} for i := int64(0); i < length; i++ { - blobLength := length(serialisedData) - data := rawtx[i].(txblob) - refinedData := data.serializeBlob() + blobLength := int64(len(serialisedData)) + data := rawtx[i] + refinedData, err := serializeBlob(data) + if err != nil { + return nil, fmt.Errorf("Error: %v", err) + } serialisedData = append(serialisedData, refinedData...) - if txblob(serialisedData).length() > collationsizelimit { + if int64(len(serialisedData)) > collationsizelimit { log.Info(fmt.Sprintf("The total number of interfaces added to the collation body are: %d", i)) serialisedData = serialisedData[:blobLength] return serialisedData, nil @@ -107,32 +111,32 @@ func Serialize(rawtx []interface{}) ([]byte, error) { return serialisedData, nil } -// Collation body deserialised and separated into its respective interfaces - -func Deserializebody(collationbody []byte) []blob { +// Deserializebody results in the Collation body being deserialised and separated into its respective interfaces +func Deserializebody(collationbody []byte, rawtx []interface{}) error { length := int64(len(collationbody)) chunksNumber := chunkSize / length indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - txblobs := []blob{} - var tempbody txblob + var txblobs []interface{} + tempbody := []byte{} for i := int64(1); i <= chunksNumber; i++ { - if reflect.TypeOf(collationbody[(i-1)*chunkSize]) == reflect.TypeOf(indicatorByte) { + if reflect.ValueOf(collationbody[(i-1)*chunkSize]) == reflect.ValueOf(indicatorByte) { tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) } else { terminalIndex := int64(collationbody[(i-1)*chunkSize]) tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) txblobs = append(txblobs, tempbody) - tempbody = txblob{} + tempbody = []byte{} } } + rawtx = txblobs - return txblobs + return nil } From 4953c2ee389cbb8b3b40e15187064cf738e0837d Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 3 May 2018 22:34:01 +0800 Subject: [PATCH 12/37] sharding/client: Adding new comments to improve clarity(#92) --- sharding/client/utils.go | 69 ++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index a3d6113bacac..e9a144d3e79c 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -18,6 +18,7 @@ var ( totalDatasize = numberOfChunks * chunkDataSize ) +// convertInterface converts inputted interface to the required type, ex: slice. func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) { val := reflect.ValueOf(arg) if val.Kind() == kind { @@ -28,8 +29,7 @@ func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) return val, err } -// Parse blob and modify it accordingly - +// serializeBlob parses the blob and serializes it appropriately. func serializeBlob(cb interface{}) ([]byte, error) { blob, err := convertInterface(cb, reflect.Slice) @@ -39,6 +39,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { length := int64(blob.Len()) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize + finalchunkIndex := totalDatasize + (terminalLength + 1) indicatorByte := make([]byte, 1) indicatorByte[0] = 0 tempbody := []byte{} @@ -46,34 +47,56 @@ func serializeBlob(cb interface{}) ([]byte, error) { // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right if chunksNumber == 0 { - paddedbytes := make([]byte, length-terminalLength) + paddedbytes := make([]byte, (length - terminalLength)) indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(cb.([]byte), paddedbytes...)...) + tempbody = append(indicatorByte, append(blob.Bytes(), paddedbytes...)...) return tempbody, nil } //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 + // Then this chunk is returned to the main Serialize function if terminalLength == 0 { for i := int64(1); i < chunksNumber; i++ { - tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk + // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of + // 31 + tempbody = append(tempbody, + append(indicatorByte, + blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } indicatorByte[0] = byte(chunkDataSize) - tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(chunksNumber-1)*chunkDataSize:chunksNumber*chunkDataSize]...)...) + + // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob + tempbody = append(tempbody, + append(indicatorByte, + blob.Bytes()[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) + return tempbody, nil } - // Appends empty indicator bytes to non terminal-chunks + // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk + // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of + // 31 for i := int64(1); i <= chunksNumber; i++ { - tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + tempbody = append(tempbody, + append(indicatorByte, + blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } - // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk + // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. + // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. + // finalchunkIndex refers to the index of the last data byte + indicatorByte[0] = byte(terminalLength) - tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + tempbody = append(tempbody, + append(indicatorByte, + blob.Bytes()[chunkDataSize*chunksNumber:finalchunkIndex]...)...) + emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -81,7 +104,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { } -// Serialize takes a set of transactions and converts them to a single byte array +// Serialize takes a set of transaction blobs and converts them to a single byte array. func Serialize(rawtx []interface{}) ([]byte, error) { length := int64(len(rawtx)) @@ -90,6 +113,7 @@ func Serialize(rawtx []interface{}) ([]byte, error) { } serialisedData := []byte{} + //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { blobLength := int64(len(serialisedData)) @@ -111,31 +135,34 @@ func Serialize(rawtx []interface{}) ([]byte, error) { return serialisedData, nil } -// Deserializebody results in the Collation body being deserialised and separated into its respective interfaces +// Deserializebody results in the Collation body being deserialised and separated into its respective interfaces. func Deserializebody(collationbody []byte, rawtx []interface{}) error { length := int64(len(collationbody)) chunksNumber := chunkSize / length indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - var txblobs []interface{} tempbody := []byte{} + // This separates the collation body into its respective transaction blobs for i := int64(1); i <= chunksNumber; i++ { - - if reflect.ValueOf(collationbody[(i-1)*chunkSize]) == reflect.ValueOf(indicatorByte) { - tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) - + indicatorIndex := (i - 1) * chunkSize + // Tests if the chunk delimiter is zero, if it is it will append the data chunk + // to tempbody + if collationbody[indicatorIndex] == indicatorByte[0] { + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) + + // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and + // add it and append to the rawtx slice. The tempbody signifies a deserialized blob } else { - terminalIndex := int64(collationbody[(i-1)*chunkSize]) - tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) - txblobs = append(txblobs, tempbody) + terminalIndex := int64(collationbody[indicatorIndex]) + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+2+terminalIndex)]...) + rawtx = append(rawtx, tempbody) tempbody = []byte{} } } - rawtx = txblobs return nil From 1e45e348dc5256d4d23a515db14d73f549f5d545 Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 17:17:21 +0800 Subject: [PATCH 13/37] sharding/client: Adding tests and modifying utils(#92) --- sharding/client/utils.go | 32 ++++++++++++++------- sharding/client/utils_test.go | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 sharding/client/utils_test.go diff --git a/sharding/client/utils.go b/sharding/client/utils.go index e9a144d3e79c..5c113fe16404 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "reflect" + //"runtime" "github.com/ethereum/go-ethereum/log" ) @@ -15,7 +16,6 @@ var ( indicatorSize = int64(1) numberOfChunks = collationsizelimit / chunkSize chunkDataSize = chunkSize - indicatorSize - totalDatasize = numberOfChunks * chunkDataSize ) // convertInterface converts inputted interface to the required type, ex: slice. @@ -29,6 +29,16 @@ func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) return val, err } +func convertbyteToInterface(arg []byte) []interface{} { + length := int64(len(arg)) + newtype := make([]interface{}, length) + for i, v := range arg { + newtype[i] = v + } + + return newtype +} + // serializeBlob parses the blob and serializes it appropriately. func serializeBlob(cb interface{}) ([]byte, error) { @@ -39,7 +49,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { length := int64(blob.Len()) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize - finalchunkIndex := totalDatasize + (terminalLength + 1) + finalchunkIndex := length - 1 indicatorByte := make([]byte, 1) indicatorByte[0] = 0 tempbody := []byte{} @@ -81,6 +91,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of // 31 + for i := int64(1); i <= chunksNumber; i++ { tempbody = append(tempbody, @@ -91,11 +102,10 @@ func serializeBlob(cb interface{}) ([]byte, error) { // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. // finalchunkIndex refers to the index of the last data byte - indicatorByte[0] = byte(terminalLength) tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[chunkDataSize*chunksNumber:finalchunkIndex]...)...) + blob.Bytes()[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -139,17 +149,17 @@ func Serialize(rawtx []interface{}) ([]byte, error) { func Deserializebody(collationbody []byte, rawtx []interface{}) error { length := int64(len(collationbody)) - chunksNumber := chunkSize / length - indicatorByte := make([]byte, 1) - indicatorByte[0] = 0 + chunksNumber := length / chunkSize + indicatorByte := byte(0) tempbody := []byte{} + deserializedblob := []byte{} // This separates the collation body into its respective transaction blobs - for i := int64(1); i <= chunksNumber; i++ { + for i := int64(1); i <= chunksNumber+1; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody - if collationbody[indicatorIndex] == indicatorByte[0] { + if collationbody[indicatorIndex] == indicatorByte { tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and @@ -157,13 +167,15 @@ func Deserializebody(collationbody []byte, rawtx []interface{}) error { } else { terminalIndex := int64(collationbody[indicatorIndex]) tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+2+terminalIndex)]...) - rawtx = append(rawtx, tempbody) + deserializedblob = append(deserializedblob, tempbody...) tempbody = []byte{} } } + rawtx = convertbyteToInterface(deserializedblob) + return nil } diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go new file mode 100644 index 000000000000..eee961510531 --- /dev/null +++ b/sharding/client/utils_test.go @@ -0,0 +1,53 @@ +package client + +import ( + "math/rand" + "reflect" + "runtime" + "testing" +) + +var testbody []interface{} + +func buildblob() []byte { + + tempbody := make([]byte, 500) + for i := int64(0); i < 500; i++ { + tempbody[i] = byte(rand.Int()) + + } + + return tempbody + +} +func TestConvertInterface(t *testing.T) { + var slice interface{} + slice = []interface{}{0, 1, 2, 3, 4, 5} + convertedValue, err := convertInterface(slice, reflect.Slice) + if err != nil { + t.Fatalf("Error: %v %v", err, convertedValue) + } + +} + +func TestSerializeblob(t *testing.T) { + + blob := buildblob() + + serializedblob, err := serializeBlob(blob) + + if err != nil { + t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) + } + runtime.Breakpoint() + err2 := Deserializebody(serializedblob, testbody) + if err2 != nil { + t.Fatalf("Error Serializing blob:%v", err2) + } + + if !reflect.DeepEqual(blob, testbody) { + + t.Fatalf("Error Serializing blob with %v %v", blob, testbody) + } + +} From 7ac8bba69702e61016c0c91a7fd506dffebd6e8c Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 19:03:17 +0800 Subject: [PATCH 14/37] sharding/client: Fixing Tests (#92) --- sharding/client/utils.go | 6 +++--- sharding/client/utils_test.go | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 5c113fe16404..a5909a98c7d7 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -146,7 +146,7 @@ func Serialize(rawtx []interface{}) ([]byte, error) { } // Deserializebody results in the Collation body being deserialised and separated into its respective interfaces. -func Deserializebody(collationbody []byte, rawtx []interface{}) error { +func Deserializebody(collationbody []byte, rawtx interface{}) error { length := int64(len(collationbody)) chunksNumber := length / chunkSize @@ -155,7 +155,7 @@ func Deserializebody(collationbody []byte, rawtx []interface{}) error { deserializedblob := []byte{} // This separates the collation body into its respective transaction blobs - for i := int64(1); i <= chunksNumber+1; i++ { + for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody @@ -166,7 +166,7 @@ func Deserializebody(collationbody []byte, rawtx []interface{}) error { // add it and append to the rawtx slice. The tempbody signifies a deserialized blob } else { terminalIndex := int64(collationbody[indicatorIndex]) - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+2+terminalIndex)]...) + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedblob = append(deserializedblob, tempbody...) tempbody = []byte{} diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index eee961510531..2617dc50ad00 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -3,16 +3,16 @@ package client import ( "math/rand" "reflect" - "runtime" + //"runtime" "testing" ) -var testbody []interface{} +var testbody interface{} -func buildblob() []byte { +func buildblob(size int64) []byte { - tempbody := make([]byte, 500) - for i := int64(0); i < 500; i++ { + tempbody := make([]byte, size) + for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) } @@ -32,22 +32,23 @@ func TestConvertInterface(t *testing.T) { func TestSerializeblob(t *testing.T) { - blob := buildblob() + blob := buildblob(20) serializedblob, err := serializeBlob(blob) if err != nil { t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) } - runtime.Breakpoint() - err2 := Deserializebody(serializedblob, testbody) + test := &testbody + //runtime.Breakpoint() + err2 := Deserializebody(serializedblob, &testbody) if err2 != nil { t.Fatalf("Error Serializing blob:%v", err2) } if !reflect.DeepEqual(blob, testbody) { - t.Fatalf("Error Serializing blob with %v %v", blob, testbody) + t.Fatalf("Error Serializing blob with %v %v %v", blob, test, &testbody) } } From 8871e42d58dd46533db65abfb9634e4764495f71 Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 20:01:56 +0800 Subject: [PATCH 15/37] sharding/client : Adding Size Test (#92) --- sharding/client/utils.go | 4 ++-- sharding/client/utils_test.go | 39 ++++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index a5909a98c7d7..80103d6e91c0 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -130,7 +130,7 @@ func Serialize(rawtx []interface{}) ([]byte, error) { data := rawtx[i] refinedData, err := serializeBlob(data) if err != nil { - return nil, fmt.Errorf("Error: %v", err) + return nil, fmt.Errorf("Error: %v at index: %v %v %v", err, i, data, rawtx) } serialisedData = append(serialisedData, refinedData...) @@ -174,7 +174,7 @@ func Deserializebody(collationbody []byte, rawtx interface{}) error { } - rawtx = convertbyteToInterface(deserializedblob) + *rawtx.(*interface{}) = convertbyteToInterface(deserializedblob) return nil diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index 2617dc50ad00..1364313a249c 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -9,9 +9,18 @@ import ( var testbody interface{} -func buildblob(size int64) []byte { +func buildtxblobs(size int64) []interface{} { + tempbody := make([]interface{}, size) + for i := int64(0); i < size; i++ { + tempbody[i] = buildblob(size) + + } + return tempbody +} + +func buildblob(size int64) []interface{} { - tempbody := make([]byte, size) + tempbody := make([]interface{}, size) for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) @@ -29,17 +38,37 @@ func TestConvertInterface(t *testing.T) { } } +func TestSize(t *testing.T) { + size := int64(20) + blob := buildtxblobs(size) + chunksafterSerialize := size / chunkDataSize + terminalchunk := size % chunkDataSize + if terminalchunk != 0 { + chunksafterSerialize = chunksafterSerialize + 1 + } + sizeafterSerialize := chunksafterSerialize * chunkSize + serializedblob, err := Serialize(blob) + if err != nil { + t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) + } + + if int64(len(serializedblob)) != sizeafterSerialize { + + t.Fatalf("Error Serializing blob,Lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) + } + +} func TestSerializeblob(t *testing.T) { - blob := buildblob(20) + blob := buildblob(200) serializedblob, err := serializeBlob(blob) if err != nil { t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) } - test := &testbody + //test := &testbody //runtime.Breakpoint() err2 := Deserializebody(serializedblob, &testbody) if err2 != nil { @@ -48,7 +77,7 @@ func TestSerializeblob(t *testing.T) { if !reflect.DeepEqual(blob, testbody) { - t.Fatalf("Error Serializing blob with %v %v %v", blob, test, &testbody) + t.Fatalf("Error Serializing blob with %v %v", blob, testbody) } } From 85bcb3b318ec4156ba905fa2450e052eeb23a0db Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 20:40:57 +0800 Subject: [PATCH 16/37] sharding/client: Cleaning up and refining tests(#92) --- sharding/client/utils.go | 29 ++++++++++++++++++----------- sharding/client/utils_test.go | 15 ++++++--------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 80103d6e91c0..cffa0d67ff43 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -5,9 +5,6 @@ import ( "fmt" "math" "reflect" - //"runtime" - - "github.com/ethereum/go-ethereum/log" ) var ( @@ -39,14 +36,25 @@ func convertbyteToInterface(arg []byte) []interface{} { return newtype } +func interfacetoByte(arg []interface{}) []byte { + length := int64(len(arg)) + newtype := make([]byte, length) + for i, v := range arg { + newtype[i] = v.(byte) + } + + return newtype +} + // serializeBlob parses the blob and serializes it appropriately. func serializeBlob(cb interface{}) ([]byte, error) { - blob, err := convertInterface(cb, reflect.Slice) + interfaceblob, err := convertInterface(cb, reflect.Slice) if err != nil { return nil, fmt.Errorf("Error: %v", err) } - length := int64(blob.Len()) + blob := interfacetoByte(interfaceblob.Interface().([]interface{})) + length := int64(len(blob)) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize finalchunkIndex := length - 1 @@ -59,7 +67,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { if chunksNumber == 0 { paddedbytes := make([]byte, (length - terminalLength)) indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(blob.Bytes(), paddedbytes...)...) + tempbody = append(indicatorByte, append(blob, paddedbytes...)...) return tempbody, nil } @@ -74,7 +82,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { // 31 tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } indicatorByte[0] = byte(chunkDataSize) @@ -82,7 +90,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) + blob[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) return tempbody, nil @@ -96,7 +104,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. @@ -105,7 +113,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { indicatorByte[0] = byte(terminalLength) tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) + blob[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -135,7 +143,6 @@ func Serialize(rawtx []interface{}) ([]byte, error) { serialisedData = append(serialisedData, refinedData...) if int64(len(serialisedData)) > collationsizelimit { - log.Info(fmt.Sprintf("The total number of interfaces added to the collation body are: %d", i)) serialisedData = serialisedData[:blobLength] return serialisedData, nil diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index 1364313a249c..3b17b730c53a 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -3,7 +3,6 @@ package client import ( "math/rand" "reflect" - //"runtime" "testing" ) @@ -39,10 +38,10 @@ func TestConvertInterface(t *testing.T) { } func TestSize(t *testing.T) { - size := int64(20) + size := int64(84) blob := buildtxblobs(size) - chunksafterSerialize := size / chunkDataSize - terminalchunk := size % chunkDataSize + chunksafterSerialize := size * size / chunkDataSize + terminalchunk := size * size % chunkDataSize if terminalchunk != 0 { chunksafterSerialize = chunksafterSerialize + 1 } @@ -59,17 +58,15 @@ func TestSize(t *testing.T) { } } -func TestSerializeblob(t *testing.T) { +func TestSerializeAndDeserializeblob(t *testing.T) { - blob := buildblob(200) + blob := buildtxblobs(31) - serializedblob, err := serializeBlob(blob) + serializedblob, err := Serialize(blob) if err != nil { t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) } - //test := &testbody - //runtime.Breakpoint() err2 := Deserializebody(serializedblob, &testbody) if err2 != nil { t.Fatalf("Error Serializing blob:%v", err2) From eb62dc6b81db6375b5ea5dbded9e1b4d63a810c3 Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 21:03:46 +0800 Subject: [PATCH 17/37] sharding/client: Removing errors(#92) --- sharding/client/utils.go | 2 +- sharding/client/utils_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index cffa0d67ff43..948043adc5db 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -138,7 +138,7 @@ func Serialize(rawtx []interface{}) ([]byte, error) { data := rawtx[i] refinedData, err := serializeBlob(data) if err != nil { - return nil, fmt.Errorf("Error: %v at index: %v %v %v", err, i, data, rawtx) + return nil, fmt.Errorf("Error: %v at index: %v", i, err) } serialisedData = append(serialisedData, refinedData...) diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index 3b17b730c53a..9f7ef000119e 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -53,14 +53,14 @@ func TestSize(t *testing.T) { if int64(len(serializedblob)) != sizeafterSerialize { - t.Fatalf("Error Serializing blob,Lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) + t.Fatalf("Error Serializing blobs the lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) } } func TestSerializeAndDeserializeblob(t *testing.T) { - blob := buildtxblobs(31) + blob := buildtxblobs(30) serializedblob, err := Serialize(blob) @@ -74,7 +74,7 @@ func TestSerializeAndDeserializeblob(t *testing.T) { if !reflect.DeepEqual(blob, testbody) { - t.Fatalf("Error Serializing blob with %v %v", blob, testbody) + t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v %v", blob, testbody) } } From 8bfd22e2dfeedb3c5dc5f5ec78d6e9c2d1e809a2 Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 14:03:49 +0800 Subject: [PATCH 18/37] sharding/utils: Shifting to utils package(#92) --- sharding/client/utils_test.go | 6 +- sharding/utils/marshal.go | 189 +++++++++++++++++++++++++++++++++ sharding/utils/marshal_test.go | 81 ++++++++++++++ 3 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 sharding/utils/marshal.go create mode 100644 sharding/utils/marshal_test.go diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index 9f7ef000119e..d87e0af2e048 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -6,8 +6,6 @@ import ( "testing" ) -var testbody interface{} - func buildtxblobs(size int64) []interface{} { tempbody := make([]interface{}, size) for i := int64(0); i < size; i++ { @@ -38,7 +36,7 @@ func TestConvertInterface(t *testing.T) { } func TestSize(t *testing.T) { - size := int64(84) + size := int64(2) blob := buildtxblobs(size) chunksafterSerialize := size * size / chunkDataSize terminalchunk := size * size % chunkDataSize @@ -60,6 +58,8 @@ func TestSize(t *testing.T) { } func TestSerializeAndDeserializeblob(t *testing.T) { + var testbody interface{} + blob := buildtxblobs(30) serializedblob, err := Serialize(blob) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go new file mode 100644 index 000000000000..bc9e70dabd8a --- /dev/null +++ b/sharding/utils/marshal.go @@ -0,0 +1,189 @@ +package utils + +import ( + "errors" + "fmt" + "math" + "reflect" +) + +var ( + collationsizelimit = int64(math.Pow(float64(2), float64(20))) + chunkSize = int64(32) + indicatorSize = int64(1) + numberOfChunks = collationsizelimit / chunkSize + chunkDataSize = chunkSize - indicatorSize +) + +// convertInterface converts inputted interface to the required type of interface, ex: slice. +func convertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) { + val := reflect.ValueOf(arg) + if val.Kind() == kind { + + return val.Interface().([]interface{}), nil + + } + err := errors.New("Interface Conversion a failure") + return nil, err +} + +func convertbyteToInterface(arg []byte) []interface{} { + length := int64(len(arg)) + newtype := make([]interface{}, length) + for i, v := range arg { + newtype[i] = v + } + + return newtype +} + +func interfacetoByte(arg []interface{}) []byte { + length := int64(len(arg)) + newtype := make([]byte, length) + for i, v := range arg { + newtype[i] = v.(byte) + } + + return newtype +} + +// serializeBlob parses the blob and serializes it appropriately. +func serializeBlob(cb interface{}) ([]byte, error) { + + interfaceblob, err := convertInterface(cb, reflect.Slice) + if err != nil { + return nil, fmt.Errorf("Error: %v", err) + } + blob := interfacetoByte(interfaceblob) + length := int64(len(blob)) + terminalLength := length % chunkDataSize + chunksNumber := length / chunkDataSize + finalchunkIndex := length - 1 + indicatorByte := make([]byte, 1) + indicatorByte[0] = 0 + tempbody := []byte{} + + // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right + + if chunksNumber == 0 { + paddedbytes := make([]byte, (chunkDataSize - length)) + indicatorByte[0] = byte(terminalLength) + tempbody = append(indicatorByte, append(blob, paddedbytes...)...) + return tempbody, nil + } + + //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 + // Then this chunk is returned to the main Serialize function + + if terminalLength == 0 { + + for i := int64(1); i < chunksNumber; i++ { + // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk + // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of + // 31 + tempbody = append(tempbody, + append(indicatorByte, + blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + } + indicatorByte[0] = byte(chunkDataSize) + + // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob + tempbody = append(tempbody, + append(indicatorByte, + blob[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) + + return tempbody, nil + + } + + // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk + // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of + // 31 + + for i := int64(1); i <= chunksNumber; i++ { + + tempbody = append(tempbody, + append(indicatorByte, + blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + } + // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. + // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. + // finalchunkIndex refers to the index of the last data byte + indicatorByte[0] = byte(terminalLength) + tempbody = append(tempbody, + append(indicatorByte, + blob[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) + + emptyBytes := make([]byte, (chunkDataSize - terminalLength)) + tempbody = append(tempbody, emptyBytes...) + + return tempbody, nil + +} + +// Serialize takes a set of transaction blobs and converts them to a single byte array. +func Serialize(rawtx []interface{}) ([]byte, error) { + length := int64(len(rawtx)) + + if length == 0 { + return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") + } + serialisedData := []byte{} + + //Loops through all the blobs and serializes them into chunks + for i := int64(0); i < length; i++ { + + blobLength := int64(len(serialisedData)) + data := rawtx[i] + refinedData, err := serializeBlob(data) + if err != nil { + return nil, fmt.Errorf("Error: %v at index: %v", i, err) + } + serialisedData = append(serialisedData, refinedData...) + + if int64(len(serialisedData)) > collationsizelimit { + serialisedData = serialisedData[:blobLength] + return serialisedData, nil + + } + + } + return serialisedData, nil +} + +// Deserialize results in the Collation body being deserialised and separated into its respective interfaces. +func Deserialize(collationbody []byte, rawtx interface{}) error { + + length := int64(len(collationbody)) + chunksNumber := length / chunkSize + indicatorByte := byte(0) + tempbody := []byte{} + var deserializedblob []interface{} + + // This separates the collation body into its respective transaction blobs + for i := int64(1); i <= chunksNumber; i++ { + indicatorIndex := (i - 1) * chunkSize + // Tests if the chunk delimiter is zero, if it is it will append the data chunk + // to tempbody + if collationbody[indicatorIndex] == indicatorByte { + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) + + // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and + // add it and append to the rawtx slice. The tempbody signifies a deserialized blob + } else { + terminalIndex := int64(collationbody[indicatorIndex]) + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) + deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) + tempbody = []byte{} + + } + + } + + *rawtx.(*interface{}) = deserializedblob + + return nil + +} diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go new file mode 100644 index 000000000000..152531de9dad --- /dev/null +++ b/sharding/utils/marshal_test.go @@ -0,0 +1,81 @@ +package utils + +import ( + "math/rand" + "reflect" + "testing" +) + +func buildtxblobs(size int64) []interface{} { + tempbody := make([]interface{}, size) + for i := int64(0); i < size; i++ { + tempbody[i] = buildblob(size) + + } + return tempbody +} + +func buildblob(size int64) []interface{} { + + tempbody := make([]interface{}, size) + for i := int64(0); i < size; i++ { + tempbody[i] = byte(rand.Int()) + + } + + return tempbody + +} +func TestConvertInterface(t *testing.T) { + var slice interface{} + slice = []interface{}{0, 1, 2, 3, 4, 5} + convertedValue, err := convertInterface(slice, reflect.Slice) + if err != nil { + t.Fatalf("Error: %v %v", err, convertedValue) + } + +} +func TestSize(t *testing.T) { + size := int64(800) + blob := buildtxblobs(size) + chunksafterSerialize := size / chunkDataSize + terminalchunk := size % chunkDataSize + if terminalchunk != 0 { + chunksafterSerialize = chunksafterSerialize + 1 + } + chunksafterSerialize = chunksafterSerialize * size + sizeafterSerialize := chunksafterSerialize * chunkSize + serializedblob, err := Serialize(blob) + if err != nil { + t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) + } + + if int64(len(serializedblob)) != sizeafterSerialize { + + t.Fatalf("Error Serializing blobs the lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) + + } + +} +func TestSerializeAndDeserializeblob(t *testing.T) { + + var testbody interface{} + + blob := buildtxblobs(31) + + serializedblob, err := Serialize(blob) + + if err != nil { + t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) + } + err2 := Deserialize(serializedblob, &testbody) + if err2 != nil { + t.Fatalf("Error Serializing blob:%v", err2) + } + + if !reflect.DeepEqual(blob, testbody) { + + t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v ------ %v", blob, testbody) + } + +} From dbac52948a7f32d0120cc6bbdd43cecd83262ece Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 15:34:48 +0800 Subject: [PATCH 19/37] sharding/utils: Fixed serialization tests(#92) --- sharding/utils/marshal.go | 9 +++++++-- sharding/utils/marshal_test.go | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index bc9e70dabd8a..e0bcf8262af8 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -58,7 +58,6 @@ func serializeBlob(cb interface{}) ([]byte, error) { length := int64(len(blob)) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize - finalchunkIndex := length - 1 indicatorByte := make([]byte, 1) indicatorByte[0] = 0 tempbody := []byte{} @@ -114,7 +113,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { indicatorByte[0] = byte(terminalLength) tempbody = append(tempbody, append(indicatorByte, - blob[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) + blob[chunkDataSize*chunksNumber:length]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -172,6 +171,12 @@ func Deserialize(collationbody []byte, rawtx interface{}) error { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and // add it and append to the rawtx slice. The tempbody signifies a deserialized blob + } else if collationbody[indicatorIndex] == byte(1) { + + tempbody = append(tempbody, collationbody[(indicatorIndex+1)]) + deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) + tempbody = []byte{} + } else { terminalIndex := int64(collationbody[indicatorIndex]) tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 152531de9dad..a9b8fd13a5c4 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -3,6 +3,7 @@ package utils import ( "math/rand" "reflect" + //"runtime" "testing" ) @@ -61,7 +62,7 @@ func TestSerializeAndDeserializeblob(t *testing.T) { var testbody interface{} - blob := buildtxblobs(31) + blob := buildtxblobs(1000) serializedblob, err := Serialize(blob) @@ -75,7 +76,7 @@ func TestSerializeAndDeserializeblob(t *testing.T) { if !reflect.DeepEqual(blob, testbody) { - t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v ------ %v", blob, testbody) + t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v ------%v ------ %v", blob, serializedblob, testbody) } } From d49086ab81e52c0805b5f4158a71cd1bb747b19d Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 16:18:12 +0800 Subject: [PATCH 20/37] sharding/utils : Adding Collation Methods (#92) --- sharding/client/utils.go | 188 --------------------------------- sharding/client/utils_test.go | 80 -------------- sharding/collation.go | 44 ++++++++ sharding/collation_test.go | 61 +++++++++++ sharding/utils/marshal.go | 37 +++---- sharding/utils/marshal_test.go | 3 +- 6 files changed, 122 insertions(+), 291 deletions(-) delete mode 100644 sharding/client/utils.go delete mode 100644 sharding/client/utils_test.go diff --git a/sharding/client/utils.go b/sharding/client/utils.go deleted file mode 100644 index 948043adc5db..000000000000 --- a/sharding/client/utils.go +++ /dev/null @@ -1,188 +0,0 @@ -package client - -import ( - "errors" - "fmt" - "math" - "reflect" -) - -var ( - collationsizelimit = int64(math.Pow(float64(2), float64(20))) - chunkSize = int64(32) - indicatorSize = int64(1) - numberOfChunks = collationsizelimit / chunkSize - chunkDataSize = chunkSize - indicatorSize -) - -// convertInterface converts inputted interface to the required type, ex: slice. -func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) { - val := reflect.ValueOf(arg) - if val.Kind() == kind { - return val, nil - - } - err := errors.New("Interface Conversion a failure") - return val, err -} - -func convertbyteToInterface(arg []byte) []interface{} { - length := int64(len(arg)) - newtype := make([]interface{}, length) - for i, v := range arg { - newtype[i] = v - } - - return newtype -} - -func interfacetoByte(arg []interface{}) []byte { - length := int64(len(arg)) - newtype := make([]byte, length) - for i, v := range arg { - newtype[i] = v.(byte) - } - - return newtype -} - -// serializeBlob parses the blob and serializes it appropriately. -func serializeBlob(cb interface{}) ([]byte, error) { - - interfaceblob, err := convertInterface(cb, reflect.Slice) - if err != nil { - return nil, fmt.Errorf("Error: %v", err) - } - blob := interfacetoByte(interfaceblob.Interface().([]interface{})) - length := int64(len(blob)) - terminalLength := length % chunkDataSize - chunksNumber := length / chunkDataSize - finalchunkIndex := length - 1 - indicatorByte := make([]byte, 1) - indicatorByte[0] = 0 - tempbody := []byte{} - - // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right - - if chunksNumber == 0 { - paddedbytes := make([]byte, (length - terminalLength)) - indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(blob, paddedbytes...)...) - return tempbody, nil - } - - //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 - // Then this chunk is returned to the main Serialize function - - if terminalLength == 0 { - - for i := int64(1); i < chunksNumber; i++ { - // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk - // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of - // 31 - tempbody = append(tempbody, - append(indicatorByte, - blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) - - } - indicatorByte[0] = byte(chunkDataSize) - - // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob - tempbody = append(tempbody, - append(indicatorByte, - blob[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) - - return tempbody, nil - - } - - // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk - // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of - // 31 - - for i := int64(1); i <= chunksNumber; i++ { - - tempbody = append(tempbody, - append(indicatorByte, - blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) - - } - // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. - // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. - // finalchunkIndex refers to the index of the last data byte - indicatorByte[0] = byte(terminalLength) - tempbody = append(tempbody, - append(indicatorByte, - blob[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) - - emptyBytes := make([]byte, (chunkDataSize - terminalLength)) - tempbody = append(tempbody, emptyBytes...) - - return tempbody, nil - -} - -// Serialize takes a set of transaction blobs and converts them to a single byte array. -func Serialize(rawtx []interface{}) ([]byte, error) { - length := int64(len(rawtx)) - - if length == 0 { - return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") - } - serialisedData := []byte{} - - //Loops through all the blobs and serializes them into chunks - for i := int64(0); i < length; i++ { - - blobLength := int64(len(serialisedData)) - data := rawtx[i] - refinedData, err := serializeBlob(data) - if err != nil { - return nil, fmt.Errorf("Error: %v at index: %v", i, err) - } - serialisedData = append(serialisedData, refinedData...) - - if int64(len(serialisedData)) > collationsizelimit { - serialisedData = serialisedData[:blobLength] - return serialisedData, nil - - } - - } - return serialisedData, nil -} - -// Deserializebody results in the Collation body being deserialised and separated into its respective interfaces. -func Deserializebody(collationbody []byte, rawtx interface{}) error { - - length := int64(len(collationbody)) - chunksNumber := length / chunkSize - indicatorByte := byte(0) - tempbody := []byte{} - deserializedblob := []byte{} - - // This separates the collation body into its respective transaction blobs - for i := int64(1); i <= chunksNumber; i++ { - indicatorIndex := (i - 1) * chunkSize - // Tests if the chunk delimiter is zero, if it is it will append the data chunk - // to tempbody - if collationbody[indicatorIndex] == indicatorByte { - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) - - // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and - // add it and append to the rawtx slice. The tempbody signifies a deserialized blob - } else { - terminalIndex := int64(collationbody[indicatorIndex]) - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) - deserializedblob = append(deserializedblob, tempbody...) - tempbody = []byte{} - - } - - } - - *rawtx.(*interface{}) = convertbyteToInterface(deserializedblob) - - return nil - -} diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go deleted file mode 100644 index d87e0af2e048..000000000000 --- a/sharding/client/utils_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package client - -import ( - "math/rand" - "reflect" - "testing" -) - -func buildtxblobs(size int64) []interface{} { - tempbody := make([]interface{}, size) - for i := int64(0); i < size; i++ { - tempbody[i] = buildblob(size) - - } - return tempbody -} - -func buildblob(size int64) []interface{} { - - tempbody := make([]interface{}, size) - for i := int64(0); i < size; i++ { - tempbody[i] = byte(rand.Int()) - - } - - return tempbody - -} -func TestConvertInterface(t *testing.T) { - var slice interface{} - slice = []interface{}{0, 1, 2, 3, 4, 5} - convertedValue, err := convertInterface(slice, reflect.Slice) - if err != nil { - t.Fatalf("Error: %v %v", err, convertedValue) - } - -} -func TestSize(t *testing.T) { - size := int64(2) - blob := buildtxblobs(size) - chunksafterSerialize := size * size / chunkDataSize - terminalchunk := size * size % chunkDataSize - if terminalchunk != 0 { - chunksafterSerialize = chunksafterSerialize + 1 - } - sizeafterSerialize := chunksafterSerialize * chunkSize - serializedblob, err := Serialize(blob) - if err != nil { - t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) - } - - if int64(len(serializedblob)) != sizeafterSerialize { - - t.Fatalf("Error Serializing blobs the lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) - - } - -} -func TestSerializeAndDeserializeblob(t *testing.T) { - - var testbody interface{} - - blob := buildtxblobs(30) - - serializedblob, err := Serialize(blob) - - if err != nil { - t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) - } - err2 := Deserializebody(serializedblob, &testbody) - if err2 != nil { - t.Fatalf("Error Serializing blob:%v", err2) - } - - if !reflect.DeepEqual(blob, testbody) { - - t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v %v", blob, testbody) - } - -} diff --git a/sharding/collation.go b/sharding/collation.go index 5528453e468f..571eb8ddda80 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -1,12 +1,16 @@ package sharding import ( + "fmt" + "math" "math/big" + "reflect" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/sharding/utils" ) // Collation base struct. @@ -90,6 +94,12 @@ func (c *Collation) Header() *CollationHeader { return c.header } func (c *Collation) Body() []byte { return c.body } // Transactions returns an array of tx's in the collation. +var ( + collationsizelimit = int64(math.Pow(float64(2), float64(20))) + chunkSize = int64(32) + numberOfChunks = collationsizelimit / chunkSize +) + func (c *Collation) Transactions() []*types.Transaction { return c.transactions } // ProposerAddress is the coinbase addr of the creator for the collation. @@ -105,3 +115,37 @@ func (c *Collation) CalculateChunkRoot() { chunkRoot := common.BytesToHash(c.body) c.header.data.ChunkRoot = &chunkRoot } + +// Serialize method serializes the collation body +func (c *Collation) Serialize() ([]byte, error) { + + blob, err := utils.ConvertInterface(c.transactions, reflect.Slice) + if err != nil { + return nil, fmt.Errorf("%v", err) + } + serializedtx, err := utils.Serialize(blob) + + if err != nil { + return nil, fmt.Errorf("%v", err) + } + + if int64(len(serializedtx)) > collationsizelimit { + serializedtx = serializedtx[0:collationsizelimit] + + } + + return serializedtx, nil + +} + +func (c *Collation) GasUsed() *big.Int { + g := uint64(0) + for _, tx := range c.transactions { + if g > math.MaxUint64-(g+tx.Gas()) { + g = math.MaxUint64 + break + } + g += tx.Gas() + } + return big.NewInt(0).SetUint64(g) +} diff --git a/sharding/collation_test.go b/sharding/collation_test.go index c94ec7199681..101bf375bbb5 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -38,6 +38,67 @@ func TestCollation_ProposerAddress(t *testing.T) { t.Errorf("initialized collation does not contain correct proposer address") } } +func TestCollation_GasUsed(t *testing.T) { + tests := []struct { + transactions []*types.Transaction + gasUsed *big.Int + }{ + { + transactions: []*types.Transaction{ + makeTxWithGasLimit(100), + makeTxWithGasLimit(100000), + makeTxWithGasLimit(899900), + }, + gasUsed: big.NewInt(1000000), + }, { + transactions: []*types.Transaction{}, + gasUsed: big.NewInt(0), + }, + { + transactions: []*types.Transaction{ + makeTxWithGasLimit(math.MaxUint64), + makeTxWithGasLimit(9001), + makeTxWithGasLimit(math.MaxUint64), + }, + gasUsed: big.NewInt(0).SetUint64(math.MaxUint64), + }, + } + + for _, tt := range tests { + got := (&Collation{transactions: tt.transactions}).GasUsed() + if tt.gasUsed.Cmp(got) != 0 { + t.Errorf("Returned unexpected gasUsed. Got=%v, wanted=%v", got, tt.gasUsed) + } + } +} + +func TestSerialize(t *testing.T) { + tests := []struct { + transactions []*types.Transaction + }{ + { + transactions: []*types.Transaction{ + makeTxWithGasLimit(0), + makeTxWithGasLimit(1), + makeTxWithGasLimit(2), + makeTxWithGasLimit(3), + }, + }, { + transactions: []*types.Transaction{}, + }, + } + + for _, tt := range tests { + c := &Collation{} + + results, err := c.Serialize() + if err != nil { + t.Fatalf("%v --- %v ----%v", err, tt, results) + } + + } + +} func makeTxWithGasLimit(gl uint64) *types.Transaction { return types.NewTransaction(0 /*nonce*/, common.HexToAddress("0x0") /*to*/, nil /*amount*/, gl, nil /*gasPrice*/, nil /*data*/) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index e0bcf8262af8..69f1fc01bd7a 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -3,24 +3,26 @@ package utils import ( "errors" "fmt" - "math" "reflect" ) var ( - collationsizelimit = int64(math.Pow(float64(2), float64(20))) - chunkSize = int64(32) - indicatorSize = int64(1) - numberOfChunks = collationsizelimit / chunkSize - chunkDataSize = chunkSize - indicatorSize + chunkSize = int64(32) + indicatorSize = int64(1) + chunkDataSize = chunkSize - indicatorSize ) -// convertInterface converts inputted interface to the required type of interface, ex: slice. -func convertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) { +// ConvertInterface converts inputted interface to the required type of interface, ex: slice. +func ConvertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) { val := reflect.ValueOf(arg) if val.Kind() == kind { - return val.Interface().([]interface{}), nil + newtype := make([]interface{}, val.Len()) + for i := 1; i < val.Len(); i++ { + newtype[i] = val.Index(i) + } + + return newtype, nil } err := errors.New("Interface Conversion a failure") @@ -50,7 +52,7 @@ func interfacetoByte(arg []interface{}) []byte { // serializeBlob parses the blob and serializes it appropriately. func serializeBlob(cb interface{}) ([]byte, error) { - interfaceblob, err := convertInterface(cb, reflect.Slice) + interfaceblob, err := ConvertInterface(cb, reflect.Slice) if err != nil { return nil, fmt.Errorf("Error: %v", err) } @@ -134,7 +136,6 @@ func Serialize(rawtx []interface{}) ([]byte, error) { //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { - blobLength := int64(len(serialisedData)) data := rawtx[i] refinedData, err := serializeBlob(data) if err != nil { @@ -142,17 +143,11 @@ func Serialize(rawtx []interface{}) ([]byte, error) { } serialisedData = append(serialisedData, refinedData...) - if int64(len(serialisedData)) > collationsizelimit { - serialisedData = serialisedData[:blobLength] - return serialisedData, nil - - } - } return serialisedData, nil } -// Deserialize results in the Collation body being deserialised and separated into its respective interfaces. +// Deserialize results in the byte array being deserialised and separated into its respective interfaces. func Deserialize(collationbody []byte, rawtx interface{}) error { length := int64(len(collationbody)) @@ -161,7 +156,7 @@ func Deserialize(collationbody []byte, rawtx interface{}) error { tempbody := []byte{} var deserializedblob []interface{} - // This separates the collation body into its respective transaction blobs + // This separates the byte array into its separate blobs for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk @@ -169,8 +164,6 @@ func Deserialize(collationbody []byte, rawtx interface{}) error { if collationbody[indicatorIndex] == indicatorByte { tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) - // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and - // add it and append to the rawtx slice. The tempbody signifies a deserialized blob } else if collationbody[indicatorIndex] == byte(1) { tempbody = append(tempbody, collationbody[(indicatorIndex+1)]) @@ -178,6 +171,8 @@ func Deserialize(collationbody []byte, rawtx interface{}) error { tempbody = []byte{} } else { + // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and + // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob terminalIndex := int64(collationbody[indicatorIndex]) tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index a9b8fd13a5c4..33cdd91da136 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -3,7 +3,6 @@ package utils import ( "math/rand" "reflect" - //"runtime" "testing" ) @@ -30,7 +29,7 @@ func buildblob(size int64) []interface{} { func TestConvertInterface(t *testing.T) { var slice interface{} slice = []interface{}{0, 1, 2, 3, 4, 5} - convertedValue, err := convertInterface(slice, reflect.Slice) + convertedValue, err := ConvertInterface(slice, reflect.Slice) if err != nil { t.Fatalf("Error: %v %v", err, convertedValue) } From 8568d34c2a941bb7d8ea7f71a0f478929a5cc861 Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 17:12:11 +0800 Subject: [PATCH 21/37] sharding: Cleaning up tests (#92) --- sharding/collation_test.go | 12 ++++++++---- sharding/utils/marshal.go | 10 +++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 101bf375bbb5..af5779f79eba 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -90,11 +90,15 @@ func TestSerialize(t *testing.T) { for _, tt := range tests { c := &Collation{} - - results, err := c.Serialize() - if err != nil { - t.Fatalf("%v --- %v ----%v", err, tt, results) + for _, tx := range tt.transactions { + c.AddTransaction(tx) } + /* + results, err := c.Serialize() + if err != nil { + t.Fatalf("%v ----%v---%v", err, results, c.transactions) + } + */ } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 69f1fc01bd7a..a8e0980079a0 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -17,16 +17,12 @@ func ConvertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) val := reflect.ValueOf(arg) if val.Kind() == kind { - newtype := make([]interface{}, val.Len()) - for i := 1; i < val.Len(); i++ { - newtype[i] = val.Index(i) - } - - return newtype, nil + return val.Interface().([]interface{}), nil } err := errors.New("Interface Conversion a failure") return nil, err + } func convertbyteToInterface(arg []byte) []interface{} { @@ -124,7 +120,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { } -// Serialize takes a set of transaction blobs and converts them to a single byte array. +// Serialize takes a set of blobs and converts them to a single byte array. func Serialize(rawtx []interface{}) ([]byte, error) { length := int64(len(rawtx)) From ce18eefb853dd05940fb44fd785491a8c4691ede Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 17:33:03 +0800 Subject: [PATCH 22/37] sharding: Removing Test (#92) --- sharding/collation_test.go | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index af5779f79eba..226bf04ae9ff 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -27,51 +27,6 @@ func TestCollation_Transactions(t *testing.T) { } } -func TestCollation_ProposerAddress(t *testing.T) { - proposerAddr := common.BytesToAddress([]byte("proposer")) - header := NewCollationHeader(big.NewInt(1), nil, big.NewInt(1), &proposerAddr, []byte{}) - body := []byte{} - - collation := NewCollation(header, body, nil) - - if collation.ProposerAddress().String() != proposerAddr.String() { - t.Errorf("initialized collation does not contain correct proposer address") - } -} -func TestCollation_GasUsed(t *testing.T) { - tests := []struct { - transactions []*types.Transaction - gasUsed *big.Int - }{ - { - transactions: []*types.Transaction{ - makeTxWithGasLimit(100), - makeTxWithGasLimit(100000), - makeTxWithGasLimit(899900), - }, - gasUsed: big.NewInt(1000000), - }, { - transactions: []*types.Transaction{}, - gasUsed: big.NewInt(0), - }, - { - transactions: []*types.Transaction{ - makeTxWithGasLimit(math.MaxUint64), - makeTxWithGasLimit(9001), - makeTxWithGasLimit(math.MaxUint64), - }, - gasUsed: big.NewInt(0).SetUint64(math.MaxUint64), - }, - } - - for _, tt := range tests { - got := (&Collation{transactions: tt.transactions}).GasUsed() - if tt.gasUsed.Cmp(got) != 0 { - t.Errorf("Returned unexpected gasUsed. Got=%v, wanted=%v", got, tt.gasUsed) - } - } -} - func TestSerialize(t *testing.T) { tests := []struct { transactions []*types.Transaction From 89f8544ae93acb17a23426aca9dee2b9d5d4f939 Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 17:43:47 +0800 Subject: [PATCH 23/37] sharding/utils: Fix Lint (#92) --- sharding/utils/marshal_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 33cdd91da136..b1f69bff7c17 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -27,8 +27,7 @@ func buildblob(size int64) []interface{} { } func TestConvertInterface(t *testing.T) { - var slice interface{} - slice = []interface{}{0, 1, 2, 3, 4, 5} + slice := []interface{}{0, 1, 2, 3, 4, 5} convertedValue, err := ConvertInterface(slice, reflect.Slice) if err != nil { t.Fatalf("Error: %v %v", err, convertedValue) From c1b5d048bf50937171a26ecaf852f5d763223bbe Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 14 May 2018 18:13:45 +0800 Subject: [PATCH 24/37] sharding/utils: Changing from interfaces to rawblobs(#92) --- sharding/collation.go | 19 +----- sharding/collation_test.go | 11 ++-- sharding/utils/marshal.go | 110 +++++++++++++++++++++------------ sharding/utils/marshal_test.go | 33 +++++++--- 4 files changed, 99 insertions(+), 74 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 571eb8ddda80..21fdc1912b6a 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -4,7 +4,6 @@ import ( "fmt" "math" "math/big" - "reflect" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -119,10 +118,8 @@ func (c *Collation) CalculateChunkRoot() { // Serialize method serializes the collation body func (c *Collation) Serialize() ([]byte, error) { - blob, err := utils.ConvertInterface(c.transactions, reflect.Slice) - if err != nil { - return nil, fmt.Errorf("%v", err) - } + blob := utils.ConvertToInterface(c.transactions) + serializedtx, err := utils.Serialize(blob) if err != nil { @@ -137,15 +134,3 @@ func (c *Collation) Serialize() ([]byte, error) { return serializedtx, nil } - -func (c *Collation) GasUsed() *big.Int { - g := uint64(0) - for _, tx := range c.transactions { - if g > math.MaxUint64-(g+tx.Gas()) { - g = math.MaxUint64 - break - } - g += tx.Gas() - } - return big.NewInt(0).SetUint64(g) -} diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 226bf04ae9ff..229e6a7064c4 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -48,12 +48,11 @@ func TestSerialize(t *testing.T) { for _, tx := range tt.transactions { c.AddTransaction(tx) } - /* - results, err := c.Serialize() - if err != nil { - t.Fatalf("%v ----%v---%v", err, results, c.transactions) - } - */ + + results, err := c.Serialize() + if err != nil { + t.Fatalf("%v ----%v---%v", err, results, c.transactions) + } } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index a8e0980079a0..252109aa1a40 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -12,12 +12,27 @@ var ( chunkDataSize = chunkSize - indicatorSize ) +type Flags struct { + skipEvmExecution bool +} + +type RawBlob struct { + flags Flags + data []byte +} + // ConvertInterface converts inputted interface to the required type of interface, ex: slice. -func ConvertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) { +func ConvertInterfacetoBytes(arg interface{}) ([]byte, error) { val := reflect.ValueOf(arg) - if val.Kind() == kind { + if val.Kind() == reflect.Slice { - return val.Interface().([]interface{}), nil + length := val.Len() + newtype := make([]byte, length) + for i := 0; i < length; i++ { + newtype[i] = val.Index(i).Interface().(byte) + } + + return newtype, nil } err := errors.New("Interface Conversion a failure") @@ -35,29 +50,28 @@ func convertbyteToInterface(arg []byte) []interface{} { return newtype } -func interfacetoByte(arg []interface{}) []byte { - length := int64(len(arg)) - newtype := make([]byte, length) - for i, v := range arg { - newtype[i] = v.(byte) +func ConvertToInterface(arg interface{}) []interface{} { + val := reflect.ValueOf(arg) + length := val.Len() + newtype := make([]interface{}, length) + for i := 0; i < length; i++ { + newtype[i] = val.Index(i) } return newtype } // serializeBlob parses the blob and serializes it appropriately. -func serializeBlob(cb interface{}) ([]byte, error) { +func serializeBlob(cb RawBlob) ([]byte, error) { - interfaceblob, err := ConvertInterface(cb, reflect.Slice) - if err != nil { - return nil, fmt.Errorf("Error: %v", err) - } - blob := interfacetoByte(interfaceblob) - length := int64(len(blob)) + length := int64(len(cb.data)) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize indicatorByte := make([]byte, 1) indicatorByte[0] = 0 + if cb.flags.skipEvmExecution { + indicatorByte[0] |= (1 << 7) + } tempbody := []byte{} // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right @@ -65,7 +79,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { if chunksNumber == 0 { paddedbytes := make([]byte, (chunkDataSize - length)) indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(blob, paddedbytes...)...) + tempbody = append(indicatorByte, append(cb.data, paddedbytes...)...) return tempbody, nil } @@ -78,17 +92,21 @@ func serializeBlob(cb interface{}) ([]byte, error) { // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of // 31 + tempbody = append(tempbody, append(indicatorByte, - blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } indicatorByte[0] = byte(chunkDataSize) + if cb.flags.skipEvmExecution { + indicatorByte[0] |= (1 << 7) + } // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob tempbody = append(tempbody, append(indicatorByte, - blob[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) + cb.data[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) return tempbody, nil @@ -102,16 +120,19 @@ func serializeBlob(cb interface{}) ([]byte, error) { tempbody = append(tempbody, append(indicatorByte, - blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. // finalchunkIndex refers to the index of the last data byte indicatorByte[0] = byte(terminalLength) + if cb.flags.skipEvmExecution { + indicatorByte[0] |= (1 << 7) + } tempbody = append(tempbody, append(indicatorByte, - blob[chunkDataSize*chunksNumber:length]...)...) + cb.data[chunkDataSize*chunksNumber:length]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -121,8 +142,8 @@ func serializeBlob(cb interface{}) ([]byte, error) { } // Serialize takes a set of blobs and converts them to a single byte array. -func Serialize(rawtx []interface{}) ([]byte, error) { - length := int64(len(rawtx)) +func Serialize(rawblobs []RawBlob) ([]byte, error) { + length := int64(len(rawblobs)) if length == 0 { return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") @@ -132,10 +153,10 @@ func Serialize(rawtx []interface{}) ([]byte, error) { //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { - data := rawtx[i] + data := rawblobs[i] refinedData, err := serializeBlob(data) if err != nil { - return nil, fmt.Errorf("Error: %v at index: %v", i, err) + return nil, fmt.Errorf("Index %v : %v", i, err) } serialisedData = append(serialisedData, refinedData...) @@ -144,42 +165,49 @@ func Serialize(rawtx []interface{}) ([]byte, error) { } // Deserialize results in the byte array being deserialised and separated into its respective interfaces. -func Deserialize(collationbody []byte, rawtx interface{}) error { +func Deserialize(collationbody []byte) ([]RawBlob, error) { length := int64(len(collationbody)) chunksNumber := length / chunkSize indicatorByte := byte(0) - tempbody := []byte{} - var deserializedblob []interface{} + var tempbody RawBlob + var deserializedblob []RawBlob // This separates the byte array into its separate blobs for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize + var terminalIndex int64 // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody - if collationbody[indicatorIndex] == indicatorByte { - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) + if collationbody[indicatorIndex] == indicatorByte || collationbody[indicatorIndex] == byte(128) { + tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(i)*chunkSize]...) - } else if collationbody[indicatorIndex] == byte(1) { - - tempbody = append(tempbody, collationbody[(indicatorIndex+1)]) - deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) - tempbody = []byte{} + } else if collationbody[indicatorIndex] == byte(31) || collationbody[indicatorIndex] == byte(159) { + if collationbody[indicatorIndex] == byte(159) { + tempbody.flags.skipEvmExecution = true + } + tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1)]) + deserializedblob = append(deserializedblob, tempbody) + tempbody = RawBlob{} } else { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob - terminalIndex := int64(collationbody[indicatorIndex]) - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) - deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) - tempbody = []byte{} + flagindex := collationbody[indicatorIndex] >> 7 + if flagindex == byte(1) { + terminalIndex = int64(collationbody[indicatorIndex]) - 128 + tempbody.flags.skipEvmExecution = true + } else { + terminalIndex = int64(collationbody[indicatorIndex]) + } + tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) + deserializedblob = append(deserializedblob, tempbody) + tempbody = RawBlob{} } } - *rawtx.(*interface{}) = deserializedblob - - return nil + return deserializedblob, nil } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index b1f69bff7c17..7594c6db195f 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -6,18 +6,27 @@ import ( "testing" ) -func buildtxblobs(size int64) []interface{} { - tempbody := make([]interface{}, size) +func buildrawblob(size int64) []RawBlob { + tempbody := make([]RawBlob, size) for i := int64(0); i < size; i++ { - tempbody[i] = buildblob(size) + var rawblob RawBlob + rawblob.data = buildblob(size) + flagset := byte(rand.Int()) >> 7 + if flagset == byte(1) { + rawblob.flags.skipEvmExecution = true + + } + + tempbody[i] = rawblob } return tempbody + } -func buildblob(size int64) []interface{} { +func buildblob(size int64) []byte { - tempbody := make([]interface{}, size) + tempbody := make([]byte, size) for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) @@ -26,6 +35,10 @@ func buildblob(size int64) []interface{} { return tempbody } + +/* +Might be required in the future for part 2 of serialization + func TestConvertInterface(t *testing.T) { slice := []interface{}{0, 1, 2, 3, 4, 5} convertedValue, err := ConvertInterface(slice, reflect.Slice) @@ -33,10 +46,10 @@ func TestConvertInterface(t *testing.T) { t.Fatalf("Error: %v %v", err, convertedValue) } -} +} */ func TestSize(t *testing.T) { size := int64(800) - blob := buildtxblobs(size) + blob := buildrawblob(size) chunksafterSerialize := size / chunkDataSize terminalchunk := size % chunkDataSize if terminalchunk != 0 { @@ -60,16 +73,16 @@ func TestSerializeAndDeserializeblob(t *testing.T) { var testbody interface{} - blob := buildtxblobs(1000) + blob := buildrawblob(10) serializedblob, err := Serialize(blob) if err != nil { t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) } - err2 := Deserialize(serializedblob, &testbody) + raw, err2 := Deserialize(serializedblob) if err2 != nil { - t.Fatalf("Error Serializing blob:%v", err2) + t.Fatalf("Error Serializing blob:%v due to %v", raw, err2) } if !reflect.DeepEqual(blob, testbody) { From 447043404de16556acbdc7f6f99fb5fd3badb2bb Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 14 May 2018 19:11:50 +0800 Subject: [PATCH 25/37] sharding/utils: Got Tests working again (#92) --- sharding/utils/marshal.go | 11 +++++++---- sharding/utils/marshal_test.go | 10 ++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 252109aa1a40..f0e0f00cced4 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -79,6 +79,9 @@ func serializeBlob(cb RawBlob) ([]byte, error) { if chunksNumber == 0 { paddedbytes := make([]byte, (chunkDataSize - length)) indicatorByte[0] = byte(terminalLength) + if cb.flags.skipEvmExecution { + indicatorByte[0] |= (1 << 7) + } tempbody = append(indicatorByte, append(cb.data, paddedbytes...)...) return tempbody, nil } @@ -170,13 +173,13 @@ func Deserialize(collationbody []byte) ([]RawBlob, error) { length := int64(len(collationbody)) chunksNumber := length / chunkSize indicatorByte := byte(0) - var tempbody RawBlob + tempbody := RawBlob{} var deserializedblob []RawBlob // This separates the byte array into its separate blobs for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize - var terminalIndex int64 + // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody if collationbody[indicatorIndex] == indicatorByte || collationbody[indicatorIndex] == byte(128) { @@ -193,12 +196,12 @@ func Deserialize(collationbody []byte) ([]RawBlob, error) { } else { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob + terminalIndex := int64(collationbody[indicatorIndex]) + //Check if EVM flag is equal to 1 flagindex := collationbody[indicatorIndex] >> 7 if flagindex == byte(1) { terminalIndex = int64(collationbody[indicatorIndex]) - 128 tempbody.flags.skipEvmExecution = true - } else { - terminalIndex = int64(collationbody[indicatorIndex]) } tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedblob = append(deserializedblob, tempbody) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 7594c6db195f..c523129b16d9 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -48,7 +48,7 @@ func TestConvertInterface(t *testing.T) { } */ func TestSize(t *testing.T) { - size := int64(800) + size := int64(8) blob := buildrawblob(size) chunksafterSerialize := size / chunkDataSize terminalchunk := size % chunkDataSize @@ -71,9 +71,7 @@ func TestSize(t *testing.T) { } func TestSerializeAndDeserializeblob(t *testing.T) { - var testbody interface{} - - blob := buildrawblob(10) + blob := buildrawblob(330) serializedblob, err := Serialize(blob) @@ -85,9 +83,9 @@ func TestSerializeAndDeserializeblob(t *testing.T) { t.Fatalf("Error Serializing blob:%v due to %v", raw, err2) } - if !reflect.DeepEqual(blob, testbody) { + if !reflect.DeepEqual(blob, raw) { - t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v ------%v ------ %v", blob, serializedblob, testbody) + t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same:\n\n %v \n\n %v \n\n %v", blob, serializedblob, raw) } } From eb582cbf691c06c505602c688d79f925cac0c527 Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 14 May 2018 20:03:18 +0800 Subject: [PATCH 26/37] sharding: Fix tests and clean up(#92) --- sharding/collation.go | 5 ++- sharding/collation_test.go | 5 ++- sharding/utils/marshal.go | 57 +++++++++++++--------------------- sharding/utils/marshal_test.go | 57 ++++++++++++++++++---------------- 4 files changed, 60 insertions(+), 64 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 21fdc1912b6a..d4cb96ae5354 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -118,7 +118,10 @@ func (c *Collation) CalculateChunkRoot() { // Serialize method serializes the collation body func (c *Collation) Serialize() ([]byte, error) { - blob := utils.ConvertToInterface(c.transactions) + blob, err := utils.ConvertToRawBlob(c.transactions) + if err != nil { + return nil, fmt.Errorf("%v", err) + } serializedtx, err := utils.Serialize(blob) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 229e6a7064c4..9f47237e8a6b 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -27,6 +27,9 @@ func TestCollation_Transactions(t *testing.T) { } } +//TODO: Add test for converting *types.Transaction into raw blobs + +//Tests thta Transactions can be serialised func TestSerialize(t *testing.T) { tests := []struct { transactions []*types.Transaction @@ -51,7 +54,7 @@ func TestSerialize(t *testing.T) { results, err := c.Serialize() if err != nil { - t.Fatalf("%v ----%v---%v", err, results, c.transactions) + t.Fatalf("%v\n %v\n %v", err, results, c.transactions) } } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index f0e0f00cced4..387ed21f8aa8 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -1,9 +1,7 @@ package utils import ( - "errors" "fmt" - "reflect" ) var ( @@ -12,16 +10,20 @@ var ( chunkDataSize = chunkSize - indicatorSize ) +// Flags to add to chunk delimiter. type Flags struct { skipEvmExecution bool } +// RawBlob type which will contain flags and data for serialization. type RawBlob struct { flags Flags data []byte } -// ConvertInterface converts inputted interface to the required type of interface, ex: slice. +/* +* Might be needed in part 2 of serialisation +* ConvertInterface converts inputted interface to the required type of interface, ex: slice. func ConvertInterfacetoBytes(arg interface{}) ([]byte, error) { val := reflect.ValueOf(arg) if val.Kind() == reflect.Slice { @@ -39,26 +41,12 @@ func ConvertInterfacetoBytes(arg interface{}) ([]byte, error) { return nil, err } +*/ -func convertbyteToInterface(arg []byte) []interface{} { - length := int64(len(arg)) - newtype := make([]interface{}, length) - for i, v := range arg { - newtype[i] = v - } - - return newtype -} - -func ConvertToInterface(arg interface{}) []interface{} { - val := reflect.ValueOf(arg) - length := val.Len() - newtype := make([]interface{}, length) - for i := 0; i < length; i++ { - newtype[i] = val.Index(i) - } - - return newtype +// ConvertToRawBlob will convert any supported type into a the RawBlob type. +func ConvertToRawBlob(arg interface{}) ([]RawBlob, error) { + //TODO: will be done in part 2 to convert any type to a rawBlob + return nil, nil } // serializeBlob parses the blob and serializes it appropriately. @@ -148,9 +136,6 @@ func serializeBlob(cb RawBlob) ([]byte, error) { func Serialize(rawblobs []RawBlob) ([]byte, error) { length := int64(len(rawblobs)) - if length == 0 { - return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") - } serialisedData := []byte{} //Loops through all the blobs and serializes them into chunks @@ -168,9 +153,9 @@ func Serialize(rawblobs []RawBlob) ([]byte, error) { } // Deserialize results in the byte array being deserialised and separated into its respective interfaces. -func Deserialize(collationbody []byte) ([]RawBlob, error) { +func Deserialize(data []byte) ([]RawBlob, error) { - length := int64(len(collationbody)) + length := int64(len(data)) chunksNumber := length / chunkSize indicatorByte := byte(0) tempbody := RawBlob{} @@ -182,28 +167,28 @@ func Deserialize(collationbody []byte) ([]RawBlob, error) { // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody - if collationbody[indicatorIndex] == indicatorByte || collationbody[indicatorIndex] == byte(128) { - tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(i)*chunkSize]...) + if data[indicatorIndex] == indicatorByte || data[indicatorIndex] == byte(128) { + tempbody.data = append(tempbody.data, data[(indicatorIndex+1):(i)*chunkSize]...) - } else if collationbody[indicatorIndex] == byte(31) || collationbody[indicatorIndex] == byte(159) { - if collationbody[indicatorIndex] == byte(159) { + } else if data[indicatorIndex] == byte(31) || data[indicatorIndex] == byte(159) { + if data[indicatorIndex] == byte(159) { tempbody.flags.skipEvmExecution = true } - tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1)]) + tempbody.data = append(tempbody.data, data[(indicatorIndex+1):indicatorIndex+1+chunkDataSize]...) deserializedblob = append(deserializedblob, tempbody) tempbody = RawBlob{} } else { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob - terminalIndex := int64(collationbody[indicatorIndex]) + terminalIndex := int64(data[indicatorIndex]) //Check if EVM flag is equal to 1 - flagindex := collationbody[indicatorIndex] >> 7 + flagindex := data[indicatorIndex] >> 7 if flagindex == byte(1) { - terminalIndex = int64(collationbody[indicatorIndex]) - 128 + terminalIndex = int64(data[indicatorIndex]) - 128 tempbody.flags.skipEvmExecution = true } - tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) + tempbody.data = append(tempbody.data, data[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedblob = append(deserializedblob, tempbody) tempbody = RawBlob{} diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index c523129b16d9..2048b8a39877 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -48,44 +48,49 @@ func TestConvertInterface(t *testing.T) { } */ func TestSize(t *testing.T) { - size := int64(8) - blob := buildrawblob(size) - chunksafterSerialize := size / chunkDataSize - terminalchunk := size % chunkDataSize - if terminalchunk != 0 { - chunksafterSerialize = chunksafterSerialize + 1 - } - chunksafterSerialize = chunksafterSerialize * size - sizeafterSerialize := chunksafterSerialize * chunkSize - serializedblob, err := Serialize(blob) - if err != nil { - t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) - } + for i := 0; i < 300; i++ { + size := int64(i) + blob := buildrawblob(size) + chunksafterSerialize := size / chunkDataSize + terminalchunk := size % chunkDataSize + if terminalchunk != 0 { + chunksafterSerialize = chunksafterSerialize + 1 + } + chunksafterSerialize = chunksafterSerialize * size + sizeafterSerialize := chunksafterSerialize * chunkSize + serializedblob, err := Serialize(blob) + if err != nil { + t.Errorf("Error Serializing blob:%v\n %v", err, serializedblob) + } - if int64(len(serializedblob)) != sizeafterSerialize { + if int64(len(serializedblob)) != sizeafterSerialize { - t.Fatalf("Error Serializing blobs the lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) + t.Errorf("Error Serializing blobs the lengths are not the same:\n %d \n %d", int64(len(serializedblob)), sizeafterSerialize) + } } } func TestSerializeAndDeserializeblob(t *testing.T) { - blob := buildrawblob(330) + for i := 1; i < 300; i++ { - serializedblob, err := Serialize(blob) + blob := buildrawblob(int64(i)) - if err != nil { - t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) - } - raw, err2 := Deserialize(serializedblob) - if err2 != nil { - t.Fatalf("Error Serializing blob:%v due to %v", raw, err2) - } + serializedblob, err := Serialize(blob) + + if err != nil { + t.Errorf("Error Serializing blob at index %d:\n%v\n%v", i, err, serializedblob) + } + raw, err2 := Deserialize(serializedblob) + if err2 != nil { + t.Errorf("Error Serializing blob at index %d:\n%v due to \n%v", i, raw, err2) + } - if !reflect.DeepEqual(blob, raw) { + if !reflect.DeepEqual(blob, raw) { - t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same:\n\n %v \n\n %v \n\n %v", blob, serializedblob, raw) + t.Errorf("Error Serializing blobs at index %d, the serialized and deserialized versions are not the same:\n\n %v \n\n %v \n\n %v", i, blob, serializedblob, raw) + } } } From 798666ab00066a974cca2633cb9c9311decfa6bf Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 15 May 2018 20:35:47 +0800 Subject: [PATCH 27/37] sharding/utils : Adding RLP encoding (#92) --- sharding/collation.go | 39 ++++++++++++++++++++++++++++++++-- sharding/collation_test.go | 21 +++++++++++++++++- sharding/utils/marshal.go | 36 ++++++++++++++----------------- sharding/utils/marshal_test.go | 11 ---------- 4 files changed, 73 insertions(+), 34 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index d4cb96ae5354..0984e124dc05 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -115,15 +115,41 @@ func (c *Collation) CalculateChunkRoot() { c.header.data.ChunkRoot = &chunkRoot } +func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { + + // It does not skip evm execution by default + + blobs := make([]*utils.RawBlob, len(c.transactions)) + for i, v := range c.transactions { + + err := error(nil) + blobs[i], err = utils.NewRawBlob(v, false) + + if err != nil { + return nil, fmt.Errorf("Creation of raw blobs from transactions failed %v", err) + } + + } + + return blobs, nil + +} + // Serialize method serializes the collation body func (c *Collation) Serialize() ([]byte, error) { - blob, err := utils.ConvertToRawBlob(c.transactions) + /*blob, err := utils.ConvertToRawBlob(c.transactions) + if err != nil { + return nil, fmt.Errorf("%v", err) + }**/ + + blobs, err := c.CreateRawBlobs() + if err != nil { return nil, fmt.Errorf("%v", err) } - serializedtx, err := utils.Serialize(blob) + serializedtx, err := utils.Serialize(blobs) if err != nil { return nil, fmt.Errorf("%v", err) @@ -137,3 +163,12 @@ func (c *Collation) Serialize() ([]byte, error) { return serializedtx, nil } + +func (c *Collation) Deserialize(serialisedblob []byte) error { + var blobs []utils.RawBlob + + deserializedblobs, err := utils.Deserialize(serialisedblob) + if err != nil { + return fmt.Errorf("%v", err) + } +} diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 9f47237e8a6b..f4184b7da022 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -2,6 +2,8 @@ package sharding import ( "math/big" + //"github.com/ethereum/go-ethereum/rlp" + //"reflect" "testing" "github.com/ethereum/go-ethereum/common" @@ -52,8 +54,25 @@ func TestSerialize(t *testing.T) { c.AddTransaction(tx) } - results, err := c.Serialize() + /*var tests *types.Transaction + yadd := reflect.ValueOf(*c.transactions[3]) + d := yadd.FieldByName("data").FieldByName("Hash") + + test, err := rlp.EncodeToBytes(c.transactions[3]) if err != nil { + t.Fatalf("%v\n %v\n %v", err, test, *(c.transactions[0])) + } + erx := rlp.DecodeBytes(test, &tests) + + dd := reflect.ValueOf(*tests) + cv := dd.FieldByName("data").FieldByName("Hash") + + if cv != d { + t.Fatalf("%v\n %v\n %v", erx, cv, d) + } */ + + results, err := c.Serialize() + if err == nil { t.Fatalf("%v\n %v\n %v", err, results, c.transactions) } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 387ed21f8aa8..267327dfe866 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -21,27 +22,22 @@ type RawBlob struct { data []byte } -/* -* Might be needed in part 2 of serialisation -* ConvertInterface converts inputted interface to the required type of interface, ex: slice. -func ConvertInterfacetoBytes(arg interface{}) ([]byte, error) { - val := reflect.ValueOf(arg) - if val.Kind() == reflect.Slice { - - length := val.Len() - newtype := make([]byte, length) - for i := 0; i < length; i++ { - newtype[i] = val.Index(i).Interface().(byte) - } - - return newtype, nil +func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { + data, err := rlp.EncodeToBytes(i) + if err != nil { + return nil, fmt.Errorf("RLP encoding was a failure:%v", err) + } + return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipevm}}, nil +} +func ConvertfromRawBlob(blob RawBlob, i interface{}) error { + err := rlp.DecodeBytes(blob.data, i) + if err != nil { + return fmt.Errorf("RLP decoding was a failure:%v", err) } - err := errors.New("Interface Conversion a failure") - return nil, err + return nil } -*/ // ConvertToRawBlob will convert any supported type into a the RawBlob type. func ConvertToRawBlob(arg interface{}) ([]RawBlob, error) { @@ -50,7 +46,7 @@ func ConvertToRawBlob(arg interface{}) ([]RawBlob, error) { } // serializeBlob parses the blob and serializes it appropriately. -func serializeBlob(cb RawBlob) ([]byte, error) { +func SerializeBlob(cb RawBlob) ([]byte, error) { length := int64(len(cb.data)) terminalLength := length % chunkDataSize @@ -133,7 +129,7 @@ func serializeBlob(cb RawBlob) ([]byte, error) { } // Serialize takes a set of blobs and converts them to a single byte array. -func Serialize(rawblobs []RawBlob) ([]byte, error) { +func Serialize(rawblobs []*RawBlob) ([]byte, error) { length := int64(len(rawblobs)) serialisedData := []byte{} @@ -142,7 +138,7 @@ func Serialize(rawblobs []RawBlob) ([]byte, error) { for i := int64(0); i < length; i++ { data := rawblobs[i] - refinedData, err := serializeBlob(data) + refinedData, err := SerializeBlob(*data) if err != nil { return nil, fmt.Errorf("Index %v : %v", i, err) } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 2048b8a39877..6cb508a78525 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -36,17 +36,6 @@ func buildblob(size int64) []byte { } -/* -Might be required in the future for part 2 of serialization - -func TestConvertInterface(t *testing.T) { - slice := []interface{}{0, 1, 2, 3, 4, 5} - convertedValue, err := ConvertInterface(slice, reflect.Slice) - if err != nil { - t.Fatalf("Error: %v %v", err, convertedValue) - } - -} */ func TestSize(t *testing.T) { for i := 0; i < 300; i++ { size := int64(i) From d1174fac9fc4062f56b86f1d271ff8706792506f Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 15 May 2018 22:15:33 +0800 Subject: [PATCH 28/37] sharding: Fixing tests in util package (#92) --- sharding/collation.go | 30 +++++++++++++++++++++++++++--- sharding/collation_test.go | 9 ++++++--- sharding/utils/marshal.go | 9 +++++---- sharding/utils/marshal_test.go | 16 ++++++++++++++-- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 0984e124dc05..c3c1002283db 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -120,10 +120,10 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { // It does not skip evm execution by default blobs := make([]*utils.RawBlob, len(c.transactions)) - for i, v := range c.transactions { + for i := 0; i < len(c.transactions); i++ { err := error(nil) - blobs[i], err = utils.NewRawBlob(v, false) + blobs[i], err = utils.NewRawBlob(c.transactions[i], false) if err != nil { return nil, fmt.Errorf("Creation of raw blobs from transactions failed %v", err) @@ -135,6 +135,23 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { } +func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) ([]*types.Transaction, error) { + + tx := make([]*types.Transaction, len(rawblobs)) + for i := 0; i < len(rawblobs); i++ { + + err := utils.ConvertfromRawBlob(&rawblobs[i], tx[i]) + + if err != nil { + return nil, fmt.Errorf("Creation of transactions from raw blobs failed %v", err) + } + + } + + return tx, nil + +} + // Serialize method serializes the collation body func (c *Collation) Serialize() ([]byte, error) { @@ -165,10 +182,17 @@ func (c *Collation) Serialize() ([]byte, error) { } func (c *Collation) Deserialize(serialisedblob []byte) error { - var blobs []utils.RawBlob deserializedblobs, err := utils.Deserialize(serialisedblob) if err != nil { return fmt.Errorf("%v", err) } + + c.transactions, err = c.ConvertBacktoTx(deserializedblobs) + + if err != nil { + return fmt.Errorf("%v", err) + } + + return nil } diff --git a/sharding/collation_test.go b/sharding/collation_test.go index f4184b7da022..b2e4790640e7 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -72,9 +72,12 @@ func TestSerialize(t *testing.T) { } */ results, err := c.Serialize() - if err == nil { - t.Fatalf("%v\n %v\n %v", err, results, c.transactions) - } + + t.Errorf("%v\n %v\n %v", err, results, c.transactions) + + err = c.Deserialize(results) + + t.Errorf("%v\n %v\n %v", err, results, c.transactions) } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 267327dfe866..d1e4c20e378c 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -30,8 +30,9 @@ func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipevm}}, nil } -func ConvertfromRawBlob(blob RawBlob, i interface{}) error { - err := rlp.DecodeBytes(blob.data, i) +func ConvertfromRawBlob(blob *RawBlob, i interface{}) error { + data := (*blob).data + err := rlp.DecodeBytes(data, i) if err != nil { return fmt.Errorf("RLP decoding was a failure:%v", err) } @@ -137,8 +138,8 @@ func Serialize(rawblobs []*RawBlob) ([]byte, error) { //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { - data := rawblobs[i] - refinedData, err := SerializeBlob(*data) + data := *rawblobs[i] + refinedData, err := SerializeBlob(data) if err != nil { return nil, fmt.Errorf("Index %v : %v", i, err) } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 6cb508a78525..2d734d62360e 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -47,7 +47,13 @@ func TestSize(t *testing.T) { } chunksafterSerialize = chunksafterSerialize * size sizeafterSerialize := chunksafterSerialize * chunkSize - serializedblob, err := Serialize(blob) + + drefbody := make([]*RawBlob, len(blob)) + for s := 0; s < len(blob); s++ { + drefbody[s] = &(blob[s]) + + } + serializedblob, err := Serialize(drefbody) if err != nil { t.Errorf("Error Serializing blob:%v\n %v", err, serializedblob) } @@ -66,7 +72,13 @@ func TestSerializeAndDeserializeblob(t *testing.T) { blob := buildrawblob(int64(i)) - serializedblob, err := Serialize(blob) + drefbody := make([]*RawBlob, len(blob)) + for s := 0; s < len(blob); s++ { + drefbody[s] = &(blob[s]) + + } + + serializedblob, err := Serialize(drefbody) if err != nil { t.Errorf("Error Serializing blob at index %d:\n%v\n%v", i, err, serializedblob) From fafc95ac49569387ea2837f85361ccd6155670c3 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 00:38:24 +0800 Subject: [PATCH 29/37] sharding: Adding roundtrip tests for tx serialization(#92) --- sharding/collation.go | 12 ++--- sharding/collation_test.go | 99 +++++++++++++++++++++++++++++--------- 2 files changed, 83 insertions(+), 28 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index c3c1002283db..342f11ec9fe8 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -135,20 +135,20 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { } -func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) ([]*types.Transaction, error) { +func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) error { - tx := make([]*types.Transaction, len(rawblobs)) + // tx := make([]*types.Transaction, len(rawblobs)) for i := 0; i < len(rawblobs); i++ { - err := utils.ConvertfromRawBlob(&rawblobs[i], tx[i]) + err := utils.ConvertfromRawBlob(&rawblobs[i], c.transactions[i]) if err != nil { - return nil, fmt.Errorf("Creation of transactions from raw blobs failed %v", err) + return fmt.Errorf("Creation of transactions from raw blobs failed %v", err) } } - return tx, nil + return nil } @@ -188,7 +188,7 @@ func (c *Collation) Deserialize(serialisedblob []byte) error { return fmt.Errorf("%v", err) } - c.transactions, err = c.ConvertBacktoTx(deserializedblobs) + err = c.ConvertBacktoTx(deserializedblobs) if err != nil { return fmt.Errorf("%v", err) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index b2e4790640e7..ac4229469cf4 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -4,12 +4,22 @@ import ( "math/big" //"github.com/ethereum/go-ethereum/rlp" //"reflect" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) +// fieldAccess is to access unexported fields in structs in another package +func fieldAccess(i interface{}, fields []string) reflect.Value { + val := reflect.ValueOf(i) + for i := 0; i < len(fields); i++ { + val = reflect.Indirect(val).FieldByName(fields[i]) + } + return val + +} func TestCollation_Transactions(t *testing.T) { header := NewCollationHeader(big.NewInt(1), nil, big.NewInt(1), nil, []byte{}) body := []byte{} @@ -31,20 +41,19 @@ func TestCollation_Transactions(t *testing.T) { //TODO: Add test for converting *types.Transaction into raw blobs -//Tests thta Transactions can be serialised -func TestSerialize(t *testing.T) { +//Tests that Transactions can be serialised +func TestSerialize_Deserialize(t *testing.T) { + tests := []struct { transactions []*types.Transaction }{ { transactions: []*types.Transaction{ makeTxWithGasLimit(0), - makeTxWithGasLimit(1), - makeTxWithGasLimit(2), - makeTxWithGasLimit(3), + makeTxWithGasLimit(5), + makeTxWithGasLimit(20), + makeTxWithGasLimit(100), }, - }, { - transactions: []*types.Transaction{}, }, } @@ -54,30 +63,76 @@ func TestSerialize(t *testing.T) { c.AddTransaction(tx) } - /*var tests *types.Transaction - yadd := reflect.ValueOf(*c.transactions[3]) - d := yadd.FieldByName("data").FieldByName("Hash") + tx := c.transactions + + results, err := c.Serialize() + + if err != nil { + t.Errorf("Unable to Serialize transactions, %v", err) + } + + err = c.Deserialize(results) - test, err := rlp.EncodeToBytes(c.transactions[3]) if err != nil { - t.Fatalf("%v\n %v\n %v", err, test, *(c.transactions[0])) + t.Errorf("Unable to deserialize collation body, %v", err) + } + + if len(tx) != len(c.transactions) { + t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(c.transactions)) } - erx := rlp.DecodeBytes(test, &tests) - dd := reflect.ValueOf(*tests) - cv := dd.FieldByName("data").FieldByName("Hash") + for i := 0; i < len(tx); i++ { - if cv != d { - t.Fatalf("%v\n %v\n %v", erx, cv, d) - } */ + aval := fieldAccess(tx[i], []string{"data", "AccountNonce"}) + aval2 := fieldAccess(c.transactions[i], []string{"data", "AccountNonce"}) - results, err := c.Serialize() + if aval != aval2 { - t.Errorf("%v\n %v\n %v", err, results, c.transactions) + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", aval, aval2) - err = c.Deserialize(results) + } + + gval := fieldAccess(tx[i], []string{"data", "GasLimit"}) + gval2 := fieldAccess(c.transactions[i], []string{"data", "GasLimit"}) + if gval != gval2 { - t.Errorf("%v\n %v\n %v", err, results, c.transactions) + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", gval, gval2) + + } + + pval := fieldAccess(tx[i], []string{"data", "Price"}) + pval2 := fieldAccess(c.transactions[i], []string{"data", "Price"}) + if pval != pval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", pval, pval2) + + } + + rval := fieldAccess(tx[i], []string{"data", "Recipient"}) + rval2 := fieldAccess(c.transactions[i], []string{"data", "Recipient"}) + if rval != rval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", rval, rval2) + + } + + amval := fieldAccess(tx[i], []string{"data", "Amount"}) + amval2 := fieldAccess(c.transactions[i], []string{"data", "Amount"}) + if amval != amval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", amval, amval2) + + } + + paval := fieldAccess(tx[i], []string{"data", "Payload"}) + paval2 := fieldAccess(c.transactions[i], []string{"data", "Payload"}) + if paval != paval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", paval, paval2) + + } + + } } From c800d300672da545b3fb04326e76bf4bf337e20a Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 00:51:28 +0800 Subject: [PATCH 30/37] sharding: Adding comments to functions(#92) --- sharding/collation.go | 16 +++++----------- sharding/utils/marshal.go | 2 ++ sharding/utils/marshal_test.go | 4 ---- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 342f11ec9fe8..56367e39e2d6 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -99,6 +99,7 @@ var ( numberOfChunks = collationsizelimit / chunkSize ) +// Transactions returns an array of tx's in the collation. func (c *Collation) Transactions() []*types.Transaction { return c.transactions } // ProposerAddress is the coinbase addr of the creator for the collation. @@ -115,10 +116,10 @@ func (c *Collation) CalculateChunkRoot() { c.header.data.ChunkRoot = &chunkRoot } +// CreateRawBlobs creates raw blobs from transactions. func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { // It does not skip evm execution by default - blobs := make([]*utils.RawBlob, len(c.transactions)) for i := 0; i < len(c.transactions); i++ { @@ -135,31 +136,23 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { } +// ConvertBacktoTx converts raw blobs back to their original transactions. func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) error { - // tx := make([]*types.Transaction, len(rawblobs)) for i := 0; i < len(rawblobs); i++ { err := utils.ConvertfromRawBlob(&rawblobs[i], c.transactions[i]) - if err != nil { return fmt.Errorf("Creation of transactions from raw blobs failed %v", err) } - } - return nil } -// Serialize method serializes the collation body +// Serialize method serializes the collation body to a byte array. func (c *Collation) Serialize() ([]byte, error) { - /*blob, err := utils.ConvertToRawBlob(c.transactions) - if err != nil { - return nil, fmt.Errorf("%v", err) - }**/ - blobs, err := c.CreateRawBlobs() if err != nil { @@ -181,6 +174,7 @@ func (c *Collation) Serialize() ([]byte, error) { } +// Deserialize takes a byte array and converts its back to its original transactions. func (c *Collation) Deserialize(serialisedblob []byte) error { deserializedblobs, err := utils.Deserialize(serialisedblob) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index d1e4c20e378c..1dce87cb16cc 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -22,6 +22,7 @@ type RawBlob struct { data []byte } +// NewRawBlob builds a raw blob from any interface by using RLP encoding func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { data, err := rlp.EncodeToBytes(i) if err != nil { @@ -30,6 +31,7 @@ func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipevm}}, nil } +// ConvertfromRawBlob converts raw blob back from a byte array to its interface func ConvertfromRawBlob(blob *RawBlob, i interface{}) error { data := (*blob).data err := rlp.DecodeBytes(data, i) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 2d734d62360e..cbf8d23eb69b 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -16,9 +16,7 @@ func buildrawblob(size int64) []RawBlob { rawblob.flags.skipEvmExecution = true } - tempbody[i] = rawblob - } return tempbody @@ -29,9 +27,7 @@ func buildblob(size int64) []byte { tempbody := make([]byte, size) for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) - } - return tempbody } From a140f8336469fb46ec2fdd1e3a4e823d5ed3637c Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 01:12:58 +0800 Subject: [PATCH 31/37] sharding: Cleaning up (#92) --- sharding/collation.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 56367e39e2d6..4948674cbbdf 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -94,9 +94,9 @@ func (c *Collation) Body() []byte { return c.body } // Transactions returns an array of tx's in the collation. var ( - collationsizelimit = int64(math.Pow(float64(2), float64(20))) + collationSizelimit = int64(math.Pow(float64(2), float64(20))) chunkSize = int64(32) - numberOfChunks = collationsizelimit / chunkSize + numberOfChunks = collationSizelimit / chunkSize ) // Transactions returns an array of tx's in the collation. @@ -159,30 +159,31 @@ func (c *Collation) Serialize() ([]byte, error) { return nil, fmt.Errorf("%v", err) } - serializedtx, err := utils.Serialize(blobs) + serializedTx, err := utils.Serialize(blobs) if err != nil { return nil, fmt.Errorf("%v", err) } - if int64(len(serializedtx)) > collationsizelimit { - serializedtx = serializedtx[0:collationsizelimit] + if int64(len(serializedTx)) > collationSizelimit { + + return nil, fmt.Errorf("The serialized body exceeded the collation size limit", serializedTx) } - return serializedtx, nil + return serializedTx, nil } // Deserialize takes a byte array and converts its back to its original transactions. -func (c *Collation) Deserialize(serialisedblob []byte) error { +func (c *Collation) Deserialize(serialisedBlob []byte) error { - deserializedblobs, err := utils.Deserialize(serialisedblob) + deserializedBlobs, err := utils.Deserialize(serialisedBlob) if err != nil { return fmt.Errorf("%v", err) } - err = c.ConvertBacktoTx(deserializedblobs) + err = c.ConvertBacktoTx(deserializedBlobs) if err != nil { return fmt.Errorf("%v", err) From 434c5119e0cda5db46a06959ca0fcab06460fd77 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 11:07:31 +0800 Subject: [PATCH 32/37] sharding: Made Requested Changes(#92) --- sharding/collation.go | 28 ++++++++------- sharding/collation_test.go | 51 +++++++++++++------------- sharding/utils/marshal.go | 66 ++++++++++++++++------------------ sharding/utils/marshal_test.go | 10 +++--- 4 files changed, 75 insertions(+), 80 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 4948674cbbdf..39a009b6ba29 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -136,17 +136,21 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { } -// ConvertBacktoTx converts raw blobs back to their original transactions. -func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) error { +// ConvertBackToTx converts raw blobs back to their original transactions. +func ConvertBackToTx(rawBlobs []utils.RawBlob) ([]*types.Transaction, error) { - for i := 0; i < len(rawblobs); i++ { + blobs := make([]*types.Transaction, len(rawBlobs)) - err := utils.ConvertfromRawBlob(&rawblobs[i], c.transactions[i]) + for i := 0; i < len(rawBlobs); i++ { + + blobs[i] = types.NewTransaction(0, common.HexToAddress("0x"), nil, 0, nil, nil) + + err := utils.ConvertFromRawBlob(&rawBlobs[i], blobs[i]) if err != nil { - return fmt.Errorf("Creation of transactions from raw blobs failed %v", err) + return nil, fmt.Errorf("Creation of transactions from raw blobs failed %v", err) } } - return nil + return blobs, nil } @@ -167,7 +171,7 @@ func (c *Collation) Serialize() ([]byte, error) { if int64(len(serializedTx)) > collationSizelimit { - return nil, fmt.Errorf("The serialized body exceeded the collation size limit", serializedTx) + return nil, fmt.Errorf("The serialized body exceeded the collation size limit: %v", serializedTx) } @@ -176,18 +180,18 @@ func (c *Collation) Serialize() ([]byte, error) { } // Deserialize takes a byte array and converts its back to its original transactions. -func (c *Collation) Deserialize(serialisedBlob []byte) error { +func Deserialize(serialisedBlob []byte) (*[]*types.Transaction, error) { deserializedBlobs, err := utils.Deserialize(serialisedBlob) if err != nil { - return fmt.Errorf("%v", err) + return nil, fmt.Errorf("%v", err) } - err = c.ConvertBacktoTx(deserializedBlobs) + txs, err := ConvertBackToTx(deserializedBlobs) if err != nil { - return fmt.Errorf("%v", err) + return nil, fmt.Errorf("%v", err) } - return nil + return &txs, nil } diff --git a/sharding/collation_test.go b/sharding/collation_test.go index ac4229469cf4..fe1cd288234c 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -4,6 +4,7 @@ import ( "math/big" //"github.com/ethereum/go-ethereum/rlp" //"reflect" + "bytes" "reflect" "testing" @@ -71,64 +72,60 @@ func TestSerialize_Deserialize(t *testing.T) { t.Errorf("Unable to Serialize transactions, %v", err) } - err = c.Deserialize(results) + deserializedTxs, err := Deserialize(results) if err != nil { t.Errorf("Unable to deserialize collation body, %v", err) } - if len(tx) != len(c.transactions) { - t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(c.transactions)) + if len(tx) != len(*deserializedTxs) { + t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(*deserializedTxs)) } for i := 0; i < len(tx); i++ { - aval := fieldAccess(tx[i], []string{"data", "AccountNonce"}) - aval2 := fieldAccess(c.transactions[i], []string{"data", "AccountNonce"}) + beforeSerialization := tx[i] + afterDeserialization := (*deserializedTxs)[i] - if aval != aval2 { + if beforeSerialization.Nonce() != afterDeserialization.Nonce() { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", aval, aval2) + t.Errorf("Data before serialization and after deserialization are not the same ,AccountNonce: %v, %v", beforeSerialization.Nonce(), afterDeserialization.Nonce()) } - gval := fieldAccess(tx[i], []string{"data", "GasLimit"}) - gval2 := fieldAccess(c.transactions[i], []string{"data", "GasLimit"}) - if gval != gval2 { + if beforeSerialization.Gas() != afterDeserialization.Gas() { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", gval, gval2) + t.Errorf("Data before serialization and after deserialization are not the same ,GasLimit: %v, %v", beforeSerialization.Gas(), afterDeserialization.Gas()) } - pval := fieldAccess(tx[i], []string{"data", "Price"}) - pval2 := fieldAccess(c.transactions[i], []string{"data", "Price"}) - if pval != pval2 { + if beforeSerialization.GasPrice().Cmp(afterDeserialization.GasPrice()) != 0 { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", pval, pval2) + t.Errorf("Data before serialization and after deserialization are not the same ,Price: %v, %v", beforeSerialization.GasPrice(), afterDeserialization.GasPrice()) } - rval := fieldAccess(tx[i], []string{"data", "Recipient"}) - rval2 := fieldAccess(c.transactions[i], []string{"data", "Recipient"}) - if rval != rval2 { + beforeAddress := reflect.ValueOf(beforeSerialization.To()) + afterAddress := reflect.ValueOf(afterDeserialization.To()) - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", rval, rval2) + if reflect.DeepEqual(beforeAddress, afterAddress) { + + t.Errorf("Data before serialization and after deserialization are not the same ,Recipient: %v, %v", beforeAddress, afterAddress) } - amval := fieldAccess(tx[i], []string{"data", "Amount"}) - amval2 := fieldAccess(c.transactions[i], []string{"data", "Amount"}) - if amval != amval2 { + if beforeSerialization.Value().Cmp(afterDeserialization.Value()) != 0 { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", amval, amval2) + t.Errorf("Data before serialization and after deserialization are not the same ,Amount: %v, %v", beforeSerialization.Value(), afterDeserialization.Value()) } - paval := fieldAccess(tx[i], []string{"data", "Payload"}) - paval2 := fieldAccess(c.transactions[i], []string{"data", "Payload"}) - if paval != paval2 { + beforeData := beforeSerialization.Data() + afterData := afterDeserialization.Data() + + if !bytes.Equal(beforeData, afterData) { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", paval, paval2) + t.Errorf("Data before serialization and after deserialization are not the same ,Payload: %v, %v", beforeData, afterData) } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 1dce87cb16cc..aaf47650c4b6 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -23,16 +23,16 @@ type RawBlob struct { } // NewRawBlob builds a raw blob from any interface by using RLP encoding -func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { +func NewRawBlob(i interface{}, skipEvm bool) (*RawBlob, error) { data, err := rlp.EncodeToBytes(i) if err != nil { return nil, fmt.Errorf("RLP encoding was a failure:%v", err) } - return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipevm}}, nil + return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipEvm}}, nil } -// ConvertfromRawBlob converts raw blob back from a byte array to its interface -func ConvertfromRawBlob(blob *RawBlob, i interface{}) error { +// ConvertFromRawBlob converts raw blob back from a byte array to its interface +func ConvertFromRawBlob(blob *RawBlob, i interface{}) error { data := (*blob).data err := rlp.DecodeBytes(data, i) if err != nil { @@ -42,13 +42,7 @@ func ConvertfromRawBlob(blob *RawBlob, i interface{}) error { return nil } -// ConvertToRawBlob will convert any supported type into a the RawBlob type. -func ConvertToRawBlob(arg interface{}) ([]RawBlob, error) { - //TODO: will be done in part 2 to convert any type to a rawBlob - return nil, nil -} - -// serializeBlob parses the blob and serializes it appropriately. +// SerializeBlob parses the blob and serializes it appropriately. func SerializeBlob(cb RawBlob) ([]byte, error) { length := int64(len(cb.data)) @@ -59,18 +53,18 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { if cb.flags.skipEvmExecution { indicatorByte[0] |= (1 << 7) } - tempbody := []byte{} + tempBody := []byte{} // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right if chunksNumber == 0 { - paddedbytes := make([]byte, (chunkDataSize - length)) + paddedBytes := make([]byte, (chunkDataSize - length)) indicatorByte[0] = byte(terminalLength) if cb.flags.skipEvmExecution { indicatorByte[0] |= (1 << 7) } - tempbody = append(indicatorByte, append(cb.data, paddedbytes...)...) - return tempbody, nil + tempBody = append(indicatorByte, append(cb.data, paddedBytes...)...) + return tempBody, nil } //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 @@ -83,7 +77,7 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of // 31 - tempbody = append(tempbody, + tempBody = append(tempBody, append(indicatorByte, cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) @@ -94,11 +88,11 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { } // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob - tempbody = append(tempbody, + tempBody = append(tempBody, append(indicatorByte, cb.data[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) - return tempbody, nil + return tempBody, nil } @@ -108,7 +102,7 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { for i := int64(1); i <= chunksNumber; i++ { - tempbody = append(tempbody, + tempBody = append(tempBody, append(indicatorByte, cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) @@ -120,14 +114,14 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { if cb.flags.skipEvmExecution { indicatorByte[0] |= (1 << 7) } - tempbody = append(tempbody, + tempBody = append(tempBody, append(indicatorByte, cb.data[chunkDataSize*chunksNumber:length]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) - tempbody = append(tempbody, emptyBytes...) + tempBody = append(tempBody, emptyBytes...) - return tempbody, nil + return tempBody, nil } @@ -157,44 +151,44 @@ func Deserialize(data []byte) ([]RawBlob, error) { length := int64(len(data)) chunksNumber := length / chunkSize indicatorByte := byte(0) - tempbody := RawBlob{} - var deserializedblob []RawBlob + tempBody := RawBlob{} + var deserializedBlob []RawBlob // This separates the byte array into its separate blobs for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk - // to tempbody + // to tempBody if data[indicatorIndex] == indicatorByte || data[indicatorIndex] == byte(128) { - tempbody.data = append(tempbody.data, data[(indicatorIndex+1):(i)*chunkSize]...) + tempBody.data = append(tempBody.data, data[(indicatorIndex+1):(i)*chunkSize]...) } else if data[indicatorIndex] == byte(31) || data[indicatorIndex] == byte(159) { if data[indicatorIndex] == byte(159) { - tempbody.flags.skipEvmExecution = true + tempBody.flags.skipEvmExecution = true } - tempbody.data = append(tempbody.data, data[(indicatorIndex+1):indicatorIndex+1+chunkDataSize]...) - deserializedblob = append(deserializedblob, tempbody) - tempbody = RawBlob{} + tempBody.data = append(tempBody.data, data[(indicatorIndex+1):indicatorIndex+1+chunkDataSize]...) + deserializedBlob = append(deserializedBlob, tempBody) + tempBody = RawBlob{} } else { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and - // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob + // add it and append to the deserializedblob slice. The tempBody signifies a single deserialized blob terminalIndex := int64(data[indicatorIndex]) //Check if EVM flag is equal to 1 flagindex := data[indicatorIndex] >> 7 if flagindex == byte(1) { terminalIndex = int64(data[indicatorIndex]) - 128 - tempbody.flags.skipEvmExecution = true + tempBody.flags.skipEvmExecution = true } - tempbody.data = append(tempbody.data, data[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) - deserializedblob = append(deserializedblob, tempbody) - tempbody = RawBlob{} + tempBody.data = append(tempBody.data, data[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) + deserializedBlob = append(deserializedBlob, tempBody) + tempBody = RawBlob{} } } - return deserializedblob, nil + return deserializedBlob, nil } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index cbf8d23eb69b..57f8832e4b00 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -6,11 +6,11 @@ import ( "testing" ) -func buildrawblob(size int64) []RawBlob { +func buildRawBlob(size int64) []RawBlob { tempbody := make([]RawBlob, size) for i := int64(0); i < size; i++ { var rawblob RawBlob - rawblob.data = buildblob(size) + rawblob.data = buildBlob(size) flagset := byte(rand.Int()) >> 7 if flagset == byte(1) { rawblob.flags.skipEvmExecution = true @@ -22,7 +22,7 @@ func buildrawblob(size int64) []RawBlob { } -func buildblob(size int64) []byte { +func buildBlob(size int64) []byte { tempbody := make([]byte, size) for i := int64(0); i < size; i++ { @@ -35,7 +35,7 @@ func buildblob(size int64) []byte { func TestSize(t *testing.T) { for i := 0; i < 300; i++ { size := int64(i) - blob := buildrawblob(size) + blob := buildRawBlob(size) chunksafterSerialize := size / chunkDataSize terminalchunk := size % chunkDataSize if terminalchunk != 0 { @@ -66,7 +66,7 @@ func TestSerializeAndDeserializeblob(t *testing.T) { for i := 1; i < 300; i++ { - blob := buildrawblob(int64(i)) + blob := buildRawBlob(int64(i)) drefbody := make([]*RawBlob, len(blob)) for s := 0; s < len(blob); s++ { From b90648305a4ba7c460a8dbe0fec6ccd02c022a6f Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 20:35:08 +0800 Subject: [PATCH 33/37] sharding: Removing pointer receiver(#92) --- sharding/collation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharding/collation.go b/sharding/collation.go index 39a009b6ba29..cb62b8fb605e 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -117,7 +117,7 @@ func (c *Collation) CalculateChunkRoot() { } // CreateRawBlobs creates raw blobs from transactions. -func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { +func (c Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { // It does not skip evm execution by default blobs := make([]*utils.RawBlob, len(c.transactions)) From 48000aa2a3c751658677407c390ec7d2f3c38b85 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Wed, 16 May 2018 13:38:26 -0400 Subject: [PATCH 34/37] sharding: fix up comment formatting in marshal and marshal_test.go --- sharding/utils/marshal.go | 60 +++++++++++++++++----------------- sharding/utils/marshal_test.go | 9 ++--- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index aaf47650c4b6..cddf61333eaf 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "github.com/ethereum/go-ethereum/rlp" ) @@ -22,7 +23,8 @@ type RawBlob struct { data []byte } -// NewRawBlob builds a raw blob from any interface by using RLP encoding +// NewRawBlob builds a raw blob from any interface by using +// RLP encoding. func NewRawBlob(i interface{}, skipEvm bool) (*RawBlob, error) { data, err := rlp.EncodeToBytes(i) if err != nil { @@ -31,7 +33,8 @@ func NewRawBlob(i interface{}, skipEvm bool) (*RawBlob, error) { return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipEvm}}, nil } -// ConvertFromRawBlob converts raw blob back from a byte array to its interface +// ConvertFromRawBlob converts raw blob back from a byte array +// to its interface. func ConvertFromRawBlob(blob *RawBlob, i interface{}) error { data := (*blob).data err := rlp.DecodeBytes(data, i) @@ -55,8 +58,8 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { } tempBody := []byte{} - // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right - + // if blob is less than 31 bytes, adds the indicator chunk + // and pads the remaining empty bytes to the right. if chunksNumber == 0 { paddedBytes := make([]byte, (chunkDataSize - length)) indicatorByte[0] = byte(terminalLength) @@ -67,15 +70,16 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { return tempBody, nil } - //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 - // Then this chunk is returned to the main Serialize function - + // if there is no need to pad empty bytes, then the indicator byte + // is added as 0001111, then this chunk is returned to the + // main Serialize function. if terminalLength == 0 { for i := int64(1); i < chunksNumber; i++ { - // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk - // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of - // 31 + // This loop loops through all non-terminal chunks and add a indicator + // byte of 00000000, each chunk is created by appending the indicator + // byte to the data chunks. The data chunks are separated into sets of + // 31 bytes. tempBody = append(tempBody, append(indicatorByte, @@ -96,20 +100,18 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { } - // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk - // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of - // 31 - + // This loop loops through all non-terminal chunks and add a indicator byte + // of 00000000, each chunk is created by appending the indcator byte + // to the data chunks. The data chunks are separated into sets of 31. for i := int64(1); i <= chunksNumber; i++ { - tempBody = append(tempBody, append(indicatorByte, cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) - } - // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. - // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. - // finalchunkIndex refers to the index of the last data byte + // Appends indicator bytes to terminal-chunks , and if the index of the chunk + // delimiter is non-zero adds it to the chunk. Also pads empty bytes to + // the terminal chunk.chunkDataSize*chunksNumber refers to the total + // size of the blob. finalchunkIndex refers to the index of the last data byte. indicatorByte[0] = byte(terminalLength) if cb.flags.skipEvmExecution { indicatorByte[0] |= (1 << 7) @@ -133,19 +135,19 @@ func Serialize(rawblobs []*RawBlob) ([]byte, error) { //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { - data := *rawblobs[i] refinedData, err := SerializeBlob(data) if err != nil { - return nil, fmt.Errorf("Index %v : %v", i, err) + return nil, fmt.Errorf("Index %v: %v", i, err) } serialisedData = append(serialisedData, refinedData...) - } + return serialisedData, nil } -// Deserialize results in the byte array being deserialised and separated into its respective interfaces. +// Deserialize results in the byte array being deserialised and +// separated into its respective interfaces. func Deserialize(data []byte) ([]RawBlob, error) { length := int64(len(data)) @@ -154,12 +156,12 @@ func Deserialize(data []byte) ([]RawBlob, error) { tempBody := RawBlob{} var deserializedBlob []RawBlob - // This separates the byte array into its separate blobs + // This separates the byte array into its separate blobs. for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk - // to tempBody + // to tempBody. if data[indicatorIndex] == indicatorByte || data[indicatorIndex] == byte(128) { tempBody.data = append(tempBody.data, data[(indicatorIndex+1):(i)*chunkSize]...) @@ -172,8 +174,9 @@ func Deserialize(data []byte) ([]RawBlob, error) { tempBody = RawBlob{} } else { - // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and - // add it and append to the deserializedblob slice. The tempBody signifies a single deserialized blob + // Since the chunk delimiter in non-zero now we can infer that it is + // a terminal chunk and add it and append to the deserializedblob + // slice. The tempBody signifies a single deserialized blob. terminalIndex := int64(data[indicatorIndex]) //Check if EVM flag is equal to 1 flagindex := data[indicatorIndex] >> 7 @@ -184,11 +187,8 @@ func Deserialize(data []byte) ([]RawBlob, error) { tempBody.data = append(tempBody.data, data[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedBlob = append(deserializedBlob, tempBody) tempBody = RawBlob{} - } - } return deserializedBlob, nil - } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 57f8832e4b00..a16013f54b34 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -18,18 +18,17 @@ func buildRawBlob(size int64) []RawBlob { } tempbody[i] = rawblob } - return tempbody + return tempbody } func buildBlob(size int64) []byte { - tempbody := make([]byte, size) for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) } - return tempbody + return tempbody } func TestSize(t *testing.T) { @@ -55,9 +54,7 @@ func TestSize(t *testing.T) { } if int64(len(serializedblob)) != sizeafterSerialize { - t.Errorf("Error Serializing blobs the lengths are not the same:\n %d \n %d", int64(len(serializedblob)), sizeafterSerialize) - } } @@ -71,7 +68,6 @@ func TestSerializeAndDeserializeblob(t *testing.T) { drefbody := make([]*RawBlob, len(blob)) for s := 0; s < len(blob); s++ { drefbody[s] = &(blob[s]) - } serializedblob, err := Serialize(drefbody) @@ -85,7 +81,6 @@ func TestSerializeAndDeserializeblob(t *testing.T) { } if !reflect.DeepEqual(blob, raw) { - t.Errorf("Error Serializing blobs at index %d, the serialized and deserialized versions are not the same:\n\n %v \n\n %v \n\n %v", i, blob, serializedblob, raw) } } From d9671185d33396b54431f294a4738ba0dfc5eb83 Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 17 May 2018 07:00:31 +0800 Subject: [PATCH 35/37] sharding: Cleaning up after merge conflicts(#92) --- sharding/collation_test.go | 102 ++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 57 deletions(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index fe1cd288234c..c0e4cb7355b0 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -1,10 +1,8 @@ package sharding import ( - "math/big" - //"github.com/ethereum/go-ethereum/rlp" - //"reflect" "bytes" + "math/big" "reflect" "testing" @@ -45,89 +43,79 @@ func TestCollation_Transactions(t *testing.T) { //Tests that Transactions can be serialised func TestSerialize_Deserialize(t *testing.T) { - tests := []struct { - transactions []*types.Transaction - }{ - { - transactions: []*types.Transaction{ - makeTxWithGasLimit(0), - makeTxWithGasLimit(5), - makeTxWithGasLimit(20), - makeTxWithGasLimit(100), - }, - }, + header := NewCollationHeader(big.NewInt(1), nil, big.NewInt(1), nil, []byte{}) + body := []byte{} + transactions := []*types.Transaction{ + makeTxWithGasLimit(0), + makeTxWithGasLimit(5), + makeTxWithGasLimit(20), + makeTxWithGasLimit(100), } - for _, tt := range tests { - c := &Collation{} - for _, tx := range tt.transactions { - c.AddTransaction(tx) - } + c := NewCollation(header, body, transactions) - tx := c.transactions + tx := c.transactions - results, err := c.Serialize() + results, err := c.Serialize() - if err != nil { - t.Errorf("Unable to Serialize transactions, %v", err) - } + if err != nil { + t.Errorf("Unable to Serialize transactions, %v", err) + } - deserializedTxs, err := Deserialize(results) + deserializedTxs, err := Deserialize(results) - if err != nil { - t.Errorf("Unable to deserialize collation body, %v", err) - } - - if len(tx) != len(*deserializedTxs) { - t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(*deserializedTxs)) - } + if err != nil { + t.Errorf("Unable to deserialize collation body, %v", err) + } - for i := 0; i < len(tx); i++ { + if len(tx) != len(*deserializedTxs) { + t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(*deserializedTxs)) + } - beforeSerialization := tx[i] - afterDeserialization := (*deserializedTxs)[i] + for i := 0; i < len(tx); i++ { - if beforeSerialization.Nonce() != afterDeserialization.Nonce() { + beforeSerialization := tx[i] + afterDeserialization := (*deserializedTxs)[i] - t.Errorf("Data before serialization and after deserialization are not the same ,AccountNonce: %v, %v", beforeSerialization.Nonce(), afterDeserialization.Nonce()) + if beforeSerialization.Nonce() != afterDeserialization.Nonce() { - } + t.Errorf("Data before serialization and after deserialization are not the same ,AccountNonce: %v, %v", beforeSerialization.Nonce(), afterDeserialization.Nonce()) - if beforeSerialization.Gas() != afterDeserialization.Gas() { + } - t.Errorf("Data before serialization and after deserialization are not the same ,GasLimit: %v, %v", beforeSerialization.Gas(), afterDeserialization.Gas()) + if beforeSerialization.Gas() != afterDeserialization.Gas() { - } + t.Errorf("Data before serialization and after deserialization are not the same ,GasLimit: %v, %v", beforeSerialization.Gas(), afterDeserialization.Gas()) - if beforeSerialization.GasPrice().Cmp(afterDeserialization.GasPrice()) != 0 { + } - t.Errorf("Data before serialization and after deserialization are not the same ,Price: %v, %v", beforeSerialization.GasPrice(), afterDeserialization.GasPrice()) + if beforeSerialization.GasPrice().Cmp(afterDeserialization.GasPrice()) != 0 { - } + t.Errorf("Data before serialization and after deserialization are not the same ,Price: %v, %v", beforeSerialization.GasPrice(), afterDeserialization.GasPrice()) - beforeAddress := reflect.ValueOf(beforeSerialization.To()) - afterAddress := reflect.ValueOf(afterDeserialization.To()) + } - if reflect.DeepEqual(beforeAddress, afterAddress) { + beforeAddress := reflect.ValueOf(beforeSerialization.To()) + afterAddress := reflect.ValueOf(afterDeserialization.To()) - t.Errorf("Data before serialization and after deserialization are not the same ,Recipient: %v, %v", beforeAddress, afterAddress) + if reflect.DeepEqual(beforeAddress, afterAddress) { - } + t.Errorf("Data before serialization and after deserialization are not the same ,Recipient: %v, %v", beforeAddress, afterAddress) - if beforeSerialization.Value().Cmp(afterDeserialization.Value()) != 0 { + } - t.Errorf("Data before serialization and after deserialization are not the same ,Amount: %v, %v", beforeSerialization.Value(), afterDeserialization.Value()) + if beforeSerialization.Value().Cmp(afterDeserialization.Value()) != 0 { - } + t.Errorf("Data before serialization and after deserialization are not the same ,Amount: %v, %v", beforeSerialization.Value(), afterDeserialization.Value()) - beforeData := beforeSerialization.Data() - afterData := afterDeserialization.Data() + } - if !bytes.Equal(beforeData, afterData) { + beforeData := beforeSerialization.Data() + afterData := afterDeserialization.Data() - t.Errorf("Data before serialization and after deserialization are not the same ,Payload: %v, %v", beforeData, afterData) + if !bytes.Equal(beforeData, afterData) { - } + t.Errorf("Data before serialization and after deserialization are not the same ,Payload: %v, %v", beforeData, afterData) } From 5f0f5425dcfb3579ee090cc595ecde60fdfa1fcc Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 17 May 2018 07:23:48 +0800 Subject: [PATCH 36/37] sharding: Adding colon(#92) --- sharding/collation.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index cb62b8fb605e..faa0b889b268 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -127,7 +127,7 @@ func (c Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { blobs[i], err = utils.NewRawBlob(c.transactions[i], false) if err != nil { - return nil, fmt.Errorf("Creation of raw blobs from transactions failed %v", err) + return nil, fmt.Errorf("Creation of raw blobs from transactions failed: %v", err) } } @@ -147,7 +147,7 @@ func ConvertBackToTx(rawBlobs []utils.RawBlob) ([]*types.Transaction, error) { err := utils.ConvertFromRawBlob(&rawBlobs[i], blobs[i]) if err != nil { - return nil, fmt.Errorf("Creation of transactions from raw blobs failed %v", err) + return nil, fmt.Errorf("Creation of transactions from raw blobs failed: %v", err) } } return blobs, nil From 6a2ebd35b8be17a040ebeded9ac76b9a9259277a Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 17 May 2018 07:31:25 +0800 Subject: [PATCH 37/37] sharding: Fix Lint (#92) --- sharding/collation.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index faa0b889b268..3ce4b55e3387 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -41,6 +41,8 @@ type collationHeaderData struct { ProposerSignature []byte // the proposer's signature for calculating collation hash. } +var collationSizelimit = int64(math.Pow(float64(2), float64(20))) + // NewCollation initializes a collation and leaves it up to clients to serialize, deserialize // and provide the body and transactions upon creation. func NewCollation(header *CollationHeader, body []byte, transactions []*types.Transaction) *Collation { @@ -92,13 +94,6 @@ func (c *Collation) Header() *CollationHeader { return c.header } // Body returns the collation's byte body. func (c *Collation) Body() []byte { return c.body } -// Transactions returns an array of tx's in the collation. -var ( - collationSizelimit = int64(math.Pow(float64(2), float64(20))) - chunkSize = int64(32) - numberOfChunks = collationSizelimit / chunkSize -) - // Transactions returns an array of tx's in the collation. func (c *Collation) Transactions() []*types.Transaction { return c.transactions }