From c482a5d4cf25e694909aec2eeeb1a5027bf38ec7 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 26 Feb 2020 13:54:12 +0100 Subject: [PATCH 1/8] Read Fixed --- src/neo/Consensus/Commit.cs | 3 ++- .../RecoveryMessage.CommitPayloadCompact.cs | 2 +- src/neo/Consensus/RecoveryMessage.cs | 2 +- src/neo/IO/Helper.cs | 17 ++++++++++++++--- .../P2P/Payloads/NetworkAddressWithTime.cs | 3 +-- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/neo/Consensus/Commit.cs b/src/neo/Consensus/Commit.cs index 84585e1ee4..b29cefc3c4 100644 --- a/src/neo/Consensus/Commit.cs +++ b/src/neo/Consensus/Commit.cs @@ -1,3 +1,4 @@ +using Neo.IO; using System.IO; namespace Neo.Consensus @@ -13,7 +14,7 @@ public class Commit : ConsensusMessage public override void Deserialize(BinaryReader reader) { base.Deserialize(reader); - Signature = reader.ReadBytes(64); + Signature = reader.ReadFixed(64); } public override void Serialize(BinaryWriter writer) diff --git a/src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs b/src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs index 72962f92b3..a78fd07043 100644 --- a/src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs +++ b/src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs @@ -23,7 +23,7 @@ void ISerializable.Deserialize(BinaryReader reader) { ViewNumber = reader.ReadByte(); ValidatorIndex = reader.ReadUInt16(); - Signature = reader.ReadBytes(64); + Signature = reader.ReadFixed(64); InvocationScript = reader.ReadVarBytes(1024); } diff --git a/src/neo/Consensus/RecoveryMessage.cs b/src/neo/Consensus/RecoveryMessage.cs index a5e1c09ac6..5c555050d3 100644 --- a/src/neo/Consensus/RecoveryMessage.cs +++ b/src/neo/Consensus/RecoveryMessage.cs @@ -39,7 +39,7 @@ public override void Deserialize(BinaryReader reader) { int preparationHashSize = UInt256.Zero.Size; if (preparationHashSize == (int)reader.ReadVarInt((ulong)preparationHashSize)) - PreparationHash = new UInt256(reader.ReadBytes(preparationHashSize)); + PreparationHash = new UInt256(reader.ReadFixed(preparationHashSize)); } PreparationMessages = reader.ReadSerializableArray(Blockchain.MaxValidators).ToDictionary(p => (int)p.ValidatorIndex); diff --git a/src/neo/IO/Helper.cs b/src/neo/IO/Helper.cs index 3acc4b2c2f..9ad85ce3fe 100644 --- a/src/neo/IO/Helper.cs +++ b/src/neo/IO/Helper.cs @@ -151,7 +151,7 @@ public static byte[] ReadBytesWithGrouping(this BinaryReader reader) int count; do { - byte[] group = reader.ReadBytes(GroupingSizeInBytes); + byte[] group = reader.ReadFixed(GroupingSizeInBytes); count = reader.ReadByte(); if (count > GroupingSizeInBytes) throw new FormatException(); @@ -162,9 +162,20 @@ public static byte[] ReadBytesWithGrouping(this BinaryReader reader) } } + public static byte[] ReadFixed(this BinaryReader reader, int size) + { + var data = new byte[size]; + if (reader.Read(data, 0, size) != size) + { + throw new FormatException(); + } + + return data; + } + public static string ReadFixedString(this BinaryReader reader, int length) { - byte[] data = reader.ReadBytes(length); + byte[] data = reader.ReadFixed(length); return Encoding.UTF8.GetString(data.TakeWhile(p => p != 0).ToArray()); } @@ -196,7 +207,7 @@ public static T[] ReadSerializableArray(this BinaryReader reader, int max = 0 public static byte[] ReadVarBytes(this BinaryReader reader, int max = 0x1000000) { - return reader.ReadBytes((int)reader.ReadVarInt((ulong)max)); + return reader.ReadFixed((int)reader.ReadVarInt((ulong)max)); } public static ulong ReadVarInt(this BinaryReader reader, ulong max = ulong.MaxValue) diff --git a/src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs b/src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs index 6fcda2f809..12e889cb9b 100644 --- a/src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs +++ b/src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs @@ -31,8 +31,7 @@ void ISerializable.Deserialize(BinaryReader reader) Timestamp = reader.ReadUInt32(); // Address - byte[] data = reader.ReadBytes(16); - if (data.Length != 16) throw new FormatException(); + byte[] data = reader.ReadFixed(16); Address = new IPAddress(data).Unmap(); // Capabilities From e956d57104ff24f092cf03d8cf9c566eb8bdc06b Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 26 Feb 2020 14:06:02 +0100 Subject: [PATCH 2/8] format --- src/neo/Consensus/Commit.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Consensus/Commit.cs b/src/neo/Consensus/Commit.cs index b29cefc3c4..12988bde31 100644 --- a/src/neo/Consensus/Commit.cs +++ b/src/neo/Consensus/Commit.cs @@ -1,4 +1,4 @@ -using Neo.IO; +using Neo.IO; using System.IO; namespace Neo.Consensus From 87a33a8b0d06e6cf30a1597b4f68b23f8ddc5110 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Thu, 27 Feb 2020 14:20:17 +0800 Subject: [PATCH 3/8] Rename --- src/neo/Consensus/Commit.cs | 2 +- src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs | 2 +- src/neo/Consensus/RecoveryMessage.cs | 2 +- src/neo/IO/Helper.cs | 8 ++++---- src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/neo/Consensus/Commit.cs b/src/neo/Consensus/Commit.cs index 12988bde31..fce9b8caaf 100644 --- a/src/neo/Consensus/Commit.cs +++ b/src/neo/Consensus/Commit.cs @@ -14,7 +14,7 @@ public class Commit : ConsensusMessage public override void Deserialize(BinaryReader reader) { base.Deserialize(reader); - Signature = reader.ReadFixed(64); + Signature = reader.ReadFixedBytes(64); } public override void Serialize(BinaryWriter writer) diff --git a/src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs b/src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs index a78fd07043..5d783c6796 100644 --- a/src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs +++ b/src/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs @@ -23,7 +23,7 @@ void ISerializable.Deserialize(BinaryReader reader) { ViewNumber = reader.ReadByte(); ValidatorIndex = reader.ReadUInt16(); - Signature = reader.ReadFixed(64); + Signature = reader.ReadFixedBytes(64); InvocationScript = reader.ReadVarBytes(1024); } diff --git a/src/neo/Consensus/RecoveryMessage.cs b/src/neo/Consensus/RecoveryMessage.cs index 5c555050d3..c25816ee7f 100644 --- a/src/neo/Consensus/RecoveryMessage.cs +++ b/src/neo/Consensus/RecoveryMessage.cs @@ -39,7 +39,7 @@ public override void Deserialize(BinaryReader reader) { int preparationHashSize = UInt256.Zero.Size; if (preparationHashSize == (int)reader.ReadVarInt((ulong)preparationHashSize)) - PreparationHash = new UInt256(reader.ReadFixed(preparationHashSize)); + PreparationHash = new UInt256(reader.ReadFixedBytes(preparationHashSize)); } PreparationMessages = reader.ReadSerializableArray(Blockchain.MaxValidators).ToDictionary(p => (int)p.ValidatorIndex); diff --git a/src/neo/IO/Helper.cs b/src/neo/IO/Helper.cs index 9ad85ce3fe..55d4bd9662 100644 --- a/src/neo/IO/Helper.cs +++ b/src/neo/IO/Helper.cs @@ -151,7 +151,7 @@ public static byte[] ReadBytesWithGrouping(this BinaryReader reader) int count; do { - byte[] group = reader.ReadFixed(GroupingSizeInBytes); + byte[] group = reader.ReadFixedBytes(GroupingSizeInBytes); count = reader.ReadByte(); if (count > GroupingSizeInBytes) throw new FormatException(); @@ -162,7 +162,7 @@ public static byte[] ReadBytesWithGrouping(this BinaryReader reader) } } - public static byte[] ReadFixed(this BinaryReader reader, int size) + public static byte[] ReadFixedBytes(this BinaryReader reader, int size) { var data = new byte[size]; if (reader.Read(data, 0, size) != size) @@ -175,7 +175,7 @@ public static byte[] ReadFixed(this BinaryReader reader, int size) public static string ReadFixedString(this BinaryReader reader, int length) { - byte[] data = reader.ReadFixed(length); + byte[] data = reader.ReadFixedBytes(length); return Encoding.UTF8.GetString(data.TakeWhile(p => p != 0).ToArray()); } @@ -207,7 +207,7 @@ public static T[] ReadSerializableArray(this BinaryReader reader, int max = 0 public static byte[] ReadVarBytes(this BinaryReader reader, int max = 0x1000000) { - return reader.ReadFixed((int)reader.ReadVarInt((ulong)max)); + return reader.ReadFixedBytes((int)reader.ReadVarInt((ulong)max)); } public static ulong ReadVarInt(this BinaryReader reader, ulong max = ulong.MaxValue) diff --git a/src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs b/src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs index 12e889cb9b..a65e28c846 100644 --- a/src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs +++ b/src/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs @@ -31,7 +31,7 @@ void ISerializable.Deserialize(BinaryReader reader) Timestamp = reader.ReadUInt32(); // Address - byte[] data = reader.ReadFixed(16); + byte[] data = reader.ReadFixedBytes(16); Address = new IPAddress(data).Unmap(); // Capabilities From 1a839b9b3dbbb2ec7b3cd9bc62d64b445308e3a7 Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 27 Feb 2020 13:59:38 +0100 Subject: [PATCH 4/8] Fix Consensus UT --- tests/neo.UnitTests/Consensus/UT_Consensus.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/neo.UnitTests/Consensus/UT_Consensus.cs b/tests/neo.UnitTests/Consensus/UT_Consensus.cs index c7a0687729..4bdb0da2c5 100644 --- a/tests/neo.UnitTests/Consensus/UT_Consensus.cs +++ b/tests/neo.UnitTests/Consensus/UT_Consensus.cs @@ -531,8 +531,8 @@ public void TestSerializeAndDeserializeConsensusContext() consensusContext.CommitPayloads = new ConsensusPayload[consensusContext.Validators.Length]; using (SHA256 sha256 = SHA256.Create()) { - consensusContext.CommitPayloads[3] = MakeSignedPayload(consensusContext, new Commit { Signature = sha256.ComputeHash(testTx1.Hash.ToArray()) }, 3, new[] { (byte)'3', (byte)'4' }); - consensusContext.CommitPayloads[6] = MakeSignedPayload(consensusContext, new Commit { Signature = sha256.ComputeHash(testTx2.Hash.ToArray()) }, 3, new[] { (byte)'6', (byte)'7' }); + consensusContext.CommitPayloads[3] = MakeSignedPayload(consensusContext, new Commit { Signature = sha256.ComputeHash(testTx1.Hash.ToArray()).Concat(sha256.ComputeHash(testTx1.Hash.ToArray())).ToArray() }, 3, new[] { (byte)'3', (byte)'4' }); + consensusContext.CommitPayloads[6] = MakeSignedPayload(consensusContext, new Commit { Signature = sha256.ComputeHash(testTx2.Hash.ToArray()).Concat(sha256.ComputeHash(testTx2.Hash.ToArray())).ToArray() }, 3, new[] { (byte)'6', (byte)'7' }); } consensusContext.Block.Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS(); From 424084c0d287c835fbe854f18b946bf3aae2991d Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 27 Feb 2020 19:52:06 +0100 Subject: [PATCH 5/8] Add ut, and allow buffered streams --- tests/neo.UnitTests/IO/UT_IOHelper.cs | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/neo.UnitTests/IO/UT_IOHelper.cs b/tests/neo.UnitTests/IO/UT_IOHelper.cs index d0f63e5c1c..3b36695b2d 100644 --- a/tests/neo.UnitTests/IO/UT_IOHelper.cs +++ b/tests/neo.UnitTests/IO/UT_IOHelper.cs @@ -23,6 +23,40 @@ public void TestAsSerializableGeneric() Assert.AreEqual(UInt160.Zero, result); } + [TestMethod] + public void TestReadFixedBytes() + { + byte[] data = new byte[] { 0x01, 0x02, 0x03, 0x04 }; + + // Less data + + using (var reader = new BinaryReader(new MemoryStream(data), Encoding.UTF8, false)) + { + byte[] result = Neo.IO.Helper.ReadFixedBytes(reader, 3); + + Assert.AreEqual("010203", result.ToHexString()); + Assert.AreEqual(3, reader.BaseStream.Position); + } + + // Same data + + using (var reader = new BinaryReader(new MemoryStream(data), Encoding.UTF8, false)) + { + byte[] result = Neo.IO.Helper.ReadFixedBytes(reader, 4); + + Assert.AreEqual("01020304", result.ToHexString()); + Assert.AreEqual(4, reader.BaseStream.Position); + } + + // More data + + using (var reader = new BinaryReader(new MemoryStream(data), Encoding.UTF8, false)) + { + Assert.ThrowsException(() => Neo.IO.Helper.ReadFixedBytes(reader, 5)); + Assert.AreEqual(4, reader.BaseStream.Position); + } + } + [TestMethod] public void TestNullableArray() { From b71c64d68318ef5deffd280d127f0047f5022d9f Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 27 Feb 2020 19:52:17 +0100 Subject: [PATCH 6/8] Add ut, and allow buffered streams --- src/neo/IO/Helper.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/neo/IO/Helper.cs b/src/neo/IO/Helper.cs index 55d4bd9662..7f367797d4 100644 --- a/src/neo/IO/Helper.cs +++ b/src/neo/IO/Helper.cs @@ -164,10 +164,20 @@ public static byte[] ReadBytesWithGrouping(this BinaryReader reader) public static byte[] ReadFixedBytes(this BinaryReader reader, int size) { + var index = 0; var data = new byte[size]; - if (reader.Read(data, 0, size) != size) + + while (size > 0) { - throw new FormatException(); + var readed = reader.Read(data, index, size); + + if (readed <= 0 && size > 0) + { + throw new FormatException(); + } + + size -= readed; + index += readed; } return data; From a3be929411b7a5629b0c0e4530cc32b99131c06f Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 9 Mar 2020 09:51:41 +0100 Subject: [PATCH 7/8] Change name --- src/neo/IO/Helper.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/neo/IO/Helper.cs b/src/neo/IO/Helper.cs index 7f367797d4..f50e0d8b11 100644 --- a/src/neo/IO/Helper.cs +++ b/src/neo/IO/Helper.cs @@ -169,15 +169,15 @@ public static byte[] ReadFixedBytes(this BinaryReader reader, int size) while (size > 0) { - var readed = reader.Read(data, index, size); + var bytesRead = reader.Read(data, index, size); - if (readed <= 0 && size > 0) + if (bytesRead <= 0 && size > 0) { throw new FormatException(); } - size -= readed; - index += readed; + size -= bytesRead; + index += bytesRead; } return data; From 45a86a5ace0a19bc90346158267b97e5f4ea8df6 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 9 Mar 2020 09:52:39 +0100 Subject: [PATCH 8/8] Update Helper.cs --- src/neo/IO/Helper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/IO/Helper.cs b/src/neo/IO/Helper.cs index f50e0d8b11..e44e5bbe0a 100644 --- a/src/neo/IO/Helper.cs +++ b/src/neo/IO/Helper.cs @@ -171,7 +171,7 @@ public static byte[] ReadFixedBytes(this BinaryReader reader, int size) { var bytesRead = reader.Read(data, index, size); - if (bytesRead <= 0 && size > 0) + if (bytesRead <= 0) { throw new FormatException(); }