From b1ae626218b04f38c50b7a08a648bd2531a93fd1 Mon Sep 17 00:00:00 2001 From: dangershony Date: Sun, 30 Jan 2022 20:28:41 +0000 Subject: [PATCH 1/2] Fix strax testnet --- src/Blockcore/NBitcoin/Bloom.cs | 299 ++++++++++++++++++ src/Blockcore/NBitcoin/BloomFilter.cs | 209 ++++++++++++ .../StraxConsensusFactory.cs | 75 ++++- 3 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 src/Blockcore/NBitcoin/Bloom.cs create mode 100644 src/Blockcore/NBitcoin/BloomFilter.cs diff --git a/src/Blockcore/NBitcoin/Bloom.cs b/src/Blockcore/NBitcoin/Bloom.cs new file mode 100644 index 000000000..10c79aa92 --- /dev/null +++ b/src/Blockcore/NBitcoin/Bloom.cs @@ -0,0 +1,299 @@ +using System; +using System.Linq; +using HashLib; +using NBitcoin.DataEncoders; + +namespace NBitcoin +{ + /// + /// Type representation of data used in a bloom filter. + /// + public class Bloom : IBitcoinSerializable + { + /// + /// Length of the bloom data in bytes. 2048 bits. + /// + public const int BloomLength = 256; + + /// + /// The actual bloom value represented as a byte array. + /// + private byte[] data; + + public Bloom() + { + this.data = new byte[BloomLength]; + } + + public Bloom(byte[] data) + { + if (data?.Length != BloomLength) + throw new ArgumentException($"Bloom byte array must be {BloomLength} bytes long.", nameof(data)); + + this.data = CopyBloom(data); + } + + /// + /// Given this and another bloom, bitwise-OR all the data to get a bloom filter representing a range of data. + /// + public void Or(Bloom bloom) + { + for (int i = 0; i < this.data.Length; ++i) + { + this.data[i] |= bloom.data[i]; + } + } + + /// + /// Add some input to the bloom filter. + /// + /// + /// From the Ethereum yellow paper (yellowpaper.io): + /// M3:2048 is a specialised Bloom filter that sets three bits + /// out of 2048, given an arbitrary byte series. It does this through + /// taking the low-order 11 bits of each of the first three pairs of + /// bytes in a Keccak-256 hash of the byte series. + /// + public void Add(byte[] input) + { + byte[] hashBytes = Keccak256(input); + // for first 3 pairs, calculate value of first 11 bits + for (int i = 0; i < 6; i += 2) + { + uint low8Bits = (uint)hashBytes[i + 1]; + uint high3Bits = ((uint)hashBytes[i] << 8) & 2047; // AND with 2047 wipes any bits higher than our desired 11. + uint index = low8Bits + high3Bits; + this.SetBit((int)index); + } + } + + /// + /// Returns a 32-byte Keccak256 hash of the given bytes. + /// + /// + /// + public static byte[] Keccak256(byte[] input) + { + return HashFactory.Crypto.SHA3.CreateKeccak256().ComputeBytes(input).GetBytes(); + } + + /// + /// Determine whether some input is possibly contained within the filter. + /// + /// The byte array to test. + /// Whether this data could be contained within the filter. + public bool Test(byte[] test) + { + var compare = new Bloom(); + compare.Add(test); + return this.Test(compare); + } + + /// + /// Determine whether a second bloom is possibly contained within the filter. + /// + /// The second bloom to test. + /// Whether this data could be contained within the filter. + public bool Test(Bloom bloom) + { + var copy = new Bloom(bloom.ToBytes()); + copy.Or(this); + return this.Equals(copy); + } + + /// + /// Sets the specific bit to 1 within our 256-byte array. + /// + /// Index (0-2047) of the bit to assign to 1. + private void SetBit(int index) + { + int byteIndex = index / 8; + int bitInByteIndex = index % 8; + byte mask = (byte)(1 << bitInByteIndex); + this.data[byteIndex] |= mask; + } + + public void ReadWrite(BitcoinStream stream) + { + if (stream.Serializing) + { + byte[] b = CopyBloom(this.data); + stream.ReadWrite(ref b); + } + else + { + var b = new byte[BloomLength]; + stream.ReadWrite(ref b); + this.data = b; + } + } + + /// + /// Returns the raw bytes of this filter. + /// + /// + public byte[] ToBytes() + { + return CopyBloom(this.data); + } + + public override string ToString() + { + return Encoders.Hex.EncodeData(this.data); + } + + public static bool operator ==(Bloom obj1, Bloom obj2) + { + if (object.ReferenceEquals(obj1, null)) + return object.ReferenceEquals(obj2, null); + + return Enumerable.SequenceEqual(obj1.data, obj2.data); + } + + public static bool operator !=(Bloom obj1, Bloom obj2) + { + return !(obj1 == obj2); + } + + public override bool Equals(object obj) + { + return this.Equals(obj as Bloom); + } + + public bool Equals(Bloom obj) + { + if (object.ReferenceEquals(obj, null)) + return false; + + if (object.ReferenceEquals(this, obj)) + return true; + + return (obj == this); + } + + public override int GetHashCode() + { + return HashCode.Combine(this.data); + } + + private static byte[] CopyBloom(byte[] bloom) + { + var result = new byte[BloomLength]; + Buffer.BlockCopy(bloom, 0, result, 0, BloomLength); + return result; + } + + /// + /// Compresses a bloom filter to the following encoding: + /// (length of encoding) [[(number of zeros)(explicit byte) ...] + /// + /// The maximum size of the compressed bytes. + /// The compressed bytes or null if is exceeded. + public byte[] GetCompressedBloom() + { + // The compressed version should be shorter than the uncompressed version. + int maxSize = BloomLength - 1; + + var b = this.ToBytes(); + var c = new byte[maxSize]; + byte zeros = 0; + int j = 0; + for (int i = 0; i < b.Length; i++) + { + if (b[i] != 0 || zeros == byte.MaxValue) + { + if (j >= (maxSize - 1)) + return b; + + c[j++] = (byte)zeros; + c[j++] = b[i]; + zeros = 0; + } + else if (zeros < byte.MaxValue) + { + zeros++; + } + } + + if (zeros != 0) + { + if (j >= maxSize) + return b; + + c[j++] = zeros; + } + + var res = new byte[j]; + + Array.Copy(c, res, j); + + return res; + } + + /// + /// Derives a bloom filter by decompressing the following encoding: + /// (length of encoding) [[(number of zeros)(explicit byte) ...] + /// + /// The data to decompress. + /// The bloom object. + public static Bloom GetDecompressedBloom(byte[] data) + { + if (data.Length == Bloom.BloomLength) + return new Bloom(data); + + var b = new byte[Bloom.BloomLength]; + int j = 0; + for (int i = 0; i < data.Length; i += 2) + { + int zeros = data[i]; + while (zeros-- > 0) + b[j++] = 0; + + if ((i + 1) < data.Length) + b[j++] = data[i + 1]; + } + + if (j != Bloom.BloomLength) + throw new InvalidOperationException("The decompressed bloom filter is not the expected length."); + + return new Bloom(b); + } + + public int GetCompressedSize() + { + return this.GetCompressedBloom().Length + 1; + } + } + + public static class BloomStreamExt + { + public static void ReadWriteCompressed(this BitcoinStream stream, ref Bloom bloom) + { + // Ensure that the length can be serialized using a single byte. + const int maxSerializedSize = byte.MaxValue + 1; + if (Bloom.BloomLength > maxSerializedSize) + throw new InvalidOperationException($"'{nameof(ReadWriteCompressed)}' does not support bloom filters greater than {maxSerializedSize} bytes in length."); + + if (stream.Serializing) + { // Writing to stream. + byte[] ser = bloom.GetCompressedBloom(); + byte len = (byte)(ser.Length - 1); + stream.ReadWrite(ref len); + stream.ReadWrite(ref ser); + } + else + { // Reading from stream. + byte len = 0; + stream.ReadWrite(ref len); + + // A value of 0 can be used to support larger blooms in the future. For now its not supported. + if (len == 0) + throw new NotImplementedException("The bloom compression format is not supported."); + + var c = new byte[(int)len + 1]; + stream.ReadWrite(ref c); + bloom = Bloom.GetDecompressedBloom(c); + } + } + } +} diff --git a/src/Blockcore/NBitcoin/BloomFilter.cs b/src/Blockcore/NBitcoin/BloomFilter.cs new file mode 100644 index 000000000..cff110501 --- /dev/null +++ b/src/Blockcore/NBitcoin/BloomFilter.cs @@ -0,0 +1,209 @@ +using System; +using Blockcore.Consensus.ScriptInfo; +using Blockcore.Consensus.TransactionInfo; +using NBitcoin.Crypto; + +namespace NBitcoin +{ + [Flags] + public enum BloomFlags : byte + { + UPDATE_NONE = 0, + UPDATE_ALL = 1, + // Only adds outpoints to the filter if the output is a pay-to-pubkey/pay-to-multisig script + UPDATE_P2PUBKEY_ONLY = 2, + UPDATE_MASK = 3, + }; + + /// + /// Used by SPV client, represent the set of interesting addresses tracked by SPV client with plausible deniability + /// + public class BloomFilter : IBitcoinSerializable + { + public BloomFilter() + { + + } + // 20,000 items with fp rate < 0.1% or 10,000 items and <0.0001% + private const uint MAX_BLOOM_FILTER_SIZE = 36000; // bytes + private const uint MAX_HASH_FUNCS = 50; + private const decimal LN2SQUARED = 0.4804530139182014246671025263266649717305529515945455M; + private const decimal LN2 = 0.6931471805599453094172321214581765680755001343602552M; + + + private byte[] vData; + private uint nHashFuncs; + private uint nTweak; + private byte nFlags; + private bool isFull = false; + private bool isEmpty; + + + public BloomFilter(int nElements, double nFPRate, BloomFlags nFlagsIn = BloomFlags.UPDATE_ALL) + : this(nElements, nFPRate, RandomUtils.GetUInt32(), nFlagsIn) + { + } + + + public BloomFilter(int nElements, double nFPRate, uint nTweakIn, BloomFlags nFlagsIn = BloomFlags.UPDATE_ALL) + { + // The ideal size for a bloom filter with a given number of elements and false positive rate is: + // - nElements * log(fp rate) / ln(2)^2 + // We ignore filter parameters which will create a bloom filter larger than the protocol limits + this.vData = new byte[Math.Min((uint)(-1 / LN2SQUARED * nElements * (decimal)Math.Log(nFPRate)), MAX_BLOOM_FILTER_SIZE) / 8]; + //vData(min((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)), MAX_BLOOM_FILTER_SIZE * 8) / 8), + // The ideal number of hash functions is filter size * ln(2) / number of elements + // Again, we ignore filter parameters which will create a bloom filter with more hash functions than the protocol limits + // See http://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas + + this.nHashFuncs = Math.Min((uint)(this.vData.Length * 8 / nElements * LN2), MAX_HASH_FUNCS); + this.nTweak = nTweakIn; + this.nFlags = (byte)nFlagsIn; + + + } + + private uint Hash(uint nHashNum, byte[] vDataToHash) + { + // 0xFBA4C795 chosen as it guarantees a reasonable bit difference between nHashNum values. + return (uint)(Hashes.MurmurHash3(nHashNum * 0xFBA4C795 + this.nTweak, vDataToHash) % (this.vData.Length * 8)); + } + + public void Insert(byte[] vKey) + { + if(this.isFull) + return; + for(uint i = 0; i < this.nHashFuncs; i++) + { + uint nIndex = Hash(i, vKey); + // Sets bit nIndex of vData + this.vData[nIndex >> 3] |= (byte)(1 << (7 & (int)nIndex)); + } + + this.isEmpty = false; + } + + public bool Contains(byte[] vKey) + { + if(this.isFull) + return true; + if(this.isEmpty) + return false; + for(uint i = 0; i < this.nHashFuncs; i++) + { + uint nIndex = Hash(i, vKey); + // Checks bit nIndex of vData + if((this.vData[nIndex >> 3] & (byte)(1 << (7 & (int)nIndex))) == 0) + return false; + } + return true; + } + public bool Contains(OutPoint outPoint) + { + if(outPoint == null) + throw new ArgumentNullException("outPoint"); + return Contains(outPoint.ToBytes()); + } + + public bool Contains(uint256 hash) + { + if(hash == null) + throw new ArgumentNullException("hash"); + return Contains(hash.ToBytes()); + } + + public void Insert(OutPoint outPoint) + { + if(outPoint == null) + throw new ArgumentNullException("outPoint"); + Insert(outPoint.ToBytes()); + } + + public void Insert(uint256 value) + { + if(value == null) + throw new ArgumentNullException("value"); + Insert(value.ToBytes()); + } + + public bool IsWithinSizeConstraints() + { + return this.vData.Length <= MAX_BLOOM_FILTER_SIZE && this.nHashFuncs <= MAX_HASH_FUNCS; + } + + #region IBitcoinSerializable Members + + public void ReadWrite(BitcoinStream stream) + { + stream.ReadWriteAsVarString(ref this.vData); + stream.ReadWrite(ref this.nHashFuncs); + stream.ReadWrite(ref this.nTweak); + stream.ReadWrite(ref this.nFlags); + } + + #endregion + + + + public bool IsRelevantAndUpdate(Transaction tx) + { + if(tx == null) + throw new ArgumentNullException("tx"); + uint256 hash = tx.GetHash(); + bool fFound = false; + // Match if the filter contains the hash of tx + // for finding tx when they appear in a block + if(this.isFull) + return true; + if(this.isEmpty) + return false; + if(Contains(hash)) + fFound = true; + + for(uint i = 0; i < tx.Outputs.Count; i++) + { + TxOut txout = tx.Outputs[(int)i]; + // Match if the filter contains any arbitrary script data element in any scriptPubKey in tx + // If this matches, also add the specific output that was matched. + // This means clients don't have to update the filter themselves when a new relevant tx + // is discovered in order to find spending transactions, which avoids round-tripping and race conditions. + foreach(Op op in txout.ScriptPubKey.ToOps()) + { + if(op.PushData != null && op.PushData.Length != 0 && Contains(op.PushData)) + { + fFound = true; + if((this.nFlags & (byte)BloomFlags.UPDATE_MASK) == (byte)BloomFlags.UPDATE_ALL) + Insert(new OutPoint(hash, i)); + else if((this.nFlags & (byte)BloomFlags.UPDATE_MASK) == (byte)BloomFlags.UPDATE_P2PUBKEY_ONLY) + { + ScriptTemplate template = StandardScripts.GetTemplateFromScriptPubKey(txout.ScriptPubKey); // this is only valid for Bitcoin. + if(template != null && + (template.Type == TxOutType.TX_PUBKEY || template.Type == TxOutType.TX_MULTISIG)) + Insert(new OutPoint(hash, i)); + } + break; + } + } + } + + if(fFound) + return true; + + foreach(TxIn txin in tx.Inputs) + { + // Match if the filter contains an outpoint tx spends + if(Contains(txin.PrevOut)) + return true; + + // Match if the filter contains any arbitrary script data element in any scriptSig in tx + foreach(Op op in txin.ScriptSig.ToOps()) + { + if(op.PushData != null && op.PushData.Length != 0 && Contains(op.PushData)) + return true; + } + } + + return false; + } + } +} diff --git a/src/Networks/Blockcore.Networks.Strax/StraxConsensusFactory.cs b/src/Networks/Blockcore.Networks.Strax/StraxConsensusFactory.cs index 5c927c3a6..0110a6864 100644 --- a/src/Networks/Blockcore.Networks.Strax/StraxConsensusFactory.cs +++ b/src/Networks/Blockcore.Networks.Strax/StraxConsensusFactory.cs @@ -29,6 +29,79 @@ public override bool IsProtocolTransaction() } } + public class StraxPosBlockHeader : PosBlockHeader + { + // Indicates that the header contains additional fields. + // The first field is a uint "Size" field to indicate the serialized size of additional fields. + public const int ExtendedHeaderBit = 0x10000000; + + // Determines whether this object should serialize the new fields associated with smart contracts. + public bool HasSmartContractFields => (this.version & ExtendedHeaderBit) != 0; + + /// + public override int CurrentVersion => 7; + + private ushort extendedHeaderSize => (ushort)(hashStateRootSize + receiptRootSize + this.logsBloom.GetCompressedSize()); + + /// + /// Root of the state trie after execution of this block. + /// + private uint256 hashStateRoot; + public uint256 HashStateRoot { get { return this.hashStateRoot; } set { this.hashStateRoot = value; } } + private static int hashStateRootSize = 32; + + /// + /// Root of the receipt trie after execution of this block. + /// + private uint256 receiptRoot; + public uint256 ReceiptRoot { get { return this.receiptRoot; } set { this.receiptRoot = value; } } + private static int receiptRootSize = 32; + + /// + /// Bitwise-OR of all the blooms generated from all of the smart contract transactions in the block. + /// + private Bloom logsBloom; + public Bloom LogsBloom { get { return this.logsBloom; } set { this.logsBloom = value; } } + + public StraxPosBlockHeader() + { + this.hashStateRoot = 0; + this.receiptRoot = 0; + this.logsBloom = new Bloom(); + } + + #region IBitcoinSerializable Members + + public override void ReadWrite(BitcoinStream stream) + { + base.ReadWrite(stream); + if (this.HasSmartContractFields) + { + stream.ReadWrite(ref this.hashStateRoot); + stream.ReadWrite(ref this.receiptRoot); + stream.ReadWriteCompressed(ref this.logsBloom); + } + } + + #endregion + + /// Populates stream with items that will be used during hash calculation. + protected override void ReadWriteHashingStream(BitcoinStream stream) + { + base.ReadWriteHashingStream(stream); + if (this.HasSmartContractFields) + { + stream.ReadWrite(ref this.hashStateRoot); + stream.ReadWrite(ref this.receiptRoot); + stream.ReadWriteCompressed(ref this.logsBloom); + } + } + + /// Gets the total header size - including the - in bytes. + public override long HeaderSize => this.HasSmartContractFields ? Size + this.extendedHeaderSize : Size; + } + + public class StraxConsensusFactory : PosConsensusFactory { /// @@ -51,7 +124,7 @@ public override Block CreateBlock() /// public override BlockHeader CreateBlockHeader() { - return new PosBlockHeader(); + return new StraxPosBlockHeader(); } /// From 5d74db24af31c8634301fd27453c25fa08c6c3f4 Mon Sep 17 00:00:00 2001 From: dangershony Date: Tue, 1 Feb 2022 20:32:02 +0000 Subject: [PATCH 2/2] Add sig count handling for STRAX federations --- .../Rules/UtxosetRules/CheckUtxosetRule.cs | 4 +- .../Rules/StraxCoinviewRule.cs | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/Features/Blockcore.Features.Consensus/Rules/UtxosetRules/CheckUtxosetRule.cs b/src/Features/Blockcore.Features.Consensus/Rules/UtxosetRules/CheckUtxosetRule.cs index 604664f0c..5ca8f84e6 100644 --- a/src/Features/Blockcore.Features.Consensus/Rules/UtxosetRules/CheckUtxosetRule.cs +++ b/src/Features/Blockcore.Features.Consensus/Rules/UtxosetRules/CheckUtxosetRule.cs @@ -368,7 +368,7 @@ private long CountWitnessSignatureOperation(Script scriptPubKey, WitScript witne /// Transaction for which we are computing the cost. /// Map of previous transactions that have outputs we're spending. /// Signature operation cost for transaction. - private uint GetP2SHSignatureOperationsCount(Transaction transaction, UnspentOutputSet inputs) + protected virtual uint GetP2SHSignatureOperationsCount(Transaction transaction, UnspentOutputSet inputs) { if (transaction.IsCoinBase) return 0; @@ -389,7 +389,7 @@ private uint GetP2SHSignatureOperationsCount(Transaction transaction, UnspentOut /// /// Transaction for which we are computing the cost. /// Legacy signature operation cost for transaction. - private long GetLegacySignatureOperationsCount(Transaction transaction) + protected virtual long GetLegacySignatureOperationsCount(Transaction transaction) { long sigOps = 0; foreach (TxIn txin in transaction.Inputs) diff --git a/src/Networks/Blockcore.Networks.Strax/Rules/StraxCoinviewRule.cs b/src/Networks/Blockcore.Networks.Strax/Rules/StraxCoinviewRule.cs index 464b31a5c..91eb622c0 100644 --- a/src/Networks/Blockcore.Networks.Strax/Rules/StraxCoinviewRule.cs +++ b/src/Networks/Blockcore.Networks.Strax/Rules/StraxCoinviewRule.cs @@ -139,5 +139,60 @@ private void AllowSpend(TxOut prevOut, Transaction tx) // Otherwise allow the spend (do nothing). } + + protected override uint GetP2SHSignatureOperationsCount(Transaction transaction, UnspentOutputSet inputs) + { + if (transaction.IsCoinBase) + return 0; + + uint sigOps = 0; + for (int i = 0; i < transaction.Inputs.Count; i++) + { + TxOut prevout = inputs.GetOutputFor(transaction.Inputs[i]); + if (prevout.ScriptPubKey.IsScriptType(ScriptType.P2SH)) + sigOps += this.GetSigOpCount(prevout.ScriptPubKey, this.Parent.Network, transaction.Inputs[i].ScriptSig); + } + + return sigOps; + } + + private uint GetSigOpCount(Script script, Network network, Script scriptSig) + { + if (!script.IsScriptType(ScriptType.P2SH)) + return script.GetSigOpCount(true); + // This is a pay-to-script-hash scriptPubKey; + // get the last item that the scriptSig + // pushes onto the stack: + bool validSig = new PayToScriptHashTemplate().CheckScriptSig(network, scriptSig, script); + return !validSig ? 0 : this.GetSigOpCount(new Script(scriptSig.ToOps().Last().PushData), true, network); + // ... and return its opcount: + } + + private uint GetSigOpCount(Script script, bool fAccurate, Network network = null) + { + if (network is not StraxBaseNetwork straxNetwork) + { + throw new System.InvalidCastException($"Expected type {nameof(StraxBaseNetwork)}"); + } + + uint n = 0; + Op lastOpcode = null; + foreach (Op op in script.ToOps()) + { + if (op.Code == OpcodeType.OP_CHECKSIG || op.Code == OpcodeType.OP_CHECKSIGVERIFY) + n++; + else if (op.Code == OpcodeType.OP_CHECKMULTISIG || op.Code == OpcodeType.OP_CHECKMULTISIGVERIFY) + { + if (fAccurate && straxNetwork?.Federations != null && lastOpcode.Code == OpcodeType.OP_NOP9) // OpcodeType.OP_FEDERATION) + n += (uint)straxNetwork.Federations.GetOnlyFederation().GetFederationDetails().transactionSigningKeys.Length; + else if (fAccurate && lastOpcode != null && lastOpcode.Code >= OpcodeType.OP_1 && lastOpcode.Code <= OpcodeType.OP_16) + n += (lastOpcode.PushData == null || lastOpcode.PushData.Length == 0) ? 0U : (uint)lastOpcode.PushData[0]; + else + n += 20; + } + lastOpcode = op; + } + return n; + } } } \ No newline at end of file