Add BIP 34 enforcement

This patch primarily enforces block height being present in the coinbase
transaction input, altough it introduces a number of other fixes and
changes to support this.

* VersionTally now returns the number of blocks at or above a version, rather than just at, to enable forward-compatible support (i.e. v3 blocks include all v2 block tests)
* Block version is now explicitely provided in most tests which generate blocks, in order to ensure correct tests are applied
* Block height is now used when generating coinbase transactions
* Added support for the chain parameters to determine which tests apply to a block, so altcoins can override the defaults if needed.
* Added initial checks ahead of full BIP 66 validation checks

core/src/main/java/org/bitcoinj/core/AbstractBlockChain.java

@@ -436,22 +436,32 @@ private boolean add(Block block, boolean tryConnecting,
                 return true;
             }
 
+            final StoredBlock storedPrev;
+            final int height;
+            final EnumSet<VerificationFlags> flags;
+
             // Prove the block is internally valid: hash is lower than target, etc. This only checks the block contents
             // if there is a tx sending or receiving coins using an address in one of our wallets. And those transactions
             // are only lightly verified: presence in a valid connecting block is taken as proof of validity. See the
             // article here for more details: http://code.google.com/p/bitcoinj/wiki/SecurityModel
             try {
                 block.verifyHeader();
+                storedPrev = getStoredBlockInCurrentScope(block.getPrevBlockHash());
+                if (storedPrev != null) {
+                    height = storedPrev.getHeight() + 1;
+                } else {
+                    height = Block.BLOCK_HEIGHT_UNKNOWN;
+                }
+                flags = params.getValidationFlags(block, versionTally, height);
                 if (shouldVerifyTransactions())
-                    block.verifyTransactions();
+                    block.verifyTransactions(height, flags);
             } catch (VerificationException e) {
                 log.error("Failed to verify block: ", e);
                 log.error(block.getHashAsString());
                 throw e;
             }
 
             // Try linking it to a place in the currently known blocks.
-            StoredBlock storedPrev = getStoredBlockInCurrentScope(block.getPrevBlockHash());
 
             if (storedPrev == null) {
                 // We can't find the previous block. Probably we are still in the process of downloading the chain and a
@@ -526,7 +536,7 @@ private void connectBlock(final Block block, StoredBlock storedPrev, boolean exp
             // net, less on test) in order to be applied. It is also limited to
             // stopping addition of new v2 blocks to the tip of the chain.
             if (block.getVersion() == Block.BLOCK_VERSION_BIP34) {
-                final Integer count = versionTally.getCount(Block.BLOCK_VERSION_BIP66);
+                final Integer count = versionTally.getCountAtOrAbove(Block.BLOCK_VERSION_BIP66);
                 if (count != null
                     && count >= params.getMajorityRejectBlockOutdated()) {
                     throw new VerificationException.BlockVersionOutOfDate(block.getVersion());

core/src/main/java/org/bitcoinj/core/Block.java

@@ -30,13 +30,18 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.math.BigInteger;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
+import java.util.EnumSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 
 import static org.bitcoinj.core.Coin.FIFTY_COINS;
 import static org.bitcoinj.core.Sha256Hash.hashTwice;
+import org.bitcoinj.script.ScriptChunk;
 
 /**
  * <p>A block is a group of transactions, and is one of the fundamental data structures of the Bitcoin system.
@@ -72,6 +77,11 @@ public class Block extends Message {
     /** A value for difficultyTarget (nBits) that allows half of all possible hash solutions. Used in unit testing. */
     public static final long EASIEST_DIFFICULTY_TARGET = 0x207fFFFFL;
 
+    /** Value to use if the block height is unknown */
+    public static final int BLOCK_HEIGHT_UNKNOWN = -1;
+    /** Height of the first block */
+    public static final int BLOCK_HEIGHT_GENESIS = 0;
+
     public static final long BLOCK_VERSION_GENESIS = 1;
     /** Block version introduced in BIP 34: Height in coinbase */
     public static final long BLOCK_VERSION_BIP34 = 2;
@@ -610,10 +620,21 @@ private List<byte[]> buildMerkleTree() {
         return tree;
     }
 
-    private void checkTransactions() throws VerificationException {
+    /**
+     * Verify the transactions on a block.
+     *
+     * @param height block height, if known, or -1 otherwise. If provided, used
+     * to validate the coinbase input script of v2 and above blocks.
+     * @throws VerificationException if there was an error verifying the block.
+     */
+    private void checkTransactions(final int height, final EnumSet<VerificationFlags> flags)
+            throws VerificationException {
         // The first transaction in a block must always be a coinbase transaction.
         if (!transactions.get(0).isCoinBase())
             throw new VerificationException("First tx is not coinbase");
+        if (flags.contains(VerificationFlags.HEIGHT_IN_COINBASE) && height >= BLOCK_HEIGHT_GENESIS) {
+            transactions.get(0).checkCoinBaseHeight(height);
+        }
         // The rest must not be.
         for (int i = 1; i < transactions.size(); i++) {
             if (transactions.get(i).isCoinBase())
@@ -642,9 +663,13 @@ public void verifyHeader() throws VerificationException {
     /**
      * Checks the block contents
      *
-     * @throws VerificationException
+     * @param height block height, if known, or -1 otherwise. If valid, used
+     * to validate the coinbase input script of v2 and above blocks.
+     * @param flags flags to indicate which tests should be applied (i.e.
+     * whether to test for height in the coinbase transaction).
+     * @throws VerificationException if there was an error verifying the block.
      */
-    public void verifyTransactions() throws VerificationException {
+    public void verifyTransactions(final int height, final EnumSet<VerificationFlags> flags) throws VerificationException {
         // Now we need to check that the body of the block actually matches the headers. The network won't generate
         // an invalid block, but if we didn't validate this then an untrusted man-in-the-middle could obtain the next
         // valid block from the network and simply replace the transactions in it with their own fictional
@@ -653,7 +678,7 @@ public void verifyTransactions() throws VerificationException {
             throw new VerificationException("Block had no transactions");
         if (this.getOptimalEncodingMessageSize() > MAX_BLOCK_SIZE)
             throw new VerificationException("Block larger than MAX_BLOCK_SIZE");
-        checkTransactions();
+        checkTransactions(height, flags);
         checkMerkleRoot();
         checkSigOps();
         for (Transaction transaction : transactions)
@@ -662,10 +687,15 @@ public void verifyTransactions() throws VerificationException {
 
     /**
      * Verifies both the header and that the transactions hash to the merkle root.
+     *
+     * @param height block height, if known, or -1 otherwise.
+     * @param flags flags to indicate which tests should be applied (i.e.
+     * whether to test for height in the coinbase transaction).
+     * @throws VerificationException if there was an error verifying the block.
      */
-    public void verify() throws VerificationException {
+    public void verify(final int height, final EnumSet<VerificationFlags> flags) throws VerificationException {
         verifyHeader();
-        verifyTransactions();
+        verifyTransactions(height, flags);
     }
 
     @Override
@@ -808,19 +838,30 @@ public List<Transaction> getTransactions() {
     // Used to make transactions unique.
     private static int txCounter;
 
-    /** Adds a coinbase transaction to the block. This exists for unit tests. */
+    /** Adds a coinbase transaction to the block. This exists for unit tests.
+     * 
+     * @param height block height, if known, or -1 otherwise.
+     */
     @VisibleForTesting
-    void addCoinbaseTransaction(byte[] pubKeyTo, Coin value) {
+    void addCoinbaseTransaction(byte[] pubKeyTo, Coin value, final int height) {
         unCacheTransactions();
         transactions = new ArrayList<Transaction>();
         Transaction coinbase = new Transaction(params);
+        final ScriptBuilder inputBuilder = new ScriptBuilder();
+
+        if (height >= Block.BLOCK_HEIGHT_GENESIS) {
+            final byte[] blockHeightBytes = ScriptBuilder.createHeightScriptData(height);
+            inputBuilder.data(blockHeightBytes);
+        }
+        inputBuilder.data(new byte[]{(byte) txCounter, (byte) (txCounter++ >> 8)});
+
         // A real coinbase transaction has some stuff in the scriptSig like the extraNonce and difficulty. The
         // transactions are distinguished by every TX output going to a different key.
         //
         // Here we will do things a bit differently so a new address isn't needed every time. We'll put a simple
         // counter in the scriptSig so every transaction has a different hash.
         coinbase.addInput(new TransactionInput(params, coinbase,
-                new ScriptBuilder().data(new byte[]{(byte) txCounter, (byte) (txCounter++ >> 8)}).build().getProgram()));
+                inputBuilder.build().getProgram()));
         coinbase.addOutput(new TransactionOutput(params, coinbase, value,
                 ScriptBuilder.createOutputScript(ECKey.fromPublicOnly(pubKeyTo)).getProgram()));
         transactions.add(coinbase);
@@ -838,19 +879,24 @@ void addCoinbaseTransaction(byte[] pubKeyTo, Coin value) {
      * Returns a solved block that builds on top of this one. This exists for unit tests.
      */
     @VisibleForTesting
-    public Block createNextBlock(Address to, long version, long time) {
-        return createNextBlock(to, version, null, time, pubkeyForTesting, FIFTY_COINS);
+    public Block createNextBlock(Address to, long version, long time, Integer blockHeight) {
+        return createNextBlock(to, version, (TransactionOutPoint) null, time,
+            pubkeyForTesting, FIFTY_COINS, blockHeight);
     }
 
     /**
      * Returns a solved block that builds on top of this one. This exists for unit tests.
      * In this variant you can specify a public key (pubkey) for use in generating coinbase blocks.
+     * 
+     * @param height block height, if known, or -1 otherwise.
      */
-    Block createNextBlock(@Nullable Address to, long version, @Nullable TransactionOutPoint prevOut,
-                          long time, byte[] pubKey, Coin coinbaseValue) {
+    Block createNextBlock(@Nullable final Address to, final long version,
+                          @Nullable TransactionOutPoint prevOut, final long time,
+                          final byte[] pubKey, final Coin coinbaseValue,
+                          final int height) {
         Block b = new Block(params, version);
         b.setDifficultyTarget(difficultyTarget);
-        b.addCoinbaseTransaction(pubKey, coinbaseValue);
+        b.addCoinbaseTransaction(pubKey, coinbaseValue, height);
 
         if (to != null) {
             // Add a transaction paying 50 coins to the "to" address.
@@ -885,17 +931,20 @@ Block createNextBlock(@Nullable Address to, long version, @Nullable TransactionO
         } catch (VerificationException e) {
             throw new RuntimeException(e); // Cannot happen.
         }
+        if (b.getVersion() != version) {
+            throw new RuntimeException();
+        }
         return b;
     }
 
     @VisibleForTesting
     public Block createNextBlock(@Nullable Address to, TransactionOutPoint prevOut) {
-        return createNextBlock(to, 1, prevOut, getTimeSeconds() + 5, pubkeyForTesting, FIFTY_COINS);
+        return createNextBlock(to, BLOCK_VERSION_GENESIS, prevOut, getTimeSeconds() + 5, pubkeyForTesting, FIFTY_COINS, BLOCK_HEIGHT_UNKNOWN);
     }
 
     @VisibleForTesting
     public Block createNextBlock(@Nullable Address to, Coin value) {
-        return createNextBlock(to, 1, null, getTimeSeconds() + 5, pubkeyForTesting, value);
+        return createNextBlock(to, BLOCK_VERSION_GENESIS, null, getTimeSeconds() + 5, pubkeyForTesting, value, BLOCK_HEIGHT_UNKNOWN);
     }
 
     @VisibleForTesting
@@ -904,17 +953,19 @@ public Block createNextBlock(@Nullable Address to) {
     }
 
     @VisibleForTesting
-    public Block createNextBlockWithCoinbase(byte[] pubKey, Coin coinbaseValue) {
-        return createNextBlock(null, 1, null, Utils.currentTimeSeconds(), pubKey, coinbaseValue);
+    public Block createNextBlockWithCoinbase(long version, byte[] pubKey, Coin coinbaseValue, final int height) {
+        return createNextBlock(null, version, (TransactionOutPoint) null,
+                               Utils.currentTimeSeconds(), pubKey, coinbaseValue, height);
     }
 
     /**
      * Create a block sending 50BTC as a coinbase transaction to the public key specified.
      * This method is intended for test use only.
      */
     @VisibleForTesting
-    Block createNextBlockWithCoinbase(byte[] pubKey) {
-        return createNextBlock(null, 1, null, Utils.currentTimeSeconds(), pubKey, FIFTY_COINS);
+    Block createNextBlockWithCoinbase(long version, byte[] pubKey, final int height) {
+        return createNextBlock(null, version, (TransactionOutPoint) null,
+                               Utils.currentTimeSeconds(), pubKey, FIFTY_COINS, height);
     }
 
     @VisibleForTesting

core/src/main/java/org/bitcoinj/core/NetworkParameters.java

@@ -35,6 +35,7 @@
 import java.util.*;
 
 import static org.bitcoinj.core.Coin.*;
+import org.bitcoinj.utils.VersionTally;
 
 /**
  * <p>NetworkParameters contains the data needed for working with an instantiation of a Bitcoin chain.</p>
@@ -476,4 +477,32 @@ public int getMajorityRejectBlockOutdated() {
     public int getMajorityWindow() {
         return majorityWindow;
     }
+
+    /**
+     * The flags indicating which block validation tests should be applied to
+     * the given block. Enables support for alternative blockchains which enable
+     * tests based on different criteria.
+     * 
+     * @param block block to determine flags for.
+     * @param height height of the block, if known, null otherwise. Returned
+     * tests should be a safe subset if block height is unknown.
+     */
+    public EnumSet<VerificationFlags> getValidationFlags(final Block block,
+            final VersionTally tally, final Integer height) {
+        final EnumSet<VerificationFlags> flags = EnumSet.noneOf(VerificationFlags.class);
+
+        if (block.getVersion() >= Block.BLOCK_VERSION_BIP34) {
+            final Integer count = tally.getCountAtOrAbove(Block.BLOCK_VERSION_BIP34);
+            if (null != count && count >= getMajorityEnforceBlockUpgrade()) {
+                flags.add(VerificationFlags.HEIGHT_IN_COINBASE);
+            }
+        }
+        if (block.getVersion() >= Block.BLOCK_VERSION_BIP66) {
+            final Integer count = tally.getCountAtOrAbove(Block.BLOCK_VERSION_BIP66);
+            if (null != count && count >= getMajorityEnforceBlockUpgrade()) {
+                flags.add(VerificationFlags.DER_SIGNATURE_FORMAT);
+            }
+        }
+        return flags;
+    }
 }

core/src/main/java/org/bitcoinj/core/Transaction.java

@@ -36,6 +36,9 @@
 
 import static org.bitcoinj.core.Utils.*;
 import static com.google.common.base.Preconditions.checkState;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.bitcoinj.script.ScriptChunk;
 
 /**
  * <p>A transaction represents the movement of coins from some addresses to some other addresses. It can also represent
@@ -1163,6 +1166,39 @@ public int getSigOpCount() throws ScriptException {
         return sigOps;
     }
 
+    /**
+     * Check block height is in coinbase input script, for use after BIP 34
+     * enforcement is enabled.
+     */
+    public void checkCoinBaseHeight(final int height)
+            throws VerificationException {
+        assert height >= Block.BLOCK_HEIGHT_GENESIS;
+        assert isCoinBase();
+
+        // Check block height is in coinbase input script
+        final TransactionInput in = this.getInputs().get(0);
+        final List<ScriptChunk> chunks;
+
+        try {
+            final Script scriptSig = in.getScriptSig();
+            chunks = scriptSig.getChunks();
+        } catch(ScriptException e) {
+            throw new VerificationException("Coinbase input script signature is invalid.", e);
+        }
+        if (chunks.isEmpty()) {
+            throw new VerificationException("Coinbase input script signature is empty.");
+        }
+        final ScriptChunk chunk = chunks.get(0);
+        if (!chunk.isPushData()) {
+            throw new VerificationException("First element of coinbase input script signature is not pushdata.");
+        }
+        final byte[] data = chunk.data;
+        final byte[] expected = ScriptBuilder.createHeightScriptData(height);
+        if (!Arrays.equals(data, expected)) {
+            throw new VerificationException.CoinbaseHeightMismatch("Coinbase height mismatch.");
+        }
+    }
+
     /**
      * <p>Checks the transaction contents for sanity, in ways that can be done in a standalone manner.
      * Does <b>not</b> perform all checks on a transaction such as whether the inputs are already spent.

core/src/main/java/org/bitcoinj/core/VerificationException.java

@@ -80,4 +80,10 @@ public UnexpectedCoinbaseInput() {
             super("Coinbase input as input in non-coinbase transaction");
         }
     }
+
+    public static class CoinbaseHeightMismatch extends VerificationException {
+        public CoinbaseHeightMismatch(final String message) {
+            super(message);
+        }
+    }
 }

core/src/main/java/org/bitcoinj/core/VerificationFlags.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2015 Ross Nicoll
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.bitcoinj.core;
+
+/**
+ * Flags used to control which elements of block validation are performed on
+ * received blocks.
+ */
+public enum VerificationFlags {
+    /** Check that block height is in coinbase transaction (BIP 34) */
+    HEIGHT_IN_COINBASE,
+    /** Check DER signature format is exact (BIP 66) */
+    DER_SIGNATURE_FORMAT
+}

core/src/main/java/org/bitcoinj/params/AbstractBitcoinNetParams.java

@@ -18,6 +18,8 @@
 package org.bitcoinj.params;
 
 import java.math.BigInteger;
+import java.util.EnumSet;
+import java.util.Set;
 
 import org.bitcoinj.core.Block;
 import org.bitcoinj.core.Coin;
@@ -33,6 +35,8 @@
 import org.slf4j.LoggerFactory;
 
 import org.bitcoinj.core.BitcoinSerializer;
+import org.bitcoinj.core.VerificationFlags;
+import org.bitcoinj.utils.VersionTally;
 
 /**
  * Parameters for Bitcoin-like networks.