Merge #2715

2715: feat(hardfork): ckb2021 hardfork features r=yangby-cryptape a=yangby-cryptape

### Suggestions for Review

**The best way to review this PR is reviewing each commit independently.**

### Brief introduction for Commits

#### New hardfork Features

**All following features can be found in RFC PRs.**

- feat(hardfork): in the "since epoch", the index should be less than length

  Ref: [CKB-RFCs PR 223: Ensure that index < length in input since field using epoch](nervosnetwork/rfcs#223)

- feat(hardfork): use block timestamp of input cells as relative since start timestamp

  Ref: [CKB-RFCs PR 221: Use Block Timestamp as Start Timestamp in Since](nervosnetwork/rfcs#221)

- **[Reverted]** ~~feat(hardfork): allow unknown block versions and transactions versions~~

  ~~Ref: [CKB-RFCs PR 230: Allow unknown tx & block version](nervosnetwork/rfcs#230

- feat(hardfork): allow script multiple matches on identical data for type hash-type scripts

  Ref: [CKB-RFCs PR 222: Allow script multiple matches on identical code](nervosnetwork/rfcs#222)

- feat(hardfork): reuse the uncles hash in the header as the extra hash

  Ref: [CKB-RFCs PR 224: Add a variable length field in the block header](nervosnetwork/rfcs#224)

- **[Reverted]** ~~feat(hardfork): allow loading uncommitted cell data hashes from tx pool~~

  ~~Ref: [CKB-RFCs PR 228: ckb2021: fix load_cell_data_hash syscall](nervosnetwork/rfcs#228

#### Other Important Commits

- feat(hardfork): setup the components for hard fork features

- refactor: let verifiers know the real environment that the transaction is in

  Almost all features require this refactor commit.

- refactor: remove useless parameter "with_data" because it always be true (tricky)

  So I can change less APIs and less code to apply the feature: allow loading uncommitted cell data hashes from tx pool.

### About Tests

Almost all features have detailed integration tests (or unit tests):
- Many blocks before hardfork;
- Only one block before hardfork;
- The block at hardfork;
- Many blocks after hardfork.

All commits can passed all integration tests and unit tests.

Co-authored-by: zhangsoledad <787953403@qq.com>
Co-authored-by: Boyu Yang <yangby@cryptape.com>

.travis.yml

@@ -64,17 +64,17 @@ matrix:
   include:
     # We don't run tests, linters and quck check in fork branch, since they will be covered in PR.
     - name: Tests on macOS
-      if: 'tag IS NOT present AND (type = pull_request OR branch IN (master, staging, staging2, trying))'
+      if: 'tag IS NOT present AND (type = pull_request OR branch IN (master, staging, staging2, trying) OR branch =~ /^ckb2021/)'
       os: osx
     - name: Tests on Linux
-      if: 'tag IS NOT present AND (type = pull_request OR branch IN (master, staging, staging2, trying) OR repo != nervosnetwork/ckb)'
+      if: 'tag IS NOT present AND (type = pull_request OR branch IN (master, staging, staging2, trying) OR branch =~ /^ckb2021/ OR repo != nervosnetwork/ckb)'
       os: linux
     - name: PR Integration
       if: 'tag IS NOT present AND branch != develop AND branch !~ /^rc\// AND (type = pull_request OR repo != nervosnetwork/ckb)'
       os: linux
       script: make CKB_TEST_SEC_COEFFICIENT=5 CKB_TEST_ARGS="-c 4 --no-report" integration
     - name: Linters
-      if: 'tag IS NOT present AND (type = pull_request OR branch in (master, staging, staging2, trying) OR repo != nervosnetwork/ckb)'
+      if: 'tag IS NOT present AND (type = pull_request OR branch in (master, staging, staging2, trying) OR branch =~ /^ckb2021/ OR repo != nervosnetwork/ckb)'
       os: linux
       install:
         - cargo fmt --version || travis_retry rustup component add rustfmt
@@ -99,7 +99,7 @@ matrix:
         - make clippy
         - mv rust-toolchain.bak rust-toolchain
     - name: Quick Check
-      if: 'tag IS NOT present AND (type = pull_request OR branch in (master, staging, staging2, trying) OR repo != nervosnetwork/ckb)'
+      if: 'tag IS NOT present AND (type = pull_request OR branch in (master, staging, staging2, trying) OR branch =~ /^ckb2021/ OR repo != nervosnetwork/ckb)'
       os: linux
       cache: false
       addons: { apt: { packages: [] } }
@@ -119,7 +119,7 @@ matrix:
       script:
         - devtools/ci/check-cyclic-dependencies.py --dev
     - name: Security Audit & Licenses
-      if: 'tag IS NOT present AND (type = pull_request OR branch in (master, staging, staging2, trying) OR repo != nervosnetwork/ckb)'
+      if: 'tag IS NOT present AND (type = pull_request OR branch in (master, staging, staging2, trying) OR branch =~ /^ckb2021/ OR repo != nervosnetwork/ckb)'
       os: linux
       install:
         - cargo deny --version || travis_retry cargo install cargo-deny --locked
@@ -129,7 +129,7 @@ matrix:
         - make check-licenses
 
     - name: WASM build
-      if: 'tag IS NOT present AND (type = pull_request OR branch in (master, staging, staging2, trying) OR repo != nervosnetwork/ckb)'
+      if: 'tag IS NOT present AND (type = pull_request OR branch in (master, staging, staging2, trying) OR branch =~ /^ckb2021/ OR repo != nervosnetwork/ckb)'
       os: linux
       script:
         - export PATH=/usr/lib/llvm-8/bin:$PATH
@@ -141,11 +141,11 @@ matrix:
       os: linux
       script: make bench-test
     - name: Integration on macOS
-      if: 'tag IS NOT present AND type != pull_request AND (branch IN (master, staging, staging2, trying) OR branch =~ /^rc\// OR (branch = develop AND commit_message !~ /^Merge #\d+/))'
+      if: 'tag IS NOT present AND type != pull_request AND (branch IN (master, staging, staging2, trying) OR branch =~ /^ckb2021/ OR branch =~ /^rc\// OR (branch = develop AND commit_message !~ /^Merge #\d+/))'
       os: osx
       script: make CKB_TEST_ARGS="-c 1 --no-report" integration
     - name: Integration on Linux
-      if: 'tag IS NOT present AND type != pull_request AND (branch IN (master, staging, staging2, trying) OR branch =~ /^rc\// OR (branch = develop AND commit_message !~ /^Merge #\d+/))'
+      if: 'tag IS NOT present AND type != pull_request AND (branch IN (master, staging, staging2, trying) OR branch =~ /^ckb2021/ OR branch =~ /^rc\// OR (branch = develop AND commit_message !~ /^Merge #\d+/))'
       os: linux
       script: make CKB_TEST_ARGS="-c 1 --no-report" integration
     - name: Code Coverage

azure-pipelines.yml

@@ -19,7 +19,8 @@ jobs:
             eq(variables['Build.Reason'], 'PullRequest'),
             ne(variables['System.PullRequest.SourceBranch'], 'develop')
           ),
-          eq(variables['Build.SourceBranch'], 'refs/heads/master')
+          eq(variables['Build.SourceBranch'], 'refs/heads/master'),
+          startsWith(variables['Build.SourceBranch'], 'refs/heads/ckb2021')
         )
       )
     pool:
@@ -41,7 +42,8 @@ jobs:
         ne(variables['Build.Reason'], 'PullRequest'),
         or(
           startsWith(variables['Build.SourceBranch'], 'refs/heads/rc/'),
-          in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/develop', 'refs/heads/staging2', 'refs/heads/trying')
+          in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/develop', 'refs/heads/staging2', 'refs/heads/trying'),
+          startsWith(variables['Build.SourceBranch'], 'refs/heads/ckb2021')
         )
       )
     pool:

db-schema/src/lib.rs

@@ -3,7 +3,7 @@
 /// Column families alias type
 pub type Col = &'static str;
 /// Total column number
-pub const COLUMNS: u32 = 15;
+pub const COLUMNS: u32 = 16;
 /// Column store chain index
 pub const COLUMN_INDEX: Col = "0";
 /// Column store block's header
@@ -36,6 +36,8 @@ pub const COLUMN_CELL_DATA: Col = "12";
 pub const COLUMN_NUMBER_HASH: Col = "13";
 /// Column store cell data hash
 pub const COLUMN_CELL_DATA_HASH: Col = "14";
+/// Column store block extension data
+pub const COLUMN_BLOCK_EXTENSION: Col = "15";
 
 /// META_TIP_HEADER_KEY tracks the latest known best block header
 pub const META_TIP_HEADER_KEY: &[u8] = b"TIP_HEADER";

freezer/src/freezer.rs

@@ -55,9 +55,12 @@ impl Freezer {
                 .retrieve(freezer_number - 1)
                 .map_err(internal_error)?
                 .ok_or_else(|| internal_error("freezer inconsistent"))?;
-            let block = packed::BlockReader::from_slice(&raw_block)
+            let block = packed::BlockReader::from_compatible_slice(&raw_block)
                 .map_err(internal_error)?
                 .to_entity();
+            if block.count_extra_fields() > 1 {
+                return Err(internal_error("block has more than one extra fields"));
+            }
             tip = Some(block.header().into_view());
         }
 
@@ -147,9 +150,12 @@ impl Freezer {
                 .retrieve(item)
                 .map_err(internal_error)?
                 .expect("frozen number sync with files");
-            let block = packed::BlockReader::from_slice(&raw_block)
+            let block = packed::BlockReader::from_compatible_slice(&raw_block)
                 .map_err(internal_error)?
                 .to_entity();
+            if block.count_extra_fields() > 1 {
+                return Err(internal_error("block has more than one extra fields"));
+            }
             inner.tip = Some(block.header().into_view());
         }
         Ok(())

rpc/src/module/experiment.rs

@@ -13,7 +13,7 @@ use ckb_types::{
     prelude::*,
     H256,
 };
-use ckb_verification::ScriptVerifier;
+use ckb_verification::{ScriptVerifier, TxVerifyEnv};
 use jsonrpc_core::Result;
 use jsonrpc_derive::rpc;
 use std::collections::HashSet;
@@ -248,8 +248,15 @@ impl<'a> DryRunner<'a> {
             Ok(resolved) => {
                 let consensus = snapshot.consensus();
                 let max_cycles = consensus.max_block_cycles;
-                match ScriptVerifier::new(&resolved, &snapshot.as_data_provider())
-                    .verify(max_cycles)
+                let tip_header = snapshot.tip_header();
+                let tx_env = TxVerifyEnv::new_submit(&tip_header);
+                match ScriptVerifier::new(
+                    &resolved,
+                    consensus,
+                    &snapshot.as_data_provider(),
+                    &tx_env,
+                )
+                .verify(max_cycles)
                 {
                     Ok(cycles) => Ok(DryRunResult {
                         cycles: cycles.into(),

script/src/lib.rs

@@ -6,8 +6,10 @@ mod syscalls;
 mod type_id;
 mod types;
 mod verify;
+mod verify_env;
 
 pub use crate::error::{ScriptError, TransactionScriptError};
 pub use crate::ill_transaction_checker::IllTransactionChecker;
 pub use crate::types::{ScriptGroup, ScriptGroupType};
 pub use crate::verify::TransactionScriptsVerifier;
+pub use crate::verify_env::TxVerifyEnv;