merge: next-costs into next

.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests

@@ -1,32 +1,12 @@
-FROM rust:bullseye
+FROM stacks-node:integrations
 
-WORKDIR /src/
-
-COPY . .
-
-WORKDIR /src/testnet/stacks-node
-RUN cargo test --no-run
-
-RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
-RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
+ARG test_name
+ENV BITCOIND_TEST 1
 
-RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
+RUN cargo build && \
+    cargo test -- --test-threads 1 --ignored "$test_name"
 
-ENV BITCOIND_TEST 1
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::microblock_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_check_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::cost_voting_integration
-RUN cargo test -- --test-threads 1 --ignored tests::integrations::integration_test_get_info
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::liquid_ustx_integration
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::stx_transfer_btc_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_forking_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::should_fix_2771
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::pox_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::bitcoin_regtest::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::should_succeed_handling_malformed_and_valid_txs
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::antientropy_integration_test
+RUN grcov . --binary-path ../../target/debug/ -s ../.. -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info && \
+    curl -Os https://uploader.codecov.io/latest/linux/codecov && \
+    chmod +x codecov && \
+    ./codecov --name "$test_name"

.github/actions/bitcoin-int-tests/Dockerfile.code-cov

@@ -16,4 +16,6 @@ RUN cargo build && \
 
 # Generate coverage report and upload it to codecov
 RUN grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info && \
-    bash -c "bash <(curl -s https://codecov.io/bash)"
+    curl -Os https://uploader.codecov.io/latest/linux/codecov && \
+    chmod +x codecov && \
+    ./codecov --name "unit_tests"

.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests

@@ -0,0 +1,24 @@
+FROM rust:bullseye
+
+WORKDIR /src/
+
+COPY . .
+
+WORKDIR /src/testnet/stacks-node
+
+RUN rustup override set nightly && \
+    rustup component add llvm-tools-preview && \
+    cargo install grcov
+
+ENV RUSTFLAGS="-Zinstrument-coverage" \
+    LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
+
+RUN cargo test --no-run && \
+    cargo build
+
+RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
+RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
+
+RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
+
+ENV BITCOIND_TEST 1

.github/actions/bitcoin-int-tests/Dockerfile.large-genesis

@@ -4,12 +4,25 @@ WORKDIR /src
 
 COPY . .
 
-RUN cargo test --no-run --workspace
-
 RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 
 RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
 
+RUN rustup override set nightly && \
+    rustup component add llvm-tools-preview && \
+    cargo install grcov
+
+ENV RUSTFLAGS="-Zinstrument-coverage" \
+    LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
+
+RUN cargo test --no-run --workspace && \
+    cargo build --workspace
+
 ENV BITCOIND_TEST 1
 RUN cd testnet/stacks-node && cargo test --release --features prod-genesis-chainstate -- --test-threads 1 --ignored neon_integrations::bitcoind_integration_test
+
+RUN grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info && \
+    curl -Os https://uploader.codecov.io/latest/linux/codecov && \
+    chmod +x codecov && \
+    ./codecov --name "large_genesis"

.github/stale.yml

@@ -1,11 +1,14 @@
 # Number of days of inactivity before an issue becomes stale
-daysUntilStale: 180
+daysUntilStale: 365
 # Number of days of inactivity before a stale issue is closed
-daysUntilClose: 7
+daysUntilClose: 30
 # Issues with these labels will never be considered stale
 exemptLabels:
   - pinned
   - security
+  - stacks-2.1
+  - consensus-critical
+  - clarity
 # Label to use when marking an issue as stale
 staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable

.github/workflows/bitcoin-tests.yml

@@ -6,22 +6,93 @@ on:
   pull_request:
 
 jobs:
-  # Run sampled genesis tests
+  build-integration-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build bitcoin integration testing image
+        env:
+          DOCKER_BUILDKIT: 1
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests -t stacks-node:integrations .
+      - name: Export docker image as tarball
+        run: docker save -o integration-image.tar stacks-node:integrations
+      - name: Upload built docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: integration-image.tar
+          path: integration-image.tar
+  # Run integration tests using sampled genesis block
   sampled-genesis:
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      fail-fast: false
+      matrix:
+        test-name:
+          - tests::neon_integrations::microblock_integration_test
+          - tests::neon_integrations::size_check_integration_test
+          - tests::neon_integrations::cost_voting_integration
+          - tests::integrations::integration_test_get_info
+          - tests::neon_integrations::bitcoind_integration_test
+          - tests::neon_integrations::liquid_ustx_integration
+          - tests::neon_integrations::stx_transfer_btc_integration_test
+          - tests::neon_integrations::bitcoind_forking_test
+          - tests::neon_integrations::should_fix_2771
+          - tests::neon_integrations::pox_integration_test
+          - tests::bitcoin_regtest::bitcoind_integration_test
+          - tests::should_succeed_handling_malformed_and_valid_txs
+          - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
+          - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::antientropy_integration_test
+          - tests::neon_integrations::filter_low_fee_tx_integration_test
+          - tests::neon_integrations::filter_long_runtime_tx_integration_test
+          - tests::neon_integrations::mining_transactions_is_fair
+          - tests::epoch_205::test_dynamic_db_method_costs
+          - tests::epoch_205::transition_empty_blocks
+          - tests::epoch_205::test_cost_limit_switch_version205
+          - tests::epoch_205::test_exact_block_costs
     steps:
       - uses: actions/checkout@v2
+      - name: Download docker image
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
       - name: All integration tests with sampled genesis
+        timeout-minutes: 30
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
   atlas-test:
-    # disable this job/test for now, as the atlas endpoints are currently disabled.
     if: ${{ true }}
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      fail-fast: false
+      matrix:
+        test-name:
+          - tests::neon_integrations::atlas_integration_test
+          - tests::neon_integrations::atlas_stress_integration_test
     steps:
       - uses: actions/checkout@v2
-      - name: All integration tests with sampled genesis
+      - name: Download docker image
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
+      - name: Atlas integration tests
+        timeout-minutes: 40
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.atlas-test .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .

.github/workflows/stacks-blockchain.yml

@@ -39,7 +39,10 @@ jobs:
       - name: Single full genesis integration test
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.large-genesis .
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.large-genesis .
 
   # Run unit tests with code coverage
   unit-tests:
@@ -49,7 +52,10 @@ jobs:
       - name: Run units tests (with coverage)
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.code-cov .
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.code-cov .
 
   open-api-validation:
     runs-on: ubuntu-latest
@@ -218,7 +224,7 @@ jobs:
           tag_name: ${{ github.event.inputs.tag || github.ref }}
           release_name: Release ${{ github.event.inputs.tag || github.ref }}
           draft: false
-          prerelease: false
+          prerelease: true
 
   # Upload distributables to a new release if we're building a tag or a tag was passed in
   upload-dist:

CHANGELOG.md

@@ -13,6 +13,76 @@ This release will contain consensus-breaking changes.
 
 - Clarity function `stx-transfer?` now takes a 4th optional argument, which is a memo.
 
+## [2.05.0.0.0]
+
+This software update is a consensus changing release and the
+implementation of the proposed cost changes in SIP-012. This release's
+chainstate directory is compatible with chainstate directories from
+2.0.11.4.0. However, this release is only compatible with chainstate
+directories before the 2.05 consensus changes activate (Bitcoin height
+713,000). If you run a 2.00 stacks-node beyond this point, and wish to
+run a 2.05 node afterwards, you must start from a new chainstate
+directory.
+
+## Added
+
+- At height 713,000 a new `costs-2` contract will be launched by the
+  Stacks boot address.
+
+## Changed
+
+- Stacks blocks whose parents are mined >= 713,000 will use default costs
+  from the new `costs-2` contract.
+- Stacks blocks whose parents are mined >= 713,000 will use the real
+  serialized length of Clarity values as the cost inputs to several methods
+  that previously used the maximum possible size for the associated types.
+- Stacks blocks whose parents are mined >= 713,000 will use the new block
+  limit defined in SIP-012.
+
+## Fixed
+
+- Miners are now more aggressive in calculating their block limits
+  when confirming microblocks (#2916)
+
+## [2.0.11.4.0]
+
+This software update is a point-release to change the transaction
+selection logic in the default miner to prioritize by an estimated fee
+rate instead of raw fee. This release's chainstate directory is
+compatible with chainstate directories from 2.0.11.3.0.
+
+## Added
+
+- FeeEstimator and CostEstimator interfaces. These can be controlled
+  via node configuration options. See the `README.md` for more
+  information on the configuration.
+- New fee rate estimation endpoint `/v2/fees/transaction` (#2872). See
+  `docs/rpc/openapi.yaml` for more information.
+
+## Changed
+
+- Prioritize transaction inclusion in blocks by estimated fee rates (#2859).
+- MARF sqlite connections will now use `mmap`'ed connections with up to 256MB
+  space (#2869).
+
+## [2.0.11.3.0]
+
+This software update is a point-release to change the transaction selection
+logic in the default miner to prioritize by fee instead of nonce sequence.  This
+release's chainstate directory is compatible with chainstate directories from
+2.0.11.2.0.
+
+## Added
+
+- The node will enforce a soft deadline for mining a block, so that a node
+  operator can control how frequently their node attempts to mine a block
+regardless of how congested the mempool is.  The timeout parameters are
+controlled in the `[miner]` section of the node's config file (#2823).
+
+## Changed
+
+- Prioritize transaction inclusion in the mempool by transaction fee (#2823).
+
 ## [2.0.11.2.0]
 
 NOTE: This change resets the `testnet`. Users running a testnet node will need