Merge branch stacks/next into feat/costs-4-benchmark

stacks-network · Mar 22, 2024 · c152ced · c152ced
2 parents 0dbf0d6 + 80adc27
commit c152ced
Show file tree

Hide file tree

Showing 394 changed files with 48,423 additions and 16,883 deletions.
diff --git a/.cargo/config b/.cargo/config
@@ -2,7 +2,16 @@
 stacks-node = "run --package stacks-node --"
 fmt-stacks = "fmt -- --config group_imports=StdExternalCrate,imports_granularity=Module"
 
+# For x86_64 CPUs, default to `native` and override in CI for release builds
+# This makes it slightly faster for users running locally built binaries.
+# This can cause trouble when building "portable" binaries, such as for docker,
+# so disable it with the "portable" feature.
+# TODO: Same for other targets?
+[target.'cfg(all(target_arch = "x86_64", not(feature = portable))']
+rustflags = ["-Ctarget-cpu=native"]
+
 # Needed by perf to generate flamegraphs.
 #[target.x86_64-unknown-linux-gnu]
 #linker = "/usr/bin/clang"
 #rustflags = ["-Clink-arg=-fuse-ld=lld", "-Clink-arg=-Wl,--no-rosegment"]
+
diff --git a/.github/actions/dockerfiles/Dockerfile.alpine-binary b/.github/actions/dockerfiles/Dockerfile.alpine-binary
@@ -7,16 +7,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG TARGETVARIANT
-ARG REPO=stacks-network/stacks-core
+ARG REPO
 
-RUN case ${TARGETARCH} in \
-    "amd64")  BIN_ARCH=linux-musl-x64  ;; \
-    "arm64")  BIN_ARCH=linux-musl-arm64  ;; \
-    "arm")  BIN_ARCH=linux-musl-armv7  ;; \
-    "*") exit 1 ;; \
+RUN case ${TARGETPLATFORM} in \
+    linux/amd64/v2) BIN_ARCH=linux-glibc-x64-v2 ;; \
+    linux/amd64*) BIN_ARCH=linux-glibc-x64 ;; \
+    linux/arm64*) BIN_ARCH=linux-glibc-arm64 ;; \
+    linux/arm/v7) BIN_ARCH=linux-glibc-armv7 ;; \
+    *) exit 1 ;; \
     esac \
-    && echo "wget  -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
-    && wget  -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
+    && echo "TARGETPLATFORM: $TARGETPLATFORM" \
+    && echo "BIN_ARCH: $BIN_ARCH" \
+    && echo "wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
+    && wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
     && unzip ${BIN_ARCH}.zip -d /out
 
 FROM --platform=${TARGETPLATFORM} alpine

diff --git a/.github/actions/dockerfiles/Dockerfile.debian-binary b/.github/actions/dockerfiles/Dockerfile.debian-binary
@@ -7,16 +7,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG TARGETVARIANT
-ARG REPO=stacks-network/stacks-core
+ARG REPO
 
-RUN case ${TARGETARCH} in \
-    "amd64")  BIN_ARCH=linux-musl-x64  ;; \
-    "arm64")  BIN_ARCH=linux-musl-arm64  ;; \
-    "arm")  BIN_ARCH=linux-musl-armv7  ;; \
-    "*") exit 1 ;; \
+RUN case ${TARGETPLATFORM} in \
+    linux/amd64/v2) BIN_ARCH=linux-glibc-x64-v2 ;; \
+    linux/amd64*) BIN_ARCH=linux-glibc-x64 ;; \
+    linux/arm64*) BIN_ARCH=linux-glibc-arm64 ;; \
+    linux/arm/v7) BIN_ARCH=linux-glibc-armv7 ;; \
+    *) exit 1 ;; \
     esac \
-    && echo "wget  -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
-    && wget  -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
+    && echo "TARGETPLATFORM: $TARGETPLATFORM" \
+    && echo "BIN_ARCH: $BIN_ARCH" \
+    && echo "wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
+    && wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
     && unzip ${BIN_ARCH}.zip -d /out
 
 FROM --platform=${TARGETPLATFORM} debian:bookworm

diff --git a/.github/actions/dockerfiles/Dockerfile.debian-source b/.github/actions/dockerfiles/Dockerfile.debian-source
@@ -1,25 +1,28 @@
-FROM rust:bullseye as build
+FROM rust:bookworm as build
 
 ARG STACKS_NODE_VERSION="No Version Info"
 ARG GIT_BRANCH='No Branch Info'
 ARG GIT_COMMIT='No Commit Info'
 ARG BUILD_DIR=/build
 ARG TARGET=x86_64-unknown-linux-gnu
+# Allow us to override the default `--target-cpu` for the given target triplet
+ARG TARGET_CPU
+ENV RUSTFLAGS="${TARGET_CPU:+${RUSTFLAGS} -Ctarget-cpu=${TARGET_CPU}}"
 WORKDIR /src
 
 COPY . .
 
 RUN apt-get update && apt-get install -y git libclang-dev
 
 # Run all the build steps in ramdisk in an attempt to speed things up
-RUN target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
+RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
     && cd ${BUILD_DIR} \
     && rustup target add ${TARGET} \
     && rustup component add rustfmt \
-    && cargo build --features monitoring_prom,slog_json --release --workspace --target ${TARGET} \
+    && cargo build --features monitoring_prom,slog_json,portable --release --workspace --target ${TARGET} \
     && mkdir -p /out \
     && cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out
 
 FROM --platform=${TARGETPLATFORM} debian:bookworm
-COPY --from=build /out/stacks-node /bin/
+COPY --from=build /out/stacks-node /out/stacks-signer /bin/
 CMD ["stacks-node", "mainnet"]
diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml
@@ -31,7 +31,7 @@ jobs:
         test-name:
           - tests::bitcoin_regtest::bitcoind_integration_test
           - tests::integrations::integration_test_get_info
-          - tests::neon_integrations::antientropy_integration_test 
+          - tests::neon_integrations::antientropy_integration_test
           - tests::neon_integrations::bad_microblock_pubkey
           - tests::neon_integrations::bitcoind_forking_test
           - tests::neon_integrations::bitcoind_integration_test
@@ -69,19 +69,28 @@ jobs:
           - tests::neon_integrations::test_problematic_txs_are_not_stored
           - tests::neon_integrations::use_latest_tip_integration_test
           - tests::neon_integrations::confirm_unparsed_ongoing_ops
+          - tests::neon_integrations::min_txs
+          - tests::neon_integrations::vote_for_aggregate_key_burn_op_test
           - tests::should_succeed_handling_malformed_and_valid_txs
           - tests::nakamoto_integrations::simple_neon_integration
           - tests::nakamoto_integrations::mine_multiple_per_tenure_integration
           - tests::nakamoto_integrations::block_proposal_api_endpoint
           - tests::nakamoto_integrations::miner_writes_proposed_block_to_stackerdb
+          - tests::nakamoto_integrations::correct_burn_outs
+          - tests::nakamoto_integrations::vote_for_aggregate_key_burn_op
+          - tests::signer::stackerdb_dkg
+          - tests::signer::stackerdb_sign
+          - tests::signer::stackerdb_block_proposal
+          - tests::signer::stackerdb_filter_bad_transactions
+          - tests::signer::stackerdb_mine_2_nakamoto_reward_cycles
     steps:
       ## Setup test environment
       - name: Setup Test Environment
         id: setup_tests
         uses: stacks-network/actions/stacks-core/testenv@main
         with:
           btc-version: "25.0"
-          
+
       ## Run test matrix using restored cache of archive file
       ##   - Test will timeout after env.TEST_TIMEOUT minutes
       - name: Run Tests

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,6 +2,9 @@
 name: CI
 
 on:
+  merge_group:
+    types:
+      - checks_requested
   push:
     branches:
       - master
@@ -77,7 +80,9 @@ jobs:
 
       - name: Rustfmt
         id: rustfmt
-        uses: actions-rust-lang/rustfmt@2d1d4e9f72379428552fa1def0b898733fb8472d # v1.1.0
+        uses: stacks-network/actions/rustfmt@main
+        with:
+          alias: "fmt-stacks"
 
   ######################################################################################
   ## Create a tagged github release
@@ -145,6 +150,7 @@ jobs:
           ) ||
           github.event_name == 'workflow_dispatch' ||
           github.event_name == 'pull_request' ||
+          github.event_name == 'merge_group' ||
           (
             contains('
               refs/heads/master

diff --git a/.github/workflows/create-source-binary.yml b/.github/workflows/create-source-binary.yml
@@ -9,12 +9,6 @@ on:
         description: "Tag name of this release (x.y.z)"
         required: true
         type: string
-      arch:
-        description: "Stringified JSON object listing of platform matrix"
-        required: false
-        type: string
-        default: >-
-          ["linux-glibc-x64", "linux-musl-x64", "linux-glibc-arm64", "linux-glibc-armv7", "linux-musl-arm64", "linux-musl-armv7", "macos-x64", "macos-arm64", "windows-x64"]
 
 ## change the display name to the tag being built
 run-name: ${{ inputs.tag }}
@@ -30,41 +24,40 @@ jobs:
   ##  - workflow is building default branch (master)
   artifact:
     if: |
-      inputs.tag != '' && 
+      inputs.tag != '' &&
       github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
     name: Build Binaries
     runs-on: ubuntu-latest
     strategy:
       ## Run a maximum of 10 builds concurrently, using the matrix defined in inputs.arch
       max-parallel: 10
       matrix:
-        platform: ${{ fromJson(inputs.arch) }}
-    steps:
-      ## Setup Docker for the builds
-      - name: Docker setup
-        uses: stacks-network/actions/docker@main
+        arch:
+          - linux-musl
+          - linux-glibc
+          - macos
+          - windows
+        cpu:
+          - arm64
+          - armv7
+          - x86-64 ## defaults to x86-64-v3 variant - intel haswell (2013) and newer
+          # - x86-64-v2 ## intel nehalem (2008) and newer
+          # - x86-64-v3 ## intel haswell (2013) and newer
+          # - x86-64-v4 ## intel skylake (2017) and newer
+        exclude:
+          - arch: windows # excludes windows-arm64
+            cpu: arm64
+          - arch: windows # excludes windows-armv7
+            cpu: armv7
+          - arch: macos # excludes macos-armv7
+            cpu: armv7
 
-      ## Build the binaries using defined dockerfiles
-      - name: Build Binary (${{ matrix.platform }})
-        id: build_binaries
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # 5.0.0
+    steps:
+      - name: Build Binary (${{ matrix.arch }}_${{ matrix.cpu }})
+        id: build_binary
+        uses: stacks-network/actions/stacks-core/create-source-binary@main
         with:
-          file: build-scripts/Dockerfile.${{ matrix.platform }}
-          outputs: type=local,dest=./release/${{ matrix.platform }}
-          build-args: |
-            STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }}
-            OS_ARCH=${{ matrix.platform }}
-            GIT_BRANCH=${{ env.GITHUB_REF_SHORT }}
-            GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }}
+          arch: ${{ matrix.arch }}
+          cpu: ${{ matrix.cpu }}
+          tag: ${{ inputs.tag }}
 
-      ## Compress the binary artifact
-      - name: Compress artifact
-        id: compress_artifact
-        run: zip --junk-paths ${{ matrix.platform }} ./release/${{ matrix.platform }}/*
-
-      ## Upload the binary artifact to the github action (used in `github-release.yml` to create a release)
-      - name: Upload artifact
-        id: upload_artifact
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
-        with:
-          path: ${{ matrix.platform }}.zip
diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml
@@ -66,12 +66,12 @@ jobs:
           git add src/_data/boot-contracts-reference.json
           if $(git diff --staged --quiet --exit-code); then
             echo "No reference.json changes, stopping"
-            echo "::set-output name=open_pr::0"
+            echo "open_pr=0" >> "$GITHUB_OUTPUT"
           else
             git remote add robot https://github.com/$ROBOT_OWNER/$ROBOT_REPO
             git commit -m "auto: update Clarity references JSONs from stacks-core@${GITHUB_SHA}"
             git push robot $ROBOT_BRANCH
-            echo "::set-output name=open_pr::1"
+            echo "open_pr=1" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Open PR

diff --git a/.github/workflows/image-build-binary.yml b/.github/workflows/image-build-binary.yml
@@ -8,16 +8,11 @@ on:
       tag:
         required: true
         type: string
-        description: "Version tag for alpine images"
-      docker-org:
-        required: false
-        type: string
-        description: "Docker repo org for uploading images (defaults to github org)"
-        default: "${GITHUB_REPOSITORY_OWNER}"
+        description: "Version tag for docker images"
 
 ## Define which docker arch to build for
 env:
-  docker_platforms: "linux/arm64, linux/arm/v7, linux/amd64, linux/amd64/v2, linux/amd64/v3"
+  docker_platforms: "linux/arm64, linux/arm/v7, linux/amd64, linux/amd64/v3"
   docker-org: blockstack
 
 concurrency:
@@ -48,28 +43,39 @@ jobs:
     steps:
       ## Setup Docker for the builds
       - name: Docker setup
+        id: docker_setup
         uses: stacks-network/actions/docker@main
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      ## if the repo owner is not `stacks-network`, default to a docker-org of the repo owner (i.e. github user id)
+      ## this allows forks to run the docker push workflows without having to hardcode a dockerhub org (but it does require docker hub user to match github username)
+      - name: Set Local env vars
+        id: set_env
+        if: |
+          github.repository_owner != 'stacks-network'
+        run: |
+          echo "docker-org=${{ github.repository_owner }}" >> "$GITHUB_ENV"
+
       ## Set docker metatdata
       ## - depending on the matrix.dist, different tags will be enabled
-      ## ex. alpine will have this tag: `type=ref,event=tag,enable=${{ matrix.dist == 'alpine' }}`
+      ## ex. debian will have this tag: `type=ref,event=tag,enable=${{ matrix.dist == 'debian' }}`
       - name: Docker Metadata ( ${{matrix.dist}} )
         id: docker_metadata
         uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 #v5.0.0
         with:
+          ## tag images with current repo name `stacks-core` as well as legacy `stacks-blockchain`
           images: |
             ${{env.docker-org}}/${{ github.event.repository.name }}
             ${{env.docker-org}}/stacks-blockchain
           tags: |
-            type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'alpine' }}
-            type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine'}}
-            type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine' }}
-            type=ref,event=tag,enable=${{ matrix.dist == 'alpine' }}
-            type=raw,value=latest-${{ matrix.dist }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'debian' }}
-            type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'debian' }}
+            type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'debian' }}
+            type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'debian'}}
+            type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' && matrix.dist == 'debian' }}
+            type=ref,event=tag,enable=${{ matrix.dist == 'debian' }}
+            type=raw,value=latest-${{ matrix.dist }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'alpine' }}
+            type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine' }}
 
       ## Build docker image for release
       - name: Build and Push ( ${{matrix.dist}} )

diff --git a/.github/workflows/image-build-source.yml b/.github/workflows/image-build-source.yml
@@ -8,7 +8,7 @@ on:
 
 ## Define which docker arch to build for
 env:
-  docker_platforms: linux/amd64
+  docker_platforms: "linux/amd64"
   docker-org: blockstack
 
 concurrency:
@@ -31,11 +31,21 @@ jobs:
     steps:
       ## Setup Docker for the builds
       - name: Docker setup
+        id: docker_setup
         uses: stacks-network/actions/docker@main
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      ## if the repo owner is not `stacks-network`, default to a docker-org of the repo owner (i.e. github user id)
+      ## this allows forks to run the docker push workflows without having to hardcode a dockerhub org (but it does require docker hub user to match github username)
+      - name: Set Local env vars
+        id: set_env
+        if: |
+          github.repository_owner != 'stacks-network'
+        run: |
+          echo "docker-org=${{ github.repository_owner }}" >> "$GITHUB_ENV"
+
       ## Set docker metatdata
       - name: Docker Metadata ( ${{matrix.dist}} )
         id: docker_metadata
@@ -58,8 +68,8 @@ jobs:
           tags: ${{ steps.docker_metadata.outputs.tags }}
           labels: ${{ steps.docker_metadata.outputs.labels }}
           build-args: |
-            REPO=${{ github.repository_owner }}/${{ github.event.repository.name }}
             STACKS_NODE_VERSION=${{ env.GITHUB_SHA_SHORT }}
             GIT_BRANCH=${{ env.GITHUB_REF_SHORT }}
             GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }}
+            TARGET_CPU=x86-64-v3
           push: ${{ env.DOCKER_PUSH }}