diff --git a/.github/workflows/vcpkg_ci_aws_amd64.yml b/.github/workflows/vcpkg_ci_amd64.yml similarity index 73% rename from .github/workflows/vcpkg_ci_aws_amd64.yml rename to .github/workflows/vcpkg_ci_amd64.yml index d4773aa0..6fe3e7e8 100644 --- a/.github/workflows/vcpkg_ci_aws_amd64.yml +++ b/.github/workflows/vcpkg_ci_amd64.yml @@ -1,4 +1,4 @@ -name: Linux CI on AWS (AMD64) +name: Linux CI env: # "Source" is set in the vcpkg install step @@ -15,52 +15,20 @@ on: - 'docker/**' - '.github/**' - '**.md' - - '!.github/workflows/vcpkg_ci_aws_amd64*' + - '!.github/workflows/vcpkg_ci_amd64*' push: paths-ignore: - 'docker/**' - '.github/**' - '**.md' - - '!.github/workflows/vcpkg_ci_aws_amd64*' + - '!.github/workflows/vcpkg_ci_amd64*' tags-ignore: - 'v*' branches: - 'master' jobs: - start-runner: - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - strategy: - matrix: - ec2: - - { ami: ami-0610b26d76319237e, instance-type: m6i.8xlarge} - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.GHA_PAT }} - ec2-image-id: ${{ matrix.ec2.ami }} - ec2-instance-type: ${{ matrix.ec2.instance-type }} - subnet-id: subnet-0deb935f0bbfe1a5d - security-group-id: sg-0f6a02eb80fafb982 - aws-resource-tags: > # optional, requires additional permissions - [ - {"Key": "Name", "Value": "ec2-github-runner"}, - {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} - ] - do-the-job: + build: strategy: fail-fast: false matrix: @@ -81,8 +49,7 @@ jobs: VCPKG_DEFAULT_HOST_TRIPLET: ${{ matrix.host.triplet }} name: Do the job on the runner - needs: start-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + runs-on: ubuntu-latest # run the job on the newly created runner steps: - uses: actions/checkout@v3 with: @@ -229,25 +196,3 @@ jobs: run: | rm -rf vcpkg/{buildtrees,installed,packages} ccache --show-stats - - stop-runner: - name: Stop self-hosted EC2 runner - needs: - - start-runner # required to get output from the start-runner job - - do-the-job # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.GHA_PAT }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/vcpkg_ci_aws_arm64.yml b/.github/workflows/vcpkg_ci_aws_arm64.yml deleted file mode 100644 index 8a725dca..00000000 --- a/.github/workflows/vcpkg_ci_aws_arm64.yml +++ /dev/null @@ -1,261 +0,0 @@ -name: Linux CI on AWS (ARM64) - -env: - # "Source" is set in the vcpkg install step - VCPKG_BINARY_SOURCES: 'clear;nuget,Source,readwrite;nugettimeout,3601' - # Self-hosted runners can hopefully reuse already downloaded packages - VCPKG_USE_NUGET_CACHE: 1 - -on: - release: - types: - - published - pull_request: - paths-ignore: - - 'docker/**' - - '.github/**' - - '**.md' - - '!.github/workflows/vcpkg_ci_aws_arm64*' - push: - paths-ignore: - - 'docker/**' - - '.github/**' - - '**.md' - - '!.github/workflows/vcpkg_ci_aws_arm64*' - tags-ignore: - - 'v*' - branches: - - 'master' - -jobs: - start-runner: - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - strategy: - matrix: - ec2: - - { ami: ami-0128e20cf90406205, instance-type: r6g.8xlarge} - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.GHA_PAT }} - ec2-image-id: ${{ matrix.ec2.ami }} - ec2-instance-type: ${{ matrix.ec2.instance-type }} - subnet-id: subnet-0deb935f0bbfe1a5d - security-group-id: sg-0f6a02eb80fafb982 - aws-resource-tags: > # optional, requires additional permissions - [ - {"Key": "Name", "Value": "ec2-github-runner"}, - {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} - ] - do-the-job: - strategy: - fail-fast: false - matrix: - host: - - { name: 'ubuntu-20.04', arch: 'arm64', triplet: 'arm64-linux-rel', arch_docker: '_arm64' } - image: - - { name: 'ubuntu', tag: '20.04' } - llvm: [ 'llvm-14', 'llvm-15' ] - - container: - image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }}${{ matrix.host.arch_docker }} - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - env: - ARTIFACT_NAME: vcpkg_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.llvm }}_${{ matrix.host.arch }} - VCPKG_DEFAULT_HOST_TRIPLET: ${{ matrix.host.triplet }} - - name: Do the job on the runner - needs: start-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - fetch-depth: 2 - - # Used to get commit message since PRs are on a merge commit - - name: Get commit message - shell: bash - run: | - echo 'COMMIT_MESSAGE<> "$GITHUB_ENV" - if [[ '${{ github.event_name }}' == 'push' ]]; then - echo "$(git log --format=%B -n 1 HEAD)" >> "$GITHUB_ENV" - elif [[ '${{ github.event_name }}' == 'pull_request' ]]; then - echo "$(git log --format=%B -n 1 HEAD^2)" >> "$GITHUB_ENV" - fi - echo "EOF" >> "$GITHUB_ENV" - - - name: Clear prior vcpkg directory - run: | - rm -rf "${{ github.workspace }}/vcpkg" - - - name: Initialize vcpkg - shell: bash - run: | - { read -r vcpkg_repo_url && read -r vcpkg_commit; } <./vcpkg_info.txt || exit 1 - git clone "${vcpkg_repo_url}" - git -C vcpkg checkout "${vcpkg_commit}" - export "CMAKE_C_COMPILER_LAUNCHER=$(which ccache)" - export "CMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)" - ./vcpkg/bootstrap-vcpkg.sh - echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV - - - name: 'vcpkg install dependencies' - shell: 'bash' - run: | - export VCPKG_DISABLE_METRICS=1 - export VCPKG_FORCE_SYSTEM_BINARIES=1 - - # Setup NuGet authentication - # Need to manually download nuget. Keep updated with URL listed in - # https://github.com/microsoft/vcpkg/blob/master/scripts/vcpkgTools.xml - export NUGET_EXE=/usr/local/bin/nuget - curl -L https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe > "${NUGET_EXE}" - chmod +x "${NUGET_EXE}" - mono "${NUGET_EXE}" sources add \ - -source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \ - -storepasswordincleartext \ - -name "Source" \ - -username "${{ github.repository_owner }}" \ - -password "${{ secrets.GITHUB_TOKEN }}" || true - - mono "${NUGET_EXE}" sources update \ - -source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \ - -storepasswordincleartext \ - -name "Source" \ - -username "${{ github.repository_owner }}" \ - -password "${{ secrets.GITHUB_TOKEN }}" - - mono "${NUGET_EXE}" setapikey \ - -source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \ - "${{ secrets.GITHUB_TOKEN }}" - - ./build_dependencies.sh --release --export-dir ./${{ env.ARTIFACT_NAME }} ${{ matrix.llvm }} --clean-after-build --debug - - echo "VCPKG_ROOT=$(pwd)/${{ env.ARTIFACT_NAME }}" >> $GITHUB_ENV - - - name: 'Export Packages' - if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release - shell: 'bash' - run: | - apt-get update - apt-get install -y pixz - tar --use-compress-program pixz -cf "${{ env.ARTIFACT_NAME }}.tar.xz" ./${{ env.ARTIFACT_NAME }} - - - name: Publish Release Assets - if: github.event.release - uses: softprops/action-gh-release@v1 - with: - files: ${{ env.ARTIFACT_NAME }}.tar.xz - prerelease: ${{ contains(github.ref, 'pre') || contains(github.ref, 'rc') }} - token: ${{ secrets.RELEASE_TOKEN }} - - - uses: actions/upload-artifact@v3 - if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release - with: - name: ${{ env.ARTIFACT_NAME }}.tar.xz - path: ${{ env.ARTIFACT_NAME }}.tar.xz - - - name: Prepare ccache - id: ccache_prep - shell: bash - run: | - echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV - echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV - echo "CMAKE_C_COMPILER_LAUNCHER=$(which ccache)" >> $GITHUB_ENV - echo "CMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)" >> $GITHUB_ENV - echo ::set-output name=timestamp::$(date +"%Y-%m-%d-%H:%M:%S" --utc) - - - name: ccache cache files - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/.ccache - key: ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.host.arch }}-${{ matrix.llvm }}-${{ steps.ccache_prep.outputs.timestamp }} - restore-keys: | - ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.host.arch }}-${{ matrix.llvm }}- - - - name: ccache Initial stats - shell: bash - run: | - ccache --show-stats - - - name: 'Make install directory' - shell: bash - run: | - export INSTALL_DIR="${GITHUB_WORKSPACE}/workspace" - echo "INSTALL_DIR=${INSTALL_DIR}" >> ${GITHUB_ENV} - mkdir -p "${INSTALL_DIR}" - - - name: 'Rellic build' - shell: 'bash' - if: ${{ matrix.llvm == 'llvm-14' }} - run: | - cd rellic - scripts/build-preset.sh debug - scripts/roundtrip.py ${INSTALL_DIR}/install/bin/rellic-decomp* $(pwd)/tests/tools/decomp "${VCPKG_ROOT}/installed/${{ matrix.host.triplet }}/tools/llvm/clang" - - - name: 'Remill build' - if: ${{ matrix.llvm == 'llvm-14' }} - shell: 'bash' - run: | - cd remill - scripts/build-preset.sh debug - pushd ${INSTALL_DIR}/build/remill - cmake --build . --parallel --target test_dependencies - # See issue https://github.com/lifting-bits/remill/issues/518 for - # failing aarch64 tests - env CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --parallel --target test || true - popd - - - name: 'Anvill build' - if: ${{ matrix.llvm == 'llvm-14' }} - shell: 'bash' - run: | - cd anvill - scripts/build-preset.sh debug - ${INSTALL_DIR}/install/bin/anvill-decompile-json* -spec $(pwd)/bin/Decompile/tests/specs/ret0.json -bc_out ./ret0.bc -ir_out ret0.ir - - - name: Cache cleanup and reporting - shell: 'bash' - run: | - rm -rf vcpkg/{buildtrees,installed,packages} - ccache --show-stats - - stop-runner: - name: Stop self-hosted EC2 runner - needs: - - start-runner # required to get output from the start-runner job - - do-the-job # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.GHA_PAT }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/vcpkg_docker_arm64.yml b/.github/workflows/vcpkg_docker_arm64.yml deleted file mode 100644 index 307663fc..00000000 --- a/.github/workflows/vcpkg_docker_arm64.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Docker Build Images (ARM64) - -on: - schedule: - # Once every Wednesday at 00:00 - - cron: '0 0 * * 3' - push: - branches: - - master - paths: - - 'docker/**' - - '.github/workflows/vcpkg_docker_arm64.yml' - pull_request: - paths: - - 'docker/**' - - '.github/workflows/vcpkg_docker_arm64.yml' - - -jobs: - start-runner: - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - strategy: - matrix: - ec2: - - { ami: ami-0128e20cf90406205, instance-type: r6g.8xlarge} - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.GHA_PAT }} - ec2-image-id: ${{ matrix.ec2.ami }} - ec2-instance-type: ${{ matrix.ec2.instance-type }} - subnet-id: subnet-0deb935f0bbfe1a5d - security-group-id: sg-0f6a02eb80fafb982 - aws-resource-tags: > # optional, requires additional permissions - [ - {"Key": "Name", "Value": "ec2-github-runner"}, - {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} - ] - - do-the-job: - strategy: - fail-fast: false - matrix: - host: - - { name: 'ubuntu-20.04', arch: 'arm64', triplet: 'arm64-linux-rel', arch_docker: '_arm64' } - container: - - { distro: 'ubuntu', version: '20.04', codename: 'focal' } - - { distro: 'ubuntu', version: '22.04', codename: 'jammy' } - - name: Do the job on the runner - needs: start-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner - - steps: - - name: Cleanup working directory with container root - run: | - docker run --rm -v $(pwd):/workspace -w /workspace ubuntu:latest find . -name . -o -prune -exec rm -rf -- {} + || true - - uses: actions/checkout@v3 - - name: Build and Push - working-directory: docker - run: | - # Pull freshest ubuntu Docker image - docker pull ubuntu:${{ matrix.container.version}} - - docker build -f Dockerfile.ubuntu.vcpkg --target caching --no-cache --build-arg "DISTRO_VERSION=${{ matrix.container.codename }}" -t "docker.pkg.github.com/${DOCKER_TAG}" . - # Docker image with NuGet support goes to github packages for CI use only - if [[ "${GITHUB_REF}" == "refs/heads/master" ]] ; then - docker login docker.pkg.github.com -u publisher -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}" - for i in 1 2 3; do docker push "docker.pkg.github.com/${DOCKER_TAG}" && break || sleep 10; done - fi - - # NOTE: Docker Hub only allows one slash in tag - docker build -f Dockerfile.ubuntu.vcpkg --target base --build-arg "DISTRO_VERSION=${{ matrix.container.codename }}" -t "trailofbits/cxx-common-vcpkg-builder-${{ matrix.container.distro }}:${{ matrix.container.version }}${{ matrix.host.arch_docker }}" . - # Smaller Docker image without NuGet support goes to Docker Hub for users - if [[ "${GITHUB_REF}" == "refs/heads/master" ]] ; then - docker login -u "${DOCKER_HUB_USER}" -p "${DOCKER_HUB_TOKEN}" - for i in 1 2 3; do docker push "trailofbits/cxx-common-vcpkg-builder-${{ matrix.container.distro }}:${{ matrix.container.version }}${{ matrix.host.arch_docker }}" && break || sleep 10; done - fi - env: - DOCKER_TAG: lifting-bits/cxx-common/vcpkg-builder-${{ matrix.container.distro }}:${{ matrix.container.version }}${{ matrix.host.arch_docker }} - GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }} - DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }} - DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} - - stop-runner: - name: Stop self-hosted EC2 runner - needs: - - start-runner # required to get output from the start-runner job - - do-the-job # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.GHA_PAT }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}