From 106058c076e44d9dbe3404b1f15428a28531915e Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Thu, 21 Dec 2023 15:40:21 -0800 Subject: [PATCH] Initial CircleCI -> GitHub Actions migration (#12163) Summary: * Largely based on https://github.com/facebook/rocksdb/issues/12085 but grouped into one large workflow because of bad GHA UI design (see comments). * Windows job details consolidated into an action file so that those jobs can easily move between per-pr-push and nightly. * Simplify some handling of "CIRCLECI" environment and add "GITHUB_ACTIONS" in the same places * For jobs that we want to go in pr-jobs or nightly there are disabled "candidate" workflows with draft versions of those jobs. * ARM jobs are disabled waiting on full GHA support. * build-linux-java-static needed some special attention to work, due to GLIBC compatibility issues (see comments). Pull Request resolved: https://github.com/facebook/rocksdb/pull/12163 Test Plan: Nightly jobs can be seen passing between these two links: https://github.com/facebook/rocksdb/actions/runs/7266835435/job/19799390061?pr=12163 https://github.com/facebook/rocksdb/actions/runs/7269697823/job/19807724471?pr=12163 And per-PR jobs of course passing on this PR. Reviewed By: hx235 Differential Revision: D52335810 Pulled By: pdillinger fbshipit-source-id: bbb95196f33eabad8cddf3c6b52f4413c80e034d --- .circleci/config.yml | 66 +-- .github/actions/build-folly/action.yml | 7 + .../actions/build-for-benchmarks/action.yml | 8 + .../action.yml | 10 + .../install-gflags-on-macos/action.yml | 7 + .github/actions/install-gflags/action.yml | 7 + .../actions/install-jdk8-on-macos/action.yml | 9 + .github/actions/install-maven/action.yml | 7 + .github/actions/perform-benchmarks/action.yml | 22 + .github/actions/post-benchmarks/action.yml | 14 + .github/actions/post-pmd-steps/action.yml | 10 + .github/actions/post-steps/action.yml | 25 + .github/actions/pre-steps-macos/action.yml | 5 + .github/actions/pre-steps/action.yml | 18 + .github/actions/setup-folly/action.yml | 7 + .github/actions/setup-upstream/action.yml | 20 + .../actions/windows-build-steps/action.yml | 44 ++ .github/workflows/benchmark-linux.yml | 14 + .github/workflows/nightly-candidate.yml | 17 + .github/workflows/nightly.yml | 91 +++ .github/workflows/pr-jobs-candidate.yml | 44 ++ .github/workflows/pr-jobs.yml | 542 ++++++++++++++++++ .github/workflows/sanity_check.yml | 45 -- CMakeLists.txt | 5 - build_tools/gnu_parallel | 2 +- util/rate_limiter_test.cc | 4 +- .../write_prepared_transaction_test.cc | 2 +- 27 files changed, 935 insertions(+), 117 deletions(-) create mode 100644 .github/actions/build-folly/action.yml create mode 100644 .github/actions/build-for-benchmarks/action.yml create mode 100644 .github/actions/increase-max-open-files-on-macos/action.yml create mode 100644 .github/actions/install-gflags-on-macos/action.yml create mode 100644 .github/actions/install-gflags/action.yml create mode 100644 .github/actions/install-jdk8-on-macos/action.yml create mode 100644 .github/actions/install-maven/action.yml create mode 100644 .github/actions/perform-benchmarks/action.yml create mode 100644 .github/actions/post-benchmarks/action.yml create mode 100644 .github/actions/post-pmd-steps/action.yml create mode 100644 .github/actions/post-steps/action.yml create mode 100644 .github/actions/pre-steps-macos/action.yml create mode 100644 .github/actions/pre-steps/action.yml create mode 100644 .github/actions/setup-folly/action.yml create mode 100644 .github/actions/setup-upstream/action.yml create mode 100644 .github/actions/windows-build-steps/action.yml create mode 100644 .github/workflows/benchmark-linux.yml create mode 100644 .github/workflows/nightly-candidate.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/pr-jobs-candidate.yml create mode 100644 .github/workflows/pr-jobs.yml delete mode 100644 .github/workflows/sanity_check.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 370ad301a8d..cebae35ec7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -244,7 +244,7 @@ jobs: - increase-max-open-files-on-macos - install-gflags-on-macos - pre-steps-macos - - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=16 -j16 all + - run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j16 all - post-steps build-macos-cmake: @@ -877,66 +877,11 @@ jobs: - perform-benchmarks - post-benchmarks -workflows: +workflows: # Only jobs that haven't been successfully migrated to GitHub Actions version: 2 - jobs-linux-run-tests: - jobs: - - build-linux - - build-linux-cmake-with-folly - - build-linux-cmake-with-folly-lite-no-test - - build-linux-gcc-7-with-folly - - build-linux-gcc-7-with-folly-lite-no-test - - build-linux-cmake-with-folly-coroutines - - build-linux-cmake-with-benchmark - - build-linux-encrypted_env-no_compression - jobs-linux-run-tests-san: - jobs: - - build-linux-clang10-asan - - build-linux-clang10-ubsan - - build-linux-clang10-mini-tsan - - build-linux-static_lib-alt_namespace-status_checked - jobs-linux-no-test-run: - jobs: - - build-linux-release - - build-linux-release-rtti - - build-examples - - build-fuzzers - - build-linux-clang-no_test_run - - build-linux-clang-13-no_test_run - - build-linux-gcc-8-no_test_run - - build-linux-gcc-10-cxx20-no_test_run - - build-linux-gcc-11-no_test_run - - build-linux-arm-cmake-no_test_run - jobs-linux-other-checks: - jobs: - - build-linux-clang10-clang-analyze - - build-linux-unity-and-headers - - build-linux-mini-crashtest - jobs-windows: - jobs: - - build-windows-vs2019 - - build-cmake-mingw - jobs-java: - jobs: - - build-linux-java - - build-linux-java-static - - build-macos-java - - build-macos-java-static - - build-macos-java-static-universal - - build-linux-java-pmd - jobs-macos: - jobs: - - build-macos - - build-macos-cmake: - run_even_tests: true - - build-macos-cmake: - run_even_tests: false jobs-linux-arm: jobs: - build-linux-arm - build-fuzzers: - jobs: - - build-fuzzers benchmark-linux: triggers: - schedule: @@ -956,11 +901,4 @@ workflows: only: - main jobs: - - build-format-compatible - build-linux-arm-test-full - - build-linux-run-microbench - - build-linux-non-shm - - build-linux-clang-13-asan-ubsan-with-folly - - build-linux-valgrind - - build-windows-vs2022-avx2 - - build-windows-vs2022 diff --git a/.github/actions/build-folly/action.yml b/.github/actions/build-folly/action.yml new file mode 100644 index 00000000000..70229199958 --- /dev/null +++ b/.github/actions/build-folly/action.yml @@ -0,0 +1,7 @@ +name: build-folly +runs: + using: composite + steps: + - name: Build folly and dependencies + run: make build_folly + shell: bash diff --git a/.github/actions/build-for-benchmarks/action.yml b/.github/actions/build-for-benchmarks/action.yml new file mode 100644 index 00000000000..20e892e8fe8 --- /dev/null +++ b/.github/actions/build-for-benchmarks/action.yml @@ -0,0 +1,8 @@ +name: build-for-benchmarks +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" + - name: Linux build for benchmarks + run: make V=1 J=8 -j8 release + shell: bash diff --git a/.github/actions/increase-max-open-files-on-macos/action.yml b/.github/actions/increase-max-open-files-on-macos/action.yml new file mode 100644 index 00000000000..59bd5cb71d0 --- /dev/null +++ b/.github/actions/increase-max-open-files-on-macos/action.yml @@ -0,0 +1,10 @@ +name: increase-max-open-files-on-macos +runs: + using: composite + steps: + - name: Increase max open files + run: |- + sudo sysctl -w kern.maxfiles=1048576 + sudo sysctl -w kern.maxfilesperproc=1048576 + sudo launchctl limit maxfiles 1048576 + shell: bash diff --git a/.github/actions/install-gflags-on-macos/action.yml b/.github/actions/install-gflags-on-macos/action.yml new file mode 100644 index 00000000000..2a864cd35c2 --- /dev/null +++ b/.github/actions/install-gflags-on-macos/action.yml @@ -0,0 +1,7 @@ +name: install-gflags-on-macos +runs: + using: composite + steps: + - name: Install gflags on macos + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags + shell: bash diff --git a/.github/actions/install-gflags/action.yml b/.github/actions/install-gflags/action.yml new file mode 100644 index 00000000000..9cec707062a --- /dev/null +++ b/.github/actions/install-gflags/action.yml @@ -0,0 +1,7 @@ +name: install-gflags +runs: + using: composite + steps: + - name: Install gflags + run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + shell: bash diff --git a/.github/actions/install-jdk8-on-macos/action.yml b/.github/actions/install-jdk8-on-macos/action.yml new file mode 100644 index 00000000000..3f670bb882e --- /dev/null +++ b/.github/actions/install-jdk8-on-macos/action.yml @@ -0,0 +1,9 @@ +name: install-jdk8-on-macos +runs: + using: composite + steps: + - name: Install JDK 8 on macos + run: |- + HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8 + shell: bash diff --git a/.github/actions/install-maven/action.yml b/.github/actions/install-maven/action.yml new file mode 100644 index 00000000000..be54927f77a --- /dev/null +++ b/.github/actions/install-maven/action.yml @@ -0,0 +1,7 @@ +name: install-maven +runs: + using: composite + steps: + - name: Install maven + run: sudo apt-get update -y && sudo apt-get install -y maven + shell: bash diff --git a/.github/actions/perform-benchmarks/action.yml b/.github/actions/perform-benchmarks/action.yml new file mode 100644 index 00000000000..502db796c71 --- /dev/null +++ b/.github/actions/perform-benchmarks/action.yml @@ -0,0 +1,22 @@ +name: perform-benchmarks +runs: + using: composite + steps: + - name: Test low-variance benchmarks + run: "./tools/benchmark_ci.py --db_dir ${{ runner.temp }}/rocksdb-benchmark-datadir --output_dir ${{ runner.temp }}/benchmark-results --num_keys 20000000" + env: + LD_LIBRARY_PATH: "/usr/local/lib" + DURATION_RO: 300 + DURATION_RW: 500 + NUM_THREADS: 1 + MAX_BACKGROUND_JOBS: 4 + CI_TESTS_ONLY: 'true' + WRITE_BUFFER_SIZE_MB: 16 + TARGET_FILE_SIZE_BASE_MB: 16 + MAX_BYTES_FOR_LEVEL_BASE_MB: 64 + COMPRESSION_TYPE: none + CACHE_INDEX_AND_FILTER_BLOCKS: 1 + MIN_LEVEL_TO_COMPRESS: 3 + CACHE_SIZE_MB: 10240 + MB_WRITE_PER_SEC: 2 + shell: bash diff --git a/.github/actions/post-benchmarks/action.yml b/.github/actions/post-benchmarks/action.yml new file mode 100644 index 00000000000..e2466f46060 --- /dev/null +++ b/.github/actions/post-benchmarks/action.yml @@ -0,0 +1,14 @@ +name: post-benchmarks +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/benchmark-results" + - name: Send benchmark report to visualisation + run: |- + set +e + set +o pipefail + ./build_tools/benchmark_log_tool.py --tsvfile ${{ runner.temp }}/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc + true + shell: bash diff --git a/.github/actions/post-pmd-steps/action.yml b/.github/actions/post-pmd-steps/action.yml new file mode 100644 index 00000000000..6c58fa49ec2 --- /dev/null +++ b/.github/actions/post-pmd-steps/action.yml @@ -0,0 +1,10 @@ +name: post-pmd-steps +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ github.workspace }}/java/target/pmd.xml" + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ github.workspace }}/java/target/site" diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml new file mode 100644 index 00000000000..8318ff7656a --- /dev/null +++ b/.github/actions/post-steps/action.yml @@ -0,0 +1,25 @@ +name: post-steps +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/test-results" + - uses: actions/upload-artifact@v3.1.3 + with: + path: LOG + - name: Compress Test Logs + run: tar -cvzf t.tar.gz t + if: ${{ failure() }} + shell: bash + - uses: actions/upload-artifact@v3.1.3 + with: + path: t.tar.gz + - run: |- + mkdir -p ${{ runner.temp }}/core_dumps + cp core.* ${{ runner.temp }}/core_dumps + if: ${{ failure() }} + shell: bash + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/core_dumps" diff --git a/.github/actions/pre-steps-macos/action.yml b/.github/actions/pre-steps-macos/action.yml new file mode 100644 index 00000000000..d485b610050 --- /dev/null +++ b/.github/actions/pre-steps-macos/action.yml @@ -0,0 +1,5 @@ +name: pre-steps-macos +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" diff --git a/.github/actions/pre-steps/action.yml b/.github/actions/pre-steps/action.yml new file mode 100644 index 00000000000..1cc586ddbae --- /dev/null +++ b/.github/actions/pre-steps/action.yml @@ -0,0 +1,18 @@ +name: pre-steps +runs: + using: composite + steps: + - name: Setup Environment Variables + run: |- + echo "GTEST_THROW_ON_FAILURE=0" >> "$GITHUB_ENV" + echo "GTEST_OUTPUT=\"xml:${{ runner.temp }}/test-results/\"" >> "$GITHUB_ENV" + echo "SKIP_FORMAT_BUCK_CHECKS=1" >> "$GITHUB_ENV" + echo "GTEST_COLOR=1" >> "$GITHUB_ENV" + echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV" + echo "CTEST_TEST_TIMEOUT=300" >> "$GITHUB_ENV" + echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> "$GITHUB_ENV" + echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> "$GITHUB_ENV" + echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> "$GITHUB_ENV" + echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> "$GITHUB_ENV" + echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> "$GITHUB_ENV" + shell: bash diff --git a/.github/actions/setup-folly/action.yml b/.github/actions/setup-folly/action.yml new file mode 100644 index 00000000000..41cec847ce6 --- /dev/null +++ b/.github/actions/setup-folly/action.yml @@ -0,0 +1,7 @@ +name: setup-folly +runs: + using: composite + steps: + - name: Checkout folly sources + run: make checkout_folly + shell: bash diff --git a/.github/actions/setup-upstream/action.yml b/.github/actions/setup-upstream/action.yml new file mode 100644 index 00000000000..2a409f7601c --- /dev/null +++ b/.github/actions/setup-upstream/action.yml @@ -0,0 +1,20 @@ +name: build-folly +runs: + using: composite + steps: + - name: Fix repo ownership + # Needed in some cases, as safe.directory setting doesn't take effect + # under env -i + run: chown `whoami` . || true + shell: bash + - name: Set upstream + run: git remote add upstream https://github.com/facebook/rocksdb.git + shell: bash + - name: Fetch upstream + run: git fetch upstream + shell: bash + - name: Git status + # NOTE: some old branch builds under check_format_compatible.sh invoke + # git under env -i + run: git status && git remote -v && env -i git branch + shell: bash diff --git a/.github/actions/windows-build-steps/action.yml b/.github/actions/windows-build-steps/action.yml new file mode 100644 index 00000000000..0f76b12d0a8 --- /dev/null +++ b/.github/actions/windows-build-steps/action.yml @@ -0,0 +1,44 @@ +name: windows-build-steps +runs: + using: composite + steps: + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@v1.3.1 + - name: Custom steps + env: + THIRDPARTY_HOME: ${{ github.workspace }}/thirdparty + CMAKE_HOME: C:/Program Files/CMake + CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe + JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8 + SNAPPY_HOME: ${{ github.workspace }}/thirdparty/snappy-1.1.8 + SNAPPY_INCLUDE: ${{ github.workspace }}/thirdparty/snappy-1.1.8;${{ github.workspace }}/thirdparty/snappy-1.1.8/build + SNAPPY_LIB_DEBUG: ${{ github.workspace }}/thirdparty/snappy-1.1.8/build/Debug/snappy.lib + run: |- + echo ===================== Install Dependencies ===================== + choco install liberica8jdk -y + mkdir $Env:THIRDPARTY_HOME + cd $Env:THIRDPARTY_HOME + echo "Building Snappy dependency..." + curl -Lo snappy-1.1.8.zip https://github.com/google/snappy/archive/refs/tags/1.1.8.zip + unzip -q snappy-1.1.8.zip + cd snappy-1.1.8 + mkdir build + cd build + & cmake -G "$Env:CMAKE_GENERATOR" .. + msbuild Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + echo ======================== Build RocksDB ========================= + cd ${{ github.workspace }} + $env:Path = $env:JAVA_HOME + ";" + $env:Path + mkdir build + cd build + & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 .. + cd .. + echo "Building with VS version: $Env:CMAKE_GENERATOR" + msbuild build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + echo ========================= Test RocksDB ========================= + build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16 + echo ======================== Test RocksJava ======================== + cd build\java + & ctest -C Debug -j 16 + shell: pwsh diff --git a/.github/workflows/benchmark-linux.yml b/.github/workflows/benchmark-linux.yml new file mode 100644 index 00000000000..e675489bd05 --- /dev/null +++ b/.github/workflows/benchmark-linux.yml @@ -0,0 +1,14 @@ +name: facebook/rocksdb/benchmark-linux +on: workflow_dispatch +jobs: + # FIXME: when this job is fixed, it should be given a cron schedule like + # schedule: + # - cron: 0 * * * * + # workflow_dispatch: + benchmark-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/build-for-benchmarks" + - uses: "./.github/actions/perform-benchmarks" + - uses: "./.github/actions/post-benchmarks" diff --git a/.github/workflows/nightly-candidate.yml b/.github/workflows/nightly-candidate.yml new file mode 100644 index 00000000000..ae1cb954939 --- /dev/null +++ b/.github/workflows/nightly-candidate.yml @@ -0,0 +1,17 @@ +name: facebook/rocksdb/nightly +on: workflow_dispatch +jobs: + # These jobs would be in nightly but are failing or otherwise broken for + # some reason. + build-linux-arm-test-full: + runs-on: + labels: arm64large + container: + image: ubuntu-2004:202111-02 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: make V=1 J=4 -j4 check + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000000..3c1119c9e6e --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,91 @@ +name: facebook/rocksdb/nightly +on: + schedule: + - cron: 0 9 * * * + workflow_dispatch: +jobs: + build-format-compatible: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + with: + fetch-depth: 0 # Need full repo history + fetch-tags: true + - uses: "./.github/actions/setup-upstream" + - uses: "./.github/actions/pre-steps" + - name: test + run: |- + export TEST_TMPDIR=/dev/shm/rocksdb + rm -rf /dev/shm/rocksdb + mkdir /dev/shm/rocksdb + git config --global --add safe.directory /__w/rocksdb/rocksdb + tools/check_format_compatible.sh + - uses: "./.github/actions/post-steps" + build-linux-run-microbench: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: DEBUG_LEVEL=0 make -j32 run_microbench + - uses: "./.github/actions/post-steps" + build-linux-non-shm: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + env: + TEST_TMPDIR: "/tmp/rocksdb_test_tmp" + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-clang-13-asan-ubsan-with-folly: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check + - uses: "./.github/actions/post-steps" + build-linux-valgrind: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: PORTABLE=1 make V=1 -j32 valgrind_test + - uses: "./.github/actions/post-steps" + build-windows-vs2022-avx2: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: AVX2 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + build-windows-vs2022: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" diff --git a/.github/workflows/pr-jobs-candidate.yml b/.github/workflows/pr-jobs-candidate.yml new file mode 100644 index 00000000000..2b288262979 --- /dev/null +++ b/.github/workflows/pr-jobs-candidate.yml @@ -0,0 +1,44 @@ +name: facebook/rocksdb/pr-jobs-candidate +on: workflow_dispatch +jobs: + # These jobs would be in pr-jobs but are failing or otherwise broken for + # some reason. + # =========================== ARM Jobs ============================ # + build-linux-arm: + runs-on: + labels: arm64large # GitHub hosted ARM runners do not yet exist + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some + - uses: "./.github/actions/post-steps" + build-linux-arm-cmake-no_test_run: + runs-on: + labels: arm64large # GitHub hosted ARM runners do not yet exist + env: + JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-arm64" + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - name: Build with cmake + run: |- + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. + make -j4 + - name: Build Java with cmake + run: |- + rm -rf build + mkdir build + cd build + cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 .. + make -j4 rocksdb rocksdbjni + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/pr-jobs.yml b/.github/workflows/pr-jobs.yml new file mode 100644 index 00000000000..3aab19f8925 --- /dev/null +++ b/.github/workflows/pr-jobs.yml @@ -0,0 +1,542 @@ +name: facebook/rocksdb/pr-jobs +on: [push, pull_request] +jobs: + # NOTE: multiple workflows would be recommended, but the current GHA UI in + # PRs doesn't make it clear when there's an overall error with a workflow, + # making it easy to overlook something broken. Grouping everything into one + # workflow minimizes the problem because it will be suspicious if there are + # no GHA results. + # + # DEBUGGING WITH SSH: Temporarily add this as a job step, either before the + # step of interest without the "if:" line or after the failing step with the + # "if:" line. Then use ssh command printed in CI output. + # - name: Setup tmate session # TEMPORARY! + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # with: + # limit-access-to-actor: true + + # ======================== Fast Initial Checks ====================== # + check-format-and-targets: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.0 + with: + fetch-depth: 0 # Need full checkout to determine merge base + fetch-tags: true + - uses: "./.github/actions/setup-upstream" + - name: Setup Python + uses: actions/setup-python@v1 + - name: Install Dependencies + run: python -m pip install --upgrade pip + - name: Install argparse + run: pip install argparse + - name: Download clang-format-diff.py + run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py + - name: Check format + run: VERBOSE_CHECK=1 make check-format + - name: Compare buckify output + run: make check-buck-targets + - name: Simple source code checks + run: make check-sources + # ========================= Linux With Tests ======================== # + build-linux: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: make V=1 J=32 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-cmake-mingw: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix + - name: Build cmake-mingw + run: |- + export PATH=$JAVA_HOME/bin:$PATH + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly-lite-no-test: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)" + - uses: "./.github/actions/post-steps" + build-linux-gcc-7-with-folly: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check + - uses: "./.github/actions/post-steps" + build-linux-gcc-7-with-folly-lite-no-test: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly-coroutines: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + env: + CC: gcc-10 + CXX: g++-10 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-benchmark: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 + - uses: "./.github/actions/post-steps" + build-linux-encrypted_env-no_compression: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check + - run: "./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression\n" + - uses: "./.github/actions/post-steps" + # ======================== Linux No Test Runs ======================= # + build-linux-release: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: "./db_stress --version" + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: "./db_stress --version" + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: if ./db_stress --version; then false; else true; fi + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: if ./db_stress --version; then false; else true; fi + - uses: "./.github/actions/post-steps" + build-linux-release-rtti: + runs-on: + labels: 8-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: "./db_stress --version" + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: if ./db_stress --version; then false; else true; fi + build-examples: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Build examples + run: make V=1 -j4 static_lib && cd examples && make V=1 -j4 + - uses: "./.github/actions/post-steps" + build-fuzzers: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Build rocksdb lib + run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib + - name: Build fuzzers + run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer + - uses: "./.github/actions/post-steps" + build-linux-clang-no_test_run: + runs-on: + labels: 8-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all + - uses: "./.github/actions/post-steps" + build-linux-clang-13-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench + - uses: "./.github/actions/post-steps" + build-linux-gcc-8-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-gcc-10-cxx20-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-gcc-11-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench + - uses: "./.github/actions/post-steps" + # ======================== Linux Other Checks ======================= # + build-linux-clang10-clang-analyze: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze + - uses: "./.github/actions/post-steps" + - name: compress test report + run: tar -cvzf scan_build_report.tar.gz scan_build_report + if: failure() + - uses: actions/upload-artifact@v3.1.3 + with: + path: scan_build_report.tar.gz + build-linux-unity-and-headers: + runs-on: + labels: 4-core-ubuntu + container: + image: gcc:latest + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: apt-get update -y && apt-get install -y libgflags-dev + - name: Unity build + run: make V=1 -j8 unity_test + - run: make V=1 -j8 -k check-headers + - uses: "./.github/actions/post-steps" + build-linux-mini-crashtest: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush + - uses: "./.github/actions/post-steps" + # ======================= Linux with Sanitizers ===================== # + build-linux-clang10-asan: + runs-on: + labels: 32-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-clang10-ubsan: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check + - uses: "./.github/actions/post-steps" + build-linux-clang10-mini-tsan: + runs-on: + labels: 32-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-static_lib-alt_namespace-status_checked: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check + - uses: "./.github/actions/post-steps" + # ========================= MacOS with Tests ======================== # + build-macos: + runs-on: macos-13 + env: + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Run all tests + run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j16 all + - uses: "./.github/actions/post-steps" + build-macos-cmake: + runs-on: macos-13 + strategy: + matrix: + run_even_tests: [true, false] + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: cmake generate project file + run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. + - name: Build tests + run: cd build && make V=1 -j16 + - name: Run even tests + run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2 + if: ${{ matrix.run_even_tests }} + - name: Run odd tests + run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2 + if: ${{ ! matrix.run_even_tests }} + - uses: "./.github/actions/post-steps" + # ======================== Windows with Tests ======================= # + # NOTE: some windows jobs are in "nightly" to save resources + build-windows-vs2019: + runs-on: windows-2019 + env: + CMAKE_GENERATOR: Visual Studio 16 2019 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + # ============================ Java Jobs ============================ # + build-linux-java: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: make V=1 J=8 -j8 jtest + - uses: "./.github/actions/post-steps" + build-linux-java-static: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + # The docker image is based on such an old OS that it has a GLIBC + # incompatibility with actions/checkout and other actions. Thus we + # implement a manual checkout step. + - name: Checkout + env: + GH_TOKEN: ${{ github.token }} + run: | + chown `whoami` . || true + git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git . + git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }} + git checkout --progress --force ${{ github.ref }} + git log -1 --format='%H' + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + # NOTE: post-steps skipped because of compatibility issues with docker image + build-macos-java: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: make V=1 J=16 -j16 jtest + - uses: "./.github/actions/post-steps" + build-macos-java-static: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava x86 and ARM Static Libraries + run: make V=1 J=16 -j16 rocksdbjavastaticosx + - uses: "./.github/actions/post-steps" + build-macos-java-static-universal: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Universal Binary Static Library + run: make V=1 J=16 -j16 rocksdbjavastaticosx_ub + - uses: "./.github/actions/post-steps" + build-linux-java-pmd: + runs-on: + labels: 4-core-ubuntu + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/install-maven" + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: PMD RocksDBJava + run: make V=1 J=8 -j8 jpmd + - uses: "./.github/actions/post-pmd-steps" diff --git a/.github/workflows/sanity_check.yml b/.github/workflows/sanity_check.yml deleted file mode 100644 index efc9d99cf37..00000000000 --- a/.github/workflows/sanity_check.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Check buck targets and code format -on: [push, pull_request] -permissions: - contents: read - -jobs: - check: - name: Check TARGETS file and code format - runs-on: ubuntu-latest - steps: - - name: Checkout feature branch - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fetch from upstream - run: | - git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream - - - name: Where am I - run: | - echo git status && git status - echo "git remote -v" && git remote -v - echo git branch && git branch - - - name: Setup Python - uses: actions/setup-python@v1 - - - name: Install Dependencies - run: python -m pip install --upgrade pip - - - name: Install argparse - run: pip install argparse - - - name: Download clang-format-diff.py - run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py - - - name: Check format - run: VERBOSE_CHECK=1 make check-format - - - name: Compare buckify output - run: make check-buck-targets - - - name: Simple source code checks - run: make check-sources diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f7f9ccada4..49868be8940 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,11 +76,6 @@ if (WITH_WINDOWS_UTF8_FILENAMES) endif() option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON) -if ($ENV{CIRCLECI}) - message(STATUS "Build for CircieCI env, a few tests may be disabled") - add_definitions(-DCIRCLECI) -endif() - if( NOT DEFINED CMAKE_CXX_STANDARD ) set(CMAKE_CXX_STANDARD 17) endif() diff --git a/build_tools/gnu_parallel b/build_tools/gnu_parallel index 3365f46ba19..52e607a0041 100755 --- a/build_tools/gnu_parallel +++ b/build_tools/gnu_parallel @@ -1561,7 +1561,7 @@ sub save_stdin_stdout_stderr { ::die_bug("Can't dup STDERR: $!"); open $Global::original_stdin, "<&", "STDIN" or ::die_bug("Can't dup STDIN: $!"); - $Global::is_terminal = (-t $Global::original_stderr) && !$ENV{'CIRCLECI'} && !$ENV{'TRAVIS'}; + $Global::is_terminal = (-t $Global::original_stderr) && !$ENV{'CIRCLECI'}&& !$ENV{'GITHUB_ACTIONS'} && !$ENV{'TRAVIS'}; } sub enough_file_handles { diff --git a/util/rate_limiter_test.cc b/util/rate_limiter_test.cc index 16e7623ac8d..3b17e8a4b2a 100644 --- a/util/rate_limiter_test.cc +++ b/util/rate_limiter_test.cc @@ -347,8 +347,10 @@ TEST_F(RateLimiterTest, Rate) { // This can fail due to slow execution speed, like when using valgrind or in // heavily loaded CI environments bool skip_minimum_rate_check = -#if (defined(CIRCLECI) && defined(OS_MACOSX)) || defined(ROCKSDB_VALGRIND_RUN) +#if defined(ROCKSDB_VALGRIND_RUN) true; +#elif defined(OS_MACOSX) + getenv("CIRCLECI") || getenv("GITHUB_ACTIONS"); #else getenv("SANDCASTLE"); #endif diff --git a/utilities/transactions/write_prepared_transaction_test.cc b/utilities/transactions/write_prepared_transaction_test.cc index 8427e2d7c8e..09e5b268941 100644 --- a/utilities/transactions/write_prepared_transaction_test.cc +++ b/utilities/transactions/write_prepared_transaction_test.cc @@ -4058,7 +4058,7 @@ TEST_P(WritePreparedTransactionTest, WC_WP_WALForwardIncompatibility) { int main(int argc, char** argv) { ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); ::testing::InitGoogleTest(&argc, argv); - if (getenv("CIRCLECI")) { + if (getenv("CIRCLECI") || getenv("GITHUB_ACTIONS")) { // Looking for backtrace on "Resource temporarily unavailable" exceptions ::testing::FLAGS_gtest_catch_exceptions = false; }