diff --git a/.circleci/config.yml b/.circleci/config.yml index 370ad301a8d..2206f1f1b8b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -879,58 +879,6 @@ jobs: workflows: version: 2 - jobs-linux-run-tests: - jobs: - - build-linux - - build-linux-cmake-with-folly - - build-linux-cmake-with-folly-lite-no-test - - build-linux-gcc-7-with-folly - - build-linux-gcc-7-with-folly-lite-no-test - - build-linux-cmake-with-folly-coroutines - - build-linux-cmake-with-benchmark - - build-linux-encrypted_env-no_compression - jobs-linux-run-tests-san: - jobs: - - build-linux-clang10-asan - - build-linux-clang10-ubsan - - build-linux-clang10-mini-tsan - - build-linux-static_lib-alt_namespace-status_checked - jobs-linux-no-test-run: - jobs: - - build-linux-release - - build-linux-release-rtti - - build-examples - - build-fuzzers - - build-linux-clang-no_test_run - - build-linux-clang-13-no_test_run - - build-linux-gcc-8-no_test_run - - build-linux-gcc-10-cxx20-no_test_run - - build-linux-gcc-11-no_test_run - - build-linux-arm-cmake-no_test_run - jobs-linux-other-checks: - jobs: - - build-linux-clang10-clang-analyze - - build-linux-unity-and-headers - - build-linux-mini-crashtest - jobs-windows: - jobs: - - build-windows-vs2019 - - build-cmake-mingw - jobs-java: - jobs: - - build-linux-java - - build-linux-java-static - - build-macos-java - - build-macos-java-static - - build-macos-java-static-universal - - build-linux-java-pmd - jobs-macos: - jobs: - - build-macos - - build-macos-cmake: - run_even_tests: true - - build-macos-cmake: - run_even_tests: false jobs-linux-arm: jobs: - build-linux-arm diff --git a/.github/actions/build-folly/action.yml b/.github/actions/build-folly/action.yml new file mode 100644 index 00000000000..909ca549b69 --- /dev/null +++ b/.github/actions/build-folly/action.yml @@ -0,0 +1,7 @@ +name: build-folly +runs: + using: composite + steps: + - name: Build folly and dependencies + run: make build_folly + shell: bash \ No newline at end of file diff --git a/.github/actions/build-for-benchmarks/action.yml b/.github/actions/build-for-benchmarks/action.yml new file mode 100644 index 00000000000..da78fdae43e --- /dev/null +++ b/.github/actions/build-for-benchmarks/action.yml @@ -0,0 +1,8 @@ +name: build-for-benchmarks +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" + - name: Linux build for benchmarks + run: make V=1 J=8 -j8 release + shell: bash \ No newline at end of file diff --git a/.github/actions/increase-max-open-files-on-macos/action.yml b/.github/actions/increase-max-open-files-on-macos/action.yml new file mode 100644 index 00000000000..0a217ff5516 --- /dev/null +++ b/.github/actions/increase-max-open-files-on-macos/action.yml @@ -0,0 +1,10 @@ +name: increase-max-open-files-on-macos +runs: + using: composite + steps: + - name: Increase max open files + run: |- + sudo sysctl -w kern.maxfiles=1048576 + sudo sysctl -w kern.maxfilesperproc=1048576 + sudo launchctl limit maxfiles 1048576 + shell: bash \ No newline at end of file diff --git a/.github/actions/install-gflags-on-macos/action.yml b/.github/actions/install-gflags-on-macos/action.yml new file mode 100644 index 00000000000..4ad14474065 --- /dev/null +++ b/.github/actions/install-gflags-on-macos/action.yml @@ -0,0 +1,7 @@ +name: install-gflags-on-macos +runs: + using: composite + steps: + - name: Install gflags on macos + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags + shell: bash \ No newline at end of file diff --git a/.github/actions/install-gflags/action.yml b/.github/actions/install-gflags/action.yml new file mode 100644 index 00000000000..b3695aed5da --- /dev/null +++ b/.github/actions/install-gflags/action.yml @@ -0,0 +1,7 @@ +name: install-gflags +runs: + using: composite + steps: + - name: Install gflags + run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + shell: bash \ No newline at end of file diff --git a/.github/actions/install-jdk8-on-macos/action.yml b/.github/actions/install-jdk8-on-macos/action.yml new file mode 100644 index 00000000000..c6cd991ddc6 --- /dev/null +++ b/.github/actions/install-jdk8-on-macos/action.yml @@ -0,0 +1,9 @@ +name: install-jdk8-on-macos +runs: + using: composite + steps: + - name: Install JDK 8 on macos + run: |- + HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8 + shell: bash \ No newline at end of file diff --git a/.github/actions/install-maven/action.yml b/.github/actions/install-maven/action.yml new file mode 100644 index 00000000000..0924f5058bb --- /dev/null +++ b/.github/actions/install-maven/action.yml @@ -0,0 +1,7 @@ +name: install-maven +runs: + using: composite + steps: + - name: Install maven + run: sudo apt-get update -y && sudo apt-get install -y maven + shell: bash \ No newline at end of file diff --git a/.github/actions/perform-benchmarks/action.yml b/.github/actions/perform-benchmarks/action.yml new file mode 100644 index 00000000000..3eac7c7b3be --- /dev/null +++ b/.github/actions/perform-benchmarks/action.yml @@ -0,0 +1,22 @@ +name: perform-benchmarks +runs: + using: composite + steps: + - name: Test low-variance benchmarks + run: "./tools/benchmark_ci.py --db_dir ${{ runner.temp }}/rocksdb-benchmark-datadir --output_dir ${{ runner.temp }}/benchmark-results --num_keys 20000000" + env: + LD_LIBRARY_PATH: "/usr/local/lib" + DURATION_RO: 300 + DURATION_RW: 500 + NUM_THREADS: 1 + MAX_BACKGROUND_JOBS: 4 + CI_TESTS_ONLY: 'true' + WRITE_BUFFER_SIZE_MB: 16 + TARGET_FILE_SIZE_BASE_MB: 16 + MAX_BYTES_FOR_LEVEL_BASE_MB: 64 + COMPRESSION_TYPE: none + CACHE_INDEX_AND_FILTER_BLOCKS: 1 + MIN_LEVEL_TO_COMPRESS: 3 + CACHE_SIZE_MB: 10240 + MB_WRITE_PER_SEC: 2 + shell: bash \ No newline at end of file diff --git a/.github/actions/post-benchmarks/action.yml b/.github/actions/post-benchmarks/action.yml new file mode 100644 index 00000000000..9d9b07827d7 --- /dev/null +++ b/.github/actions/post-benchmarks/action.yml @@ -0,0 +1,14 @@ +name: post-benchmarks +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/benchmark-results" + - name: Send benchmark report to visualisation + run: |- + set +e + set +o pipefail + ./build_tools/benchmark_log_tool.py --tsvfile ${{ runner.temp }}/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc + true + shell: bash \ No newline at end of file diff --git a/.github/actions/post-pmd-steps/action.yml b/.github/actions/post-pmd-steps/action.yml new file mode 100644 index 00000000000..a9313895b9d --- /dev/null +++ b/.github/actions/post-pmd-steps/action.yml @@ -0,0 +1,10 @@ +name: post-pmd-steps +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ github.workspace }}/java/target/pmd.xml" + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ github.workspace }}/java/target/site" \ No newline at end of file diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml new file mode 100644 index 00000000000..b41bfc30860 --- /dev/null +++ b/.github/actions/post-steps/action.yml @@ -0,0 +1,31 @@ +name: post-steps +runs: + using: composite + steps: + - uses: actions/checkout@v3 + - name: Setup tmate session #TEMPORARY! + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/test-results" + - uses: actions/upload-artifact@v3.1.3 + with: + path: LOG + - name: Compress Test Logs + run: tar -cvzf t.tar.gz t + if: ${{ failure() }} + shell: bash + - uses: actions/upload-artifact@v3.1.3 + with: + path: t.tar.gz + - run: |- + mkdir -p ${{ runner.temp }}/core_dumps + cp core.* ${{ runner.temp }}/core_dumps + if: ${{ failure() }} + shell: bash + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/core_dumps" diff --git a/.github/actions/pre-steps-macos/action.yml b/.github/actions/pre-steps-macos/action.yml new file mode 100644 index 00000000000..2cc9195c884 --- /dev/null +++ b/.github/actions/pre-steps-macos/action.yml @@ -0,0 +1,5 @@ +name: pre-steps-macos +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" \ No newline at end of file diff --git a/.github/actions/pre-steps/action.yml b/.github/actions/pre-steps/action.yml new file mode 100644 index 00000000000..b192c87ab92 --- /dev/null +++ b/.github/actions/pre-steps/action.yml @@ -0,0 +1,18 @@ +name: pre-steps +runs: + using: composite + steps: + - name: Setup Environment Variables + run: |- + echo "GTEST_THROW_ON_FAILURE=0" >> "$GITHUB_ENV" + echo "GTEST_OUTPUT=\"xml:${{ runner.temp }}/test-results/\"" >> "$GITHUB_ENV" + echo "SKIP_FORMAT_BUCK_CHECKS=1" >> "$GITHUB_ENV" + echo "GTEST_COLOR=1" >> "$GITHUB_ENV" + echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV" + echo "CTEST_TEST_TIMEOUT=300" >> "$GITHUB_ENV" + echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> "$GITHUB_ENV" + echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> "$GITHUB_ENV" + echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> "$GITHUB_ENV" + echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> "$GITHUB_ENV" + echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> "$GITHUB_ENV" + shell: bash \ No newline at end of file diff --git a/.github/actions/setup-folly/action.yml b/.github/actions/setup-folly/action.yml new file mode 100644 index 00000000000..dd19977e3aa --- /dev/null +++ b/.github/actions/setup-folly/action.yml @@ -0,0 +1,7 @@ +name: setup-folly +runs: + using: composite + steps: + - name: Checkout folly sources + run: make checkout_folly + shell: bash \ No newline at end of file diff --git a/.github/actions/windows-build-steps/action.yml b/.github/actions/windows-build-steps/action.yml new file mode 100644 index 00000000000..5923a54bc30 --- /dev/null +++ b/.github/actions/windows-build-steps/action.yml @@ -0,0 +1,38 @@ +name: windows-build-steps +runs: + using: composite + steps: + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@v1.3.1 + - name: Install thirdparty dependencies + run: |- + choco install liberica8jdk -y + mkdir $Env:THIRDPARTY_HOME + cd $Env:THIRDPARTY_HOME + echo "Building Snappy dependency..." + curl -Lo snappy-1.1.8.zip https://github.com/google/snappy/archive/refs/tags/1.1.8.zip + unzip -q snappy-1.1.8.zip + cd snappy-1.1.8 + mkdir build + cd build + & cmake -G "$Env:CMAKE_GENERATOR" .. + msbuild Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + shell: pwsh + - name: Build RocksDB + run: |- + $env:Path = $env:JAVA_HOME + ";" + $env:Path + mkdir build + cd build + & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 .. + cd .. + echo "Building with VS version: $Env:CMAKE_GENERATOR" + msbuild build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + shell: pwsh + - name: Test RocksDB + run: build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16 + shell: pwsh + - name: Test RocksJava + run: |- + cd build\java + & ctest -C Debug -j 16 + shell: pwsh \ No newline at end of file diff --git a/.github/workflows/benchmark-linux.yml b/.github/workflows/benchmark-linux.yml new file mode 100644 index 00000000000..8e190eed94e --- /dev/null +++ b/.github/workflows/benchmark-linux.yml @@ -0,0 +1,13 @@ +name: facebook/rocksdb/benchmark-linux +on: + schedule: + - cron: 0 * * * * + workflow_dispatch: +jobs: + benchmark-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/build-for-benchmarks" + - uses: "./.github/actions/perform-benchmarks" + - uses: "./.github/actions/post-benchmarks" diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml new file mode 100644 index 00000000000..a3a95565b6a --- /dev/null +++ b/.github/workflows/jobs-java.yml @@ -0,0 +1,185 @@ +name: facebook/rocksdb/jobs-java +on: [push, pull_request] +syntax-error: foo +jobs: + build-linux-java: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: make V=1 J=8 -j8 jtest + - uses: "./.github/actions/post-steps" + build-linux-java-static27: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v2.7.0 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-linux-java-static35: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v3.5.3 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-linux-java-static36: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v3.6.0 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-linux-java-static40: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.0.0 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-linux-java-static41: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.1 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-macos-java: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: make V=1 J=16 -j16 jtest + - uses: "./.github/actions/post-steps" + build-macos-java-static: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava x86 and ARM Static Libraries + run: make V=1 J=16 -j16 rocksdbjavastaticosx + - uses: "./.github/actions/post-steps" + build-macos-java-static-universal: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Universal Binary Static Library + run: make V=1 J=16 -j16 rocksdbjavastaticosx_ub + - uses: "./.github/actions/post-steps" + build-linux-java-pmd: + runs-on: + labels: 4-core-ubuntu + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/install-maven" + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: PMD RocksDBJava + run: make V=1 J=8 -j8 jpmd + - uses: "./.github/actions/post-pmd-steps" diff --git a/.github/workflows/jobs-linux-arm.yml b/.github/workflows/jobs-linux-arm.yml new file mode 100644 index 00000000000..c420e967f11 --- /dev/null +++ b/.github/workflows/jobs-linux-arm.yml @@ -0,0 +1,42 @@ +name: facebook/rocksdb/jobs-linux-arm +on: [push, pull_request] +jobs: + build-linux-arm: + syntax-error: foo + runs-on: + labels: arm64large # GitHub hosted ARM runners do not yet exist + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some + - uses: "./.github/actions/post-steps" + build-linux-arm-cmake-no_test_run: + runs-on: + labels: arm64large # GitHub hosted ARM runners do not yet exist + env: + JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-arm64" + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - name: Build with cmake + run: |- + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. + make -j4 + - name: Build Java with cmake + run: |- + rm -rf build + mkdir build + cd build + cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 .. + make -j4 rocksdb rocksdbjni + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml new file mode 100644 index 00000000000..217dbcb2ea6 --- /dev/null +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -0,0 +1,122 @@ +name: facebook/rocksdb/jobs-linux-no-test-run +on: [push, pull_request] +jobs: + build-linux-release: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: "./db_stress --version" + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: "./db_stress --version" + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: if ./db_stress --version; then false; else true; fi + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: if ./db_stress --version; then false; else true; fi + - uses: "./.github/actions/post-steps" + build-linux-release-rtti: + runs-on: + labels: 8-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: "./db_stress --version" + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: if ./db_stress --version; then false; else true; fi + build-examples: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Build examples + run: make V=1 -j4 static_lib && cd examples && make V=1 -j4 + - uses: "./.github/actions/post-steps" + build-fuzzers: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Build rocksdb lib + run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib + - name: Build fuzzers + run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer + - uses: "./.github/actions/post-steps" + build-linux-clang-no_test_run: + runs-on: + labels: 8-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all + - uses: "./.github/actions/post-steps" + build-linux-clang-13-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench + - uses: "./.github/actions/post-steps" + build-linux-gcc-8-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-gcc-10-cxx20-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-gcc-11-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-linux-other-checks.yml b/.github/workflows/jobs-linux-other-checks.yml new file mode 100644 index 00000000000..8f85ef52569 --- /dev/null +++ b/.github/workflows/jobs-linux-other-checks.yml @@ -0,0 +1,47 @@ +name: facebook/rocksdb/jobs-linux-other-checks +on: [push, pull_request] +jobs: + build-linux-clang10-clang-analyze: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze + - uses: "./.github/actions/post-steps" + - name: compress test report + run: tar -cvzf scan_build_report.tar.gz scan_build_report + if: failure() + - uses: actions/upload-artifact@v3.1.3 + with: + path: scan_build_report.tar.gz + build-linux-unity-and-headers: + runs-on: + labels: 4-core-ubuntu + container: + image: gcc:latest + options: --shm-size=16gb + env: + EXTRA_CXXFLAGS: "-mno-avx512f" + steps: + - uses: actions/checkout@v4.1.0 + - run: apt-get update -y && apt-get install -y libgflags-dev + - name: Unity build + run: make V=1 -j8 unity_test + syntax-error: foo + - run: make V=1 -j8 -k check-headers + - uses: "./.github/actions/post-steps" + build-linux-mini-crashtest: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml new file mode 100644 index 00000000000..ac6b757503f --- /dev/null +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -0,0 +1,48 @@ +name: facebook/rocksdb/jobs-linux-run-tests-san +on: [push, pull_request] +jobs: + syntax-error: foo + build-linux-clang10-asan: + runs-on: + labels: 32-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-clang10-ubsan: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check + - uses: "./.github/actions/post-steps" + build-linux-clang10-mini-tsan: + runs-on: + labels: 32-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-static_lib-alt_namespace-status_checked: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-linux-run-tests.yml b/.github/workflows/jobs-linux-run-tests.yml new file mode 100644 index 00000000000..8c207f56664 --- /dev/null +++ b/.github/workflows/jobs-linux-run-tests.yml @@ -0,0 +1,121 @@ +name: facebook/rocksdb/jobs-linux-run-tests +on: [push, pull_request] +jobs: + build-linux: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: make V=1 J=32 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-cmake-mingw: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix + - name: Build cmake-mingw + run: |- + export PATH=$JAVA_HOME/bin:$PATH + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly-lite-no-test: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)" + - uses: "./.github/actions/post-steps" + build-linux-gcc-7-with-folly: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check + - uses: "./.github/actions/post-steps" + build-linux-gcc-7-with-folly-lite-no-test: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly-coroutines: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + env: + CC: gcc-10 + CXX: g++-10 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-benchmark: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 + - uses: "./.github/actions/post-steps" + build-linux-encrypted_env-no_compression: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check + - run: "./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression\n" + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-macos.yml b/.github/workflows/jobs-macos.yml new file mode 100644 index 00000000000..265e3d501b2 --- /dev/null +++ b/.github/workflows/jobs-macos.yml @@ -0,0 +1,43 @@ +name: facebook/rocksdb/jobs-macos +on: [push, pull_request] +jobs: + build-macos: + syntax-error: foo + runs-on: macos-13 + env: + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Run all tests + run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=16 -j16 all + - uses: "./.github/actions/post-steps" + build-macos-cmake: + runs-on: macos-13 + strategy: + matrix: + run_even_tests: [true, false] + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: cmake generate project file + run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. + - name: Build tests + run: cd build && make V=1 -j16 + - name: Run even tests + run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2 + if: ${{ matrix.run_even_tests }} + - name: Run odd tests + run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2 + if: ${{ ! matrix.run_even_tests }} + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-windows.yml b/.github/workflows/jobs-windows.yml new file mode 100644 index 00000000000..4f452a2c48f --- /dev/null +++ b/.github/workflows/jobs-windows.yml @@ -0,0 +1,22 @@ +name: facebook/rocksdb/jobs-windows +on: [push, pull_request] +env: + THIRDPARTY_HOME: ${{ github.workspace }}/thirdparty + CMAKE_HOME: C:/Program Files/CMake + CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe + JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8 + SNAPPY_HOME: ${{ github.workspace }}/thirdparty/snappy-1.1.8 + SNAPPY_INCLUDE: ${{ github.workspace }}/thirdparty/snappy-1.1.8;${{ github.workspace }}/thirdparty/snappy-1.1.8/build + SNAPPY_LIB_DEBUG: ${{ github.workspace }}/thirdparty/snappy-1.1.8/build/Debug/snappy.lib +jobs: + # NOTE: some windows jobs are in "nightly" to save resources + build-windows-vs2019: + syntax-error: foo + runs-on: windows-2019 + env: + CMAKE_GENERATOR: Visual Studio 16 2019 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000000..cfdd3d85d5e --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,97 @@ +name: facebook/rocksdb/nightly +on: [push, pull_request] # TEMPORARY: test on PR +jobs: + build-format-compatible: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: test + run: |- + export TEST_TMPDIR=/dev/shm/rocksdb + rm -rf /dev/shm/rocksdb + mkdir /dev/shm/rocksdb + git config --global --add safe.directory /__w/rocksdb/rocksdb + tools/check_format_compatible.sh + - uses: "./.github/actions/post-steps" + build-linux-arm-test-full: + syntax-error: blah + runs-on: + group: arm64large + container: + image: ubuntu-2004:202111-02 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: make V=1 J=4 -j4 check + - uses: "./.github/actions/post-steps" + build-linux-run-microbench: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: DEBUG_LEVEL=0 make -j32 run_microbench + - uses: "./.github/actions/post-steps" + build-linux-non-shm: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + env: + TEST_TMPDIR: "/tmp/rocksdb_test_tmp" + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-clang-13-asan-ubsan-with-folly: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check + - uses: "./.github/actions/post-steps" + build-linux-valgrind: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: PORTABLE=1 make V=1 -j32 valgrind_test + - uses: "./.github/actions/post-steps" + build-windows-vs2022-avx2: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: AVX2 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + build-windows-vs2022: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps"