[Tester] Add --require <name>
#10878
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LinuxRelease | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| push: | |
| branches: | |
| - '**' | |
| - '!main' | |
| - '!feature' | |
| tags: | |
| - '**' | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/odbc/**' | |
| - '!tools/shell/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/LinuxRelease.yml' | |
| pull_request: | |
| types: [opened, reopened, ready_for_review] | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/odbc/**' | |
| - '!tools/shell/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/LinuxRelease.yml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| jobs: | |
| linux-release-64: | |
| # Builds binaries for linux_amd64_gcc4 | |
| name: Linux (x64) | |
| runs-on: ubuntu-latest | |
| container: quay.io/pypa/manylinux2014_x86_64 | |
| env: | |
| EXTENSION_CONFIGS: '${GITHUB_WORKSPACE}/.github/config/bundled_extensions.cmake' | |
| ENABLE_EXTENSION_AUTOLOADING: 1 | |
| ENABLE_EXTENSION_AUTOINSTALL: 1 | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| BUILD_ODBC: 1 | |
| DEBUG_STACKTRACE: 1 | |
| FORCE_WARN_UNUSED: 1 | |
| DUCKDB_RUN_PARALLEL_CSV_TESTS: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/manylinux_2014_setup | |
| with: | |
| ninja-build: 1 | |
| ccache: 1 | |
| odbc: 1 | |
| python_alias: 1 | |
| - name: Install pytest | |
| run: | | |
| python3 -m pip install pytest | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Print platform | |
| shell: bash | |
| run: ./build/release/duckdb -c "PRAGMA platform;" | |
| - name: Test | |
| shell: bash | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| run: | | |
| make | |
| python3 scripts/run_tests_one_by_one.py build/release/test/unittest "*" --time_execution --order lex --list | |
| - name: Tools Tests | |
| shell: bash | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| run: | | |
| python3 -m pytest tools/shell/tests --shell-binary build/release/duckdb | |
| - name: Examples | |
| shell: bash | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| run: | | |
| (cd examples/embedded-c; make) | |
| (cd examples/embedded-c++; make) | |
| build/release/benchmark/benchmark_runner benchmark/tpch/sf1/q01.benchmark | |
| build/release/duckdb -c "COPY (SELECT 42) TO '/dev/stdout' (FORMAT PARQUET)" | cat | |
| - name: Deploy | |
| shell: bash | |
| run: | | |
| python3 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-amd64.zip build/release/duckdb | |
| zip -j libduckdb-linux-amd64.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| zip -j libduckdb-src.zip src/amalgamation/duckdb.hpp src/amalgamation/duckdb.cpp src/include/duckdb.h | |
| zip -j duckdb_odbc-linux-amd64.zip build/release/tools/odbc/libduckdb_odbc.so tools/odbc/linux_setup/unixodbc_setup.sh | |
| python3 scripts/asset-upload-gha.py libduckdb-src.zip libduckdb-linux-amd64.zip duckdb_cli-linux-amd64.zip duckdb_odbc-linux-amd64.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux | |
| path: | | |
| libduckdb-linux-amd64.zip | |
| duckdb_cli-linux-amd64.zip | |
| duckdb_odbc-linux-amd64.zip | |
| linux-release-aarch64: | |
| # Builds binaries for linux_arm64 | |
| name: Linux (aarch64) | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' | |
| runs-on: ubuntu-latest | |
| needs: linux-release-64 | |
| container: ubuntu:18.04 | |
| env: | |
| EXTENSION_CONFIGS: '${GITHUB_WORKSPACE}/.github/config/bundled_extensions.cmake' | |
| ENABLE_EXTENSION_AUTOLOADING: 1 | |
| ENABLE_EXTENSION_AUTOINSTALL: 1 | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| TREAT_WARNINGS_AS_ERRORS: 1 | |
| FORCE_WARN_UNUSED: 1 | |
| BUILD_ODBC: 1 | |
| ODBC_CONFIG: ../../build/unixodbc/build/bin/odbc_config | |
| DUCKDB_PLATFORM: linux_arm64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| ccache: 1 | |
| aarch64_cross_compile: 1 | |
| - name: Install unixODBC | |
| shell: bash | |
| run: | # we need an x86 odbc_config tool to run cmake. fun. | |
| apt-get remove -y unixodbc unixodbc-dev | |
| CC=gcc ./scripts/install_unixodbc.sh | |
| cp build/unixodbc/build/bin/odbc_config . | |
| CC=aarch64-linux-gnu-gcc ./scripts/install_unixodbc.sh --host aarch64-linux-gnu | |
| cp odbc_config build/unixodbc/build/bin/odbc_config | |
| - name: Build | |
| shell: bash | |
| run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ make | |
| - name: Deploy | |
| shell: bash | |
| run: | | |
| python3 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-aarch64.zip build/release/duckdb | |
| zip -j duckdb_odbc-linux-aarch64.zip build/release/tools/odbc/libduckdb_odbc.so | |
| zip -j libduckdb-linux-aarch64.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| python3 scripts/asset-upload-gha.py libduckdb-linux-aarch64.zip duckdb_cli-linux-aarch64.zip duckdb_odbc-linux-aarch64.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux-aarch64 | |
| path: | | |
| libduckdb-linux-aarch64.zip | |
| duckdb_odbc-linux-aarch64.zip | |
| duckdb_cli-linux-aarch64.zip | |
| # Linux extensions for builds that use C++11 ABI, currently these are all linux builds based on ubuntu >= 18 (e.g. NodeJS) | |
| # note that the linux-release-64 is based on the manylinux-based extensions, which are built in .github/workflows/Python.yml | |
| linux-extensions-64: | |
| # Builds extensions for linux_amd64 | |
| name: Linux Extensions (x64) | |
| runs-on: ubuntu-latest | |
| container: ubuntu:18.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| vcpkg: 1 | |
| openssl: 1 | |
| ccache: 1 | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| vcpkg_target_triplet: x64-linux | |
| post_install: rm build/release/src/libduckdb* | |
| deploy_as: linux_amd64 | |
| treat_warn_as_error: 0 | |
| run_autoload_tests: ${{ startsWith(github.ref, 'refs/tags/v') && 0 || 1 }} | |
| run_tests: ${{ startsWith(github.ref, 'refs/tags/v') && 0 || 1 }} | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| ninja: 1 | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: linux-extensions-64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| linux-extensions-64-aarch64: | |
| # Builds extensions for linux_arm64 | |
| name: Linux Extensions (aarch64) | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| container: ubuntu:18.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| vcpkg: 1 | |
| openssl: 1 | |
| aarch64_cross_compile: 1 | |
| ccache: 1 | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| vcpkg_target_triplet: arm64-linux | |
| deploy_as: linux_arm64 | |
| duckdb_arch: linux_arm64 | |
| treat_warn_as_error: 0 | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| aarch64_cross_compile: 1 | |
| run_tests: 0 # Cannot run tests here due to cross-compiling | |
| run_autoload_tests: 0 | |
| ninja: 1 | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: linux-extensions-64-aarch64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| check-load-install-extensions: | |
| name: Checks extension entries | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| runs-on: ubuntu-20.04 | |
| needs: linux-extensions-64 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| env: | |
| DISABLE_BUILTIN_EXTENSIONS: 1 | |
| run: | | |
| make | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: linux-extensions-64 | |
| path: /tmp | |
| - name: Check if extension_entries.hpp is up to date | |
| shell: bash | |
| env: | |
| EXTENSION_CONFIGS: '.github/config/in_tree_extensions.cmake;.github/config/out_of_tree_extensions.cmake' | |
| run: | | |
| make extension_configuration | |
| python scripts/generate_extensions_function.py --validate | |
| symbol-leakage: | |
| name: Symbol Leakage | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Symbol Leakage Test | |
| shell: bash | |
| run: python3 scripts/exported_symbols_check.py build/release/src/libduckdb*.so | |
| linux-httpfs: | |
| name: Linux HTTPFS | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| env: | |
| BUILD_HTTPFS: 1 | |
| BUILD_TPCH: 1 | |
| BUILD_TPCDS: 1 | |
| BUILD_PARQUET: 1 | |
| BUILD_JSON: 1 | |
| S3_TEST_SERVER_AVAILABLE: 1 | |
| AWS_DEFAULT_REGION: eu-west-1 | |
| AWS_ACCESS_KEY_ID: minio_duckdb_user | |
| AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password | |
| DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000 | |
| DUCKDB_S3_USE_SSL: false | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install Ninja | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Start S3/HTTP test server | |
| shell: bash | |
| run: | | |
| sudo ./scripts/install_s3_test_server.sh | |
| ./scripts/generate_presigned_url.sh | |
| source ./scripts/run_s3_test_server.sh | |
| sleep 60 | |
| - name: Test | |
| shell: bash | |
| run: | | |
| python3 scripts/get_test_list.py --file-contains 'require httpfs' --list '"*"' > test.list | |
| python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '-f test.list' | |
| python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '[secret]' | |
| amalgamation-tests: | |
| name: Amalgamation Tests | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| needs: linux-release-64 | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CC: clang | |
| CXX: clang++ | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install LLVM and Clang | |
| uses: KyleMayes/install-llvm-action@v1 | |
| with: | |
| version: "10.0" | |
| - name: Generate Amalgamation | |
| shell: bash | |
| run: | | |
| python scripts/amalgamation.py --extended | |
| clang++ -std=c++17 -Isrc/amalgamation src/amalgamation/duckdb.cpp -emit-llvm -S -O0 |