Skip to content
Build X86

Build X86 #9

Sign in to view logs

Build X86

Build X86 #9

Workflow file for this run

.github/workflows/build_wheels.yml at 50ec0e0
name: Build X86
on:
workflow_dispatch:
inputs:
TAG_NAME:
description: 'Release Version Tag'
required: true
release:
types: [created]
jobs:
build_wheels_linux:
name: ${{ matrix.os }} py${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-20.04 ]
python-version: [ "3.11"]
#python-version: [ "3.8", "3.9", "3.10", "3.11"]
# python-version: [ "3.7" ]
env:
RUNNER_OS: ${{ matrix.os }}
PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- name: Install python
uses: actions/setup-python@v4
with:
python-version: "${{ matrix.python-version }}"
# - name: Install clang++ for Ubuntu
# if: matrix.os == 'ubuntu-20.04'
# run: |
# pwd
# uname -a
# wget https://apt.llvm.org/llvm.sh
# chmod +x llvm.sh
# sudo ./llvm.sh 15
# which clang++-15
# clang++-15 --version
# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget nasm lsb-release software-properties-common gnupg
# ccache -s
# - name: Update git
# run: |
# sudo add-apt-repository ppa:git-core/ppa -y
# sudo apt-get update
# sudo apt-get install -y git
# git --version
# - uses: actions/checkout@v3
# with:
# fetch-depth: 0
# - name: Restore submodules cache
# uses: actions/cache/restore@v3
# id: cache
# with:
# path: |
# contrib/**
# key: |
# submodule-${{ hashFiles('.gitmodules') }}
# - name: Update submodules if cache miss
# if: steps.cache.outputs.cache-hit != 'true'
# run: |
# git submodule update --init --recursive --jobs 4
# - name: Save submodules cache
# if: steps.cache.outputs.cache-hit != 'true'
# uses: actions/cache/save@v3
# with:
# path: |
# contrib/**
# key: |
# submodule-${{ hashFiles('.gitmodules') }}
# - name: ccache
# uses: hendrikmuhs/ccache-action@v1.2
# with:
# key: ${{ matrix.os }}
# max-size: 5G
# append-timestamp: true
# - name: remove old clang and link clang-15 to clang
# if: matrix.os == 'ubuntu-20.04'
# run: |
# sudo rm /usr/bin/clang
# sudo ln -s /usr/bin/clang-15 /usr/bin/clang
# sudo rm /usr/bin/clang++
# sudo ln -s /usr/bin/clang++-15 /usr/bin/clang++
# which clang++
# clang++ --version
# - name: Run chdb/build.sh
# run: |
# python3 -m pip install pybind11
# export CC=/usr/bin/clang
# export CXX=/usr/bin/clang++
# bash ./chdb/build.sh
# python3 -m pip install pandas pyarrow
# bash -x ./chdb/test_smoke.sh
# continue-on-error: false
# - name: Check ccache statistics
# run: |
# ccache -s
# ls -lh chdb
# df -h
# - name: Install dependencies for building wheels
# run: python3 -m pip install -U pip tox pybind11 twine setuptools wheel
# - name: Build wheels
# run: |
# export CC=/usr/bin/clang
# export CXX=/usr/bin/clang++
# make wheel
# - name: Install patchelf from github
# run: |
# wget https://github.com/NixOS/patchelf/releases/download/0.17.2/patchelf-0.17.2-x86_64.tar.gz -O patchelf.tar.gz
# tar -xvf patchelf.tar.gz
# sudo cp bin/patchelf /usr/bin/
# sudo chmod +x /usr/bin/patchelf
# patchelf --version
# - name: Audit wheels
# run: |
# python3 -m pip install auditwheel
# auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl
# continue-on-error: false
# - name: Show files
# run: |
# sudo rm -f dist/*-linux_x86_64.whl
# ls -lh dist
# shell: bash
# - name: Run tests
# run: |
# python3 -m pip install dist/*.whl
# python3 -m pip install pandas pyarrow psutil
# python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)"
# make test
# continue-on-error: false
# - uses: actions/upload-artifact@v3
# with:
# path: ./dist/*.whl
#- name: Upload pypi
#if: startsWith(github.ref, 'refs/tags/v')
#run: |
#python3 -m pip install twine
#python3 -m twine upload dist/*.whl
#env:
#TWINE_USERNAME: __token__
#TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
build_wheels_macos_12:
name: ${{ matrix.os }} py${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ macos-12 ]
# python-version: [ "3.8", "3.9", "3.10", "3.11"]
python-version: [ "3.11" ]
env:
RUNNER_OS: ${{ matrix.os }}
PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- name: Install python
uses: actions/setup-python@v4
with:
python-version: "${{ matrix.python-version }}"
- name: Remove /usr/local/bin/python3
run: |
sudo rm -f /usr/local/bin/python3
- name: Install clang++ for macOS
run: |
pwd
uname -a
export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
brew update
brew install git ccache ninja libtool gettext llvm@15 gcc binutils grep findutils zstd nasm
export PATH=$(brew --prefix llvm@15)/bin:$PATH
which clang++
clang++ --version
ccache -s
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Restore submodules cache
uses: actions/cache/restore@v3
id: cache
with:
path: |
contrib/**
key: |
submodule-${{ hashFiles('.gitmodules') }}
- name: Update submodules if cache miss
if: steps.cache.outputs.cache-hit != 'true'
run: |
git submodule update --init --recursive --jobs 4
- name: Save submodules cache
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v3
with:
path: |
contrib/**
key: |
submodule-${{ hashFiles('.gitmodules') }}
- name: ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: ${{ matrix.os }}
max-size: 5G
append-timestamp: true
- name: Run chdb/build.sh
timeout-minutes: 1000
run: |
python3 -m pip install pybind11
export PATH=$(brew --prefix llvm@15)/bin:$(brew --prefix)/opt/grep/libexec/gnubin:$(brew --prefix)/opt/binutils/bin:$PATH:$(brew --prefix)/opt/findutils/libexec/gnubin
export CC=$(brew --prefix llvm@15)/bin/clang
export CXX=$(brew --prefix llvm@15)/bin/clang++
bash gen_manifest.sh
bash ./chdb/build.sh
python3 -m pip install pandas pyarrow
bash -x ./chdb/test_smoke.sh
continue-on-error: false
- name: Keep killall ccache and wait for ccache to finish
if: always()
run: |
sleep 60
while ps -ef | grep ccache | grep -v grep; do \
killall ccache; \
sleep 10; \
done
- name: Check ccache statistics
run: |
ccache -s
ls -lh chdb
df -h
env:
CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@15)/bin/clang CXX=$(brew --prefix llvm@15)/bin/clang++"
- name: Install dependencies for building wheels
run: |
python3 -m pip install -U pip tox pybind11 twine setuptools wheel>=0.40.0
- name: Build wheels
run: |
export PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
export CC=$(brew --prefix llvm@15)/bin/clang
export CXX=$(brew --prefix llvm@15)/bin/clang++
make wheel
- name: Fix wheel platform tag
run: |
python3 -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl
- name: Run tests
run: |
python3 -m pip install dist/*.whl
python3 -m pip install pandas pyarrow psutil
python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)"
make test
continue-on-error: false
- name: Show files
run: ls -lh dist
shell: bash
- uses: actions/upload-artifact@v3
with:
path: ./dist/*.whl
#- name: Upload pypi
#if: startsWith(github.ref, 'refs/tags/v')
#run: |
#python3 -m pip install twine
#python3 -m twine upload dist/*.whl
#env:
#TWINE_USERNAME: __token__
#TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
build_wheels_macos:
name: cbw on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ macos-11 ]
# python-version: [ "3.8", "3.9", "3.10"]
python-version: [ "3.11" ]
env:
RUNNER_OS: ${{ matrix.os }}
PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- name: Install clang++ for macOS
if: matrix.os == 'macos-11' || matrix.os == 'macos-12'
run: |
pwd
uname -a
export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
brew update
brew install git ccache ninja libtool gettext llvm@15 gcc binutils grep findutils zstd nasm
export PATH=$(brew --prefix llvm@15)/bin:$PATH
which clang++
clang++ --version
ccache -s
# - uses: actions/checkout@v3
# with:
# fetch-depth: 0
# - name: Restore submodules cache
# uses: actions/cache/restore@v3
# id: cache
# with:
# path: |
# contrib/**
# key: |
# submodule-${{ hashFiles('.gitmodules') }}
# - name: Update submodules if cache miss
# if: steps.cache.outputs.cache-hit != 'true'
# run: |
# git submodule update --init --recursive --jobs 4
# - name: ccache
# uses: hendrikmuhs/ccache-action@v1.2
# with:
# key: ${{ matrix.os }}
# max-size: 5G
# append-timestamp: true
# - name: Prepare chdb/build.sh
# run: |
# python3 -m pip install pybind11
# export PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
# export CC=$(brew --prefix llvm@15)/bin/clang
# export CXX=$(brew --prefix llvm@15)/bin/clang++
# continue-on-error: false
# - name: Check ccache statistics
# run: |
# ccache -s
# ls -lh chdb
# df -h
# env:
# CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@15)/bin/clang CXX=$(brew --prefix llvm@15)/bin/clang++"
# - name: Remove /usr/local/bin/python3
# run: |
# sudo rm -f /usr/local/bin/python3
# - name: Install dependencies for building wheels
# run: |
# python3 -m pip install -U pip tox pybind11 twine setuptools wheel
# python3 -m pip install cibuildwheel==2.12.1
# - name: Build wheels
# run: python3 -m cibuildwheel --output-dir wheelhouse
# timeout-minutes: 1000
# env:
# CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@15)/bin/clang CXX=$(brew --prefix llvm@15)/bin/clang++"
# CIBW_DEBUG: 1
# CIBW_BEFORE_BUILD: "pip install -U pip tox pybind11 && bash -x gen_manifest.sh && bash chdb/build.sh"
# CIBW_BUILD_VERBOSITY: 3
# CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64"
# CIBW_TEST_REQUIRES: "pyarrow pandas psutil"
# CIBW_TEST_COMMAND: "cd {project} && make test"
# - name: Keep killall ccache and wait for ccache to finish
# if: always()
# run: |
# sleep 60
# while ps -ef | grep ccache | grep -v grep; do \
# killall ccache; \
# sleep 10; \
# done
# - name: Show files
# run: ls -lh wheelhouse
# shell: bash
# - uses: actions/upload-artifact@v3
# with:
# path: ./wheelhouse/*.whl
#- name: Upload pypi
#if: startsWith(github.ref, 'refs/tags/v')
#run: |
#python3 -m pip install twine
#python3 -m twine upload wheelhouse/*.whl
#env:
#TWINE_USERNAME: __token__
#TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
# #build for macos12 arm64(Apple Silicon)
# build_wheels_macos_arm64:
## Experimental Restart on Expire/Cancel (WIP)
## Requires: PAT Token
## the PAT token should have permissions according to this
## documentation - https://developer.github.com/v3/actions/workflow-runs/
restart_failed:
name: Retry Expired Builds
runs-on: ubuntu-latest
if: github.event_name == 'push' && cancelled()
needs: [ build_wheels_linux, build_wheels_macos, build_wheels_macos_12 ]
steps:
- name: Sleep for 20 seconds
run: sleep 20s
shell: bash
- name: Retry Failed/Expired Build
run: |
curl -i \
-X POST \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: token ${{ secrets.PAT_TOKEN }}" \
https://api.github.com/repos/chdb-io/chdb/actions/runs/${{ github.run_id }}/rerun \
-d '{"ref": "${{ github.ref }}" }'