Skip to content
TileDB-SOMA Python CI (Packaging)

Update scene component names (again) #23

Sign in to view logs

Update scene component names (again)

Update scene component names (again) #23

Workflow file for this run

.github/workflows/python-ci-packaging.yml at 71d04da
# This CI confirms the Python package continues to install properly in the many
# different required contexts
name: TileDB-SOMA Python CI (Packaging)
on:
push:
paths:
- '.github/workflows/python-ci-packaging.yml'
- 'apis/python/MANIFEST.in'
- 'apis/python/pyproject.toml'
- 'apis/python/setup.py'
- 'apis/python/src/tiledbsoma/__init__.py'
- 'libtiledbsoma/cmake/inputs/Config.cmake.in'
- 'libtiledbsoma/cmake/inputs/tiledbsoma.pc.in'
- 'libtiledbsoma/cmake/Modules/FindTileDB_EP.cmake'
- 'libtiledbsoma/cmake/Modules/TileDBCommon.cmake'
- 'libtiledbsoma/cmake/Superbuild.cmake'
- 'libtiledbsoma/CMakeLists.txt'
pull_request:
paths:
- '.github/workflows/python-ci-packaging.yml'
- 'apis/python/MANIFEST.in'
- 'apis/python/pyproject.toml'
- 'apis/python/setup.py'
- 'apis/python/src/tiledbsoma/__init__.py'
- 'libtiledbsoma/cmake/inputs/Config.cmake.in'
- 'libtiledbsoma/cmake/inputs/tiledbsoma.pc.in'
- 'libtiledbsoma/cmake/Modules/FindTileDB_EP.cmake'
- 'libtiledbsoma/cmake/Modules/TileDBCommon.cmake'
- 'libtiledbsoma/cmake/Superbuild.cmake'
- 'libtiledbsoma/CMakeLists.txt'
workflow_dispatch:
jobs:
# Confirm shared object copying when building the Python package
# https://github.com/single-cell-data/TileDB-SOMA/pull/1937
docker:
runs-on: ubuntu-latest
name: "SO copying (docker) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
strategy:
fail-fast: false
matrix:
TILEDB_EXISTS: ["no", "yes"]
TILEDBSOMA_EXISTS: ["no", "yes"]
exclude:
- TILEDB_EXISTS: "no"
TILEDBSOMA_EXISTS: "yes"
container:
image: ubuntu:22.04
defaults:
run:
shell: bash
steps:
- name: Docker image info
run: |
uname -a
cat /etc/lsb-release
- name: Setup
run: |
apt-get update
apt-get install --yes cmake git python-is-python3 python3 python3-pip python3-venv unzip wget
- uses: actions/checkout@v4
with:
fetch-depth: 0 # for setuptools-scm
- name: Configure Git
run: |
# This is a permissions quirk due to running Git as root inside of a Docker container
git config --global --add safe.directory $(pwd)
git branch
- name: Install pre-built libtiledb
if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
run: |
mkdir -p external
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-linux-x86_64-2.23.0-152093b.tar.gz
tar -C external -xzf tiledb-linux-x86_64-*.tar.gz
ls external/lib/
echo "LD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
echo "PKG_CONFIG_PATH=$(pwd)/external/lib/pkgconfig" >> $GITHUB_ENV
echo "TILEDB_PATH=$(pwd)/external" >> $GITHUB_ENV
- name: Build and install libtiledbsoma
if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }}
run: |
cmake -S libtiledbsoma -B build-libtiledbsoma \
-D CMAKE_BUILD_TYPE=Release \
-D CMAKE_PREFIX_PATH=$(pwd)/external/ \
-D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \
-D OVERRIDE_INSTALL_PREFIX=OFF \
-D DOWNLOAD_TILEDB_PREBUILT=OFF \
-D FORCE_BUILD_TILEDB=OFF
cmake --build build-libtiledbsoma -j $(nproc)
cmake --build build-libtiledbsoma --target install-libtiledbsoma
ls external/lib/
echo "TILEDBSOMA_PATH=$(pwd)/external" >> $GITHUB_ENV
- name: Setup Python
run: |
python --version
python -m venv ./venv-soma
./venv-soma/bin/python -m pip install --prefer-binary pybind11-global typeguard sparse wheel
./venv-soma/bin/python -m pip list
- name: Build wheel
run: |
echo env vars: $LD_LIBRARY_PATH $PKG_CONFIG_PATH $TILEDB_PATH $TILEDBSOMA_PATH
cd apis/python
../../venv-soma/bin/python setup.py bdist_wheel
- name: Inspect wheel
run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep '\.so'
- name: Confirm libtiledb.so is copied
if: ${{ matrix.TILEDB_EXISTS == 'no' }}
run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledb.so
- name: Confirm libtiledb.so is **not** copied when using external shared object
if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
run: |
if unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledb.so
then
echo "libtiledb.so was copied into the wheel when it was built against an external shared object"
exit 1
fi
- name: Confirm libtiledbsoma.so is copied
if: ${{ matrix.TILEDBSOMA_EXISTS == 'no' }}
run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledbsoma.so
- name: Confirm libtiledbsoma.so is **not** copied when using external shared object
if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }}
run: |
if unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledbsoma.so
then
echo "libtiledbsoma.so was copied into the wheel when it was built against an external shared object"
exit 1
fi
- name: Install wheel
run: ./venv-soma/bin/python -m pip install --prefer-binary apis/python/dist/tiledbsoma-*.whl
- name: Check linking and RPATH
run: |
ldd ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so
readelf -d ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | grep R*PATH
- name: Runtime test
run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
- name: Confirm linking to installed shared objects
run: |
rm -fr build/ build-libtiledbsoma/ dist/ apis/python/build apis/python/src/tiledbsoma/*tile*.so*
# should only show shared objects installed in virtual env or in ./external/
find . -name '*tile*.so*'
./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
# Confirm shared object copying when building the Python package
# https://github.com/single-cell-data/TileDB-SOMA/pull/1937
# Same as job above, but running on macOS instead of in a Docker container
macos:
runs-on: macos-13
name: "SO copying (macos) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
strategy:
fail-fast: false
matrix:
TILEDB_EXISTS: ["no", "yes"]
TILEDBSOMA_EXISTS: ["no", "yes"]
exclude:
- TILEDB_EXISTS: "no"
TILEDBSOMA_EXISTS: "yes"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # for setuptools-scm
- name: Check if System Integrity Protection (SIP) is enabled
run: csrutil status
- name: Install pre-built libtiledb
if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
run: |
mkdir -p external
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-macos-x86_64-2.23.0-152093b.tar.gz
tar -C external -xzf tiledb-macos-x86_64-*.tar.gz
ls external/lib/
echo "DYLD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
echo "PKG_CONFIG_PATH=$(pwd)/external/lib/pkgconfig" >> $GITHUB_ENV
echo "TILEDB_PATH=$(pwd)/external" >> $GITHUB_ENV
- name: Build and install libtiledbsoma
if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }}
run: |
cmake -S libtiledbsoma -B build-libtiledbsoma \
-D CMAKE_BUILD_TYPE=Release \
-D CMAKE_PREFIX_PATH=$(pwd)/external/ \
-D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \
-D OVERRIDE_INSTALL_PREFIX=OFF \
-D DOWNLOAD_TILEDB_PREBUILT=OFF \
-D FORCE_BUILD_TILEDB=OFF
cmake --build build-libtiledbsoma -j $(nproc)
cmake --build build-libtiledbsoma --target install-libtiledbsoma
ls external/lib/
echo "TILEDBSOMA_PATH=$(pwd)/external" >> $GITHUB_ENV
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Setup Python virtual env
run: |
python --version
python -m venv ./venv-soma
./venv-soma/bin/python -m pip install --prefer-binary pybind11-global typeguard sparse wheel setuptools
./venv-soma/bin/python -m pip list
- name: Build wheel
run: |
echo env vars: $DYLD_LIBRARY_PATH $PKG_CONFIG_PATH $TILEDB_PATH $TILEDBSOMA_PATH
cd apis/python
../../venv-soma/bin/python setup.py bdist_wheel
- name: Inspect wheel
run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -e '\.dylib' -e '\.so'
- name: Confirm libtiledb.dylib is copied
if: ${{ matrix.TILEDB_EXISTS == 'no' }}
run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledb.dylib
- name: Confirm libtiledb.dylib is **not** copied when using external shared object
if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
run: |
if unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledb.dylib
then
echo "libtiledb.dylib was copied into the wheel when it was built against an external shared object"
exit 1
fi
- name: Confirm libtiledbsoma.dylib is copied
if: ${{ matrix.TILEDBSOMA_EXISTS == 'no' }}
run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledbsoma.dylib
- name: Confirm libtiledbsoma.dylib is **not** copied when using external shared object
if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }}
run: |
if unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledbsoma.dylib
then
echo "libtiledbsoma.dylib was copied into the wheel when it was built against an external shared object"
exit 1
fi
- name: Install wheel
run: ./venv-soma/bin/python -m pip install --prefer-binary apis/python/dist/tiledbsoma-*.whl
- name: Check linking and RPATH
run: otool -L ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so
- name: Runtime test
run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
- name: Confirm linking to installed shared objects
run: |
rm -fr build/ build-libtiledbsoma/ dist/ apis/python/build apis/python/src/tiledbsoma/*tile*.dylib*
# should only show shared objects installed in virtual env or in ./external/
find . -name '*tile*.so*'
find . -name '*tile*.dylib*'
./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
# Tests that the --libtiledbsoma flag to setup.py continues working
setuptools:
runs-on: ${{ matrix.os }}
name: "setuptools (${{ matrix.os }})"
strategy:
fail-fast: false
matrix:
os: ["ubuntu-22.04", "macos-12"]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # for setuptools-scm
- name: Install pre-built libtiledb
run: |
mkdir -p external
if [ `uname -s` == "Darwin" ];
then
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-macos-x86_64-2.23.0-152093b.tar.gz
else
# Please do not edit manually -- let scripts/update-tiledb-version.py update this
wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-linux-x86_64-2.23.0-152093b.tar.gz
fi
tar -C external -xzf tiledb-*.tar.gz
ls external/lib/
- name: Build and install libtiledbsoma
run: |
cmake -S libtiledbsoma -B build-libtiledbsoma \
-D CMAKE_BUILD_TYPE=Release \
-D CMAKE_PREFIX_PATH=$(pwd)/external/ \
-D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \
-D OVERRIDE_INSTALL_PREFIX=OFF \
-D DOWNLOAD_TILEDB_PREBUILT=OFF \
-D FORCE_BUILD_TILEDB=OFF
cmake --build build-libtiledbsoma -j 2
cmake --build build-libtiledbsoma --target install-libtiledbsoma
ls external/lib/
# Delete all cmake executables from the runner. This will ensure that
# tiledbsoma-py has to use the cli flags to find the external
# libtiledbsoma.so and not build it from source by shelling out to cmake
- name: Delete cmake
run: |
echo before
which -a cmake
which -a cmake | xargs sudo rm -f
echo after
which -a cmake || echo cmake removed
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Setup Python virtual env
run: |
python --version
python -m venv ./venv-soma
./venv-soma/bin/python -m pip install --prefer-binary pybind11-global typeguard sparse wheel setuptools
./venv-soma/bin/python -m pip list
- name: Install TileDB-SOMA-Py with setuptools and --libtiledbsoma
run: |
cd apis/python
../../venv-soma/bin/python setup.py install \
--single-version-externally-managed \
--record record.txt \
--tiledb=$GITHUB_WORKSPACE/external/ \
--libtiledbsoma=$GITHUB_WORKSPACE/external/
- name: Check linking and RPATH (Linux)
if: runner.os == 'Linux'
run: |
ldd ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so
readelf -d ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | grep R*PATH
- name: Check linking and RPATH (macOS)
if: runner.os == 'macOS'
run: |
otool -L ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so
otool -l ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so
- name: Install runtime dependencies
run: |
grep -v '^\[' apis/python/src/tiledbsoma.egg-info/requires.txt >/tmp/filtered-requirements.txt
./venv-soma/bin/pip install --prefer-binary -r /tmp/filtered-requirements.txt
- name: Runtime test
run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
# Build a wheel from a source tarball (confirms all required files are
# distributed in the sdist). This is in preparation for the official building of
# the PyPI wheels in python-packaging.yml.
# https://github.com/single-cell-data/TileDB-SOMA/pull/2506
#
#
# Differences to python-packaging.yml:
#
# * Uses transparent python commands instead of the opaque action
# `pypa/cibuildwheel`, and therefore easier to debug
# * Only builds two wheels total (python 3.11 on Ubuntu and macOS), so it is
# much faster. It's not creating the wheels for distribution, but instead just
# providing a quick check that all the required files are distributed in the
# source tarball
# * Runs whenever an installation-related file is modified instead of only
# after a release
#
# In summary, the goal is to identify any potential problems with building
# the PyPI wheels when a PR is submitted, and not at release time.
sdist:
runs-on: ${{ matrix.os }}
name: "Wheel from sdist (${{ matrix.os }})"
strategy:
fail-fast: false
matrix:
os: ["ubuntu-22.04", "macos-12"]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # for setuptools-scm
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install dependencies
run: python -m pip install --prefer-binary pybind11 wheel
- name: Build source tarball (sdist)
run: |
cd apis/python
python setup.py sdist
- name: Extract source tarball
run: |
tar --list -f apis/python/dist/tiledbsoma-*.tar.gz
tar -xzf apis/python/dist/tiledbsoma-*.tar.gz
- name: Build wheel
run: |
cd tiledbsoma-*/
python setup.py bdist_wheel
- name: Install wheel
run: |
python -m pip install --prefer-binary tiledbsoma-*/dist/tiledbsoma-*.whl
python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"