From a19f32b22fc4f762eaf65a889d3ef97d0bfda258 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Mon, 30 Jan 2023 18:21:37 +0100 Subject: [PATCH 01/28] - Upgrade numpy, pyo3, maturin - WIP wheel release GH workflows --- .github/workflows/release.yml | 93 ++++++++++++++++++++++++++++++++++- Cargo.lock | 39 +++++++++------ rerun_py/Cargo.toml | 6 +-- rerun_py/README.md | 6 +-- rerun_py/pyproject.toml | 7 ++- rerun_py/src/python_bridge.rs | 2 + 6 files changed, 129 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9641de98b70e..76ff61e549cd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,20 @@ on: tags: - "*.*.*" # on release tag + workflow_dispatch: + inputs: + pyindex: + description: "PyPI Index" + required: true + default: "testpypi" + type: choice + options: + - testpypi + - pypi + +env: + PYTHON_VERSION: "3.8" + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -22,7 +36,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: ${{ env.PYTHON_VERSION }} cache: "pip" cache-dependency-path: "rerun_py/requirements-doc.txt" @@ -42,3 +56,80 @@ jobs: - name: Deploy via mike # https://github.com/jimporter/mike run: | mike deploy -F rerun_py/mkdocs.yml -p --rebase -b gh-pages -u ${{github.ref_name}} latest + + py-wheel-macos-aarch64: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + default: true + + - name: Set up Rust targets + run: rustup target add aarch64-apple-darwin + + - name: Publish wheel + uses: PyO3/maturin-action@v1 + env: + #MATURIN_PASSWORD: ${{ secrets.PYPI_PASS }} + with: + maturin-version: "0.14.10" + command: build + args: | + --manifest-path rerun_py/Cargo.toml + --release + --strip + --target aarch64-apple-darwin + #--skip-existing + #--no-sdist + #--repository ${{ inputs.pyindex }} + #--username john.rerun.io + + - name: Upload Artifact + uses: actions/upload-artifact@v3 + with: + name: rerun-sdk-wheel + path: target/wheels/*.whl + + py-wheel-manylinux-x64_64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" + + - name: Publish wheel + uses: PyO3/maturin-action@v1 + env: + #MATURIN_PASSWORD: ${{ secrets.PYPI_PASS }} + RUSTFLAGS: -C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma + with: + maturin-version: '0.14.10' + command: build + args: | + --manifest-path rerun_py/Cargo.toml + --release + --strip + #--skip-existing + #--repository ${{ inputs.pyindex }} + #--username john.rerun.io + + - name: Upload Artifact + uses: actions/upload-artifact@v3 + with: + name: rerun-sdk-wheel + path: target/wheels/*.whl diff --git a/Cargo.lock b/Cargo.lock index f2e2ac222526..aebce43f4a00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2766,6 +2766,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "memory-stats" version = "1.0.0" @@ -3168,18 +3177,18 @@ dependencies = [ [[package]] name = "numpy" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a462c1af5ba1fddec1488c4646993a23ae7931f9e170ccba23e9c7c834277797" +checksum = "96b0fee4571867d318651c24f4a570c3f18408cf95f16ccb576b3ce85496a46e" dependencies = [ - "ahash 0.7.6", - "half 1.8.2", + "half 2.1.0", "libc", "ndarray", "num-complex", "num-integer", "num-traits", "pyo3", + "rustc-hash", ] [[package]] @@ -3603,14 +3612,14 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" +checksum = "ccd4149c8c3975099622b4e1962dac27565cf5663b76452c3e2b66e0b6824277" dependencies = [ "cfg-if", "indoc", "libc", - "memoffset 0.6.5", + "memoffset 0.8.0", "parking_lot", "pyo3-build-config", "pyo3-ffi", @@ -3620,9 +3629,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" +checksum = "9cd09fe469834db21ee60e0051030339e5d361293d8cb5ec02facf7fdcf52dbf" dependencies = [ "once_cell", "target-lexicon", @@ -3630,9 +3639,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" +checksum = "0c427c9a96b9c5b12156dbc11f76b14f49e9aae8905ca783ea87c249044ef137" dependencies = [ "libc", "pyo3-build-config", @@ -3640,9 +3649,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" +checksum = "16b822bbba9d60630a44d2109bc410489bb2f439b33e3a14ddeb8a40b378a7c4" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -3652,9 +3661,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" +checksum = "84ae898104f7c99db06231160770f3e40dad6eb9021daddc0fedfa3e41dff10a" dependencies = [ "proc-macro2", "quote", diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 328b060a3bde..7370bd8202f1 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -54,14 +54,14 @@ image = { version = "0.24", default-features = false, features = [ itertools = "0.10" macaw = "0.17" mimalloc = { version = "0.1.29", features = ["local_dynamic_tls"] } -numpy = { version = "0.17", features = ["half"] } -pyo3 = "0.17" +numpy = { version = "0.18.0", features = ["half"] } +pyo3 = { version = "0.18.0", features = ["abi3-py38"] } rand = { version = "0.8", features = ["std_rng"] } tokio = { workspace = true, features = ["rt-multi-thread"] } uuid = "1.1" [build-dependencies] -pyo3-build-config = "0.17" +pyo3-build-config = "0.18.0" [package.metadata.maturin] name = "rerun.rerun_bindings" diff --git a/rerun_py/README.md b/rerun_py/README.md index 96233e02f80f..c5d602d4494e 100644 --- a/rerun_py/README.md +++ b/rerun_py/README.md @@ -3,10 +3,10 @@ Goal: an ergonomic Python library for logging rich data, over TCP, to a rerun server. ℹ️ Note: -- The rust crate is called `re_sdk_python`, while the Python library is called `rerun`. -- These instructions assume you're running from the `rerun` root folder and have Python 3.7 or later available. +- The rust crate is called `re_sdk_python`, the Python module is called `rerun`, and the package published on PyPI is `rerun-sdk`. +- These instructions assume you're running from the `rerun` root folder and have Python 3.8 or later available. -## Simply build and install +## Building from Source To build from source and install the `rerun` into your *current* Python environment run: ```sh diff --git a/rerun_py/pyproject.toml b/rerun_py/pyproject.toml index 516294b19807..6cc93d2c70c1 100644 --- a/rerun_py/pyproject.toml +++ b/rerun_py/pyproject.toml @@ -2,15 +2,18 @@ [build-system] build-backend = "maturin" -requires = ["maturin>=0.13.5,<0.14"] +requires = ["maturin>=0.14.0,<0.15"] [project] -name = "rerun" +name = "rerun-sdk" authors = [{ name = "Rerun.io", email = "opensource@rerun.io" }] classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", + "Development Status :: 3 - Alpha", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Visualization", ] description = "The Rerun Logging SDK" keywords = ["computer-vision", "logging", "rerun"] diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index 458dc5c08715..ce0f8c53ea1f 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -472,6 +472,7 @@ fn log_transform( // ---------------------------------------------------------------------------- #[pyfunction] +#[pyo3(signature = (obj_path, xyz, right_handed, timeless))] fn log_view_coordinates_xyz( obj_path: &str, xyz: &str, @@ -642,6 +643,7 @@ fn log_mesh_file( /// /// If no `img_format` is specified, we will try and guess it. #[pyfunction] +#[pyo3(signature = (obj_path, img_path, img_format = "None", timeless = false))] fn log_image_file( obj_path: &str, img_path: PathBuf, From 03dd9c20f2362bb2ec543d102da70b1b6a3547d5 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Mon, 30 Jan 2023 18:25:09 +0100 Subject: [PATCH 02/28] trigger on config changes --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 76ff61e549cd..d0c00ac85b1b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,11 @@ on: push: tags: - "*.*.*" # on release tag + pull_request: + branches: + - "main" + paths: + - ".github/workflows/release.yaml" workflow_dispatch: inputs: From a29c1da8292e4879b9ae7992dfedba54c37f6fc5 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Mon, 30 Jan 2023 18:27:06 +0100 Subject: [PATCH 03/28] fix --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d0c00ac85b1b..ab8d30b35c2f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ on: branches: - "main" paths: - - ".github/workflows/release.yaml" + - ".github/workflows/release.yml" workflow_dispatch: inputs: From bb877ebcc600396643690992a8f98e8d1f80765c Mon Sep 17 00:00:00 2001 From: John Hughes Date: Mon, 30 Jan 2023 18:44:09 +0100 Subject: [PATCH 04/28] WIP --- .github/workflows/python.yml | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index c7cdc5994d32..0984fb01ea4f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -10,6 +10,9 @@ on: branches: - "main" +env: + PYTHON_VERSION: "3.8" + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -30,7 +33,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: ${{ env.PYTHON_VERSION }} cache: "pip" cache-dependency-path: "rerun_py/requirements-lint.txt" @@ -119,10 +122,10 @@ jobs: - name: Install rerun_py run: pip install ./rerun_py - py-pip-install-macos: - name: Pip install Rerun SDK (macOS) + py-wheel-macos-aarch64: + name: Build Python Wheel for MacOS/aarch64 runs-on: macos-latest - if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes + #if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes steps: - uses: actions/checkout@v3 @@ -135,11 +138,26 @@ jobs: - name: Setup python for building wheel uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: ${{ env.PYTHON_VERSION }} cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" - - name: Install rerun_py - run: pip install ./rerun_py + - name: Build wheel + uses: PyO3/maturin-action@v1 + with: + maturin-version: "0.14.10" + command: build + args: | + --manifest-path rerun_py/Cargo.toml + --release + --strip + --target aarch64-apple-darwin + + - name: Upload Artifact + uses: actions/upload-artifact@v3 + with: + name: rerun-sdk-wheel + path: target/wheels/*.whl py-docs: name: Build and deploy docs From 4c3e31a68f76ebb8e3cb7b645f4e391ad5e57632 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 12:06:38 +0100 Subject: [PATCH 05/28] WIP --- .github/workflows/python.yml | 203 +++++++++++++++++++++++++--------- Cargo.lock | 59 +++++++++- rerun_py/Cargo.toml | 1 + rerun_py/build.rs | 42 +++++++ rerun_py/src/python_bridge.rs | 4 +- 5 files changed, 250 insertions(+), 59 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 0984fb01ea4f..0b1e5333b431 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -46,86 +46,130 @@ jobs: run: | just py-lint - py-run-examples: - name: Run Python examples (headless mode) - runs-on: ubuntu-latest + #py-run-examples: + # name: Run Python examples (headless mode) + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + + # - run: | + # sudo apt-get update + # sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs + + # - name: Set up cargo cache + # uses: Swatinem/rust-cache@v2 + # with: + # # See: https://github.com/rerun-io/rerun/pull/497 + # save-if: ${{ github.event_name == 'push'}} + + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.8" + # cache: "pip" + # cache-dependency-path: "rerun_py/requirements-build.txt" + + # - name: Install Python dependencies + # run: | + # pip install --upgrade pip + # pip install -r rerun_py/requirements-build.txt + # pip install ./rerun_py + + # - name: Run Car example + # run: | + # pip install -r examples/car/requirements.txt && ./examples/car/main.py --headless + + macos: + name: Build Python Wheels for MacOS + runs-on: macos-latest + #if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes steps: - uses: actions/checkout@v3 - - run: | - sudo apt-get update - sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs - - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: # See: https://github.com/rerun-io/rerun/pull/497 save-if: ${{ github.event_name == 'push'}} - - name: Set up Python + - name: Setup python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: ${{ env.PYTHON_VERSION }} cache: "pip" cache-dependency-path: "rerun_py/requirements-build.txt" - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-build.txt - pip install ./rerun_py + - name: Build wheel + uses: PyO3/maturin-action@v1 + with: + maturin-version: "0.14.10" + command: build + target: aarch64-apple-darwin + args: | + --manifest-path rerun_py/Cargo.toml + --release + --out dist - - name: Run Car example + - name: Install built wheel run: | - pip install -r examples/car/requirements.txt && ./examples/car/main.py --headless + pip install rerun-sdk --no-index --find-links dist --force-reinstall + pip install pytest #TODO(john) move to requirements-build.txt + cd tests && pytest - py-pip-install-linux: - name: Pip install Rerun SDK (Linux) - runs-on: ubuntu-latest + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + windows: + runs-on: windows-latest + strategy: + matrix: + target: [x64, x86] steps: - uses: actions/checkout@v3 - - run: | - sudo apt-get update - sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs - - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: # See: https://github.com/rerun-io/rerun/pull/497 save-if: ${{ github.event_name == 'push'}} - - name: Setup python for building wheel + - name: Setup python uses: actions/setup-python@v4 with: - python-version: "3.8" + architecture: ${{ matrix.target }} + python-version: ${{ env.PYTHON_VERSION }} cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" - - name: Install rerun_py - run: pip install ./rerun_py - - py-pip-install-windows: - name: Pip install Rerun SDK (Windows) - runs-on: windows-latest - steps: - - uses: actions/checkout@v3 + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: | + --manifest-path rerun_py/Cargo.toml + --release + --out dist - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 + - name: Install built wheel + run: | + pip install rerun-sdk --no-index --find-links dist --force-reinstall + pip install pytest + cd tests && pytest - - name: Setup python for building wheel - uses: actions/setup-python@v4 + - name: Upload wheels + uses: actions/upload-artifact@v3 with: - python-version: "3.8" - cache: "pip" - - - name: Install rerun_py - run: pip install ./rerun_py + name: wheels + path: dist - py-wheel-macos-aarch64: - name: Build Python Wheel for MacOS/aarch64 - runs-on: macos-latest - #if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes + linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64, i686] steps: - uses: actions/checkout@v3 @@ -135,29 +179,80 @@ jobs: # See: https://github.com/rerun-io/rerun/pull/497 save-if: ${{ github.event_name == 'push'}} - - name: Setup python for building wheel + - name: Setup python uses: actions/setup-python@v4 with: + architecture: x64 python-version: ${{ env.PYTHON_VERSION }} cache: "pip" cache-dependency-path: "rerun_py/requirements-build.txt" - - name: Build wheel + - name: Install Linux dependencies + run: | + sudo apt-get update + sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs + + - name: Build wheels uses: PyO3/maturin-action@v1 with: maturin-version: "0.14.10" - command: build + target: ${{ matrix.target }} + manylinux: auto args: | --manifest-path rerun_py/Cargo.toml --release - --strip - --target aarch64-apple-darwin + --out dist + + - name: Install built wheel + if: matrix.target == 'x86_64' + run: | + pip install rerun-sdk --no-index --find-links dist --force-reinstall + pip install pytest + cd tests && pytest - - name: Upload Artifact + - name: Upload wheels uses: actions/upload-artifact@v3 with: - name: rerun-sdk-wheel - path: target/wheels/*.whl + name: wheels + path: dist + + pre-release: + name: Pre Release + needs: [ macos, windows, linux ] + runs-on: "ubuntu-latest" + steps: + - name: Download Artifact + uses: actions/download-artifact@v3 + with: + name: wheels + + - name: Github Release + uses: marvinpinto/action-automatic-releases@v1.2.1 + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "latest" + prerelease: true + title: "Development Build" + files: dist/* + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [ macos, windows, linux ] + steps: + - name: Download Artifact + uses: actions/download-artifact@v3 + with: + name: wheels + + - name: Publish to PyPI + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASSWORD }} + uses: PyO3/maturin-action@v1 + with: + command: upload + args: --skip-existing * py-docs: name: Build and deploy docs diff --git a/Cargo.lock b/Cargo.lock index aebce43f4a00..e7d66417bfbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3028,6 +3028,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +dependencies = [ + "memchr", +] + [[package]] name = "notify" version = "5.0.0" @@ -3534,7 +3543,7 @@ checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" dependencies = [ "once_cell", "thiserror", - "toml", + "toml 0.5.9", ] [[package]] @@ -4059,6 +4068,7 @@ dependencies = [ "rerun", "rerun_sdk", "tokio", + "toml 0.7.1", "uuid", ] @@ -4636,6 +4646,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +dependencies = [ + "serde", +] + [[package]] name = "serde_test" version = "1.0.149" @@ -4957,7 +4976,7 @@ dependencies = [ "cfg-expr", "heck 0.4.0", "pkg-config", - "toml", + "toml 0.5.9", "version-compare", ] @@ -5251,6 +5270,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" +dependencies = [ + "indexmap", + "nom8", + "serde", + "serde_spanned", + "toml_datetime", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -5348,7 +5401,7 @@ dependencies = [ "serde_derive", "serde_json", "termcolor", - "toml", + "toml 0.5.9", ] [[package]] diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 7370bd8202f1..cf16dea9b667 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -62,6 +62,7 @@ uuid = "1.1" [build-dependencies] pyo3-build-config = "0.18.0" +toml = "0.7" [package.metadata.maturin] name = "rerun.rerun_bindings" diff --git a/rerun_py/build.rs b/rerun_py/build.rs index 647f3d4e8e51..7bb9fec29968 100644 --- a/rerun_py/build.rs +++ b/rerun_py/build.rs @@ -1,4 +1,46 @@ fn main() { // Required for `cargo build` to work on mac: https://pyo3.rs/v0.14.2/building_and_distribution.html#macos pyo3_build_config::add_extension_module_link_args(); + + generate_dev_version(); +} + +/// Generates a development version number for the Python package based on the Cargo package version and the current git SHA. +/// This results in a version number like `0.1.0+abcdefg`. +fn generate_dev_version() { + let new_version = { + let pkg_version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| { + eprintln!("cargo:warning=Version not set!"); + "0.0.0".to_owned() + }); + let git_sha = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .output() + .expect("Failed to run git rev-parse") + .stdout; + let git_sha = String::from_utf8(git_sha).unwrap(); + toml::Value::String(format!("{pkg_version}+{}", git_sha.trim())) + }; + + let pyproject_toml = + std::fs::read_to_string("pyproject.toml").expect("Failed to read pyproject.toml"); + let mut pyproject_toml: toml::Value = + toml::from_str(&pyproject_toml).expect("Failed to parse pyproject.toml"); + + let project = pyproject_toml + .get_mut("project") + .expect("Failed to get project"); + + match project.get_mut("version") { + Some(version) => *version = new_version, + None => { + project + .as_table_mut() + .unwrap() + .insert("version".to_owned(), new_version); + } + } + + let pyproject_toml = toml::to_string(&pyproject_toml).unwrap(); + std::fs::write("pyproject.toml", pyproject_toml).unwrap(); } diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index 9746282bfb1c..8b632684b96e 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -471,7 +471,7 @@ fn log_transform( // ---------------------------------------------------------------------------- #[pyfunction] -#[pyo3(signature = (obj_path, xyz, right_handed, timeless))] +#[pyo3(signature = (entity_path, xyz, right_handed = None, timeless = false))] fn log_view_coordinates_xyz( entity_path: &str, xyz: &str, @@ -642,7 +642,7 @@ fn log_mesh_file( /// /// If no `img_format` is specified, we will try and guess it. #[pyfunction] -#[pyo3(signature = (obj_path, img_path, img_format = "None", timeless = false))] +#[pyo3(signature = (entity_path, img_path, img_format = None, timeless = false))] fn log_image_file( entity_path: &str, img_path: PathBuf, From db6a61601b2a28b9ba3f0de51c12bba4fc11b256 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 12:11:01 +0100 Subject: [PATCH 06/28] Fixup --- .github/workflows/python.yml | 6 +- .github/workflows/rust.yml | 130 +++++++++++++++++------------------ rerun_py/pyproject.toml | 34 ++++----- 3 files changed, 83 insertions(+), 87 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 0b1e5333b431..afe795725989 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -149,9 +149,9 @@ jobs: with: target: ${{ matrix.target }} args: | - --manifest-path rerun_py/Cargo.toml - --release - --out dist + --manifest-path rerun_py/Cargo.toml + --release + --out dist - name: Install built wheel run: | diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 18b50dbd3b45..8ac23a189777 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -211,74 +211,74 @@ jobs: # --------------------------------------------------------------------------- - rs-maturin-linux: - name: Build Rust SDK Maturin Wheel (Linux) - runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.3 - steps: - - uses: actions/checkout@v2 - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # See: https://github.com/rerun-io/rerun/pull/497 - save-if: ${{ github.event_name == 'push'}} - - - name: Build wheels - run: maturin build -m rerun_py/Cargo.toml -o dist + #rs-maturin-linux: + # name: Build Rust SDK Maturin Wheel (Linux) + # runs-on: ubuntu-latest-16-cores + # container: + # image: rerunio/ci_docker:0.3 + # steps: + # - uses: actions/checkout@v2 + + # - name: Set up cargo cache + # uses: Swatinem/rust-cache@v2 + # with: + # env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # # See: https://github.com/rerun-io/rerun/pull/497 + # save-if: ${{ github.event_name == 'push'}} + + # - name: Build wheels + # run: maturin build -m rerun_py/Cargo.toml -o dist # --------------------------------------------------------------------------- - rs-maturin-windows: - name: Build Rust SDK Maturin Wheel (Windows) - runs-on: windows-latest-8-cores - steps: - - uses: actions/checkout@v3 - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - # See: https://github.com/rerun-io/rerun/pull/497 - save-if: ${{ github.event_name == 'push'}} - - - name: Setup python for building wheel - uses: actions/setup-python@v4 - with: - python-version: "3.8" - # Caching pip packages is broken for some reason - # cache: "pip" - - - uses: messense/maturin-action@v1 - with: - maturin-version: "0.13.5" - command: build - args: -m rerun_py/Cargo.toml -o dist + #rs-maturin-windows: + # name: Build Rust SDK Maturin Wheel (Windows) + # runs-on: windows-latest-8-cores + # steps: + # - uses: actions/checkout@v3 + + # - name: Set up cargo cache + # uses: Swatinem/rust-cache@v2 + # with: + # # See: https://github.com/rerun-io/rerun/pull/497 + # save-if: ${{ github.event_name == 'push'}} + + # - name: Setup python for building wheel + # uses: actions/setup-python@v4 + # with: + # python-version: "3.8" + # # Caching pip packages is broken for some reason + # # cache: "pip" + + # - uses: messense/maturin-action@v1 + # with: + # maturin-version: "0.13.5" + # command: build + # args: -m rerun_py/Cargo.toml -o dist # --------------------------------------------------------------------------- - rs-maturin-macos: - name: Build Rust SDK Maturin Wheel (macOS) - runs-on: macos-latest - if: "startsWith(github.ref, 'refs/tags/')" # slow and uses up a lot of CI minutes (charged 10x of ubuntu!) - steps: - - uses: actions/checkout@v3 - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - # See: https://github.com/rerun-io/rerun/pull/497 - save-if: ${{ github.event_name == 'push'}} - - - name: Setup python for building wheel - uses: actions/setup-python@v4 - with: - python-version: "3.8" - cache: "pip" - - - uses: messense/maturin-action@v1 - with: - maturin-version: "0.13.5" - command: build - args: -m rerun_py/Cargo.toml -o dist --universal2 +# rs-maturin-macos: +# name: Build Rust SDK Maturin Wheel (macOS) +# runs-on: macos-latest +# if: "startsWith(github.ref, 'refs/tags/')" # slow and uses up a lot of CI minutes (charged 10x of ubuntu!) +# steps: +# - uses: actions/checkout@v3 +# +# - name: Set up cargo cache +# uses: Swatinem/rust-cache@v2 +# with: +# # See: https://github.com/rerun-io/rerun/pull/497 +# save-if: ${{ github.event_name == 'push'}} +# +# - name: Setup python for building wheel +# uses: actions/setup-python@v4 +# with: +# python-version: "3.8" +# cache: "pip" +# +# - uses: messense/maturin-action@v1 +# with: +# maturin-version: "0.13.5" +# command: build +# args: -m rerun_py/Cargo.toml -o dist --universal2 diff --git a/rerun_py/pyproject.toml b/rerun_py/pyproject.toml index 6cc93d2c70c1..1f97324b8230 100644 --- a/rerun_py/pyproject.toml +++ b/rerun_py/pyproject.toml @@ -1,38 +1,34 @@ -# https://maturin.rs/metadata.html - [build-system] build-backend = "maturin" requires = ["maturin>=0.14.0,<0.15"] [project] -name = "rerun-sdk" -authors = [{ name = "Rerun.io", email = "opensource@rerun.io" }] -classifiers = [ - "Programming Language :: Rust", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Development Status :: 3 - Alpha", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Scientific/Engineering :: Visualization", -] +classifiers = ["Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Development Status :: 3 - Alpha", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Scientific/Engineering :: Visualization"] +dependencies = ["numpy>=1.23", "pyarrow==10.0.1"] description = "The Rerun Logging SDK" keywords = ["computer-vision", "logging", "rerun"] -license = { text = "MIT OR Apache-2.0" } +name = "rerun-sdk" requires-python = ">=3.8" +version = "0.1.0+4c3e31a6" -dependencies = ["numpy>=1.23", "pyarrow==10.0.1"] +[[project.authors]] +email = "opensource@rerun.io" +name = "Rerun.io" + +[project.license] +text = "MIT OR Apache-2.0" [project.optional-dependencies] tests = ["pytest==7.1.2"] [project.urls] -homepage = "https://www.rerun.io/" documentation = "https://github.com/rerun-io/rerun" +homepage = "https://www.rerun.io/" repository = "https://github.com/rerun-io/rerun" -[tool.maturin] -locked = true - [tool.black] line-length = 120 -target-version = ['py38'] +target-version = ["py38"] + +[tool.maturin] +locked = true From 43002dd61b14e50cce8ef720c030b51a6bd9e1de Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 13:01:42 +0100 Subject: [PATCH 07/28] fixes --- .github/workflows/python.yml | 15 ++++++++----- .github/workflows/rust.yml | 41 ------------------------------------ 2 files changed, 10 insertions(+), 46 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index afe795725989..2a55d0a2f15e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -114,7 +114,7 @@ jobs: run: | pip install rerun-sdk --no-index --find-links dist --force-reinstall pip install pytest #TODO(john) move to requirements-build.txt - cd tests && pytest + cd rerun_py/tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 @@ -123,7 +123,8 @@ jobs: path: dist windows: - runs-on: windows-latest + name: Build Python Wheels for Windows + runs-on: windows-latest-8-cores strategy: matrix: target: [x64, x86] @@ -157,7 +158,7 @@ jobs: run: | pip install rerun-sdk --no-index --find-links dist --force-reinstall pip install pytest - cd tests && pytest + cd rerun_py/tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 @@ -166,7 +167,10 @@ jobs: path: dist linux: - runs-on: ubuntu-latest + name: Build Python Wheels for Linux + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.3 strategy: matrix: target: [x86_64, i686] @@ -176,6 +180,7 @@ jobs: - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY # See: https://github.com/rerun-io/rerun/pull/497 save-if: ${{ github.event_name == 'push'}} @@ -208,7 +213,7 @@ jobs: run: | pip install rerun-sdk --no-index --find-links dist --force-reinstall pip install pytest - cd tests && pytest + cd rerun_py/tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8ac23a189777..157488f18673 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -211,50 +211,9 @@ jobs: # --------------------------------------------------------------------------- - #rs-maturin-linux: - # name: Build Rust SDK Maturin Wheel (Linux) - # runs-on: ubuntu-latest-16-cores - # container: - # image: rerunio/ci_docker:0.3 - # steps: - # - uses: actions/checkout@v2 - - # - name: Set up cargo cache - # uses: Swatinem/rust-cache@v2 - # with: - # env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # # See: https://github.com/rerun-io/rerun/pull/497 - # save-if: ${{ github.event_name == 'push'}} - - # - name: Build wheels - # run: maturin build -m rerun_py/Cargo.toml -o dist # --------------------------------------------------------------------------- - #rs-maturin-windows: - # name: Build Rust SDK Maturin Wheel (Windows) - # runs-on: windows-latest-8-cores - # steps: - # - uses: actions/checkout@v3 - - # - name: Set up cargo cache - # uses: Swatinem/rust-cache@v2 - # with: - # # See: https://github.com/rerun-io/rerun/pull/497 - # save-if: ${{ github.event_name == 'push'}} - - # - name: Setup python for building wheel - # uses: actions/setup-python@v4 - # with: - # python-version: "3.8" - # # Caching pip packages is broken for some reason - # # cache: "pip" - - # - uses: messense/maturin-action@v1 - # with: - # maturin-version: "0.13.5" - # command: build - # args: -m rerun_py/Cargo.toml -o dist # --------------------------------------------------------------------------- From a9534a5ca415991cde6f8dd96a1d22fb5c41f077 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 14:49:01 +0100 Subject: [PATCH 08/28] wip --- .github/workflows/python.yml | 39 +++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 2a55d0a2f15e..4c03ae7be10f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -98,21 +98,34 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} cache: "pip" cache-dependency-path: "rerun_py/requirements-build.txt" + architecture: x64 + + - name: Build wheel x86_64 + uses: PyO3/maturin-action@v1 + with: + maturin-version: "0.14.10" + command: build + target: x86_64 + args: | + --manifest-path rerun_py/Cargo.toml + --release + --out dist + --sdist - - name: Build wheel + - name: Build wheel - universal2 uses: PyO3/maturin-action@v1 with: maturin-version: "0.14.10" command: build - target: aarch64-apple-darwin args: | --manifest-path rerun_py/Cargo.toml --release --out dist + --universal2 - name: Install built wheel run: | - pip install rerun-sdk --no-index --find-links dist --force-reinstall + pip install rerun-sdk --find-links dist --force-reinstall pip install pytest #TODO(john) move to requirements-build.txt cd rerun_py/tests && pytest @@ -156,7 +169,7 @@ jobs: - name: Install built wheel run: | - pip install rerun-sdk --no-index --find-links dist --force-reinstall + pip install rerun-sdk --find-links dist --force-reinstall pip install pytest cd rerun_py/tests && pytest @@ -173,10 +186,17 @@ jobs: image: rerunio/ci_docker:0.3 strategy: matrix: - target: [x86_64, i686] + #target: [x86_64, i686] + target: [x86_64] steps: - uses: actions/checkout@v3 + - name: Install Linux dependencies + run: | + sudo apt-get update + sudo apt-get install -y lsb-release + # sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs + - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: @@ -187,16 +207,11 @@ jobs: - name: Setup python uses: actions/setup-python@v4 with: - architecture: x64 + #architecture: x64 python-version: ${{ env.PYTHON_VERSION }} cache: "pip" cache-dependency-path: "rerun_py/requirements-build.txt" - - name: Install Linux dependencies - run: | - sudo apt-get update - sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs - - name: Build wheels uses: PyO3/maturin-action@v1 with: @@ -211,7 +226,7 @@ jobs: - name: Install built wheel if: matrix.target == 'x86_64' run: | - pip install rerun-sdk --no-index --find-links dist --force-reinstall + pip install rerun-sdk --find-links dist --force-reinstall pip install pytest cd rerun_py/tests && pytest From e9638a6e9fd87704be7464b308117fb3a80c4acc Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 15:31:43 +0100 Subject: [PATCH 09/28] WIP --- .github/workflows/python.yml | 65 ++++++++------------ .github/workflows/rust.yml | 33 ---------- examples/api_demo/requirements.txt | 2 +- examples/car/requirements.txt | 2 +- examples/deep_sdf/requirements.txt | 2 +- examples/dicom/requirements.txt | 2 +- examples/tracking_hf_opencv/requirements.txt | 2 +- 7 files changed, 32 insertions(+), 76 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4c03ae7be10f..f97a9803749d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -46,43 +46,16 @@ jobs: run: | just py-lint - #py-run-examples: - # name: Run Python examples (headless mode) - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - - # - run: | - # sudo apt-get update - # sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs - - # - name: Set up cargo cache - # uses: Swatinem/rust-cache@v2 - # with: - # # See: https://github.com/rerun-io/rerun/pull/497 - # save-if: ${{ github.event_name == 'push'}} - - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version: "3.8" - # cache: "pip" - # cache-dependency-path: "rerun_py/requirements-build.txt" - - # - name: Install Python dependencies - # run: | - # pip install --upgrade pip - # pip install -r rerun_py/requirements-build.txt - # pip install ./rerun_py - - # - name: Run Car example - # run: | - # pip install -r examples/car/requirements.txt && ./examples/car/main.py --headless + # --------------------------------------------------------------------------- macos: name: Build Python Wheels for MacOS runs-on: macos-latest #if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes + strategy: + matrix: + target: ["x64", "universal2"] + steps: - uses: actions/checkout@v3 @@ -100,7 +73,8 @@ jobs: cache-dependency-path: "rerun_py/requirements-build.txt" architecture: x64 - - name: Build wheel x86_64 + - name: Build wheel - x86_64 + if: matrix.target == "x64" uses: PyO3/maturin-action@v1 with: maturin-version: "0.14.10" @@ -113,6 +87,7 @@ jobs: --sdist - name: Build wheel - universal2 + if: matrix.target == "universal2" uses: PyO3/maturin-action@v1 with: maturin-version: "0.14.10" @@ -135,12 +110,15 @@ jobs: name: wheels path: dist + # --------------------------------------------------------------------------- + windows: name: Build Python Wheels for Windows runs-on: windows-latest-8-cores strategy: matrix: - target: [x64, x86] + #target: [x64, x86] + target: [x64] steps: - uses: actions/checkout@v3 @@ -179,6 +157,8 @@ jobs: name: wheels path: dist + # --------------------------------------------------------------------------- + linux: name: Build Python Wheels for Linux runs-on: ubuntu-latest-16-cores @@ -193,9 +173,8 @@ jobs: - name: Install Linux dependencies run: | - sudo apt-get update - sudo apt-get install -y lsb-release - # sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs + apt-get update + apt-get install -y lsb-release - name: Set up cargo cache uses: Swatinem/rust-cache@v2 @@ -207,7 +186,6 @@ jobs: - name: Setup python uses: actions/setup-python@v4 with: - #architecture: x64 python-version: ${{ env.PYTHON_VERSION }} cache: "pip" cache-dependency-path: "rerun_py/requirements-build.txt" @@ -230,15 +208,22 @@ jobs: pip install pytest cd rerun_py/tests && pytest + - name: Run Car example + run: | + pip install -r examples/car/requirements.txt && ./examples/car/main.py --headless + - name: Upload wheels uses: actions/upload-artifact@v3 with: name: wheels path: dist + # --------------------------------------------------------------------------- + pre-release: name: Pre Release needs: [ macos, windows, linux ] + #if: github.ref == 'refs/heads/main' runs-on: "ubuntu-latest" steps: - name: Download Artifact @@ -255,6 +240,8 @@ jobs: title: "Development Build" files: dist/* + # --------------------------------------------------------------------------- + release: name: Release runs-on: ubuntu-latest @@ -274,6 +261,8 @@ jobs: command: upload args: --skip-existing * + # --------------------------------------------------------------------------- + py-docs: name: Build and deploy docs if: ${{ github.event_name == 'push' }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 157488f18673..39a9e7fe3cda 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -208,36 +208,3 @@ jobs: with: command: deny args: --log-level=error check - - # --------------------------------------------------------------------------- - - - # --------------------------------------------------------------------------- - - - # --------------------------------------------------------------------------- - -# rs-maturin-macos: -# name: Build Rust SDK Maturin Wheel (macOS) -# runs-on: macos-latest -# if: "startsWith(github.ref, 'refs/tags/')" # slow and uses up a lot of CI minutes (charged 10x of ubuntu!) -# steps: -# - uses: actions/checkout@v3 -# -# - name: Set up cargo cache -# uses: Swatinem/rust-cache@v2 -# with: -# # See: https://github.com/rerun-io/rerun/pull/497 -# save-if: ${{ github.event_name == 'push'}} -# -# - name: Setup python for building wheel -# uses: actions/setup-python@v4 -# with: -# python-version: "3.8" -# cache: "pip" -# -# - uses: messense/maturin-action@v1 -# with: -# maturin-version: "0.13.5" -# command: build -# args: -m rerun_py/Cargo.toml -o dist --universal2 diff --git a/examples/api_demo/requirements.txt b/examples/api_demo/requirements.txt index 04c3305dca7f..ebb847ff0d2d 100644 --- a/examples/api_demo/requirements.txt +++ b/examples/api_demo/requirements.txt @@ -1 +1 @@ -rerun +rerun-sdk diff --git a/examples/car/requirements.txt b/examples/car/requirements.txt index 403498218de7..d5c293f0f98b 100644 --- a/examples/car/requirements.txt +++ b/examples/car/requirements.txt @@ -1,4 +1,4 @@ -rerun +rerun-sdk # Avoid opencv-4.6 since it rotates images incorrectly: # See: https://github.com/opencv/opencv/issues/22088 opencv-python<4.6 diff --git a/examples/deep_sdf/requirements.txt b/examples/deep_sdf/requirements.txt index 0d854ebaab46..3234a1a3f051 100644 --- a/examples/deep_sdf/requirements.txt +++ b/examples/deep_sdf/requirements.txt @@ -1,4 +1,4 @@ -rerun +rerun-sdk mesh_to_sdf===0.0.14 requests==2.28.1 trimesh==3.15.2 diff --git a/examples/dicom/requirements.txt b/examples/dicom/requirements.txt index adcd1b001b20..262e5703b673 100644 --- a/examples/dicom/requirements.txt +++ b/examples/dicom/requirements.txt @@ -1,4 +1,4 @@ -rerun +rerun-sdk dicom_numpy==0.6.2 pydicom==2.3.0 requests==2.28.1 diff --git a/examples/tracking_hf_opencv/requirements.txt b/examples/tracking_hf_opencv/requirements.txt index fffbe725e0ca..c49c7d0edc91 100644 --- a/examples/tracking_hf_opencv/requirements.txt +++ b/examples/tracking_hf_opencv/requirements.txt @@ -4,7 +4,7 @@ opencv-contrib-python<4.6 opencv-python<4.6 pillow requests==2.28.1 -rerun +rerun-sdk timm==0.6.11 torch>=1.13.0 transformers From f44fa082cb268a4405dd85cacbd0405043cdd1ae Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 15:33:32 +0100 Subject: [PATCH 10/28] fix --- .github/workflows/python.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index f97a9803749d..e8792dcedf63 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -54,7 +54,7 @@ jobs: #if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes strategy: matrix: - target: ["x64", "universal2"] + target: [x64, universal2] steps: - uses: actions/checkout@v3 @@ -74,7 +74,7 @@ jobs: architecture: x64 - name: Build wheel - x86_64 - if: matrix.target == "x64" + if: matrix.target == 'x64' uses: PyO3/maturin-action@v1 with: maturin-version: "0.14.10" @@ -87,7 +87,7 @@ jobs: --sdist - name: Build wheel - universal2 - if: matrix.target == "universal2" + if: matrix.target == 'universal2' uses: PyO3/maturin-action@v1 with: maturin-version: "0.14.10" From 0f16eefb17de0b0f43c8d3de6f796e0cd533680f Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 16:17:22 +0100 Subject: [PATCH 11/28] blind stab --- .github/workflows/python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index e8792dcedf63..7a0e34fa0a0f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -187,6 +187,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_VERSION }} + architecture: x64 cache: "pip" cache-dependency-path: "rerun_py/requirements-build.txt" From 075892bb0f4314ada77c29d15739745d332bdbaa Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 18:19:05 +0100 Subject: [PATCH 12/28] Disable steering python veresion on Linux build --- .github/workflows/python.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 7a0e34fa0a0f..6a5978277dce 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -171,10 +171,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install Linux dependencies - run: | - apt-get update - apt-get install -y lsb-release + #- name: Install Linux dependencies + # run: | + # apt-get update + # apt-get install -y lsb-release - name: Set up cargo cache uses: Swatinem/rust-cache@v2 @@ -183,13 +183,13 @@ jobs: # See: https://github.com/rerun-io/rerun/pull/497 save-if: ${{ github.event_name == 'push'}} - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - architecture: x64 - cache: "pip" - cache-dependency-path: "rerun_py/requirements-build.txt" + #- name: Setup python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # architecture: x64 + # cache: "pip" + # cache-dependency-path: "rerun_py/requirements-build.txt" - name: Build wheels uses: PyO3/maturin-action@v1 From 1029c788cc926517841a61ed7126fc116a252702 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Tue, 31 Jan 2023 18:32:21 +0100 Subject: [PATCH 13/28] wip --- .github/workflows/python.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 6a5978277dce..7293880c8eb9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -164,10 +164,6 @@ jobs: runs-on: ubuntu-latest-16-cores container: image: rerunio/ci_docker:0.3 - strategy: - matrix: - #target: [x86_64, i686] - target: [x86_64] steps: - uses: actions/checkout@v3 From f4f46f8055c28cf48163df86a4e05d0e89f23428 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 09:46:40 +0100 Subject: [PATCH 14/28] fix --- .github/workflows/python.yml | 30 ++++++++++++++++-------------- rerun_py/pyproject.toml | 10 ++++++++-- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 7293880c8eb9..817d06a97f74 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -161,16 +161,18 @@ jobs: linux: name: Build Python Wheels for Linux - runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.3 + runs-on: ubuntu-latest #-16-cores + #container: + # image: rerunio/ci_docker:0.3 steps: - uses: actions/checkout@v3 - #- name: Install Linux dependencies - # run: | - # apt-get update - # apt-get install -y lsb-release + - name: Cache APT Packages + uses: awalsh128/cache-apt-pkgs-action@v1.2.2 + with: + packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev + version: 1.0 + #libatk-bridge2.0 libfontconfig1-dev libfreetype6-dev libglib2.0-dev - name: Set up cargo cache uses: Swatinem/rust-cache@v2 @@ -179,13 +181,13 @@ jobs: # See: https://github.com/rerun-io/rerun/pull/497 save-if: ${{ github.event_name == 'push'}} - #- name: Setup python - # uses: actions/setup-python@v4 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - # architecture: x64 - # cache: "pip" - # cache-dependency-path: "rerun_py/requirements-build.txt" + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + architecture: x64 + cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" - name: Build wheels uses: PyO3/maturin-action@v1 diff --git a/rerun_py/pyproject.toml b/rerun_py/pyproject.toml index 1f97324b8230..c3fbc2d42033 100644 --- a/rerun_py/pyproject.toml +++ b/rerun_py/pyproject.toml @@ -3,13 +3,19 @@ build-backend = "maturin" requires = ["maturin>=0.14.0,<0.15"] [project] -classifiers = ["Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Development Status :: 3 - Alpha", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Scientific/Engineering :: Visualization"] +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Development Status :: 3 - Alpha", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Visualization", +] dependencies = ["numpy>=1.23", "pyarrow==10.0.1"] description = "The Rerun Logging SDK" keywords = ["computer-vision", "logging", "rerun"] name = "rerun-sdk" requires-python = ">=3.8" -version = "0.1.0+4c3e31a6" [[project.authors]] email = "opensource@rerun.io" From 722ff9c10d8246a93f4eb0360035af0132be9b12 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 10:04:16 +0100 Subject: [PATCH 15/28] fix --- .github/workflows/python.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 817d06a97f74..68e985e8d547 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -170,9 +170,8 @@ jobs: - name: Cache APT Packages uses: awalsh128/cache-apt-pkgs-action@v1.2.2 with: - packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev + packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev version: 1.0 - #libatk-bridge2.0 libfontconfig1-dev libfreetype6-dev libglib2.0-dev - name: Set up cargo cache uses: Swatinem/rust-cache@v2 From a120a5301329870de9f9d9ec000ed5c9265b3472 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 10:22:41 +0100 Subject: [PATCH 16/28] add fontconfig to apt install --- .github/workflows/python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 68e985e8d547..d33476fa8089 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -170,7 +170,7 @@ jobs: - name: Cache APT Packages uses: awalsh128/cache-apt-pkgs-action@v1.2.2 with: - packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev + packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev fontconfig version: 1.0 - name: Set up cargo cache From d2e6f94ea6b84709c9c61416383c6c51caa28d64 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 10:40:58 +0100 Subject: [PATCH 17/28] another try --- .github/workflows/python.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d33476fa8089..a28b088c2180 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -51,7 +51,7 @@ jobs: macos: name: Build Python Wheels for MacOS runs-on: macos-latest - #if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes + if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes strategy: matrix: target: [x64, universal2] @@ -170,8 +170,13 @@ jobs: - name: Cache APT Packages uses: awalsh128/cache-apt-pkgs-action@v1.2.2 with: - packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev fontconfig + packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev version: 1.0 + execute_install_scripts: true + + - name: Test fontconfig + run: | + "pkg-config" "--libs" "--cflags" "fontconfig" "fontconfig >= 2.11.1" - name: Set up cargo cache uses: Swatinem/rust-cache@v2 From 565dbb255c6b5597503dfb3ea6591c388bf352ff Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 11:04:52 +0100 Subject: [PATCH 18/28] disable manylinux container --- .github/workflows/python.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a28b088c2180..21283e618ebe 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -174,10 +174,6 @@ jobs: version: 1.0 execute_install_scripts: true - - name: Test fontconfig - run: | - "pkg-config" "--libs" "--cflags" "fontconfig" "fontconfig >= 2.11.1" - - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: @@ -199,6 +195,7 @@ jobs: maturin-version: "0.14.10" target: ${{ matrix.target }} manylinux: auto + container: off args: | --manifest-path rerun_py/Cargo.toml --release From 8a604d9d1ef774495678ba8ab8eac684def5129f Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 11:24:52 +0100 Subject: [PATCH 19/28] fix --- .github/workflows/python.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 21283e618ebe..737ffd3f01ac 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -161,7 +161,7 @@ jobs: linux: name: Build Python Wheels for Linux - runs-on: ubuntu-latest #-16-cores + runs-on: ubuntu-latest-16-cores #container: # image: rerunio/ci_docker:0.3 steps: @@ -202,7 +202,6 @@ jobs: --out dist - name: Install built wheel - if: matrix.target == 'x86_64' run: | pip install rerun-sdk --find-links dist --force-reinstall pip install pytest @@ -222,7 +221,8 @@ jobs: pre-release: name: Pre Release - needs: [ macos, windows, linux ] + #needs: [ macos, windows, linux ] + needs: [ windows, linux ] #if: github.ref == 'refs/heads/main' runs-on: "ubuntu-latest" steps: From 9e10f397ac27048a1a7faf2a4e9c976190ca6da6 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 11:47:49 +0100 Subject: [PATCH 20/28] wip --- .github/workflows/python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 737ffd3f01ac..ca543386b753 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -171,7 +171,7 @@ jobs: uses: awalsh128/cache-apt-pkgs-action@v1.2.2 with: packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev - version: 1.0 + version: 1.1 execute_install_scripts: true - name: Set up cargo cache From d5b22aa3f0d45e5254a4eda69f16a9fd08921c5b Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 12:06:07 +0100 Subject: [PATCH 21/28] wip --- .github/workflows/python.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ca543386b753..2a271e884b23 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -49,9 +49,11 @@ jobs: # --------------------------------------------------------------------------- macos: + # This build is really slow (>30 mins); uses up a lot of CI minutes name: Build Python Wheels for MacOS runs-on: macos-latest - if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes + #if: "startsWith(github.ref, 'refs/tags/')" + #if: github.ref == 'refs/heads/main' strategy: matrix: target: [x64, universal2] @@ -209,7 +211,8 @@ jobs: - name: Run Car example run: | - pip install -r examples/car/requirements.txt && ./examples/car/main.py --headless + pip install -r examples/car/requirements.txt + ./examples/car/main.py --headless - name: Upload wheels uses: actions/upload-artifact@v3 @@ -230,6 +233,7 @@ jobs: uses: actions/download-artifact@v3 with: name: wheels + path: dist - name: Github Release uses: marvinpinto/action-automatic-releases@v1.2.1 @@ -252,6 +256,7 @@ jobs: uses: actions/download-artifact@v3 with: name: wheels + path: dist - name: Publish to PyPI env: From ca2754344b28de975a30232f0faf850db097c845 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 13:01:04 +0100 Subject: [PATCH 22/28] mostly working I think --- .github/workflows/python.yml | 9 ++-- .github/workflows/release.yml | 98 +---------------------------------- rerun_py/build.rs | 40 ++++++++------ 3 files changed, 28 insertions(+), 119 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 2a271e884b23..070eb2bbe6f0 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -52,8 +52,7 @@ jobs: # This build is really slow (>30 mins); uses up a lot of CI minutes name: Build Python Wheels for MacOS runs-on: macos-latest - #if: "startsWith(github.ref, 'refs/tags/')" - #if: github.ref == 'refs/heads/main' + #if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' strategy: matrix: target: [x64, universal2] @@ -164,8 +163,7 @@ jobs: linux: name: Build Python Wheels for Linux runs-on: ubuntu-latest-16-cores - #container: - # image: rerunio/ci_docker:0.3 + steps: - uses: actions/checkout@v3 @@ -224,8 +222,7 @@ jobs: pre-release: name: Pre Release - #needs: [ macos, windows, linux ] - needs: [ windows, linux ] + needs: [ macos, windows, linux ] #if: github.ref == 'refs/heads/main' runs-on: "ubuntu-latest" steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ab8d30b35c2f..9641de98b70e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,25 +4,6 @@ on: push: tags: - "*.*.*" # on release tag - pull_request: - branches: - - "main" - paths: - - ".github/workflows/release.yml" - - workflow_dispatch: - inputs: - pyindex: - description: "PyPI Index" - required: true - default: "testpypi" - type: choice - options: - - testpypi - - pypi - -env: - PYTHON_VERSION: "3.8" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -41,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "3.8" cache: "pip" cache-dependency-path: "rerun_py/requirements-doc.txt" @@ -61,80 +42,3 @@ jobs: - name: Deploy via mike # https://github.com/jimporter/mike run: | mike deploy -F rerun_py/mkdocs.yml -p --rebase -b gh-pages -u ${{github.ref_name}} latest - - py-wheel-macos-aarch64: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "rerun_py/requirements-build.txt" - - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - default: true - - - name: Set up Rust targets - run: rustup target add aarch64-apple-darwin - - - name: Publish wheel - uses: PyO3/maturin-action@v1 - env: - #MATURIN_PASSWORD: ${{ secrets.PYPI_PASS }} - with: - maturin-version: "0.14.10" - command: build - args: | - --manifest-path rerun_py/Cargo.toml - --release - --strip - --target aarch64-apple-darwin - #--skip-existing - #--no-sdist - #--repository ${{ inputs.pyindex }} - #--username john.rerun.io - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: rerun-sdk-wheel - path: target/wheels/*.whl - - py-wheel-manylinux-x64_64: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "rerun_py/requirements-build.txt" - - - name: Publish wheel - uses: PyO3/maturin-action@v1 - env: - #MATURIN_PASSWORD: ${{ secrets.PYPI_PASS }} - RUSTFLAGS: -C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma - with: - maturin-version: '0.14.10' - command: build - args: | - --manifest-path rerun_py/Cargo.toml - --release - --strip - #--skip-existing - #--repository ${{ inputs.pyindex }} - #--username john.rerun.io - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: rerun-sdk-wheel - path: target/wheels/*.whl diff --git a/rerun_py/build.rs b/rerun_py/build.rs index 7bb9fec29968..e0c685f1220c 100644 --- a/rerun_py/build.rs +++ b/rerun_py/build.rs @@ -2,26 +2,34 @@ fn main() { // Required for `cargo build` to work on mac: https://pyo3.rs/v0.14.2/building_and_distribution.html#macos pyo3_build_config::add_extension_module_link_args(); - generate_dev_version(); + if let Ok(ref ref_type) = std::env::var("GITHUB_REF_TYPE") { + // We're in CI + if ref_type == "branch" { + // A branch build, so we're building a development version. + + let new_version = { + let pkg_version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| { + eprintln!("cargo:warning=Version not set!"); + "0.0.0".to_owned() + }); + let git_sha = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .output() + .expect("Failed to run git rev-parse") + .stdout; + let git_sha = String::from_utf8(git_sha).unwrap(); + toml::Value::String(format!("{pkg_version}+{}", git_sha.trim())) + }; + + eprintln!("cargo:warning=Overriding packaged wheel version to a development version: {new_version}."); + generate_dev_version(new_version); + } + } } /// Generates a development version number for the Python package based on the Cargo package version and the current git SHA. /// This results in a version number like `0.1.0+abcdefg`. -fn generate_dev_version() { - let new_version = { - let pkg_version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| { - eprintln!("cargo:warning=Version not set!"); - "0.0.0".to_owned() - }); - let git_sha = std::process::Command::new("git") - .args(["rev-parse", "--short", "HEAD"]) - .output() - .expect("Failed to run git rev-parse") - .stdout; - let git_sha = String::from_utf8(git_sha).unwrap(); - toml::Value::String(format!("{pkg_version}+{}", git_sha.trim())) - }; - +fn generate_dev_version(new_version: toml::Value) { let pyproject_toml = std::fs::read_to_string("pyproject.toml").expect("Failed to read pyproject.toml"); let mut pyproject_toml: toml::Value = From 2f84eb9b475c5ccf17c61c9f9aa13a16596d9736 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 14:10:14 +0100 Subject: [PATCH 23/28] - Chain wheel builds with python lint - Fix `rerun` entrypoint --- .github/workflows/python.yml | 29 +++++++++++++++++++---------- rerun_py/pyproject.toml | 3 +++ rerun_py/requirements-build.txt | 2 +- rerun_py/rerun/__main__.py | 7 ++++++- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 070eb2bbe6f0..e7298b78adf7 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -18,7 +18,7 @@ concurrency: cancel-in-progress: true jobs: - py-lint: + lint: name: Python lints (black, mypy, flake8) runs-on: ubuntu-latest steps: @@ -49,10 +49,11 @@ jobs: # --------------------------------------------------------------------------- macos: - # This build is really slow (>30 mins); uses up a lot of CI minutes name: Build Python Wheels for MacOS runs-on: macos-latest - #if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' + # This build is really slow (>30 mins); uses up a lot of CI minutes + if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' + needs: [ lint ] strategy: matrix: target: [x64, universal2] @@ -85,7 +86,6 @@ jobs: --manifest-path rerun_py/Cargo.toml --release --out dist - --sdist - name: Build wheel - universal2 if: matrix.target == 'universal2' @@ -116,9 +116,9 @@ jobs: windows: name: Build Python Wheels for Windows runs-on: windows-latest-8-cores + needs: [ lint ] strategy: matrix: - #target: [x64, x86] target: [x64] steps: - uses: actions/checkout@v3 @@ -163,7 +163,7 @@ jobs: linux: name: Build Python Wheels for Linux runs-on: ubuntu-latest-16-cores - + needs: [ lint ] steps: - uses: actions/checkout@v3 @@ -199,6 +199,7 @@ jobs: args: | --manifest-path rerun_py/Cargo.toml --release + --sdist --out dist - name: Install built wheel @@ -223,7 +224,7 @@ jobs: pre-release: name: Pre Release needs: [ macos, windows, linux ] - #if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' runs-on: "ubuntu-latest" steps: - name: Download Artifact @@ -239,14 +240,15 @@ jobs: automatic_release_tag: "latest" prerelease: true title: "Development Build" - files: dist/* + files: dist/* CHANGELOG.md # --------------------------------------------------------------------------- - release: + # This job is run on tags starting with "v", e.g., "v0.1.0" + tagged-release: name: Release runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" + if: startsWith(github.ref, 'refs/tags/v') needs: [ macos, windows, linux ] steps: - name: Download Artifact @@ -255,6 +257,13 @@ jobs: name: wheels path: dist + - name: Github Release + uses: "marvinpinto/action-automatic-releases@1.2.1" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + prerelease: false + files: dist/* CHANGELOG.md + - name: Publish to PyPI env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASSWORD }} diff --git a/rerun_py/pyproject.toml b/rerun_py/pyproject.toml index c3fbc2d42033..5c0fe506413f 100644 --- a/rerun_py/pyproject.toml +++ b/rerun_py/pyproject.toml @@ -32,6 +32,9 @@ documentation = "https://github.com/rerun-io/rerun" homepage = "https://www.rerun.io/" repository = "https://github.com/rerun-io/rerun" +[project.scripts] +rerun = "rerun.__main__:main" + [tool.black] line-length = 120 target-version = ["py38"] diff --git a/rerun_py/requirements-build.txt b/rerun_py/requirements-build.txt index 9380d17eae35..ffb1e920f541 100644 --- a/rerun_py/requirements-build.txt +++ b/rerun_py/requirements-build.txt @@ -1 +1 @@ -maturin==0.13.5,<0.14 +maturin>=0.14,<0.15 diff --git a/rerun_py/rerun/__main__.py b/rerun_py/rerun/__main__.py index 109613b022dd..be1e5244a4c9 100644 --- a/rerun_py/rerun/__main__.py +++ b/rerun_py/rerun/__main__.py @@ -4,7 +4,12 @@ from rerun import rerun_bindings, unregister_shutdown # type: ignore[attr-defined] -if __name__ == "__main__": + +def main() -> None: # We don't need to call shutdown in this case. Rust should be handling everything unregister_shutdown() exit(rerun_bindings.main(sys.argv)) + + +if __name__ == "__main__": + main() From 6ec3f46243af2c9fd6419487325cf59ac922aad0 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 14:33:05 +0100 Subject: [PATCH 24/28] last fixes (hopefully) --- .github/workflows/python.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index e7298b78adf7..a41516613369 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -9,6 +9,8 @@ on: push: branches: - "main" + tags: + - "v*" env: PYTHON_VERSION: "3.8" @@ -266,7 +268,7 @@ jobs: - name: Publish to PyPI env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASSWORD }} + MATURIN_PYPI_TOKEN: ${{ secrets.MATURIN_PYPI_TOKEN }} uses: PyO3/maturin-action@v1 with: command: upload From cb5611627d477ed4cb6a33d2ee9375f385a9301f Mon Sep 17 00:00:00 2001 From: John Hughes Date: Wed, 1 Feb 2023 15:47:37 +0100 Subject: [PATCH 25/28] Use Rerun fork of cache-apt-pkgs-action with a fix --- .github/workflows/python.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a41516613369..5c3d7e3ef179 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -170,10 +170,12 @@ jobs: - uses: actions/checkout@v3 - name: Cache APT Packages - uses: awalsh128/cache-apt-pkgs-action@v1.2.2 + #uses: awalsh128/cache-apt-pkgs-action@v1.2.2 + #TODO(john) merge upstream + uses: rerun-io/cache-apt-pkgs-action@59534850182063abf1b2c11bb3686722a12a8397 with: packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev - version: 1.1 + version: 1.0 execute_install_scripts: true - name: Set up cargo cache From ad11af2d49acddafd690915523b53bdd4f5981a8 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Thu, 2 Feb 2023 10:50:00 +0100 Subject: [PATCH 26/28] - Unify the version tags - Rename the python crate to `rerun_py` to match the folder --- .github/workflows/python.yml | 18 ++++++++-- Cargo.lock | 64 ++++++++++++++++++------------------ rerun_py/Cargo.toml | 2 +- rerun_py/README.md | 2 +- 4 files changed, 50 insertions(+), 36 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 5c3d7e3ef179..3fd0ab22678f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -10,7 +10,7 @@ on: branches: - "main" tags: - - "v*" + - "v*.*.*" # on release tag env: PYTHON_VERSION: "3.8" @@ -54,7 +54,7 @@ jobs: name: Build Python Wheels for MacOS runs-on: macos-latest # This build is really slow (>30 mins); uses up a lot of CI minutes - if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' + if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' needs: [ lint ] strategy: matrix: @@ -107,6 +107,11 @@ jobs: pip install pytest #TODO(john) move to requirements-build.txt cd rerun_py/tests && pytest + - name: Run Car example + run: | + pip install -r examples/car/requirements.txt + ./examples/car/main.py --headless + - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -154,6 +159,11 @@ jobs: pip install pytest cd rerun_py/tests && pytest + - name: Run Car example + run: | + pip install -r examples/car/requirements.txt + ./examples/car/main.py --headless + - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -225,6 +235,7 @@ jobs: # --------------------------------------------------------------------------- + # See https://github.com/marvinpinto/action-automatic-releases#automatically-generate-a-pre-release-when-changes-land-on-master for details pre-release: name: Pre Release needs: [ macos, windows, linux ] @@ -261,6 +272,9 @@ jobs: name: wheels path: dist + #TODO(#966) https://github.com/rerun-io/rerun/issues/1054 + # Santiy check the git tag vs. the crate version. + - name: Github Release uses: "marvinpinto/action-automatic-releases@1.2.1" with: diff --git a/Cargo.lock b/Cargo.lock index f4db7712d279..aae4c52d417f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4040,38 +4040,6 @@ dependencies = [ "re_smart_channel", ] -[[package]] -name = "re_sdk_python" -version = "0.1.0" -dependencies = [ - "ahash 0.8.2", - "arrow2", - "bytemuck", - "crossbeam", - "document-features", - "glam", - "half 1.8.2", - "image", - "itertools", - "macaw", - "mimalloc", - "numpy", - "pyo3", - "pyo3-build-config", - "rand", - "re_error", - "re_log", - "re_log_types", - "re_memory", - "re_sdk_comms", - "re_tensor_ops", - "rerun", - "rerun_sdk", - "tokio", - "toml 0.7.1", - "uuid", -] - [[package]] name = "re_smart_channel" version = "0.1.0" @@ -4351,6 +4319,38 @@ dependencies = [ "webbrowser", ] +[[package]] +name = "rerun_py" +version = "0.1.0" +dependencies = [ + "ahash 0.8.2", + "arrow2", + "bytemuck", + "crossbeam", + "document-features", + "glam", + "half 1.8.2", + "image", + "itertools", + "macaw", + "mimalloc", + "numpy", + "pyo3", + "pyo3-build-config", + "rand", + "re_error", + "re_log", + "re_log_types", + "re_memory", + "re_sdk_comms", + "re_tensor_ops", + "rerun", + "rerun_sdk", + "tokio", + "toml 0.7.1", + "uuid", +] + [[package]] name = "rerun_sdk" version = "0.1.0" diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index cf16dea9b667..e3a94a75729b 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "re_sdk_python" # name of the rust crate +name = "rerun_py" # name of the rust crate edition.workspace = true license.workspace = true publish = false diff --git a/rerun_py/README.md b/rerun_py/README.md index bfa9c028f21f..11d410978bb2 100644 --- a/rerun_py/README.md +++ b/rerun_py/README.md @@ -3,7 +3,7 @@ Goal: an ergonomic Python library for logging rich data, over TCP, to a rerun server. ℹ️ Note: -- The rust crate is called `re_sdk_python`, the Python module is called `rerun`, and the package published on PyPI is `rerun-sdk`. +- The rust crate is called `rerun_py`, the Python module is called `rerun`, and the package published on PyPI is `rerun-sdk`. - These instructions assume you're running from the `rerun` root folder and have Python 3.8 or later available. ## Building from Source From 136a4e15502e518ccce9a9b20da13a54eabea069 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Thu, 2 Feb 2023 14:38:38 +0100 Subject: [PATCH 27/28] fixes --- RELEASES.md | 24 ++++++++++++++---------- rerun_py/build.rs | 11 +++++++---- rerun_py/pyproject.toml | 4 ++-- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 5374c7de20f3..17cc3edeecf0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -26,7 +26,8 @@ We tag all data files (`.rrd` files) and communication protocols with the rerun ## Releases -When doing a normal release, we create a branch off of `main`. If we are doing a patch release, we do a branch off of the latest release tag (e.g. `0.3.0`) and cherry-pick any fixes we want into that branch. +Release builds of the Python Wheels are triggered by pushing a release tag to github in the form `v0.2.0`. +When doing a normal release, we tag the release commit on the `main` branch. If we are doing a patch release, we do a branch off of the latest release tag (e.g. `v0.3.0`) and cherry-pick any fixes we want into that branch. ### Release checklist Copy this checklist to the the PR description, go through it from top to bottom, and check each item before moving onto the next. This is a living document. Strive to improve it on each new release. @@ -48,9 +49,9 @@ Copy this checklist to the the PR description, go through it from top to bottom, * [ ] Publish new Rust crates * [ ] Publish new Python wheels * [ ] Merge PR -* [ ] `git tag -a 0.x.y -m 'Release 0.x.y - summary'` +* [ ] `git tag -a v0.x.y -m 'Release 0.x.y - summary'` * [ ] `git push && git push --tags` -* [ ] Do a GitHub release: https://github.com/rerun-io/rerun/releases/new +* [ ] Wait for CI to build release artifacts and publish them on GitHub and PyPI. Verify this at https://github.com/rerun-io/rerun/releases/new. * [ ] Post on: * [ ] Community Discord * [ ] Rerun Twitter @@ -58,13 +59,16 @@ Copy this checklist to the the PR description, go through it from top to bottom, ### Testing a release -* [ ] `just py-run-all` -* [ ] Test the web viewer: - * [ ] `cargo run -p rerun --features web -- --web-viewer ../nyud.rrd` - * [ ] Test on: - * [ ] Chromium - * [ ] Firefox - * [ ] Mobile +* Before pushing the release tag: + * [ ] `just py-run-all` + * [ ] Test the web viewer: + * [ ] `cargo run -p rerun --features web -- --web-viewer ../nyud.rrd` + * [ ] Test on: + * [ ] Chromium + * [ ] Firefox + * [ ] Mobile +* After tagging and the CI has published: + * [ ] Test the Python packages from PyPI: `pip install -U rerun-sdk` ## To do before first release diff --git a/rerun_py/build.rs b/rerun_py/build.rs index e0c685f1220c..172fed19f2fd 100644 --- a/rerun_py/build.rs +++ b/rerun_py/build.rs @@ -1,3 +1,5 @@ +use std::path::Path; + fn main() { // Required for `cargo build` to work on mac: https://pyo3.rs/v0.14.2/building_and_distribution.html#macos pyo3_build_config::add_extension_module_link_args(); @@ -6,7 +8,6 @@ fn main() { // We're in CI if ref_type == "branch" { // A branch build, so we're building a development version. - let new_version = { let pkg_version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| { eprintln!("cargo:warning=Version not set!"); @@ -22,16 +23,18 @@ fn main() { }; eprintln!("cargo:warning=Overriding packaged wheel version to a development version: {new_version}."); - generate_dev_version(new_version); + generate_dev_version("pyproject.toml", new_version); } } } /// Generates a development version number for the Python package based on the Cargo package version and the current git SHA. /// This results in a version number like `0.1.0+abcdefg`. -fn generate_dev_version(new_version: toml::Value) { +/// The resulting version number is written to the `pyproject.toml` file. +fn generate_dev_version(project_file: impl AsRef, new_version: toml::Value) { let pyproject_toml = - std::fs::read_to_string("pyproject.toml").expect("Failed to read pyproject.toml"); + std::fs::read_to_string(project_file).unwrap_or_else(|e| panic!("Failed to read {e:?}")); + let mut pyproject_toml: toml::Value = toml::from_str(&pyproject_toml).expect("Failed to parse pyproject.toml"); diff --git a/rerun_py/pyproject.toml b/rerun_py/pyproject.toml index 5c0fe506413f..15127bf433c7 100644 --- a/rerun_py/pyproject.toml +++ b/rerun_py/pyproject.toml @@ -28,8 +28,8 @@ text = "MIT OR Apache-2.0" tests = ["pytest==7.1.2"] [project.urls] -documentation = "https://github.com/rerun-io/rerun" -homepage = "https://www.rerun.io/" +documentation = "https://www.rerun.io/docs" +homepage = "https://www.rerun.io" repository = "https://github.com/rerun-io/rerun" [project.scripts] From eaf2f3f7907565c598c24679cfdaa1ed86746be2 Mon Sep 17 00:00:00 2001 From: John Hughes Date: Thu, 2 Feb 2023 15:09:41 +0100 Subject: [PATCH 28/28] Updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0427616d674a..666fdd446406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ A rough time-line of major user-facing things added, removed and changed. Newest on top. +* 2023-02-02: Automate Python Wheel deployments. The Rerun python package is renamed to `rerun-sdk`. [#996](https://github.com/rerun-io/rerun/pull/996) * 2023-01-27: Update wgpu to 0.15. [#954](https://github.com/rerun-io/rerun/pull/954) * 2023-01-25: Depth images are now color mapped with turbo color map. [#912](https://github.com/rerun-io/rerun/pull/912) * 2023-01-20: Arrow Storage is now the default storage backend for rerun. Use RERUN_STORE=classic to opt out. [#822](https://github.com/rerun-io/rerun/pull/822).