Build+Deploy Python Wheels in CI

.github/workflows/python.yml

@@ -9,13 +9,18 @@ on:
   push:
     branches:
       - "main"
+    tags:
+      - "v*.*.*" # on release tag
+
+env:
+  PYTHON_VERSION: "3.8"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }} # Cancel previous CI jobs on the same branch
   cancel-in-progress: true
 
 jobs:
-  py-lint:
+  lint:
     name: Python lints (black, mypy, flake8)
     runs-on: ubuntu-latest
     steps:
@@ -30,7 +35,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: ${{ env.PYTHON_VERSION }}
           cache: "pip"
           cache-dependency-path: "rerun_py/requirements-lint.txt"
 
@@ -43,103 +48,249 @@ jobs:
         run: |
           just py-lint
 
-  py-run-examples:
-    name: Run Python examples (headless mode)
-    runs-on: ubuntu-latest
+  # ---------------------------------------------------------------------------
+
+  macos:
+    name: Build Python Wheels for MacOS
+    runs-on: macos-latest
+    # This build is really slow (>30 mins); uses up a lot of CI minutes
+    if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
+    needs: [ lint ]
+    strategy:
+      matrix:
+        target: [x64, universal2]
+
     steps:
       - uses: actions/checkout@v3
 
-      - run: |
-          sudo apt-get update
-          sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs
-
       - name: Set up cargo cache
         uses: Swatinem/rust-cache@v2
         with:
           # See: https://github.com/rerun-io/rerun/pull/497
           save-if: ${{ github.event_name == 'push'}}
 
-      - name: Set up Python
+      - name: Setup python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: ${{ env.PYTHON_VERSION }}
           cache: "pip"
           cache-dependency-path: "rerun_py/requirements-build.txt"
+          architecture: x64
 
-      - name: Install Python dependencies
+      - name: Build wheel - x86_64
+        if: matrix.target == 'x64'
+        uses: PyO3/maturin-action@v1
+        with:
+          maturin-version: "0.14.10"
+          command: build
+          target: x86_64
+          args: |
+            --manifest-path rerun_py/Cargo.toml
+            --release
+            --out dist
+
+      - name: Build wheel - universal2
+        if: matrix.target == 'universal2'
+        uses: PyO3/maturin-action@v1
+        with:
+          maturin-version: "0.14.10"
+          command: build
+          args: |
+            --manifest-path rerun_py/Cargo.toml
+            --release
+            --out dist
+            --universal2
+
+      - name: Install built wheel
         run: |
-          pip install --upgrade pip
-          pip install -r rerun_py/requirements-build.txt
-          pip install ./rerun_py
+          pip install rerun-sdk --find-links dist --force-reinstall
+          pip install pytest #TODO(john) move to requirements-build.txt
+          cd rerun_py/tests && pytest
 
       - name: Run Car example
         run: |
-          pip install -r examples/car/requirements.txt && ./examples/car/main.py --headless
+          pip install -r examples/car/requirements.txt
+          ./examples/car/main.py --headless
 
-  py-pip-install-linux:
-    name: Pip install Rerun SDK (Linux)
-    runs-on: ubuntu-latest
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+  # ---------------------------------------------------------------------------
+
+  windows:
+    name: Build Python Wheels for Windows
+    runs-on: windows-latest-8-cores
+    needs: [ lint ]
+    strategy:
+      matrix:
+        target: [x64]
     steps:
       - uses: actions/checkout@v3
 
-      - run: |
-          sudo apt-get update
-          sudo apt-get install libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev # eframe reqs
-
       - name: Set up cargo cache
         uses: Swatinem/rust-cache@v2
         with:
           # See: https://github.com/rerun-io/rerun/pull/497
           save-if: ${{ github.event_name == 'push'}}
 
-      - name: Setup python for building wheel
+      - name: Setup python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          architecture: ${{ matrix.target }}
+          python-version: ${{ env.PYTHON_VERSION }}
           cache: "pip"
+          cache-dependency-path: "rerun_py/requirements-build.txt"
 
-      - name: Install rerun_py
-        run: pip install ./rerun_py
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: |
+            --manifest-path rerun_py/Cargo.toml
+            --release
+            --out dist
 
-  py-pip-install-windows:
-    name: Pip install Rerun SDK (Windows)
-    runs-on: windows-latest
-    steps:
-      - uses: actions/checkout@v3
+      - name: Install built wheel
+        run: |
+          pip install rerun-sdk --find-links dist --force-reinstall
+          pip install pytest
+          cd rerun_py/tests && pytest
 
-      - name: Set up cargo cache
-        uses: Swatinem/rust-cache@v2
+      - name: Run Car example
+        run: |
+          pip install -r examples/car/requirements.txt
+          ./examples/car/main.py --headless
 
-      - name: Setup python for building wheel
-        uses: actions/setup-python@v4
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
         with:
-          python-version: "3.8"
-          cache: "pip"
+          name: wheels
+          path: dist
 
-      - name: Install rerun_py
-        run: pip install ./rerun_py
+  # ---------------------------------------------------------------------------
 
-  py-pip-install-macos:
-    name: Pip install Rerun SDK (macOS)
-    runs-on: macos-latest
-    if: "startsWith(github.ref, 'refs/tags/')" # slow; uses up a lot of CI minutes
+  linux:
+    name: Build Python Wheels for Linux
+    runs-on: ubuntu-latest-16-cores
+    needs: [ lint ]
     steps:
       - uses: actions/checkout@v3
 
+      - name: Cache APT Packages
+        #uses: awalsh128/cache-apt-pkgs-action@v1.2.2
+        #TODO(john) merge upstream
+        uses: rerun-io/cache-apt-pkgs-action@59534850182063abf1b2c11bb3686722a12a8397
+        with:
+          packages: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libspeechd-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev
+          version: 1.0
+          execute_install_scripts: true
+
       - name: Set up cargo cache
         uses: Swatinem/rust-cache@v2
         with:
+          env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY
           # See: https://github.com/rerun-io/rerun/pull/497
           save-if: ${{ github.event_name == 'push'}}
 
-      - name: Setup python for building wheel
+      - name: Setup python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: ${{ env.PYTHON_VERSION }}
+          architecture: x64
           cache: "pip"
+          cache-dependency-path: "rerun_py/requirements-build.txt"
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          maturin-version: "0.14.10"
+          target: ${{ matrix.target }}
+          manylinux: auto
+          container: off
+          args: |
+            --manifest-path rerun_py/Cargo.toml
+            --release
+            --sdist
+            --out dist
+
+      - name: Install built wheel
+        run: |
+          pip install rerun-sdk --find-links dist --force-reinstall
+          pip install pytest
+          cd rerun_py/tests && pytest
+
+      - name: Run Car example
+        run: |
+          pip install -r examples/car/requirements.txt
+          ./examples/car/main.py --headless
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+  # ---------------------------------------------------------------------------
+
+  # See https://github.com/marvinpinto/action-automatic-releases#automatically-generate-a-pre-release-when-changes-land-on-master for details
+  pre-release:
+    name: Pre Release
+    needs: [ macos, windows, linux ]
+    if: github.ref == 'refs/heads/main'
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Download Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+      - name: Github Release
+        uses: marvinpinto/action-automatic-releases@v1.2.1
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: "latest"
+          prerelease: true
+          title: "Development Build"
+          files: dist/* CHANGELOG.md
+
+  # ---------------------------------------------------------------------------
+
+  # This job is run on tags starting with "v", e.g., "v0.1.0"
+  tagged-release:
+    name: Release
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: [ macos, windows, linux ]
+    steps:
+      - name: Download Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+      #TODO(#966) https://github.com/rerun-io/rerun/issues/1054
+      # Santiy check the git tag vs. the crate version.
+
+      - name: Github Release
+        uses: "marvinpinto/action-automatic-releases@1.2.1"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          prerelease: false
+          files: dist/* CHANGELOG.md
+
+      - name: Publish to PyPI
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.MATURIN_PYPI_TOKEN }}
+        uses: PyO3/maturin-action@v1
+        with:
+          command: upload
+          args: --skip-existing *
 
-      - name: Install rerun_py
-        run: pip install ./rerun_py
+  # ---------------------------------------------------------------------------
 
   py-test-docs:
     name: Verify the docs build