Use Prebuilt TileDB Binaries For CI (#1474)

* Azure set TILEDB_FORCE_ALL_DEPS=ON * Use 2.13.0
TileDB-Inc · Dec 1, 2022 · d04c802 · d04c802
1 parent 1e344b7
commit d04c802
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 26 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -41,51 +41,75 @@ jobs:
         run: pip debug --verbose
 
       - name: "Install dependencies"
-        run: |
-          python -m pip install --upgrade -r misc/requirements_ci.txt
+        run: python -m pip install --upgrade -r misc/requirements_ci.txt
 
-      - name: "Check build directory"
-        run: |
-            ls -Rl
+      - name: "Get TILEDB_VERSION"
+        run: echo "LIBTILEDB_VERSION=$(python setup.py get_tiledb_version | tail -n 1)" >> $GITHUB_ENV
 
-      - name: "Build TileDB and TileDB-Py extension (Windows)"
+      - name: "Get LIBTILEDB_SHA"
+        run: echo "LIBTILEDB_SHA=$(git ls-remote https://github.com/TileDB-Inc/TileDB $LIBTILEDB_VERSION | cut -c1-7)" >> $GITHUB_ENV
+
+      - name: "Download TileDB From Zip And Build TileDB-Py (Windows)"
         run: |
-          python setup.py build_ext --inplace
-          python setup.py install
+          choco install wget --no-progress
+
+          if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-windows-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.zip; then
+            unzip tiledb-windows-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.zip -d /usr/local
+            cp /usr/local/bin/tiledb.dll tiledb
+            python setup.py develop --tiledb=/usr/local
+          else 
+            # Build from source as fallback
+            python setup.py build_ext --inplace
+            python setup.py develop
+          fi
+        env:
+          TILEDB_FORCE_ALL_DEPS: True
         if: matrix.os == 'windows-latest'
 
-      - name: "Build TileDB and TileDB-Py extension (POSIX)"
+      - name: "Download TileDB From Tarball And Build TileDB-Py (macOS)"
         run: |
           set -xeo pipefail
-          python setup.py build_ext --inplace --werror
-          python setup.py install
-        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-11'
 
-      - name: "Run tests"
+          if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-macos-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz; then
+            sudo tar -vzxf tiledb-macos-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz -C /usr/local
+            python setup.py develop --tiledb=/usr/local
+          else 
+            # Build from source as fallback
+            python setup.py build_ext --inplace
+            python setup.py install
+          fi
+        if: matrix.os == 'macos-11'
+
+      - name: "Download TileDB From Tarball And Build TileDB-Py (Linux)"
         run: |
           set -xeo pipefail
 
-          pytest -vv --showlocals
+          if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-linux-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz; then
+            sudo tar -vzxf tiledb-linux-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz -C /usr/local
+            python setup.py develop --tiledb=/usr/local
+          else 
+            # Build from source as fallback
+            python setup.py build_ext --inplace
+            python setup.py install
+          fi
+        if: matrix.os == 'ubuntu-latest'
 
-          # Test wheel build, install, and run
-          python setup.py bdist_wheel
-          mkdir /tmp/wheel_test
-          cp dist/*.whl /tmp/wheel_test
-          pushd /tmp/wheel_test
-          ls
-          python -m pip install *.whl
-          python -c 'import tiledb ; tiledb.libtiledb.version()'
+      - name: "Check build directory"
+        run: ls -Rl
+
+      - name: "Run tests"
+        run: pytest -vv --showlocals
 
       - name: "Print log files (failed build only)"
         run: |
           set -xeo pipefail
           # Display log files if the build failed
           echo 'Dumping log files for failed build'
           echo '----------------------------------'
-          for f in $(find $BUILD_REPOSITORY_LOCALPATH/build -name *.log);
+          for f in $(find build -name *.log);
             do echo '------'
                echo $f
                echo '======'
                cat $f
             done;
-        if: ${{ failure() }}
+        if: failure()
diff --git a/misc/azure-ci.yml b/misc/azure-ci.yml
@@ -48,6 +48,7 @@ stages:
             displayName: "Install dependencies"
 
           - script: |
+              set TILEDB_FORCE_ALL_DEPS=ON
               # vcvarsall is necessary so that numpy uses the correct compiler
               call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
               python setup.py build_ext --inplace

diff --git a/setup.py b/setup.py
@@ -21,7 +21,7 @@
 ### DO NOT USE ON CI
 
 # Target branch: Note that this should be set to the current core release, not `dev`
-TILEDB_VERSION = "2.12.3"
+TILEDB_VERSION = "2.13.0"
 
 # allow overriding w/ environment variable
 TILEDB_VERSION = os.environ.get("TILEDB_VERSION") or TILEDB_VERSION
@@ -423,7 +423,7 @@ class LazyCommandClass(dict):
     """
 
     def __contains__(self, key):
-        if key in ("build_ext", "bdist_wheel", "bdist_egg"):
+        if key in ("build_ext", "bdist_wheel", "bdist_egg", "get_tiledb_version"):
             return True
         return super().__contains__(key)
 
@@ -439,6 +439,9 @@ def __getitem__(self, key):
             return self.make_bdist_wheel_cmd()
         elif key == "bdist_egg":
             return self.make_bdist_egg_cmd()
+        elif key == "get_tiledb_version":
+            print(TILEDB_VERSION)
+            exit()
         else:
             return super().__getitem__(key)