Update python from 3.10 to 3.11, update vcpkg (#933)

Cascade of changes to get the latest vcpkg to work to build kart * Update vcpkg to latest (as of 2023-11-01) * Remove pdal overlay-port * Install autoconf-archive on linux * Update pyodbc to 5.0.1 * Update psycopg2 to 2.9.9 * Update pysqlite to 0.5.2 * Remove pyodbc.pyi from spec * Various python3.10 -> python3.11 changes * Update pyinstaller to 6.1.0 * Handle _internal folder in newer pyinstaller * Fix for cpack-wix-patch.xml * Find mod_spatialite inside _internal or Frameworks, further kart.spec fixes * Add fail-fast=false for macos matrix * Change get-cmake@arm64-win-linux -> @latest * Pin run-cmake at v10.6 * Fix codesign on macos * Set spatialite_path from kart * Add odbc_config dir to path while building pyodbc * pin get-cmake action to v3.25.2 in attempt to workaround glibc issue * vendor: Spatialite update, keep necessary local patches (eg ok-load-permanently.patch) * gdal: backport upstream fix wrt spatialite versioning * use run-vcpkg@v11 everywhere * use AWS docker image mirrors * Disable proj linkage from spatialite on macos/linux * Reinstate ok-load-permanently.patch * update vcpkg: - python3: explicitly disable tkinter. microsoft/vcpkg#34998 * linux package tests: python3.10 -> python3.11 * Rebuild py-requirements with python3.11 * Mostly documentation - python3.10 -> python3.11 * Update changelog --------- Co-authored-by: rcoup <robert.coup@koordinates.com>
koordinates · Nov 9, 2023 · 436a2d9 · 436a2d9
1 parent bb753cf
commit 436a2d9
Show file tree

Hide file tree

Showing 35 changed files with 232 additions and 523 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ name: Build
 on: [push, pull_request]
 
 env:
-  PY_VER: "3.10"
+  PY_VER: "3.11"
   CMAKE_VERSION: "~3.25.0"
   FORCE_COLOR: "YES"
   KART_S3_TEST_DATA_POINT_CLOUD: "s3://kart-bring-your-own-data-poc/auckland-small-laz1.2/*.laz"
@@ -55,7 +55,7 @@ jobs:
         ports:
           - 1433:1433
       mysql:
-        image: mysql
+        image: public.ecr.aws/docker/library/mysql:latest
         options: >-
           -e MYSQL_ROOT_PASSWORD=PassWord1
         ports:
@@ -71,7 +71,7 @@ jobs:
       #
 
       - name: "setup: cmake & ninja"
-        uses: lukka/get-cmake@arm64-win-linux
+        uses: lukka/get-cmake@v3.25.2  # workaround for node 20 issue
         with:
           cmakeVersion: "${{ env.CMAKE_VERSION }}"
 
@@ -84,17 +84,16 @@ jobs:
       - name: "setup: misc"
         run: |
           sudo apt-get update -q -y
-          sudo apt-get install -q -y --no-install-recommends ccache
+          sudo apt-get install -q -y --no-install-recommends autoconf-archive ccache
           mkdir -p ${{ env.CCACHE_DIR }}
           echo "/usr/lib/ccache" >> $GITHUB_PATH
 
       - name: "setup: vcpkg"
-        uses: lukka/run-vcpkg@v10
+        uses: lukka/run-vcpkg@v11.1
         with:
           vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg"
           vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json"
-          prependedCacheKey: "20221110a-ubuntu-22.04"
-          appendedCacheKey: "C${{ env.CMAKE_VERSION}}:${{ hashFiles('vcpkg-vendor/vcpkg-overlay-triplets/**', 'vcpkg-vendor/vcpkg-overlay-ports/**', 'vcpkg-vendor/vcpkg-toolchain-setup.cmake') }}"
+          doNotCache: false  # do-cache
 
       #
       # App Build
@@ -117,17 +116,12 @@ jobs:
           echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT
 
       - name: "app: configuration & vendor dependencies"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-linux
 
-      - name: "check for vcpkg errors"
-        if: "failure() || cancelled()"
-        run: |
-          echo "RUNVCPKG_NO_CACHE=1" >> $GITHUB_ENV
-
       - name: "app: build"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-linux
           buildPreset: ci-linux
@@ -161,7 +155,7 @@ jobs:
           mv fienode /usr/local/bin/
 
       - name: "test: unit tests"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-linux
           testPreset: ci-linux
@@ -175,7 +169,7 @@ jobs:
       #
 
       - name: "bundle: assemble"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-linux
           buildPreset: ci-bundle-linux
@@ -186,7 +180,7 @@ jobs:
           ./build/pyinstaller/dist/kart/kart --version
 
       - name: "bundle: e2e tests"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-linux
           testPreset: ci-e2e-linux
@@ -261,7 +255,7 @@ jobs:
         ports:
           - 1433:1433
       mysql:
-        image: mysql
+        image: public.ecr.aws/docker/library/mysql:latest
         options: >-
           -e MYSQL_ROOT_PASSWORD=PassWord1
         ports:
@@ -286,7 +280,7 @@ jobs:
       image: ${{ matrix.os.image }}
       env:
         ACLOCAL_PATH: "/usr/local/share/aclocal:/usr/share/aclocal"
-        PATH: "/opt/python/cp310-cp310/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/github/home/.cargo/bin:/opt/mssql-tools/bin"
+        PATH: "/opt/python/cp311-cp311/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/github/home/.cargo/bin:/opt/mssql-tools/bin"
 
     steps:
       - uses: actions/checkout@v3
@@ -297,7 +291,7 @@ jobs:
       # setup
       #
       - name: "setup: cmake & ninja"
-        uses: lukka/get-cmake@arm64-win-linux
+        uses: lukka/get-cmake@v3.25.2  # workaround for node 20 issue
         with:
           cmakeVersion: "${{ env.CMAKE_VERSION }}"
 
@@ -313,7 +307,7 @@ jobs:
 
           git config --global --add safe.directory ${GITHUB_WORKSPACE}
 
-          yum install -y perl-IPC-Cmd rpm-build unixODBC zip
+          yum install -y autoconf-archive perl-IPC-Cmd rpm-build unixODBC zip
 
           echo "installing golang..."
           curl -qL "https://go.dev/dl/go1.19.2.linux-${ARCH}.tar.gz" | tar xz -C /usr/local/
@@ -324,12 +318,11 @@ jobs:
           fi
 
       - name: "setup: vcpkg"
-        uses: lukka/run-vcpkg@v10
+        uses: lukka/run-vcpkg@v11.1
         with:
-          vcpkgDirectory: "vcpkg-vendor/vcpkg"
+          vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg"
           vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json"
-          prependedCacheKey: "20230113a-linux-${{ matrix.os.label }}"
-          appendedCacheKey: "C${{ env.CMAKE_VERSION}}:${{ hashFiles('vcpkg-vendor/vcpkg-overlay-triplets/**', 'vcpkg-vendor/vcpkg-overlay-ports/**', 'vcpkg-vendor/vcpkg-toolchain-setup.cmake') }}"
+          doNotCache: false  # do-cache
 
       - name: "setup: pkgconf"
         working-directory: /__w/_temp
@@ -359,19 +352,13 @@ jobs:
           echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT
 
       - name: "app: configuration & vendor dependencies"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         env:
           LD_LIBRARY_PATH: "${{ env.LD_LIBRARY_PATH }}:${{ env.GITHUB_WORKSPACE }}/build/vcpkg_installed/${{ env.ARCH_TRIPLET }}/lib"
         with:
           configurePreset: ci-linux
           configurePresetAdditionalArgs: ${{ env.OPTIONS_FOR_CMAKE }}
 
-      - name: "check for vcpkg errors"
-        if: "failure() || cancelled()"
-        shell: bash
-        run: |
-          echo "RUNVCPKG_NO_CACHE=1" >> $GITHUB_ENV
-
       - name: "app: build"
         shell: bash
         run: |
@@ -486,34 +473,34 @@ jobs:
             runner: buildjet-2vcpu-ubuntu-2204-arm
         os:
           - label: "tgz-ubuntu-22.04"
-            image: "ubuntu:jammy"
+            image: "public.ecr.aws/ubuntu/ubuntu:jammy"
             type: "tgz"
           - label: "ubuntu-22.04"
-            image: "ubuntu:jammy"
+            image: "public.ecr.aws/ubuntu/ubuntu:jammy"
             type: "deb"
           - label: "ubuntu-20.04"
-            image: "ubuntu:focal"
+            image: "public.ecr.aws/ubuntu/ubuntu:focal"
             type: "deb"
           - label: "ubuntu-18.04"
-            image: "ubuntu:bionic"
+            image: "public.ecr.aws/ubuntu/ubuntu:bionic"
             type: "deb"
           - label: "debian-stable"
-            image: "debian:stable-slim"
+            image: "public.ecr.aws/docker/library/debian:stable-slim"
             type: "deb"
           - label: "debian-oldstable"
-            image: "debian:oldstable-slim"
+            image: "public.ecr.aws/docker/library/debian:oldstable-slim"
             type: "deb"
           - label: "amazonlinux-2023"
-            image: "amazonlinux:2023"
+            image: "public.ecr.aws/amazonlinux/amazonlinux:2023"
             type: "rpm"
           - label: "fedora-latest"
-            image: "fedora:latest"
+            image: "public.ecr.aws/docker/library/fedora:latest"
             type: "rpm"
           - label: "almalinux-9"
-            image: "almalinux:9"
+            image: "public.ecr.aws/docker/library/almalinux:9"
             type: "rpm"
           - label: "almalinux-8"
-            image: "almalinux:8"
+            image: "public.ecr.aws/docker/library/almalinux:8"
             type: "rpm"
 
     runs-on: ${{ matrix.arch.runner }}
@@ -614,7 +601,7 @@ jobs:
     env:
       CCACHE_DIR: ${{ github.workspace }}/.cache/ccache
       CCACHE_COMPRESS: "1"
-      PY_VER_ID: "cp310-cp310"
+      PY_VER_ID: "cp311-cp311"
 
     steps:
       - uses: actions/checkout@v3
@@ -633,7 +620,7 @@ jobs:
           cache-dependency-path: 'requirements/*.txt'
 
       - name: "setup: cmake & ninja"
-        uses: lukka/get-cmake@arm64-win-linux
+        uses: lukka/get-cmake@v3.25.2  # workaround for node 20 issue
         with:
           cmakeVersion: "${{ env.CMAKE_VERSION }}"
 
@@ -646,7 +633,7 @@ jobs:
       - name: "setup: misc"
         run: |
           sudo apt-get update -q -y
-          sudo apt-get install -q -y --no-install-recommends ccache unixodbc file sqlite3
+          sudo apt-get install -q -y --no-install-recommends autoconf-archive ccache unixodbc file sqlite3
           mkdir -p ${{ env.CCACHE_DIR }}
           echo "/usr/lib/ccache" >> $GITHUB_PATH
 
@@ -718,6 +705,7 @@ jobs:
   #
   macOS:
     strategy:
+      fail-fast: false
       matrix:
         os:
           - id: macos-12
@@ -748,7 +736,7 @@ jobs:
       MACOS_PKGSIGN_ID: ${{ secrets.MACOS_PKGSIGN_ID }}
       MACOS_NOTARIZE_KEYCHAIN_PROFILE: "NOTARIZE_AUTH"
       # X.Y version needs to match PY_VER:
-      PY_VER_INSTALLER: "https://www.python.org/ftp/python/3.10.9/python-3.10.9-macos11.pkg"
+      PY_VER_INSTALLER: "https://www.python.org/ftp/python/3.11.6/python-3.11.6-macos11.pkg"
 
     steps:
       - uses: actions/checkout@v3
@@ -855,12 +843,12 @@ jobs:
           fi
 
       - name: "app: configuration & vendor dependencies"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-macos
 
       - name: "app: build"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-macos
           buildPreset: ci-macos
@@ -889,7 +877,7 @@ jobs:
           sudo mv clone_checker /usr/local/bin/
 
       - name: "test: unit tests"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-macos
           testPreset: ci-macos
@@ -919,7 +907,7 @@ jobs:
               --password "$MACOS_NOTARIZE_PW"
 
       - name: "bundle: assemble"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-macos
           buildPreset: ci-bundle-macos
@@ -930,7 +918,7 @@ jobs:
           ./build/pyinstaller/dist/Kart.app/Contents/MacOS/kart --version
 
       - name: "bundle: e2e tests"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-macos
           testPreset: ci-e2e-macos
@@ -1041,12 +1029,11 @@ jobs:
           cache-dependency-path: 'requirements/*.txt'
 
       - name: "setup: vcpkg"
-        uses: lukka/run-vcpkg@v10
+        uses: lukka/run-vcpkg@v11
         with:
           vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg"
           vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json"
-          prependedCacheKey: "20221201a"
-          appendedCacheKey: "C${{ env.CMAKE_VERSION}}:N${{ env.NINJA_VERSION }}:${{ hashFiles('vcpkg-vendor/vcpkg-overlay-triplets/**', 'vcpkg-vendor/vcpkg-overlay-ports/**', 'vcpkg-vendor/vcpkg-toolchain-setup.cmake') }}"
+          doNotCache: false  # do-cache
 
       - name: "setup: misc"
         shell: pwsh
@@ -1084,17 +1071,12 @@ jobs:
           fi
 
       - name: "app: configuration & vendor dependencies"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-windows
 
-      - name: "check for vcpkg errors"
-        if: failure() || cancelled()
-        run: |
-          echo "RUNVCPKG_NO_CACHE=1" >> $GITHUB_ENV
-
       - name: "app: build"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-windows
           buildPreset: ci-windows
@@ -1116,7 +1098,7 @@ jobs:
           ./build/kart.cmd --version
 
       - name: "test: unit tests"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-windows
           testPreset: ci-windows
@@ -1126,7 +1108,7 @@ jobs:
       #
 
       - name: "bundle: assemble"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         env:
           SIGN_AZURE_VAULT: ${{ secrets.WIN_SIGN_AZURE_VAULT }}
           SIGN_AZURE_CLIENTID: ${{ secrets.WIN_SIGN_AZURE_CLIENTID }}
@@ -1142,7 +1124,7 @@ jobs:
           ./build/pyinstaller/dist/kart/kart.exe --version
 
       - name: "bundle: e2e tests"
-        uses: lukka/run-cmake@v10
+        uses: lukka/run-cmake@v10.6
         with:
           configurePreset: ci-windows
           testPreset: ci-e2e-windows

diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "vcpkg-vendor/vcpkg"]
 	path = vcpkg-vendor/vcpkg
 	url = https://github.com/koordinates/vcpkg.git
-	branch = kx-master
+	branch = kx-20231101
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ _When adding new entries to the changelog, please include issue/PR numbers where
 - Fixes a bug where Kart would misidentify a non-Kart repo as a Kart V1 repo in some circumstances. [#918](https://github.com/koordinates/kart/issues/918)
 - Improve schema extraction for point cloud datasets. [#924](https://github.com/koordinates/kart/issues/924)
 - Some tweaks to `--dry-run` output of Kart LFS commands. [#932](https://github.com/koordinates/kart/pull/932)
+- Now using Python 3.11 to build Kart, and vendored dependencies have been updated to newer versions. [#933](https://github.com/koordinates/kart/pull/933)
 
 ## 0.14.2
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -355,8 +355,10 @@ if(MACOS)
           COMMAND ${XCODE_CODESIGN} --display --verbose \"\${CHECK_DIR}/\${CPACK_PACKAGE_FILE_NAME}/Kart.app\"
           COMMAND_ECHO STDERR
           COMMAND_ERROR_IS_FATAL ANY)
+        # codesign --verify --strict=all only works in macOS 15 and later,
+        # a bug in earlier versions means it fails with 'file not found'
         execute_process(
-          COMMAND ${XCODE_CODESIGN} --verify --verbose --deep --strict=all \"\${CHECK_DIR}/\${CPACK_PACKAGE_FILE_NAME}/Kart.app\"
+          COMMAND ${XCODE_CODESIGN} --verify --verbose --deep \"\${CHECK_DIR}/\${CPACK_PACKAGE_FILE_NAME}/Kart.app\"
           COMMAND_ECHO STDERR
           COMMAND_ERROR_IS_FATAL ANY)