From 5fefb1ab6692db7473a20dbcc7501daed434b51c Mon Sep 17 00:00:00 2001 From: Anindya Roy Date: Tue, 14 May 2024 12:22:32 +0530 Subject: [PATCH] NXDRIVE-2926: Update github Action Runner to use mac-latest 01 --- .github/workflows/release.yml | 23 +- docs/changes/5.5.0.md | 6 + posix-deploy_ci_agent | 451 ------------------------------ tools/deps/requirements.txt | 6 +- tools/posix/deploy_ci_agent.sh | 7 +- tools/windows/deploy_ci_agent.ps1 | 5 +- 6 files changed, 15 insertions(+), 483 deletions(-) delete mode 100644 posix-deploy_ci_agent diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index af9ff77ad1..184277234e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -104,33 +104,14 @@ jobs: #- name: "[macOS] Install Python" # if: matrix.os == 'macos-latest' - # run: | - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # brew install pyenv - # pyenv install 3.9.5 - # echo ">>>> Python Version: '$(python3 --version)'" - # pyenv global 3.9.5 - # eval "$(pyenv init -)" - # eval "$(pyenv init --path)" - # eval "$(pyenv virtualenv-init -)" - # echo "sudo installer -pkg python.pkg -target /" - # echo ">>>> Current Python Version: '$(python3 --version)'" + # run: sudo installer -pkg python.pkg -target / - name: "[macOS] Setup the environment" if: matrix.os == 'macos-latest' env: MACOSX_DEPLOYMENT_TARGET: "11.3" SYSTEM_VERSION_COMPAT: 0 - run: | - bash tools/osx/deploy_ci_agent.sh --install-release - echo "-----python-----" - python3 --version - - echo "---pwd---" - pwd - - echo "---ls-a---" - ls -a + run: bash tools/osx/deploy_ci_agent.sh --install-release - name: "[macOS] Generate the .dmg and validate against 2021" if: matrix.os == 'macos-latest' diff --git a/docs/changes/5.5.0.md b/docs/changes/5.5.0.md index 157c57adb6..cf855cb53f 100644 --- a/docs/changes/5.5.0.md +++ b/docs/changes/5.5.0.md @@ -22,6 +22,7 @@ Release date: `2024-xx-xx` ## Packaging / Build +- [NXDRIVE-2926] (https://jira.nuxeo.com/browse/NXDRIVE-2926): Update github Action Runner to use mac-latest - [NXDRIVE-2896](https://jira.nuxeo.com/browse/NXDRIVE-2896): Fix release build for upload/download artifact ## Tests @@ -36,6 +37,9 @@ Release date: `2024-xx-xx` - Added `build` 1.1.1 - Added `pyproject-hooks` 1.0.0 +- Added `PyQt5-Qt5` 5.15.13 (for MAC) +- Added `PyQt5-Qt5` 5.15.2 (for Windows and Linux) +- Added `setuptools` 69.5.1 - Added `tomli` 2.0.1 - Removed `toml` 0.10.2 - Upgraded `actions/cache` from 3 to 4 @@ -83,6 +87,8 @@ Release date: `2024-xx-xx` - Upgraded `pyobjc-framework-fsevents` from 7.3 to 10.1 - Upgraded `pyobjc-framework-scriptingbridge` from 7.3 to 10.1 - Upgraded `pyobjc-framework-systemconfiguration` from 7.3 to 10.1 +- Upgraded `pyqt5-sip` from 12.8.1 to 12.13.0 +- Upgraded `pyqt5` from 5.15.2 to 5.15.10 - Upgraded `pytest` from 7.4.0 to 7.4.4 - Upgraded `pytest-timeout` from 2.0.2 to 2.2.0 - Upgraded `pytest-xdist` from 3.3.1 to 3.5.0 diff --git a/posix-deploy_ci_agent b/posix-deploy_ci_agent deleted file mode 100644 index 3b84c57987..0000000000 --- a/posix-deploy_ci_agent +++ /dev/null @@ -1,451 +0,0 @@ -#!/bin/bash -# Shared functions for tools/$OSI/deploy_ci_agent.sh files. -# -# Usage: sh tools/$OSI/deploy_ci_agent.sh [ARG] -# -# Possible ARG: -# --build: build the package -# --check: check AppImage conformity (GNU/Linux only) -# --check-upgrade: check the auto-update works -# --install: install all dependencies -# --install-python: install only Python -# --install-release: install all but test dependencies -# --start: start Nuxeo Drive -# --tests: launch the tests suite -# -# See /docs/deployment.md for more information. -# -# --- -# -# You can tweak tests checks by setting the SKIP envar: -# - SKIP=rerun to not rerun failed test(s) -# -# There is no strict syntax about multiple skips (coma, coma + space, no separator, ... ). -# - -set -e - -# Global variables -PYTHON="python3 -Xutf8 -E -s" -PYTHON_VENV="./venv/bin/python -Xutf8 -E -s" -PYTHON_OPT="${PYTHON_VENV} -OO" -PIP="${PYTHON_OPT} -m pip install --no-cache-dir --upgrade --upgrade-strategy=only-if-needed --progress-bar=off" - -build_installer() { - local version - - echo ">>> Building the release package" - ${PYTHON_VENV} -m PyInstaller ndrive.spec --clean --noconfirm - - # Do some clean-up - ${PYTHON_VENV} tools/cleanup_application_tree.py dist/ndrive - - # Remove compiled QML files - find dist -depth -type f -name "*.qmlc" -delete - - # Remove empty folders - find dist -depth -type d -empty -delete - - if [ "${OSI}" = "osx" ]; then - ${PYTHON_VENV} tools/cleanup_application_tree.py dist/*.app/Contents/Resources - ${PYTHON_VENV} tools/cleanup_application_tree.py dist/*.app/Contents/MacOS - - # Move problematic folders out of Contents/MacOS - ${PYTHON_VENV} tools/osx/fix_app_qt_folder_names_for_codesign.py dist/*.app - - # Remove broken symlinks pointing to an inexistent target - find dist/*.app/Contents/MacOS -type l -exec sh -c 'for x; do [ -e "$x" ] || rm -v "$x"; done' _ {} + - elif [ "${OSI}" = "linux" ]; then - remove_excluded_files dist/ndrive - fi - - # Stop now if we only want the application to be frozen (for integration tests) - if [ "${FREEZE_ONLY:-0}" = "1" ]; then - exit 0 - fi - - if [ "${ZIP_NEEDED:-0}" = "1" ]; then - version="$(grep __version__ nxdrive/__init__.py | cut -d'"' -f2)" - cd dist - zip -9 -q -r "nuxeo-drive-${OSI}-${version}.zip" "ndrive" - cd - - fi - - create_package -} - -check_import() { - # Check module import to know if it must be installed - # i.e: check_import "from PyQt4 import QtWebKit" - # or: check_import "import cx_Freeze" - local import="$1" - local ret=0 - - /bin/echo -n ">>> Checking Python code: ${import} ... " - ${PYTHON_VENV} -c "${import}" 2>/dev/null || ret=1 - if [ ${ret} -ne 0 ]; then - echo "Failed." - return 1 - fi - echo "OK." -} - -check_upgrade() { - # Ensure a new version can be released by checking the auto-update process. - ${PYTHON_VENV} tools/scripts/check_update_process.py -} - -check_vars() { - # Check required variables - if [ "${PYTHON_DRIVE_VERSION:-unset}" = "unset" ]; then - export PYTHON_DRIVE_VERSION="3.9.5" # XXX_PYTHON - fi - if [ "${WORKSPACE:-unset}" = "unset" ]; then - if [ "${GITHUB_WORKSPACE:-unset}" != "unset" ]; then - # Running from GitHub Actions - WORKSPACE="$(dirname "${GITHUB_WORKSPACE}")" - export WORKSPACE - else - echo "WORKSPACE not defined. Aborting." - exit 1 - fi - fi - if [ "${OSI:-unset}" = "unset" ]; then - echo "OSI not defined. Aborting." - echo "Please do not call this script directly. Use the good one from 'tools/OS/deploy_ci_agent.sh'." - exit 1 - fi - if [ "${WORKSPACE_DRIVE:-unset}" = "unset" ]; then - if [ -d "${WORKSPACE}/sources" ]; then - export WORKSPACE_DRIVE="${WORKSPACE}/sources" - elif [ -d "${WORKSPACE}/nuxeo-drive" ]; then - export WORKSPACE_DRIVE="${WORKSPACE}/nuxeo-drive" - else - export WORKSPACE_DRIVE="${WORKSPACE}" - fi - fi - export STORAGE_DIR="${WORKSPACE}/deploy-dir" - - echo " PYTHON_DRIVE_VERSION = ${PYTHON_DRIVE_VERSION}" - echo " WORKSPACE = ${WORKSPACE}" - echo " WORKSPACE_DRIVE = ${WORKSPACE_DRIVE}" - echo " STORAGE_DIR = ${STORAGE_DIR}" - - cd "${WORKSPACE_DRIVE}" - - if [ "${SPECIFIC_TEST:-unset}" = "unset" ] || [ "${SPECIFIC_TEST}" = "" ]; then - export SPECIFIC_TEST="tests" - else - echo " SPECIFIC_TEST = ${SPECIFIC_TEST}" - export SPECIFIC_TEST="tests/${SPECIFIC_TEST}" - fi - - if [ "${SKIP:-unset}" = "unset" ]; then - export SKIP="" - else - echo " SKIP = ${SKIP}" - fi -} - -create_venv() { - # Create the isolated virtual environment - # No-op if the folder already exists. - # Note: the Python version will be checked later in the process to prevent any issues. - echo "**** [pyenv] create_venv ${version}" - echo "**** installing python" - install_python "${PYTHON_DRIVE_VERSION}" - echo "**** installing python" - ${PYTHON} -m venv --copies venv -} - -install_deps() { - echo ">>> Installing requirements" - ${PIP} -r tools/deps/requirements-pip.txt - ${PIP} -r tools/deps/requirements.txt - ${PIP} -r tools/deps/requirements-dev.txt - if [ "${INSTALL_RELEASE_ARG:-0}" != "1" ]; then - ${PIP} -r tools/deps/requirements-tests.txt - ./venv/bin/pre-commit install - fi -} - -install_pyenv() { - local url="https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer" - local venv_plugin - local venv_plugin_url="https://github.com/yyuu/pyenv-virtualenv.git" - - export PYENV_ROOT="${STORAGE_DIR}/.pyenv" - export PATH="${PYENV_ROOT}/bin:$PATH" - - venv_plugin="${PYENV_ROOT}/plugins/pyenv-virtualenv" - - if [ "${INSTALL_ARG:-0}" = "1" ]; then - if [ ! -d "${PYENV_ROOT}" ]; then - echo ">>> [pyenv] Downloading and installing" - curl -L "${url}" | bash - else - echo ">>> [pyenv] Updating" - cd "${PYENV_ROOT}" - # git is not available from the docker image, this is on purpose - git pull || true - cd - - fi - if [ ! -d "${venv_plugin}" ]; then - echo ">>> [pyenv] Installing virtualenv plugin" - git clone "${venv_plugin_url}" "${venv_plugin}" - fi - fi - - echo ">>> [pyenv] Initializing" - # Ensure pyenv shims are added to PATH, see https://github.com/pyenv/pyenv/issues/1906 - eval "$(pyenv init -)" - eval "$(pyenv init --path)" - eval "$(pyenv virtualenv-init -)" -} - -install_python() { - local version="$1" - - # To fix this error when building the package "libpythonXY.dylib does not exist" - # Some of 3rd party tool like PyInstaller might require CPython installation - # built with --enable-shared - # Source: https://github.com/pyenv/pyenv/wiki#how-to-build-cpython-with---enable-shared - export PYTHON_CONFIGURE_OPTS="--enable-shared" - - # We need to override the default python used by pyenv as we installed it manually - # See ticket: https://jira.nuxeo.com/browse/NXDRIVE-2724 - #if [ "${MACOSX_DEPLOYMENT_TARGET:-unset}" != "unset" ]; then - # echo "**** if" - # echo "**** \"${MACOSX_DEPLOYMENT_TARGET:-unset}\"" - # echo "**** python3 $(which python3)" - # echo "**** python $(which python)" - # echo "**** python v $(python --version)" - # echo "**** python3 v $(python3 --version)" - - - # ln -sf $(which python3) $(which python) - # pyenv install --skip-existing "${version}" - # pyenv global "${version}" - #else - # echo "**** else" - # pyenv install --skip-existing "${version}" - # pyenv global "${version}" - #fi - - #echo ">>> [pyenv] Using Python ${version}" - - - ln -sf $(which python3) $(which python) - - - - echo "**** pyenv versions 1: $(pyenv versions)" - #pyenv install --skip-existing "${version}" - echo "**** pyenv versions 2: $(pyenv versions)" - echo ">>> [pyenv] Using Python ${version}" - #pyenv global "${version}" - echo "**** python3 $(which python3)" - echo "**** python $(which python)" - echo "**** python version $(python --version)" - echo "**** python3 version $(python3 --version)" - -} - -junit_arg() { - local junit="tools/jenkins/junit/xml" - local path="$1" - local run="$2" - - if [ "${run}" != "" ]; then - run=".${run}" - fi - echo "--junitxml=${junit}/${path}${run}.xml" -} - -launch_test() { - # Launch tests on a specific path. On failure, retry failed tests. - local cmd="${PYTHON_VENV} -bb -Wall -m pytest" - local path="${1}" - local pytest_args="${2:-}" - - ${cmd} ${pytest_args} `junit_arg ${path} 1` "${path}" && return - - if should_run "rerun"; then - # Will return 0 if rerun is needed else 1 - ${PYTHON_VENV} tools/check_pytest_lastfailed.py || return - - # Do not fail on error as all failures will be re-run another time at the end - ${cmd} -n0 --last-failed --last-failed-no-failures none `junit_arg ${path} 2` || true - fi -} - -launch_tests() { - local ret=0 - - rm -rf .pytest_cache - - # If a specific test is asked, just run it and bypass all over checks - if [ "${SPECIFIC_TEST}" != "tests" ]; then - echo ">>> Launching the specific tests" - launch_test "${SPECIFIC_TEST}" - return - fi - - if should_run "tests"; then - echo ">>> Launching synchronization functional tests, file by file" - echo " (first, run for each test file, failures are ignored to have" - echo " a whole picture of errors)" - total="$(find tests/old_functional -name "test_*.py" | wc -l | tr -d '[:space:]')" - number=1 - for test_file in $(find tests/old_functional -name "test_*.py"); do - echo "" - echo ">>> [${number}/${total}] Testing ${test_file} ..." - launch_test "${test_file}" "-q --durations=3" - number=$(( number + 1 )) - done - - if should_run "rerun"; then - echo ">>> Re-rerun failed tests" - - ${PYTHON_VENV} -m pytest --cache-show - - set +e - # Will return 0 if rerun is needed else 1 - ${PYTHON_VENV} tools/check_pytest_lastfailed.py && \ - ${PYTHON_VENV} -bb -Wall -m pytest -n0 --last-failed --last-failed-no-failures none `junit_arg "final"` - # The above command will exit with error code 5 if there is no failure to rerun - ret=$? - set -e - fi - - # Do not fail on junit merge - export TEST_SUITE="Drive" - python tools/jenkins/junit/merge.py "tools/jenkins/junit/xml" || true - - if [ $ret -ne 0 ] && [ $ret -ne 5 ]; then - exit 1 - fi - fi -} - -should_run() { - # Return 0 if we should run the given action. - local action - - action="$1" - - if should_skip "${action}"; then - return 1 - else - return 0 - fi - -} - -should_skip() { - # Return 0 if we should skip the given action. - local action - local ret - - action="$1" - - if [ "${SKIP}" = "all" ]; then - if [ "${action}" = "tests" ]; then - # "all" does not affect "tests" - ret=1 - else - ret=0 - fi - else - case "${SKIP}" in - *"${action}"*) ret=0 ;; - *) ret=1 ;; - esac - fi - - return ${ret} -} - -start_nxdrive() { - echo ">>> Starting Nuxeo Drive" - - export PYTHONPATH="${WORKSPACE_DRIVE}" - ${PYTHON_OPT} -m nxdrive -} - -verify_python() { - local version="$1" - local cur_version - - echo ">>> Verifying Python version in use" - cur_version=$(${PYTHON_VENV} --version 2>&1 | head -n 1 | awk '{print $2}') - if [ "${cur_version}" != "${version}" ]; then - pyenv uninstall -f "${cur_version}" - install_python "${PYTHON_DRIVE_VERSION}" - fi - - cur_version=$(${PYTHON_VENV} --version 2>&1 | head -n 1 | awk '{print $2}') - if [ "${cur_version}" != "${version}" ]; then - echo ">>> Drive requires Python '${version}'." - echo ">>> Current Python version ('\$ ${PYTHON_VENV} --version'):" - ${PYTHON_VENV} --version - exit 1 - fi - - # Also, check that primary modules are present (in case of wrong build) - if ! check_import "import sqlite3"; then - echo ">>> Uninstalling wrong Python version" - pyenv uninstall -f "${PYTHON_DRIVE_VERSION}" - install_python "${PYTHON_DRIVE_VERSION}" - fi -} - -# The main function, last in the script -main() { - # Adjust PATH for Mac - [ "${OSI}" = "osx" ] && export PATH="$PATH:/usr/local/bin:/usr/sbin" - - check_vars - - # The FinderSync extension build does not require extra setup - if [ $# -eq 1 ]; then - case "$1" in - "--check") - check - exit 0 - ;; - "--install" | "--install-python") - export INSTALL_ARG="1" - ;; - "--install-release") - export INSTALL_ARG="1" - export INSTALL_RELEASE_ARG="1" - ;; - esac - fi - - # If running from a virtual environment, let's use it - if ! command -v deactivate > /dev/null; then - install_pyenv - install_python "${PYTHON_DRIVE_VERSION}" - create_venv - fi - - verify_python "${PYTHON_DRIVE_VERSION}" - - if [ $# -eq 1 ]; then - case "$1" in - "--build") build_installer ;; - "--check-upgrade") check_upgrade ;; - "--install" | "--install-release") - install_deps - if ! check_import "import PyQt5" >/dev/null; then - echo ">>> No PyQt5. Installation failed." - exit 1 - fi - ;; - "--start") start_nxdrive ;; - "--tests") launch_tests ;; - esac - fi -} diff --git a/tools/deps/requirements.txt b/tools/deps/requirements.txt index af18704c0c..f21efedb1f 100644 --- a/tools/deps/requirements.txt +++ b/tools/deps/requirements.txt @@ -324,9 +324,9 @@ pyqt5==5.15.10 \ --hash=sha256:ff99b4f91aa8eb60510d5889faad07116d3340041916e46c07d519f7cad344e1 PyQt5-Qt5==5.15.13 ; sys_platform == "darwin" \ --hash=sha256:141859f2ffe04cc6c5db970e2b6ad9f98897805d886a14c52614e3799daab6d6 -PyQt5-Qt5==5.15.2 ; sys_platform == "win32" \ - --hash=sha256:9cc7a768b1921f4b982ebc00a318ccb38578e44e45316c7a4a850e953e1dd327 -PyQt5-Qt5==5.15.2 ; sys_platform == "linux" \ +PyQt5-Qt5==5.15.2 ; sys_platform != "darwin" \ + --hash=sha256:9cc7a768b1921f4b982ebc00a318ccb38578e44e45316c7a4a850e953e1dd327 \ + --hash=sha256:750b78e4dba6bdf1607febedc08738e318ea09e9b10aea9ff0d73073f11f6962 \ --hash=sha256:1988f364ec8caf87a6ee5d5a3a5210d57539988bf8e84714c7d60972692e2f4a pyqt5-stubs==5.15.6.0 \ --hash=sha256:7fb8177c72489a8911f021b7bd7c33f12c87f6dba92dcef3fdcdb5d9400f0f3f \ diff --git a/tools/posix/deploy_ci_agent.sh b/tools/posix/deploy_ci_agent.sh index aa6e0cf199..12179a2f85 100755 --- a/tools/posix/deploy_ci_agent.sh +++ b/tools/posix/deploy_ci_agent.sh @@ -155,7 +155,6 @@ create_venv() { } install_deps() { - # brew install qt@5 echo ">>> Installing requirements" ${PIP} -r tools/deps/requirements-pip.txt ${PIP} -r tools/deps/requirements.txt @@ -211,14 +210,14 @@ install_python() { # We need to override the default python used by pyenv as we installed it manually # See ticket: https://jira.nuxeo.com/browse/NXDRIVE-2724 - echo "**** Python Version: '$(python3 --version)'" + echo ">>> Actual Python Version: '$(python3 --version)'" pyenv install --skip-existing "${version}" - echo ">>> [pyenv] Using Python ${version}" pyenv global "${version}" eval "$(pyenv init -)" eval "$(pyenv init --path)" eval "$(pyenv virtualenv-init -)" - echo "**** Current Python Version: '$(python3 --version)'" + echo "**** Python Version in use: '$(python3 --version)'" + echo ">>> [pyenv] Using Python ${version}" } diff --git a/tools/windows/deploy_ci_agent.ps1 b/tools/windows/deploy_ci_agent.ps1 index c0168464ee..6c4dff8c50 100644 --- a/tools/windows/deploy_ci_agent.ps1 +++ b/tools/windows/deploy_ci_agent.ps1 @@ -325,7 +325,7 @@ function install_deps { if ($lastExitCode -ne 0) { ExitWithCode $lastExitCode } - & $Env:STORAGE_DIR\Scripts\python.exe $global:PYTHON_OPT -OO $global:PIP_OPT -r tools\deps\requirements.txt + & $Env:STORAGE_DIR\Scripts\python.exe $global:PYTHON_OPT -OO $global:PIP_OPT --no-cache-dir -r tools\deps\requirements.txt if ($lastExitCode -ne 0) { ExitWithCode $lastExitCode } @@ -399,9 +399,6 @@ function install_python { if ($lastExitCode -ne 0) { ExitWithCode $lastExitCode } - Write-Output ">>> Drive Requires ('\$ {$Env:PYTHON_DRIVE_VERSION} '):" - $version_ = Get-Command python | fl * # Get-Command python | fl * - Write-Output ">>> Current Python version $version_" } function junit_arg($path, $run) {