DRIVERS-2497 Fix paths on Cygwin and Python package dependencies (#244)

* Use absolute Windows form for venv path in Cygwin

* Remove unnecessary --no-cache-dir

* Correctly ensure -p is used with virtualenv module

* Ensure all seed packages are installed in virtual environment

* Add --system-site-packages when creating virtual environment

* Address python package build failures

* Improve documentation justifying use of -p with virtualenv

* Simplify fallback for cryptography package

.evergreen/csfle/activate-kmstlsvenv.sh

@@ -5,24 +5,74 @@
 # Usage:
 #   . ./activate-kmstlsvenv.sh
 #
-# This file creates and activates the kmstlsvenv virtual environment in the
+# This file creates and/or activates the kmstlsvenv virtual environment in the
 # current working directory. This file must be invoked from within the
 # .evergreen/csfle directory in the Drivers Evergreen Tools repository.
+#
+# If a kmstlsvenv virtual environment already exists, it will be activated and
+# no further action will be taken. If a kmstlsvenv virtual environment must be
+# created, required packages will also be installed.
+
+# If an error occurs during creation, activation, or installation of packages,
+# the kmstlsvenv virtual environment will be deactivated and activate_kmstlsvenv
+# will return a non-zero value.
 
 # Automatically invoked by activate-kmstlsvenv.sh.
 activate_kmstlsvenv() {
   # shellcheck source=.evergreen/venv-utils.sh
   . ../venv-utils.sh || return
 
   if [[ -d kmstlsvenv ]]; then
-    venvactivate kmstlsvenv
+    venvactivate kmstlsvenv || return
   else
     # shellcheck source=.evergreen/find-python3.sh
     . ../find-python3.sh || return
 
     venvcreate "$(find_python3)" kmstlsvenv || return
 
-    CRYPTOGRAPHY_DONT_BUILD_RUST=1 python -m pip install --upgrade boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0
+    local packages=(
+      "boto3~=1.19.0"
+      "pykmip~=0.10.0"
+    )
+
+    if [[ "$OSTYPE" == darwin16 && "$HOSTTYPE" == x86_64 ]]; then
+      # Avoid `error: thread-local storage is not supported for the current
+      # target` on macos-1012.
+      packages+=("greenlet<2.0")
+    fi
+
+    if [[ "$OSTYPE" == cygwin && "$HOSTTYPE" == x86_64 ]]; then
+      local -r windows_os_name="$(systeminfo.exe /FO LIST | perl -lne 'print $1 if m/^OS Name:\s+(.*)$/' || true)"
+
+      if [[ "$windows_os_name" =~ 2016 ]]; then
+        # Avoid `RuntimeError: Could not determine home directory.` on
+        # windows-64-2016. See BUILD-16233.
+        python -m pip install -U "setuptools<65.0" || {
+          local -r ret="$?"
+          deactivate || return 1 # Deactivation should never fail!
+          return "$ret"
+        }
+      fi
+    fi
+
+    if ! python -m pip install -U "${packages[@]}"; then
+      # Avoid `error: can't find Rust compiler`.
+      # Assume install failure at this point is due to new versions of
+      # cryptography require a Rust toolchain when a cryptography wheel is not
+      # present due to the package versions available.  This is required by at
+      # least the following distros (by OS and host):
+      #  - RHEL 6.2
+      #  - All RHEL on powerpc64le or s390x.
+      #  - OpenSUSE 12 on s390x.
+      #  - Ubuntu 18.04 on powerpc64le or s390x
+      packages+=("cryptography<3.4")
+
+      python -m pip install -U "${packages[@]}" || {
+        local -r ret="$?"
+        deactivate || return 1 # Deactivation should never fail!
+        return "$ret"
+      }
+    fi
   fi
 }
 

.evergreen/find-python3.sh

@@ -42,7 +42,7 @@ is_python3() (
 
   # Expect an output of the form: "Python x.y.z".
   # Note: Python 2 binaries output to stderr rather than stdout.
-  local -r version_output="$("$bin" -V 2>&1)"
+  local -r version_output="$("$bin" -V 2>&1 | tr -d '\n')"
 
   # For diagnostic purposes.
   echo " - $bin: $version_output"
@@ -81,15 +81,24 @@ is_venv_capable() (
   local -r tmp="$(mktemp -d)"
   trap 'rm -rf "$tmp"' EXIT
 
-  "$bin" -m venv "$tmp" || return
+  if [[ "$OSTYPE" == cygwin ]]; then
+    local -r real_path="$(cygpath -aw "$tmp")" || return
+  else
+    local -r real_path="$tmp"
+  fi
+
+  "$bin" -m venv "$real_path" || return
 
   if [[ -f "$tmp/bin/activate" ]]; then
     # shellcheck source=/dev/null
     . "$tmp/bin/activate"
-  else
+  elif [[ -f "$tmp/Scripts/activate" ]]; then
     dos2unix "$tmp/Scripts/activate" || return
     # shellcheck source=/dev/null
     . "$tmp/Scripts/activate"
+  else
+    echo "Could not find an activation script in $tmp!"
+    return 1
   fi
 ) 1>&2
 
@@ -121,15 +130,27 @@ is_virtualenv_capable() (
   local -r tmp="$(mktemp -d)"
   trap 'rm -rf "$tmp"' EXIT
 
-  "$bin" -m virtualenv -p "$bin" "$tmp" || return
+  if [[ "$OSTYPE" == cygwin ]]; then
+    local -r real_path="$(cygpath -aw "$tmp")" || return
+  else
+    local -r real_path="$tmp"
+  fi
+
+  # -p: some old versions of virtualenv (e.g. installed on Debian 10) are buggy.
+  # Without -p, the created virtual environment may use the wrong Python binary
+  # (e.g. using a Python 2 binary even if it was created by a Python 3 binary).
+  "$bin" -m virtualenv -p "$bin" "$real_path" || return
 
   if [[ -f "$tmp/bin/activate" ]]; then
     # shellcheck source=/dev/null
     . "$tmp/bin/activate"
-  else
+  elif [[ -f "$tmp/Scripts/activate" ]]; then
     dos2unix "$tmp/Scripts/activate" || return
     # shellcheck source=/dev/null
     . "$tmp/Scripts/activate"
+  else
+    echo "Could not find an activation script in $tmp!"
+    return 1
   fi
 ) 1>&2
 

.evergreen/venv-utils.sh

@@ -21,38 +21,76 @@
 #   "$2": The path to the virtual environment (directory) to create.
 #
 # Return 0 (true) if the virtual environment has been successfully created,
-# activated, and the pip package upgraded.
-# Return a non-zero value (false) otherwise.
+# activated, and all seed packages are successfully installed in the new
+# virtual environment.
+# Return a non-zero value (false) in a deactivated state otherwise.
+#
+# The "seed" packages pip, setuptools, and wheel are automatically installed
+# into the virtual environment. All packages must be successfully installed for
+# venvcreate to be considered a success.
 #
 # If a file or directory exists at the given path to the virtual environment,
 # they may be deleted as part of virtual environment creation.
 venvcreate() {
   local -r bin="${1:?'venvcreate requires a Python binary to use for the virtual environment'}"
   local -r path="${2:?'venvcreate requires a path to the virtual environment to create'}"
 
+  if [[ "$OSTYPE" == cygwin ]]; then
+    local -r real_path="$(cygpath -aw "$path")" || return
+  else
+    local -r real_path="$path" || return
+  fi
+
   # Prefer venv, but fallback to virtualenv if venv fails.
   for mod in "venv" "virtualenv"; do
-    # Ensure a clean directory before attempting to create a virtual environment.
+    # Ensure a clean directory before attempting to create a virtual
+    # environment.
     rm -rf "$path"
 
-    if "$bin" -m "$mod" "$path"; then
-      # Workaround https://bugs.python.org/issue32451:
-      # mongovenv/Scripts/activate: line 3: $'\r': command not found
-      if [[ -f "$path/Scripts/activate" ]]; then
-        dos2unix "$path/Scripts/activate" || return
-      fi
+    case "$mod" in
+    venv)
+      "$bin" -m "$mod" --system-site-packages "$real_path" || continue
+      ;;
+    virtualenv)
+      # -p: some old versions of virtualenv (e.g. installed on Debian 10) are
+      # buggy. Without -p, the created virtual environment may use the wrong
+      # Python binary (e.g. using a Python 2 binary even if it was created by a
+      # Python 3 binary).
+      "$bin" -m "$mod" -p "$bin" --system-site-packages "$real_path" || continue
+      ;;
+    *)
+      echo "Unexpected virtual environment module $mod!"
+      return 1
+      ;;
+    esac
 
-      if venvactivate "$path"; then
-        # Use --no-cache-dir to ensure ensure the *actual* latest pip is
-        # correctly installed.
-        if python -m pip install --no-cache-dir --upgrade pip; then
-          # Only consider success if activation + pip upgrade was successful.
-          return
-        fi
+    # Workaround https://bugs.python.org/issue32451:
+    # mongovenv/Scripts/activate: line 3: $'\r': command not found
+    if [[ -f "$path/Scripts/activate" ]]; then
+      dos2unix "$path/Scripts/activate" || continue
+    fi
 
-        deactivate
-      fi
+    venvactivate "$path" || continue
+
+    if ! python -m pip install -U pip; then
+      deactivate || return 1 # Deactivation should never fail!
+      continue
+    fi
+
+    # Ensure setuptools and wheel are installed in the virtual environment.
+    # virtualenv only guarantees "one or more of" the seed packages are
+    # installed. venv only guarantees pip is installed via ensurepip.
+    #
+    # These packages must be upgraded *after* pip, *separately*, as some old
+    # versions of pip do not handle their simultaneous installation properly.
+    # See: https://github.com/pypa/pip/issues/4253
+    if ! python -m pip install -U setuptools wheel; then
+      deactivate || return 1 # Deactivation should never fail!
+      continue
     fi
+
+    # Success only if both activation and package upgrades are successful.
+    return 0
   done
 
   echo "Could not use either venv or virtualenv with $bin to create a virtual environment at $path!" 1>&2