Support multiple MSYS2 environments (#13649)

Refactor MSYS2 environment handling, support multiple native compilation
environments, and introduce a new `unix_path_to_native` function (the
inverse of `native_path_to_unix`).

---------

Co-authored-by: Bianca Henderson <bhenderson@anaconda.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Ken Odegard <kodegard@anaconda.com>

conda/activate.py

@@ -13,7 +13,9 @@
 
 import abc
 import json
+import ntpath
 import os
+import posixpath
 import re
 import sys
 from logging import getLogger
@@ -28,6 +30,8 @@
     join,
 )
 from pathlib import Path
+from shutil import which
+from subprocess import run
 from textwrap import dedent
 from typing import TYPE_CHECKING
 
@@ -618,6 +622,38 @@ def _get_starting_path_list(self):
     def _get_path_dirs(self, prefix):
         if on_win:  # pragma: unix no cover
             yield prefix.rstrip("\\")
+
+            # We need to stat(2) for possible environments because
+            # tests can't be told where to look!
+            #
+            # mingw-w64 is a legacy variant used by m2w64-* packages
+            #
+            # We could include clang32 and mingw32 variants
+            variants = []
+            for variant in ["ucrt64", "clang64", "mingw64", "clangarm64"]:
+                path = self.sep.join((prefix, "Library", variant))
+
+                # MSYS2 /c/
+                # cygwin /cygdrive/c/
+                if re.match("^(/[A-Za-z]/|/cygdrive/[A-Za-z]/).*", prefix):
+                    path = unix_path_to_native(path, prefix)
+
+                if isdir(path):
+                    variants.append(variant)
+
+            if len(variants) > 1:
+                print(
+                    f"WARNING: {prefix}: {variants} MSYS2 envs exist: please check your dependencies",
+                    file=sys.stderr,
+                )
+                print(
+                    f"WARNING: conda list -n {self._default_env(prefix)}",
+                    file=sys.stderr,
+                )
+
+            if variants:
+                yield self.sep.join((prefix, "Library", variants[0], "bin"))
+
             yield self.sep.join((prefix, "Library", "mingw-w64", "bin"))
             yield self.sep.join((prefix, "Library", "usr", "bin"))
             yield self.sep.join((prefix, "Library", "bin"))
@@ -830,6 +866,119 @@ def ensure_fs_path_encoding(value):
         return value
 
 
+class _Cygpath:
+    @classmethod
+    def nt_to_posix(cls, paths: str) -> str:
+        return cls.RE_UNIX.sub(cls.translate_unix, paths).replace(
+            ntpath.pathsep, posixpath.pathsep
+        )
+
+    RE_UNIX = re.compile(
+        r"""
+        (?P<drive>[A-Za-z]:)?
+        (?P<path>[\/\\]+(?:[^:*?\"<>|;]+[\/\\]*)*)
+        """,
+        flags=re.VERBOSE,
+    )
+
+    @staticmethod
+    def translate_unix(match: re.Match) -> str:
+        return "/" + (
+            ((match.group("drive") or "").lower() + match.group("path"))
+            .replace("\\", "/")
+            .replace(":", "")  # remove drive letter delimiter
+            .replace("//", "/")
+            .rstrip("/")
+        )
+
+    @classmethod
+    def posix_to_nt(cls, paths: str, prefix: str) -> str:
+        if posixpath.sep not in paths:
+            # nothing to translate
+            return paths
+
+        if posixpath.pathsep in paths:
+            return ntpath.pathsep.join(
+                cls.posix_to_nt(path, prefix) for path in paths.split(posixpath.pathsep)
+            )
+        path = paths
+
+        # Reverting a Unix path means unpicking MSYS2/Cygwin
+        # conventions -- in order!
+        # 1. drive letter forms:
+        #      /x/here/there - MSYS2
+        #      /cygdrive/x/here/there - Cygwin
+        #    transformed to X:\here\there -- note the uppercase drive letter!
+        # 2. either:
+        #    a. mount forms:
+        #         //here/there
+        #       transformed to \\here\there
+        #    b. root filesystem forms:
+        #         /here/there
+        #       transformed to {prefix}\Library\here\there
+        # 3. anything else
+
+        # continue performing substitutions until a match is found
+        path, subs = cls.RE_DRIVE.subn(cls.translation_drive, path)
+        if not subs:
+            path, subs = cls.RE_MOUNT.subn(cls.translation_mount, path)
+        if not subs:
+            path, _ = cls.RE_ROOT.subn(
+                lambda match: cls.translation_root(match, prefix), path
+            )
+
+        return re.sub(r"/+", r"\\", path)
+
+    RE_DRIVE = re.compile(
+        r"""
+        ^
+        (/cygdrive)?
+        /(?P<drive>[A-Za-z])
+        (/+(?P<path>.*)?)?
+        $
+        """,
+        flags=re.VERBOSE,
+    )
+
+    @staticmethod
+    def translation_drive(match: re.Match) -> str:
+        drive = match.group("drive").upper()
+        path = match.group("path") or ""
+        return f"{drive}:\\{path}"
+
+    RE_MOUNT = re.compile(
+        r"""
+        ^
+        //(
+            (?P<mount>[^/]+)
+            (?P<path>/+.*)?
+        )?
+        $
+        """,
+        flags=re.VERBOSE,
+    )
+
+    @staticmethod
+    def translation_mount(match: re.Match) -> str:
+        mount = match.group("mount") or ""
+        path = match.group("path") or ""
+        return f"\\\\{mount}{path}"
+
+    RE_ROOT = re.compile(
+        r"""
+        ^
+        (?P<path>/[^:]*)
+        $
+        """,
+        flags=re.VERBOSE,
+    )
+
+    @staticmethod
+    def translation_root(match: re.Match, prefix: str) -> str:
+        path = match.group("path")
+        return f"{prefix}\\Library{path}"
+
+
 def native_path_to_unix(
     paths: str | Iterable[str] | None,
 ) -> str | tuple[str, ...] | None:
@@ -844,8 +993,6 @@ def native_path_to_unix(
         return "." if isinstance(paths, str) else ()
 
     # on windows, uses cygpath to convert windows native paths to posix paths
-    from shutil import which
-    from subprocess import run
 
     # It is very easy to end up with a bash in one place and a cygpath in another due to e.g.
     # using upstream MSYS2 bash, but with a conda env that does not have bash but does have
@@ -855,33 +1002,22 @@ def native_path_to_unix(
 
     bash = which("bash")
     cygpath = str(Path(bash).parent / "cygpath") if bash else "cygpath"
-    joined = paths if isinstance(paths, str) else os.pathsep.join(paths)
+    joined = paths if isinstance(paths, str) else ntpath.pathsep.join(paths)
 
     try:
         # if present, use cygpath to convert paths since its more reliable
         unix_path = run(
-            [cygpath, "--path", joined],
+            [cygpath, "--unix", "--path", joined],
             text=True,
             capture_output=True,
             check=True,
         ).stdout.strip()
     except FileNotFoundError:
         # fallback logic when cygpath is not available
         # i.e. conda without anything else installed
-        def _translation(match):
-            return "/" + (
-                match.group(1)
-                .replace("\\", "/")
-                .replace(":", "")
-                .replace("//", "/")
-                .rstrip("/")
-            )
+        log.warning("cygpath is not available, fallback to manual path conversion")
 
-        unix_path = (
-            re.sub(r"([a-zA-Z]:[\/\\]+(?:[^:*?\"<>|;]+[\/\\]*)*)", _translation, joined)
-            .replace(";", ":")
-            .rstrip(";")
-        )
+        unix_path = _Cygpath.nt_to_posix(joined)
     except Exception as err:
         log.error("Unexpected cygpath error (%s)", err)
         raise
@@ -891,7 +1027,61 @@ def _translation(match):
     elif not unix_path:
         return ()
     else:
-        return tuple(unix_path.split(":"))
+        return tuple(unix_path.split(posixpath.pathsep))
+
+
+def unix_path_to_native(
+    paths: str | Iterable[str] | None, prefix: str
+) -> str | tuple[str, ...] | None:
+    if paths is None:
+        return None
+    elif not on_win:
+        return path_identity(paths)
+
+    # short-circuit if we don't get any paths
+    paths = paths if isinstance(paths, str) else tuple(paths)
+    if not paths:
+        return "." if isinstance(paths, str) else ()
+
+    # on windows, uses cygpath to convert posix paths to windows native paths
+
+    # It is very easy to end up with a bash in one place and a cygpath in another due to e.g.
+    # using upstream MSYS2 bash, but with a conda env that does not have bash but does have
+    # cygpath.  When this happens, we have two different virtual POSIX machines, rooted at
+    # different points in the Windows filesystem.  We do our path conversions with one and
+    # expect the results to work with the other.  It does not.
+
+    bash = which("bash")
+    cygpath = str(Path(bash).parent / "cygpath") if bash else "cygpath"
+    joined = paths if isinstance(paths, str) else posixpath.pathsep.join(paths)
+
+    try:
+        # if present, use cygpath to convert paths since its more reliable
+        win_path = run(
+            [cygpath, "--windows", "--path", joined],
+            text=True,
+            capture_output=True,
+            check=True,
+        ).stdout.strip()
+    except FileNotFoundError:
+        # fallback logic when cygpath is not available
+        # i.e. conda without anything else installed
+        log.warning("cygpath is not available, fallback to manual path conversion")
+
+        # The conda prefix can be in a drive letter form
+        prefix = _Cygpath.posix_to_nt(prefix, prefix)
+
+        win_path = _Cygpath.posix_to_nt(joined, prefix)
+    except Exception as err:
+        log.error("Unexpected cygpath error (%s)", err)
+        raise
+
+    if isinstance(paths, str):
+        return win_path
+    elif not win_path:
+        return ()
+    else:
+        return tuple(win_path.split(ntpath.pathsep))
 
 
 def path_identity(paths: str | Iterable[str] | None) -> str | tuple[str, ...] | None:

docs/source/user-guide/troubleshooting.rst

@@ -151,6 +151,7 @@ PATH folders goes from left to right. If you choose to put Anaconda's folders on
 PATH, there are several of them:
 
   * (install root)
+  * (install root)/Library/(MSYS2 env)/bin ## dependent on MSYS2 packages
   * (install root)/Library/mingw-w64/bin
   * (install root)/Library/usr/bin
   * (install root)/Library/bin

news/13649-msys2-environments

@@ -0,0 +1,3 @@
+### Enhancements
+
+* MSYS2 packages can now use the upstream installation prefixes. (#13649)