Redesign SystemPackageTool interface (#10380)

* basic implementation

* wip

* wip

* wip

* wip

* wip

* wip

* revert change

* fix indent

* raise error and add update and check args for install

* simplify a bit

* install distro just for linux

* add distro pm map

* test brew check

* separate functional/integration tests

* use two decorators

* consider build requires

* consider build requires

* mock context

* add msys2

* minor changes

* private class

* change name

* change name

* change logic

* wip

* add test

* use run

* skip py2

* change name

* skip tests

* revert skip

* add more info to message

conan/tools/system/__init__.py

@@ -0,0 +1 @@
+from conan.tools.system.package_manager import Apt, Yum, Dnf, Brew, Pkg, PkgUtil, Chocolatey, PacMan, Zypper

conan/tools/system/package_manager.py

@@ -0,0 +1,218 @@
+import platform
+
+from conans.client.graph.graph import CONTEXT_BUILD
+from conans.errors import ConanException
+
+
+class _SystemPackageManagerTool(object):
+    mode_check = "check"
+    mode_install = "install"
+    tool_name = None
+    install_command = ""
+    update_command = ""
+    check_command = ""
+
+    def __init__(self, conanfile):
+        self._conanfile = conanfile
+        self._active_tool = self._conanfile.conf["tools.system.package_manager:tool"] or self.get_default_tool()
+        self._sudo = self._conanfile.conf["tools.system.package_manager:sudo"]
+        self._sudo_askpass = self._conanfile.conf["tools.system.package_manager:sudo_askpass"]
+        self._mode = self._conanfile.conf["tools.system.package_manager:mode"] or self.mode_check
+        self._arch = self._conanfile.settings_build.get_safe('arch') \
+            if self._conanfile.context == CONTEXT_BUILD else self._conanfile.settings.get_safe('arch')
+        self._arch_names = {}
+        self._arch_separator = ""
+
+    def get_default_tool(self):
+        os_name = platform.system()
+        if os_name in ["Linux", "FreeBSD"]:
+            import distro
+            os_name = distro.id() or os_name
+        elif os_name == "Windows" and self._conanfile.conf["tools.microsoft.bash:subsystem"] == "msys2":
+            os_name = "msys2"
+        manager_mapping = {"apt-get": ["Linux", "ubuntu", "debian"],
+                           "yum": ["pidora", "scientific", "xenserver", "amazon", "oracle", "amzn",
+                                   "almalinux"],
+                           "dnf": ["fedora", "rhel", "centos", "mageia"],
+                           "brew": ["Darwin"],
+                           "pacman": ["arch", "manjaro", "msys2"],
+                           "choco": ["Windows"],
+                           "zypper": ["opensuse", "sles"],
+                           "pkg": ["freebsd"],
+                           "pkgutil": ["Solaris"]}
+        for tool, distros in manager_mapping.items():
+            if os_name in distros:
+                return tool
+
+    def get_package_name(self, package):
+        # TODO: should we only add the arch if cross-building?
+        if self._arch in self._arch_names:
+            return "{}{}{}".format(package, self._arch_separator,
+                                   self._arch_names.get(self._arch))
+        return package
+
+    @property
+    def sudo_str(self):
+        sudo = "sudo " if self._sudo else ""
+        askpass = "-A " if self._sudo and self._sudo_askpass else ""
+        return "{}{}".format(sudo, askpass)
+
+    def run(self, method, *args, **kwargs):
+        if self._active_tool == self.__class__.tool_name:
+            return method(*args, **kwargs)
+
+    def install(self, *args, **kwargs):
+        return self.run(self._install, *args, **kwargs)
+
+    def update(self, *args, **kwargs):
+        return self.run(self._update, *args, **kwargs)
+
+    def check(self, *args, **kwargs):
+        return self.run(self._check, *args, **kwargs)
+
+    def _install(self, packages, update=False, check=True, **kwargs):
+        if update:
+            self.update()
+
+        if check:
+            packages = self.check(packages)
+
+        if self._mode == self.mode_check and packages:
+            raise ConanException("System requirements: '{0}' are missing but can't install "
+                                 "because tools.system.package_manager:mode is '{1}'."
+                                 "Please update packages manually or set "
+                                 "'tools.system.package_manager:mode' "
+                                 "to '{2}' in the [conf] section of the profile, "
+                                 "or in the command line using "
+                                 "'-c tools.system.package_manager:mode={2}'".format(", ".join(packages),
+                                                                                     self.mode_check,
+                                                                                     self.mode_install))
+        elif packages:
+            packages_arch = [self.get_package_name(package) for package in packages]
+            if packages_arch:
+                command = self.install_command.format(sudo=self.sudo_str,
+                                                      tool=self.tool_name,
+                                                      packages=" ".join(packages_arch),
+                                                      **kwargs)
+                return self._conanfile.run(command)
+        else:
+            self._conanfile.output.info("System requirements: {} already "
+                                        "installed".format(" ".join(packages)))
+
+    def _update(self):
+        if self._mode == self.mode_check:
+            raise ConanException("Can't update because tools.system.package_manager:mode is '{0}'."
+                                 "Please update packages manually or set "
+                                 "'tools.system.package_manager:mode' "
+                                 "to '{1}' in the [conf] section of the profile, "
+                                 "or in the command line using "
+                                 "'-c tools.system.package_manager:mode={1}'".format(self.mode_check,
+                                                                                     self.mode_install))
+        command = self.update_command.format(sudo=self.sudo_str, tool=self.tool_name)
+        return self._conanfile.run(command)
+
+    def _check(self, packages):
+        missing = [pkg for pkg in packages if self.check_package(self.get_package_name(pkg)) != 0]
+        return missing
+
+    def check_package(self, package):
+        command = self.check_command.format(tool=self.tool_name,
+                                            package=package)
+        return self._conanfile.run(command, ignore_errors=True)
+
+
+class Apt(_SystemPackageManagerTool):
+    # TODO: apt? apt-get?
+    tool_name = "apt-get"
+    install_command = "{sudo}{tool} install -y {recommends}{packages}"
+    update_command = "{sudo}{tool} update"
+    check_command = "dpkg-query -W -f='${{Status}}' {package} | grep -q \"ok installed\""
+
+    def __init__(self, conanfile, arch_names=None):
+        super(Apt, self).__init__(conanfile)
+        self._arch_names = {"x86_64": "amd64",
+                            "x86": "i386",
+                            "ppc32": "powerpc",
+                            "ppc64le": "ppc64el",
+                            "armv7": "arm",
+                            "armv7hf": "armhf",
+                            "armv8": "arm64",
+                            "s390x": "s390x"} if arch_names is None else arch_names
+
+        self._arch_separator = ":"
+
+    def install(self, packages, update=False, check=False, recommends=False):
+        recommends_str = '' if recommends else '--no-install-recommends '
+        return super(Apt, self).install(packages, update=update, check=check,
+                                        recommends=recommends_str)
+
+
+class Yum(_SystemPackageManagerTool):
+    tool_name = "yum"
+    install_command = "{sudo}{tool} install -y {packages}"
+    update_command = "{sudo}{tool} check-update -y"
+    check_command = "rpm -q {package}"
+
+    def __init__(self, conanfile, arch_names=None):
+        super(Yum, self).__init__(conanfile)
+        self._arch_names = {"x86_64": "x86_64",
+                            "x86": "i?86",
+                            "ppc32": "powerpc",
+                            "ppc64le": "ppc64le",
+                            "armv7": "armv7",
+                            "armv7hf": "armv7hl",
+                            "armv8": "aarch64",
+                            "s390x": "s390x"} if arch_names is None else arch_names
+        self._arch_separator = "."
+
+
+class Dnf(Yum):
+    tool_name = "dnf"
+
+
+class Brew(_SystemPackageManagerTool):
+    tool_name = "brew"
+    install_command = "{sudo}{tool} install {packages}"
+    update_command = "{sudo}{tool} update"
+    check_command = 'test -n "$({tool} ls --versions {package})"'
+
+
+class Pkg(_SystemPackageManagerTool):
+    tool_name = "pkg"
+    install_command = "{sudo}{tool} install -y {packages}"
+    update_command = "{sudo}{tool} update"
+    check_command = "{tool} info {package}"
+
+
+class PkgUtil(_SystemPackageManagerTool):
+    tool_name = "pkgutil"
+    install_command = "{sudo}{tool} --install --yes {packages}"
+    update_command = "{sudo}{tool} --catalog"
+    check_command = 'test -n "`{tool} --list {package}`"'
+
+
+class Chocolatey(_SystemPackageManagerTool):
+    tool_name = "choco"
+    install_command = "{tool} --install --yes {packages}"
+    update_command = "{tool} outdated"
+    check_command = '{tool} search --local-only --exact {package} | ' \
+                    'findstr /c:"1 packages installed."'
+
+
+class PacMan(_SystemPackageManagerTool):
+    tool_name = "pacman"
+    install_command = "{sudo}{tool} -S --noconfirm {packages}"
+    update_command = "{sudo}{tool} -Syyu --noconfirm"
+    check_command = "{tool} -Qi {package}"
+
+    def __init__(self, conanfile, arch_names=None):
+        super(PacMan, self).__init__(conanfile)
+        self._arch_names = {"x86": "lib32"} if arch_names is None else arch_names
+        self._arch_separator = "-"
+
+
+class Zypper(_SystemPackageManagerTool):
+    tool_name = "zypper"
+    install_command = "{sudo}{tool} --non-interactive in {packages}"
+    update_command = "{sudo}{tool} --non-interactive ref"
+    check_command = "rpm -q {package}"

conans/requirements.txt

@@ -7,7 +7,7 @@ patch-ng>=1.17.4, <1.18
 fasteners>=0.14.1
 six>=1.10.0,<=1.16.0
 node-semver==0.6.1
-distro>=1.0.2, <=1.6.0
+distro>=1.0.2, <=1.6.0; sys_platform == 'linux' or sys_platform == 'linux2'
 pygments>=2.0, <3.0
 tqdm>=4.28.1, <5
 Jinja2>=2.9, <4.0.0

conans/test/conftest.py

@@ -110,6 +110,8 @@
                             "Darwin": '/Users/jenkins/bin'}},
     },
     'premake': {},
+    'apt_get': { "exe": "apt-get"},
+    'brew': {},
     # TODO: Intel oneAPI is not installed in CI yet. Uncomment this line whenever it's done.
     # "intel_oneapi": {
     #     "default": "2021.3",

conans/test/functional/tools/system/package_manager_test.py

@@ -0,0 +1,112 @@
+import platform
+import textwrap
+
+import pytest
+import six
+
+from conans.test.utils.tools import TestClient
+
+
+@pytest.mark.tool_apt_get
+@pytest.mark.skipif(platform.system() != "Linux", reason="Requires apt")
+@pytest.mark.skipif(six.PY2, reason="Does not pass on Py2 with Pytest")
+def test_apt_check():
+    client = TestClient()
+    client.save({"conanfile.py": textwrap.dedent("""
+        from conans import ConanFile
+        from conan.tools.system import Apt
+        class MyPkg(ConanFile):
+            settings = "arch", "os"
+            def system_requirements(self):
+                apt = Apt(self)
+                not_installed = apt.check(["non-existing1", "non-existing2"])
+                print("missing:", not_installed)
+        """)})
+    client.run("create . test/1.0@ -s:b arch=armv8 -s:h arch=x86")
+    assert "dpkg-query: no packages found matching non-existing1:i386" in client.out
+    assert "dpkg-query: no packages found matching non-existing2:i386" in client.out
+    assert "missing: ['non-existing1', 'non-existing2']" in client.out
+
+
+@pytest.mark.tool_apt_get
+@pytest.mark.skipif(platform.system() != "Linux", reason="Requires apt")
+@pytest.mark.skipif(six.PY2, reason="Does not pass on Py2 with Pytest")
+def test_build_require():
+    client = TestClient()
+    client.save({"tool_require.py": textwrap.dedent("""
+        from conans import ConanFile
+        from conan.tools.system import Apt
+        class MyPkg(ConanFile):
+            settings = "arch", "os"
+            def system_requirements(self):
+                apt = Apt(self)
+                not_installed = apt.check(["non-existing1", "non-existing2"])
+                print("missing:", not_installed)
+        """)})
+    client.run("export tool_require.py tool_require/1.0@")
+    client.save({"consumer.py": textwrap.dedent("""
+        from conans import ConanFile
+        class consumer(ConanFile):
+            settings = "arch", "os"
+            tool_requires = "tool_require/1.0"
+        """)})
+    client.run("create consumer.py consumer/1.0@ -s:b arch=armv8 -s:h arch=x86 --build=missing")
+    assert "dpkg-query: no packages found matching non-existing1:arm64" in client.out
+    assert "dpkg-query: no packages found matching non-existing2:arm64" in client.out
+    assert "missing: ['non-existing1', 'non-existing2']" in client.out
+
+
+@pytest.mark.tool_brew
+@pytest.mark.skipif(platform.system() != "Darwin", reason="Requires brew")
+@pytest.mark.skipif(six.PY2, reason="Does not pass on Py2 with Pytest")
+def test_brew_check():
+    client = TestClient()
+    client.save({"conanfile.py": textwrap.dedent("""
+        from conans import ConanFile
+        from conan.tools.system import Brew
+        class MyPkg(ConanFile):
+            settings = "arch"
+            def system_requirements(self):
+                brew = Brew(self)
+                not_installed = brew.check(["non-existing1", "non-existing2"])
+                print("missing:", not_installed)
+        """)})
+    client.run("create . test/1.0@")
+    assert "missing: ['non-existing1', 'non-existing2']" in client.out
+
+
+@pytest.mark.tool_brew
+@pytest.mark.skipif(platform.system() != "Darwin", reason="Requires brew")
+@pytest.mark.skip(reason="brew update takes a lot of time")
+def test_brew_install_check_mode():
+    client = TestClient()
+    client.save({"conanfile.py": textwrap.dedent("""
+        from conans import ConanFile
+        from conan.tools.system import Brew
+        class MyPkg(ConanFile):
+            settings = "arch"
+            def system_requirements(self):
+                brew = Brew(self)
+                brew.install(["non-existing1", "non-existing2"])
+        """)})
+    client.run("create . test/1.0@", assert_error=True)
+    assert "System requirements: 'non-existing1, non-existing2' are missing but " \
+           "can't install because tools.system.package_manager:mode is 'check'" in client.out
+
+
+@pytest.mark.tool_brew
+@pytest.mark.skipif(platform.system() != "Darwin", reason="Requires brew")
+@pytest.mark.skip(reason="brew update takes a lot of time")
+def test_brew_install_install_mode():
+    client = TestClient()
+    client.save({"conanfile.py": textwrap.dedent("""
+        from conans import ConanFile
+        from conan.tools.system import Brew
+        class MyPkg(ConanFile):
+            settings = "arch"
+            def system_requirements(self):
+                brew = Brew(self)
+                brew.install(["non-existing1", "non-existing2"])
+        """)})
+    client.run("create . test/1.0@ -c tools.system.package_manager:mode=install", assert_error=True)
+    assert "Error: No formulae found in taps." in client.out