Improvements on package discovery for setup.py (sagemath#41)

* Improvements on package discovery for setup.py

* Exclude test file from flake8

* Add stdlib-list to azure

.flake8

@@ -1,3 +1,4 @@
 [flake8]
 max-line-length=88
 extend-ignore=E203,D104,D100
+exclude=tests/data/*

.pre-commit-config.yaml

@@ -29,10 +29,12 @@ repos:
     rev: v4.3.21
     hooks:
         -   id: isort
+            exclude: tests/data
 -   repo: https://gitlab.com/pycqa/flake8
     rev: 3.7.9
     hooks:
     -   id: flake8
+        exclude: tests/data
         language_version: python3
         additional_dependencies:
             - flake8-typing-imports==1.3.0

azure-pipelines.yml

@@ -1,7 +1,7 @@
 variables:
 - group: Codecov
 - name: grayskull_deps
-  value: pytest pytest-azurepipelines pytest-xdist pytest-cov requests ruamel.yaml codecov ruamel.yaml.jinja2 "coverage<5.0"
+  value: pytest pytest-azurepipelines pytest-xdist pytest-cov requests ruamel.yaml codecov ruamel.yaml.jinja2 "coverage<5.0" stdlib-list
 
 jobs:
   - job:

grayskull/base/recipe_item.py

@@ -27,6 +27,9 @@ def __str__(self) -> str:
             val += f"  {comment.value}"
         return val
 
+    def __lt__(self, other: "RecipeItem") -> bool:
+        return self.value < other.value
+
     def __eq__(self, other: Union[str, int, "RecipeItem"]) -> bool:
         if isinstance(other, RecipeItem):
             return self.value == other.value and self.selector == other.selector

grayskull/base/section.py

@@ -105,6 +105,11 @@ def __eq__(self, other: Any) -> bool:
             return self.values[0] == other
         if isinstance(other, str):
             return self.section_name == other
+        if isinstance(other, list) and isinstance(other[0], str):
+            for pos, item in enumerate(self.values):
+                if item.value != other[pos]:
+                    return False
+            return True
         return other == self.values
 
     def __iter__(self) -> Iterator:

grayskull/pypi/pypi.py

@@ -16,6 +16,7 @@
 from requests import HTTPError
 
 from grayskull.base.base_recipe import AbstractRecipeModel
+from grayskull.utils import get_vendored_dependencies
 
 log = logging.getLogger(__name__)
 PyVer = namedtuple("PyVer", ["major", "minor"])
@@ -104,7 +105,9 @@ def __fake_distutils_setup(*args, **kwargs):
             data_dist["install_requires"] = kwargs.get("install_requires", [])
             if not data_dist.get("setup_requires"):
                 data_dist["setup_requires"] = []
-            data_dist["setup_requires"] += kwargs.get("setup_requires", [])
+            data_dist["setup_requires"] += (
+                kwargs.get("setup_requires") if kwargs.get("setup_requires") else []
+            )
             data_dist["extras_require"] = kwargs.get("extras_require", [])
             data_dist["requires_python"] = kwargs.get("requires_python", None)
             data_dist["entry_points"] = kwargs.get("entry_points", None)
@@ -136,10 +139,10 @@ def __fake_distutils_setup(*args, **kwargs):
             setup_ext.build_ext = _fake_build_ext_setuptools
             path_setup = str(path_setup)
             PyPi.__run_setup_py(path_setup, data_dist)
-            if not data_dist:
+            if not data_dist or data_dist.get("install_requires", None) is None:
                 PyPi.__run_setup_py(path_setup, data_dist, run_py=True)
             yield data_dist
-        except Exception:
+        except Exception as err:  # noqa
             yield data_dist
         core.setup = setup_core_original
         dist_ext.build_ext = original_build_ext_distutils
@@ -155,6 +158,7 @@ def __run_setup_py(path_setup: str, data_dist: dict, run_py=False):
         if os.path.dirname(path_setup) not in sys.path:
             sys.path.append(os.path.dirname(path_setup))
             sys.path.append(pip_dir)
+        PyPi._install_deps_if_necessary(path_setup, data_dist, pip_dir)
         try:
             if run_py:
                 import runpy
@@ -166,17 +170,34 @@ def __run_setup_py(path_setup: str, data_dist: dict, run_py=False):
                     path_setup, script_args=["install", f"--target={pip_dir}"]
                 )
         except ModuleNotFoundError as err:
-            if not data_dist.get("setup_requires"):
-                data_dist["setup_requires"] = []
-            data_dist["setup_requires"].append(err.name)
-            check_output(["pip", "install", err.name, f"--target={pip_dir}"])
+            PyPi._pip_install_dep(data_dist, err.name, pip_dir)
             PyPi.__run_setup_py(path_setup, data_dist, run_py)
-        except Exception:
+        except Exception as err:  # noqa
             pass
         if os.path.exists(pip_dir):
-            os.rmdir(pip_dir)
+            shutil.rmtree(pip_dir)
         sys.path = original_path
 
+    @staticmethod
+    def _install_deps_if_necessary(setup_path: str, data_dist: dict, pip_dir: str):
+        all_setup_deps = get_vendored_dependencies(setup_path)
+        for dep in all_setup_deps:
+            PyPi._pip_install_dep(data_dist, dep, pip_dir)
+
+    @staticmethod
+    def _pip_install_dep(data_dist: dict, dep_name: str, pip_dir: str):
+        if not data_dist.get("setup_requires"):
+            data_dist["setup_requires"] = []
+        if dep_name == "pkg_resources":
+            dep_name = "setuptools"
+        if (
+            dep_name.lower() not in data_dist["setup_requires"]
+            and dep_name.lower() != "setuptools"
+        ):
+            data_dist["setup_requires"].append(dep_name.lower())
+        if dep_name != "setuptools":
+            check_output(["pip", "install", dep_name, f"--target={pip_dir}"])
+
     @staticmethod
     def _merge_pypi_sdist_metadata(pypi_metadata: dict, sdist_metadata: dict) -> dict:
         """This method is responsible to merge two dictionaries and it will give
@@ -266,6 +287,7 @@ def refresh_section(self, section: str = ""):
         if not self._is_arch:
             self["build"]["noarch"] = "python"
 
+    @lru_cache(maxsize=10)
     def _get_metadata(self) -> dict:
         name = self.get_var_content(self["package"]["name"].values[0])
         pypi_metada = self._get_pypi_metadata()
@@ -365,7 +387,8 @@ def _extract_requirements(self, metadata: dict) -> dict:
         run_req = self._get_run_req_from_requires_dist(requires_dist)
 
         build_req = [f"<{{ compiler('{c}') }}}}" for c in metadata.get("compilers", [])]
-        self._is_arch = self._is_arch or build_req
+        if build_req:
+            self._is_arch = True
 
         if self._is_arch:
             version_to_selector = PyPi.py_version_to_selector(metadata)

grayskull/utils.py

@@ -0,0 +1,45 @@
+import ast
+from functools import lru_cache
+from typing import List
+
+
+@lru_cache(maxsize=10)
+def get_std_modules() -> List:
+    from stdlib_list import stdlib_list
+
+    all_libs = set()
+    for py_ver in ("2.7", "3.6", "3.7", "3.8"):
+        all_libs.update(stdlib_list(py_ver))
+    return list(all_libs)
+
+
+def get_all_modules_imported_script(script_file: str) -> set:
+    modules = set()
+
+    def visit_Import(node):
+        for name in node.names:
+            modules.add(name.name.split(".")[0])
+
+    def visit_ImportFrom(node):
+        # if node.module is missing it's a "from . import ..." statement
+        # if level > 0 it's a "from .submodule import ..." statement
+        if node.module is not None and node.level == 0:
+            modules.add(node.module.split(".")[0])
+
+    node_iter = ast.NodeVisitor()
+    node_iter.visit_Import = visit_Import
+    node_iter.visit_ImportFrom = visit_ImportFrom
+    with open(script_file, "r") as f:
+        node_iter.visit(ast.parse(f.read()))
+    return modules
+
+
+def get_vendored_dependencies(script_file: str) -> List:
+    all_std_modules = get_std_modules()
+    all_modules_used = get_all_modules_imported_script(script_file)
+    vendored_modules = []
+    for dep in all_modules_used:
+        if dep in all_std_modules:
+            continue
+        vendored_modules.append(dep.lower())
+    return vendored_modules

setup.py

@@ -7,7 +7,13 @@
     entry_points={"console_scripts": ["grayskull = grayskull.__main__:main"]},
     use_scm_version={"write_to": "grayskull/_version.py"},
     setup_requires=["setuptools-scm", "setuptools>=30.3.0"],
-    install_requires=["requests", "ruamel.yaml >=0.15.3", "ruamel.yaml.jinja2"],
+    install_requires=[
+        "requests",
+        "ruamel.yaml >=0.15.3",
+        "ruamel.yaml.jinja2",
+        "stdlib-list",
+        "pip",
+    ],
     extras_require={"testing": ["pytest"]},
     url="https://github.com/marcelotrevisani/grayskull",
     license="MIT",

tests/conftest.py

@@ -0,0 +1,8 @@
+import os
+
+from pytest import fixture
+
+
+@fixture
+def data_dir() -> str:
+    return os.path.join(os.path.dirname(__file__), "data")

tests/data/foo_imports.py

@@ -0,0 +1,12 @@
+import os
+import sys
+
+import numpy as np
+
+
+def foo():
+    import pandas
+
+
+def bar():
+    from requests import get

tests/test_base_recipe.py

@@ -15,8 +15,8 @@ def refresh_section(self, section="", **kwargs):
 
 
 @fixture
-def data_recipes() -> str:
-    return os.path.join(os.path.dirname(__file__), "data", "recipes")
+def data_recipes(data_dir: str) -> str:
+    return os.path.join(data_dir, "recipes")
 
 
 def test_update_all_recipe(data_recipes):

tests/test_pypi.py

@@ -190,7 +190,9 @@ def test_merge_pypi_sdist_metadata():
     sdist_metadata = recipe._get_sdist_metadata(pypi_metadata["sdist_url"])
     merged_data = PyPi._merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata)
     assert merged_data["compilers"] == ["c"]
-    assert merged_data["setup_requires"] == ["numpy"]
+    assert sorted(merged_data["setup_requires"]) == sorted(
+        ["numpy", "pip", "versioneer"]
+    )
 
 
 def test_update_requirements_with_pin():
@@ -273,3 +275,13 @@ def test_download_pkg_sdist(tmpdir):
     assert (
         pkg_sha256 == "0d5fe9189a148acc3c3eb2ac8e1ac0742cb7618c084f3d228baaec0c254b318d"
     )
+
+
+def test_ciso_recipe():
+    recipe = PyPi(name="ciso", version="0.1.0")
+    assert sorted(recipe["requirements"]["host"]) == sorted(
+        ["cython", "numpy", "pip", "python", "versioneer"]
+    )
+    assert sorted(recipe["requirements"]["run"]) == sorted(
+        ["cython", "python", "<{ pin_compatible('numpy') }}"]
+    )

tests/test_utils.py

@@ -0,0 +1,27 @@
+import os
+
+from grayskull.utils import (
+    get_all_modules_imported_script,
+    get_std_modules,
+    get_vendored_dependencies,
+)
+
+
+def test_get_std_modules():
+    std_modules = get_std_modules()
+    assert "sys" in std_modules
+    assert "os" in std_modules
+    assert "ast" in std_modules
+    assert "typing" in std_modules
+
+
+def test_get_all_modules_imported_script(data_dir):
+    all_imports = get_all_modules_imported_script(
+        os.path.join(data_dir, "foo_imports.py")
+    )
+    assert sorted(all_imports) == sorted(["numpy", "pandas", "requests", "os", "sys"])
+
+
+def test_get_vendored_dependencies(data_dir):
+    all_deps = get_vendored_dependencies(os.path.join(data_dir, "foo_imports.py"))
+    assert sorted(all_deps) == sorted(["numpy", "pandas", "requests"])