diff --git a/.flake8 b/.flake8 index 125076a4c5c..e9e773e74a1 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,4 @@ [flake8] max-line-length=88 extend-ignore=E203,D104,D100 +exclude=tests/data/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 627c41fe704..3276dac41cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,10 +29,12 @@ repos: rev: v4.3.21 hooks: - id: isort + exclude: tests/data - repo: https://gitlab.com/pycqa/flake8 rev: 3.7.9 hooks: - id: flake8 + exclude: tests/data language_version: python3 additional_dependencies: - flake8-typing-imports==1.3.0 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f2d5e246678..b8ef702d19b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,7 @@ variables: - group: Codecov - name: grayskull_deps - value: pytest pytest-azurepipelines pytest-xdist pytest-cov requests ruamel.yaml codecov ruamel.yaml.jinja2 "coverage<5.0" + value: pytest pytest-azurepipelines pytest-xdist pytest-cov requests ruamel.yaml codecov ruamel.yaml.jinja2 "coverage<5.0" stdlib-list jobs: - job: diff --git a/grayskull/base/recipe_item.py b/grayskull/base/recipe_item.py index 16d8202adb1..72b4cabdbdc 100644 --- a/grayskull/base/recipe_item.py +++ b/grayskull/base/recipe_item.py @@ -27,6 +27,9 @@ def __str__(self) -> str: val += f" {comment.value}" return val + def __lt__(self, other: "RecipeItem") -> bool: + return self.value < other.value + def __eq__(self, other: Union[str, int, "RecipeItem"]) -> bool: if isinstance(other, RecipeItem): return self.value == other.value and self.selector == other.selector diff --git a/grayskull/base/section.py b/grayskull/base/section.py index e201ac58ff8..d128ce5ddf4 100644 --- a/grayskull/base/section.py +++ b/grayskull/base/section.py @@ -105,6 +105,11 @@ def __eq__(self, other: Any) -> bool: return self.values[0] == other if isinstance(other, str): return self.section_name == other + if isinstance(other, list) and isinstance(other[0], str): + for pos, item in enumerate(self.values): + if item.value != other[pos]: + return False + return True return other == self.values def __iter__(self) -> Iterator: diff --git a/grayskull/pypi/pypi.py b/grayskull/pypi/pypi.py index d78bb1b4a4f..0579a0fb741 100644 --- a/grayskull/pypi/pypi.py +++ b/grayskull/pypi/pypi.py @@ -16,6 +16,7 @@ from requests import HTTPError from grayskull.base.base_recipe import AbstractRecipeModel +from grayskull.utils import get_vendored_dependencies log = logging.getLogger(__name__) PyVer = namedtuple("PyVer", ["major", "minor"]) @@ -104,7 +105,9 @@ def __fake_distutils_setup(*args, **kwargs): data_dist["install_requires"] = kwargs.get("install_requires", []) if not data_dist.get("setup_requires"): data_dist["setup_requires"] = [] - data_dist["setup_requires"] += kwargs.get("setup_requires", []) + data_dist["setup_requires"] += ( + kwargs.get("setup_requires") if kwargs.get("setup_requires") else [] + ) data_dist["extras_require"] = kwargs.get("extras_require", []) data_dist["requires_python"] = kwargs.get("requires_python", None) data_dist["entry_points"] = kwargs.get("entry_points", None) @@ -136,10 +139,10 @@ def __fake_distutils_setup(*args, **kwargs): setup_ext.build_ext = _fake_build_ext_setuptools path_setup = str(path_setup) PyPi.__run_setup_py(path_setup, data_dist) - if not data_dist: + if not data_dist or data_dist.get("install_requires", None) is None: PyPi.__run_setup_py(path_setup, data_dist, run_py=True) yield data_dist - except Exception: + except Exception as err: # noqa yield data_dist core.setup = setup_core_original dist_ext.build_ext = original_build_ext_distutils @@ -155,6 +158,7 @@ def __run_setup_py(path_setup: str, data_dist: dict, run_py=False): if os.path.dirname(path_setup) not in sys.path: sys.path.append(os.path.dirname(path_setup)) sys.path.append(pip_dir) + PyPi._install_deps_if_necessary(path_setup, data_dist, pip_dir) try: if run_py: import runpy @@ -166,17 +170,34 @@ def __run_setup_py(path_setup: str, data_dist: dict, run_py=False): path_setup, script_args=["install", f"--target={pip_dir}"] ) except ModuleNotFoundError as err: - if not data_dist.get("setup_requires"): - data_dist["setup_requires"] = [] - data_dist["setup_requires"].append(err.name) - check_output(["pip", "install", err.name, f"--target={pip_dir}"]) + PyPi._pip_install_dep(data_dist, err.name, pip_dir) PyPi.__run_setup_py(path_setup, data_dist, run_py) - except Exception: + except Exception as err: # noqa pass if os.path.exists(pip_dir): - os.rmdir(pip_dir) + shutil.rmtree(pip_dir) sys.path = original_path + @staticmethod + def _install_deps_if_necessary(setup_path: str, data_dist: dict, pip_dir: str): + all_setup_deps = get_vendored_dependencies(setup_path) + for dep in all_setup_deps: + PyPi._pip_install_dep(data_dist, dep, pip_dir) + + @staticmethod + def _pip_install_dep(data_dist: dict, dep_name: str, pip_dir: str): + if not data_dist.get("setup_requires"): + data_dist["setup_requires"] = [] + if dep_name == "pkg_resources": + dep_name = "setuptools" + if ( + dep_name.lower() not in data_dist["setup_requires"] + and dep_name.lower() != "setuptools" + ): + data_dist["setup_requires"].append(dep_name.lower()) + if dep_name != "setuptools": + check_output(["pip", "install", dep_name, f"--target={pip_dir}"]) + @staticmethod def _merge_pypi_sdist_metadata(pypi_metadata: dict, sdist_metadata: dict) -> dict: """This method is responsible to merge two dictionaries and it will give @@ -266,6 +287,7 @@ def refresh_section(self, section: str = ""): if not self._is_arch: self["build"]["noarch"] = "python" + @lru_cache(maxsize=10) def _get_metadata(self) -> dict: name = self.get_var_content(self["package"]["name"].values[0]) pypi_metada = self._get_pypi_metadata() @@ -365,7 +387,8 @@ def _extract_requirements(self, metadata: dict) -> dict: run_req = self._get_run_req_from_requires_dist(requires_dist) build_req = [f"<{{ compiler('{c}') }}}}" for c in metadata.get("compilers", [])] - self._is_arch = self._is_arch or build_req + if build_req: + self._is_arch = True if self._is_arch: version_to_selector = PyPi.py_version_to_selector(metadata) diff --git a/grayskull/utils.py b/grayskull/utils.py new file mode 100644 index 00000000000..99a4f242ab4 --- /dev/null +++ b/grayskull/utils.py @@ -0,0 +1,45 @@ +import ast +from functools import lru_cache +from typing import List + + +@lru_cache(maxsize=10) +def get_std_modules() -> List: + from stdlib_list import stdlib_list + + all_libs = set() + for py_ver in ("2.7", "3.6", "3.7", "3.8"): + all_libs.update(stdlib_list(py_ver)) + return list(all_libs) + + +def get_all_modules_imported_script(script_file: str) -> set: + modules = set() + + def visit_Import(node): + for name in node.names: + modules.add(name.name.split(".")[0]) + + def visit_ImportFrom(node): + # if node.module is missing it's a "from . import ..." statement + # if level > 0 it's a "from .submodule import ..." statement + if node.module is not None and node.level == 0: + modules.add(node.module.split(".")[0]) + + node_iter = ast.NodeVisitor() + node_iter.visit_Import = visit_Import + node_iter.visit_ImportFrom = visit_ImportFrom + with open(script_file, "r") as f: + node_iter.visit(ast.parse(f.read())) + return modules + + +def get_vendored_dependencies(script_file: str) -> List: + all_std_modules = get_std_modules() + all_modules_used = get_all_modules_imported_script(script_file) + vendored_modules = [] + for dep in all_modules_used: + if dep in all_std_modules: + continue + vendored_modules.append(dep.lower()) + return vendored_modules diff --git a/setup.py b/setup.py index 204b209b863..cc65e4e167c 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,13 @@ entry_points={"console_scripts": ["grayskull = grayskull.__main__:main"]}, use_scm_version={"write_to": "grayskull/_version.py"}, setup_requires=["setuptools-scm", "setuptools>=30.3.0"], - install_requires=["requests", "ruamel.yaml >=0.15.3", "ruamel.yaml.jinja2"], + install_requires=[ + "requests", + "ruamel.yaml >=0.15.3", + "ruamel.yaml.jinja2", + "stdlib-list", + "pip", + ], extras_require={"testing": ["pytest"]}, url="https://github.com/marcelotrevisani/grayskull", license="MIT", diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000000..a3f0bd759f3 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,8 @@ +import os + +from pytest import fixture + + +@fixture +def data_dir() -> str: + return os.path.join(os.path.dirname(__file__), "data") diff --git a/tests/data/foo_imports.py b/tests/data/foo_imports.py new file mode 100644 index 00000000000..57372d3e130 --- /dev/null +++ b/tests/data/foo_imports.py @@ -0,0 +1,12 @@ +import os +import sys + +import numpy as np + + +def foo(): + import pandas + + +def bar(): + from requests import get diff --git a/tests/test_base_recipe.py b/tests/test_base_recipe.py index dab0b9dd61c..67db445d54d 100644 --- a/tests/test_base_recipe.py +++ b/tests/test_base_recipe.py @@ -15,8 +15,8 @@ def refresh_section(self, section="", **kwargs): @fixture -def data_recipes() -> str: - return os.path.join(os.path.dirname(__file__), "data", "recipes") +def data_recipes(data_dir: str) -> str: + return os.path.join(data_dir, "recipes") def test_update_all_recipe(data_recipes): diff --git a/tests/test_pypi.py b/tests/test_pypi.py index 176ea2bc1db..c1d83094475 100644 --- a/tests/test_pypi.py +++ b/tests/test_pypi.py @@ -190,7 +190,9 @@ def test_merge_pypi_sdist_metadata(): sdist_metadata = recipe._get_sdist_metadata(pypi_metadata["sdist_url"]) merged_data = PyPi._merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata) assert merged_data["compilers"] == ["c"] - assert merged_data["setup_requires"] == ["numpy"] + assert sorted(merged_data["setup_requires"]) == sorted( + ["numpy", "pip", "versioneer"] + ) def test_update_requirements_with_pin(): @@ -273,3 +275,13 @@ def test_download_pkg_sdist(tmpdir): assert ( pkg_sha256 == "0d5fe9189a148acc3c3eb2ac8e1ac0742cb7618c084f3d228baaec0c254b318d" ) + + +def test_ciso_recipe(): + recipe = PyPi(name="ciso", version="0.1.0") + assert sorted(recipe["requirements"]["host"]) == sorted( + ["cython", "numpy", "pip", "python", "versioneer"] + ) + assert sorted(recipe["requirements"]["run"]) == sorted( + ["cython", "python", "<{ pin_compatible('numpy') }}"] + ) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000000..d2b8b041d23 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,27 @@ +import os + +from grayskull.utils import ( + get_all_modules_imported_script, + get_std_modules, + get_vendored_dependencies, +) + + +def test_get_std_modules(): + std_modules = get_std_modules() + assert "sys" in std_modules + assert "os" in std_modules + assert "ast" in std_modules + assert "typing" in std_modules + + +def test_get_all_modules_imported_script(data_dir): + all_imports = get_all_modules_imported_script( + os.path.join(data_dir, "foo_imports.py") + ) + assert sorted(all_imports) == sorted(["numpy", "pandas", "requests", "os", "sys"]) + + +def test_get_vendored_dependencies(data_dir): + all_deps = get_vendored_dependencies(os.path.join(data_dir, "foo_imports.py")) + assert sorted(all_deps) == sorted(["numpy", "pandas", "requests"])