Fix the lack of space between the package name and the constrain (sag…

…emath#37)

* Fix the lack of space between the package name and the constrain

* Add tests to cover more case scenarios

* Recipe improvements, download sdist packages using requests instead of pip

grayskull/pypi/pypi.py

@@ -9,7 +9,7 @@
 from functools import lru_cache
 from pathlib import Path
 from subprocess import check_output
-from tempfile import mktemp
+from tempfile import mkdtemp
 from typing import Dict, List, Optional, Tuple, Union
 
 import requests
@@ -36,28 +36,24 @@ def __init__(self, name=None, version=None, force_setup=False):
         super(PyPi, self).__init__(name=name, version=version)
         self["build"]["script"] = "<{ PYTHON }} -m pip install . -vv"
 
+    @staticmethod
+    def _download_sdist_pkg(sdist_url: str, dest: str):
+        response = requests.get(sdist_url, allow_redirects=True, stream=True)
+        with open(dest, "wb") as pkg_file:
+            for chunk_data in response.iter_content(chunk_size=1024 ** 2):
+                if chunk_data:
+                    pkg_file.write(chunk_data)
+
     @lru_cache(maxsize=10)
-    def _get_sdist_metadata(self, version: Optional[str] = None) -> dict:
+    def _get_sdist_metadata(self, sdist_url: str) -> dict:
         name = self.get_var_content(self["package"]["name"].values[0])
-        if not version and self["package"]["version"].values:
-            version = self.get_var_content(self["package"]["version"].values[0])
-        pkg = f"{name}=={version}" if version else name
-        temp_folder = mktemp(prefix=f"grayskull-{name}-")
-        check_output(
-            [
-                "pip",
-                "download",
-                pkg,
-                "--no-binary",
-                ":all:",
-                "--no-deps",
-                "-d",
-                str(temp_folder),
-            ]
-        )
-        shutil.unpack_archive(
-            os.path.join(temp_folder, os.listdir(temp_folder)[0]), temp_folder
-        )
+        temp_folder = mkdtemp(prefix=f"grayskull-{name}-")
+
+        pkg_name = sdist_url.split("/")[-1]
+        path_pkg = os.path.join(temp_folder, pkg_name)
+
+        self._download_sdist_pkg(sdist_url=sdist_url, dest=path_pkg)
+        shutil.unpack_archive(path_pkg, temp_folder)
         with PyPi._injection_distutils(temp_folder) as metadata:
             return metadata
 
@@ -247,7 +243,7 @@ def _merge_requires_dist(pypi_metadata: dict, sdist_metadata: dict) -> List:
         all_deps = []
         if pypi_metadata.get("requires_dist"):
             all_deps = pypi_metadata.get("requires_dist", [])
-        if sdist_metadata.get("requires_dist"):
+        if sdist_metadata.get("install_requires"):
             all_deps += sdist_metadata.get("install_requires", [])
 
         for sdist_pkg in all_deps:
@@ -273,7 +269,7 @@ def refresh_section(self, section: str = ""):
     def _get_metadata(self) -> dict:
         name = self.get_var_content(self["package"]["name"].values[0])
         pypi_metada = self._get_pypi_metadata()
-        sdist_metada = self._get_sdist_metadata()
+        sdist_metada = self._get_sdist_metadata(sdist_url=pypi_metada["sdist_url"])
         metadata = self._merge_pypi_sdist_metadata(pypi_metada, sdist_metada)
         test_imports = (
             metadata.get("packages") if metadata.get("packages") else [name.lower()]
@@ -332,8 +328,14 @@ def _get_pypi_metadata(self, version: Optional[str] = None) -> dict:
                 "{{ name }}-{{ version }}.tar.gz",
                 "sha256": PyPi.get_sha256_from_pypi_metadata(metadata),
             },
+            "sdist_url": self._get_sdist_url_from_pypi(metadata),
         }
 
+    def _get_sdist_url_from_pypi(self, metadata: dict) -> str:
+        for sdist_url in metadata["urls"]:
+            if sdist_url["packagetype"] == "sdist":
+                return sdist_url["url"]
+
     @staticmethod
     def get_sha256_from_pypi_metadata(pypi_metadata: dict) -> str:
         for pkg_info in pypi_metadata.get("urls"):
@@ -351,24 +353,21 @@ def __skip_pypi_requirement(list_extra: List) -> bool:
         return False
 
     def _extract_requirements(self, metadata: dict) -> dict:
-        requires_dist = metadata.get("requires_dist")
+        requires_dist = self._format_dependencies(metadata.get("requires_dist"))
         setup_requires = (
             metadata.get("setup_requires") if metadata.get("setup_requires") else []
         )
-        host_req = self._format_host_requirements(setup_requires)
+        host_req = self._format_dependencies(setup_requires)
 
         if not requires_dist and not host_req:
             return {"host": sorted(["python", "pip"]), "run": ["python"]}
 
-        run_req = self._get_run_req_from_requires_dist(
-            metadata.get("requires_dist", [])
-        )
+        run_req = self._get_run_req_from_requires_dist(requires_dist)
 
-        limit_python = metadata.get("requires_python", "")
         build_req = [f"<{{ compiler('{c}') }}}}" for c in metadata.get("compilers", [])]
         self._is_arch = self._is_arch or build_req
 
-        if limit_python or self._is_arch:
+        if self._is_arch:
             version_to_selector = PyPi.py_version_to_selector(metadata)
             if version_to_selector:
                 self["build"]["skip"] = True
@@ -383,27 +382,34 @@ def _extract_requirements(self, metadata: dict) -> dict:
             host_req += [f"python{limit_python}", "pip"]
 
         run_req.insert(0, f"python{limit_python}")
-        result = {"build": sorted(build_req)} if build_req else {}
-        result.update({"host": sorted(host_req), "run": sorted(run_req)})
+        result = (
+            {"build": sorted(map(lambda x: x.lower(), build_req))} if build_req else {}
+        )
+        result.update(
+            {
+                "host": sorted(map(lambda x: x.lower(), host_req)),
+                "run": sorted(map(lambda x: x.lower(), run_req)),
+            }
+        )
         self._update_requirements_with_pin(result)
         return result
 
     @staticmethod
-    def _format_host_requirements(setup_requires: List) -> List:
-        host_req = []
+    def _format_dependencies(all_dependencies: List) -> List:
+        formated_dependencies = []
         re_deps = re.compile(
             r"^\s*([\.a-zA-Z0-9_-]+)\s*(.*)\s*$", re.MULTILINE | re.DOTALL
         )
-        for req in setup_requires:
+        for req in all_dependencies:
             match_req = re_deps.match(req)
             deps_name = req
             if match_req:
                 match_req = match_req.groups()
                 deps_name = match_req[0]
                 if len(match_req) > 1:
                     deps_name = " ".join(match_req)
-            host_req.append(deps_name.strip())
-        return host_req
+            formated_dependencies.append(deps_name.strip())
+        return formated_dependencies
 
     @staticmethod
     def _update_requirements_with_pin(requirements: dict):
@@ -497,7 +503,7 @@ def _get_name_version_from_requires_dist(string_parse: str) -> Tuple[str, str]:
         :param string_parse: requires_dist value from PyPi metadata
         :return: Name and version of a package
         """
-        pkg = re.match(r"^\s*([^\s]+)\s*(\(.*\))?\s*", string_parse, re.DOTALL)
+        pkg = re.match(r"^\s*([^\s]+)\s*([\(]*.*[\)]*)?\s*", string_parse, re.DOTALL)
         pkg_name = pkg.group(1).strip()
         version = ""
         if len(pkg.groups()) > 1 and pkg.group(2):

tests/test_pypi.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 
@@ -132,7 +133,9 @@ def test_get_sha256_from_pypi_metadata():
 
 def test_injection_distutils():
     recipe = PyPi(name="hypothesis", version="5.5.1")
-    data = recipe._get_sdist_metadata()
+    data = recipe._get_sdist_metadata(
+        "https://pypi.io/packages/source/h/hypothesis/hypothesis-5.5.1.tar.gz"
+    )
     assert data["install_requires"] == [
         "attrs>=19.2.0",
         "sortedcontainers>=2.1.0,<3.0.0",
@@ -147,7 +150,9 @@ def test_injection_distutils():
 
 def test_injection_distutils_pytest():
     recipe = PyPi(name="pytest", version="5.3.2")
-    data = recipe._get_sdist_metadata()
+    data = recipe._get_sdist_metadata(
+        "https://pypi.io/packages/source/p/pytest/pytest-5.3.2.tar.gz"
+    )
     assert data["install_requires"] == [
         "py>=1.5.0",
         "packaging",
@@ -165,19 +170,24 @@ def test_injection_distutils_pytest():
         "setuptools_scm",
     ]
     assert not data.get("compilers")
+    assert recipe["build"]["skip"].values[0].value
+    assert recipe["build"]["skip"].values[0].selector == "py2k"
+    assert not recipe["build"]["noarch"]
 
 
 def test_injection_distutils_compiler_gsw():
     recipe = PyPi(name="gsw", version="3.3.1")
-    data = recipe._get_sdist_metadata()
+    data = recipe._get_sdist_metadata(
+        "https://pypi.io/packages/source/g/gsw/gsw-3.3.1.tar.gz"
+    )
     assert data.get("compilers") == ["c"]
     assert data["packages"] == ["gsw"]
 
 
 def test_merge_pypi_sdist_metadata():
     recipe = PyPi(name="gsw", version="3.3.1")
     pypi_metadata = recipe._get_pypi_metadata()
-    sdist_metadata = recipe._get_sdist_metadata()
+    sdist_metadata = recipe._get_sdist_metadata(pypi_metadata["sdist_url"])
     merged_data = PyPi._merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata)
     assert merged_data["compilers"] == ["c"]
     assert merged_data["setup_requires"] == ["numpy"]
@@ -229,7 +239,37 @@ def test_get_entry_points_from_sdist():
     ) == sorted(["gui_scripts=entrypoints", "console_scripts=entrypoints"])
 
 
+def test_build_noarch_skip():
+    recipe = PyPi(name="hypothesis", version="5.5.2")
+    assert recipe["build"]["noarch"].values[0] == "python"
+    assert not recipe["build"]["skip"].values
+
+
+def test_run_requirements_sdist():
+    recipe = PyPi(name="botocore", version="1.14.17")
+    assert recipe["requirements"]["run"].values == [
+        "docutils >=0.10,<0.16",
+        "jmespath >=0.7.1,<1.0.0",
+        "python",
+        "python-dateutil >=2.1,<3.0.0",
+        "urllib3 >=1.20,<1.26",
+    ]
+
+
 def test_format_host_requirements():
-    assert sorted(
-        PyPi._format_host_requirements(["setuptools>=40.0", "pkg2"])
-    ) == sorted(["setuptools >=40.0", "pkg2"])
+    assert sorted(PyPi._format_dependencies(["setuptools>=40.0", "pkg2"])) == sorted(
+        ["setuptools >=40.0", "pkg2"]
+    )
+
+
+def test_download_pkg_sdist(tmpdir):
+    dest_pkg = str(tmpdir / "test-download-pkg")
+    PyPi._download_sdist_pkg(
+        "https://pypi.io/packages/source/p/pytest/pytest-5.3.5.tar.gz", dest_pkg
+    )
+    with open(dest_pkg, "rb") as pkg_file:
+        content = pkg_file.read()
+        pkg_sha256 = hashlib.sha256(content).hexdigest()
+    assert (
+        pkg_sha256 == "0d5fe9189a148acc3c3eb2ac8e1ac0742cb7618c084f3d228baaec0c254b318d"
+    )