Merge pull request #50 from toabctl/fixes

Use xmlrpc_client from six

.travis.yml

@@ -8,12 +8,8 @@ matrix:
       env: TOX_ENV=pep8
     - python: 2.7
       env: TOX_ENV=py27
-    - python: 3.3
-      env: TOX_ENV=py34
-    - python: 3.4
-      env: TOX_ENV=py34
     - python: 3.5
-      env: TOX_ENV=py34
+      env: TOX_ENV=py35
 install:
   - pip install tox
 script:

py2pack/__init__.py

@@ -35,35 +35,34 @@
 import sys
 import tarfile
 import urllib
-
-try:
-    import xmlrpc.client as xmlrpclib
-except:
-    import xmlrpclib
+from six.moves import xmlrpc_client
+from six.moves import filter
+from six.moves import map
 import zipfile
-
 import jinja2
 
 import py2pack.proxy
 import py2pack.requires
 import py2pack.utils
 
 
-TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')  # absolute template path
-pypi = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')                      # XML RPC connection to PyPI
-env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR))      # Jinja2 template environment
+TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')
+pypi = xmlrpc_client.ServerProxy('https://pypi.python.org/pypi')
+
+# setup jinja2 environment with custom filters
+env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR))
 env.filters['parenthesize_version'] = \
     lambda s: re.sub('([=<>]+)(.+)', r' (\1 \2)', s)
 env.filters['basename'] = \
     lambda s: s[s.rfind('/') + 1:]
 
-SPDX_LICENSES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'spdx_license_map.p')  # absolute template path
+SPDX_LICENSES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'spdx_license_map.p')
 SDPX_LICENSES = pickle.load(open(SPDX_LICENSES_FILE, 'rb'))
 
 
 def list(args=None):
     print('listing all PyPI packages...')
-    for package in pypi.list_packages():                                    # nothing fancy
+    for package in pypi.list_packages():
         print(package)
 
 
@@ -76,19 +75,19 @@ def search(args):
 def show(args):
     check_or_set_version(args)
     print('showing package {0}...'.format(args.name))
-    data = pypi.release_data(args.name, args.version)                       # fetch all meta data
+    data = pypi.release_data(args.name, args.version)
     pprint.pprint(data)
 
 
 def fetch(args):
     check_or_set_version(args)
     url = newest_download_url(args)
     if not url:
-        print("unable to find a source release for {0}!".format(args.name))  # pass out if nothing is found
+        print("unable to find a source release for {0}!".format(args.name))
         sys.exit(1)
     print('downloading package {0}-{1}...'.format(args.name, args.version))
     print('from {0}'.format(url['url']))
-    urllib.urlretrieve(url['url'], url['filename'])                         # download the object behind the URL
+    urllib.urlretrieve(url['url'], url['filename'])
 
 
 def _parse_setup_py(filename, setup_filename, data):
@@ -117,41 +116,66 @@ def _run_setup_py(tarfile, data):
     return names
 
 
-def _canonicalize_setup_data(data):
-    def _sanitize_requirements(req):
+def _requirement_filter_by_marker(req):
+    """check if the requirement is satisfied by the marker"""
+    if req.marker:
+        # TODO (toabctl): currently we hardcode python 2.7 and linux2
+        # see https://www.python.org/dev/peps/pep-0508/#environment-markers
+        marker_env = {'python_version': '2.7', 'sys_platform': 'linux'}
+        if not req.marker.evaluate(environment=marker_env):
+            return False
+    return True
+
+
+def _requirement_find_lowest_possible(req):
         """ find lowest required version"""
         version_dep = None
         version_comp = None
-        pkg = pkg_resources.Requirement.parse(req)
-        for dep in pkg.specs:
+        for dep in req.specs:
             version = pkg_resources.parse_version(dep[1])
+            # we don't want to have a not supported version as minimal version
+            if dep[0] == '!=':
+                continue
             # try to use the lowest version available
             # i.e. for ">=0.8.4,>=0.9.7", select "0.8.4"
             if (not version_dep or
                     version < pkg_resources.parse_version(version_dep)):
                 version_dep = dep[1]
                 version_comp = dep[0]
         return filter(lambda x: x is not None,
-                      [pkg.unsafe_name, version_comp, version_dep])
+                      [req.unsafe_name, version_comp, version_dep])
+
+
+def _requirements_sanitize(req_list):
+    filtered_req_list = map(
+        _requirement_find_lowest_possible, filter(
+            _requirement_filter_by_marker,
+            map(lambda x: pkg_resources.Requirement.parse(x), req_list)
+        )
+    )
+    return [" ".join(req) for req in filtered_req_list]
 
+
+def _canonicalize_setup_data(data):
     if "install_requires" in data:
         # install_requires may be a string, convert to list of strings:
         if isinstance(data["install_requires"], str):
             data["install_requires"] = data["install_requires"].splitlines()
+        data["install_requires"] = _requirements_sanitize(data["install_requires"])
 
-        # find lowest version and take care of spaces between name and version
-        data["install_requires"] = [" ".join(_sanitize_requirements(req))
-                                    for req in data["install_requires"]]
+    if "tests_require" in data:
+        # tests_require may be a string, convert to list of strings:
+        if isinstance(data["tests_require"], str):
+            data["tests_require"] = data["tests_require"].splitlines()
+        data["tests_require"] = _requirements_sanitize(data["tests_require"])
 
     if "extras_require" in data:
         # extras_require value may be a string, convert to list of strings:
         for (key, value) in data["extras_require"].items():
             if isinstance(value, str):
                 data["extras_require"][key] = value.splitlines()
-            # find lowest version and take care of spaces between name and ver
-            data["extras_require"][key] = [
-                " ".join(_sanitize_requirements(req))
-                for req in data["extras_require"][key]]
+            data["extras_require"][key] = _requirements_sanitize(
+                data["extras_require"][key])
 
     if "data_files" in data:
         # data_files may be a sequence of files without a target directory:
@@ -181,6 +205,8 @@ def _augment_data_from_tarball(args, filename, data):
     else:
         names = _parse_setup_py(filename, setup_filename, data)
 
+    _canonicalize_setup_data(data)
+
     for name in names:
         match = re.match(docs_re, name)
         if match:

py2pack/utils.py

@@ -21,6 +21,7 @@
 import tarfile
 import tempfile
 import zipfile
+from six.moves import filter
 
 
 @contextmanager
@@ -42,7 +43,7 @@ def _extract_to_tempdir(filename):
         else:
             raise Exception("Can not extract '%s'. Not a tar or zip file" % filename)
         os.chdir(tempdir)
-        yield tempdir, names
+        yield tempdir, filter(lambda x: x != './', names)
     finally:
         os.chdir(current_cwd)
         shutil.rmtree(tempdir)

test/test_py2pack.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pkg_resources
 import unittest
 from ddt import ddt, data, unpack
 
@@ -56,23 +57,64 @@ def test_normalize_license(self, value, expected_result):
         py2pack._normalize_license(d)
         self.assertEqual(d['license'], expected_result)
 
+    @data(
+        ("pywin32>=1.0;sys_platform=='win32'  # PSF", False),
+        ("foobar", True),
+        ("foobar;python_version=='2.7'", True),
+        ("foobar;python_version=='3.5'", False),
+    )
+    @unpack
+    def test__requirement_filter_by_marker(self, req, expected):
+        pkg = pkg_resources.Requirement.parse(req)
+        self.assertEqual(py2pack._requirement_filter_by_marker(pkg), expected)
+
+    @data(
+        ("foobar>=1.0", ["foobar", ">=", "1.0"]),
+        ("foobar>=1.0,>2", ["foobar", ">=", "1.0"]),
+        ("foobar>=2,>1.0,<=3", ["foobar", ">", "1.0"]),
+        ("foobar>=2,>1.0,!=0.5", ["foobar", ">", "1.0"]),
+        ("foobar!=0.5", ["foobar"]),
+    )
+    @unpack
+    def test__requirement_find_lowest_possible(self, req, expected):
+        pkg = pkg_resources.Requirement.parse(req)
+        self.assertEqual(list(py2pack._requirement_find_lowest_possible(pkg)), expected)
+
+    @data(
+        (['six', 'monotonic>=0.1'], ['six', 'monotonic >= 0.1']),
+        (['monotonic>=1.0,>0.1'], ['monotonic > 0.1']),
+    )
+    @unpack
+    def test__requirements_sanitize(self, req_list, expected):
+        self.assertEqual(py2pack._requirements_sanitize(req_list), expected)
+
     @data(
         (
-            {'install_requires': ['six', 'monotonic>=0.1']},
-            {'install_requires': ['six', 'monotonic >= 0.1']},
+            {'install_requires': ["pywin32>=1.0;sys_platform=='win32'", 'monotonic>=0.1 #comment']},
+            {'install_requires': ['monotonic >= 0.1']},
         ),
         (
-            {'install_requires': ['six', 'foobar>=0.1,>=0.5']},
-            {'install_requires': ['six', 'foobar >= 0.1']},
+            {'install_requires': 'six  >=1.9,!=1.0  # comment\nfoobar>=0.1,>=0.5'},
+            {'install_requires': ['six >= 1.9', 'foobar >= 0.1']}
         ),
         (
-            {'install_requires': ['six  >=1.9', 'foobar>=0.1,>=0.5']},
-            {'install_requires': ['six >= 1.9', 'foobar >= 0.1']}
+            {'tests_require': ['six  >=1.9', 'foobar>=0.1,>=0.5']},
+            {'tests_require': ['six >= 1.9', 'foobar >= 0.1']}
+        ),
+        (
+            {'tests_require': 'six  >=1.9\nfoobar>=0.1,>=0.5'},
+            {'tests_require': ['six >= 1.9', 'foobar >= 0.1']}
         ),
         (
             {'extras_require': {'extra1': ['foobar<=3.0, >= 2.1']}},
             {'extras_require': {'extra1': ['foobar >= 2.1']}}
-        )
+        ),
+        (
+            {'extras_require': {'extra1': 'foobar<=3.0, >= 2.1\ntest1  # comment',
+                                'extra2': ['test2']}},
+            {'extras_require': {'extra1': ['foobar >= 2.1', 'test1'],
+                                'extra2': ['test2']}}
+        ),
     )
     @unpack
     def test_canonicalize_setup_data(self, data, expected_data):

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py33,py34,py35,pep8,cover
+envlist = py27,py35,pep8,cover
 
 [testenv]
 usedevelop = True