Skip to content

Commit

Permalink
Merge pull request #54 from toabctl/docs2
Browse files Browse the repository at this point in the history 
Fix README.rst style
  • Loading branch information
@toabctl
toabctl committed Jun 30, 2016
2 parents c56f756 + c7f7433 commit 93f3017
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 257 deletions.
6 changes: 0 additions & 6 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,6 @@ generate a spec file (named 'python-zope.interface.spec'):
The source tarball and the package recipe is all you need to generate the RPM_
(or DEB_) file.

.. note:: The "generate" operation is parsing the setup.py and tries to extract
the needed information to generate a .spec file. That doesn't work in all cases.
There is a "--run" option (:code:`py2pack generate --run`) which may give better
results.

This final step may depend on which distribution you use. Again,
for openSUSE_ (and by using the `Open Build Service`_), the complete recipe is:

Expand Down
91 changes: 18 additions & 73 deletions py2pack/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,11 @@
import pwd
import re
import sys
import tarfile
import urllib
from six.moves import xmlrpc_client
from six.moves import filter
from six.moves import map
import zipfile
import jinja2
import warnings
warnings.simplefilter('always', DeprecationWarning)

import py2pack.proxy
import py2pack.requires
Expand Down Expand Up @@ -90,92 +88,35 @@ def fetch(args):
urllib.urlretrieve(url['url'], url['filename'])


def _parse_setup_py(filename, setup_filename, data):
"""parse the given setup_filename from the filename (usually a tarball)
this is not executing the setup.py file but parsing it with regex."""
names = []
if tarfile.is_tarfile(filename):
with tarfile.open(filename) as f:
names = f.getnames()
d = py2pack.requires._requires_from_setup_py(
f.extractfile(setup_filename))
data.update(d)
elif zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename) as f:
names = f.namelist()
with f.open(setup_filename) as s:
d = py2pack.requires._requires_from_setup_py(s)
data.update(d)
return names


def _run_setup_py(tarfile, data):
with py2pack.utils._extract_to_tempdir(tarfile) as (tmp_dir, names):
d = py2pack.requires._requires_from_setup_py_run(tmp_dir)
data.update(d)
return names


def _requirement_filter_by_marker(req):
"""check if the requirement is satisfied by the marker"""
if hasattr(req, 'marker') and req.marker:
# TODO (toabctl): currently we hardcode python 2.7 and linux2
# see https://www.python.org/dev/peps/pep-0508/#environment-markers
marker_env = {'python_version': '2.7', 'sys_platform': 'linux'}
if not req.marker.evaluate(environment=marker_env):
return False
return True


def _requirement_find_lowest_possible(req):
""" find lowest required version"""
version_dep = None
version_comp = None
for dep in req.specs:
version = pkg_resources.parse_version(dep[1])
# we don't want to have a not supported version as minimal version
if dep[0] == '!=':
continue
# try to use the lowest version available
# i.e. for ">=0.8.4,>=0.9.7", select "0.8.4"
if (not version_dep or
version < pkg_resources.parse_version(version_dep)):
version_dep = dep[1]
version_comp = dep[0]
return filter(lambda x: x is not None,
[req.unsafe_name, version_comp, version_dep])


def _requirements_sanitize(req_list):
filtered_req_list = map(
_requirement_find_lowest_possible, filter(
_requirement_filter_by_marker,
map(lambda x: pkg_resources.Requirement.parse(x), req_list)
)
)
return [" ".join(req) for req in filtered_req_list]


def _canonicalize_setup_data(data):
if "install_requires" in data:
# install_requires may be a string, convert to list of strings:
if isinstance(data["install_requires"], str):
data["install_requires"] = data["install_requires"].splitlines()
data["install_requires"] = _requirements_sanitize(data["install_requires"])
data["install_requires"] = \
py2pack.requires._requirements_sanitize(data["install_requires"])

if "tests_require" in data:
# tests_require may be a string, convert to list of strings:
if isinstance(data["tests_require"], str):
data["tests_require"] = data["tests_require"].splitlines()
data["tests_require"] = _requirements_sanitize(data["tests_require"])
data["tests_require"] = \
py2pack.requires._requirements_sanitize(data["tests_require"])

if "extras_require" in data:
# extras_require value may be a string, convert to list of strings:
for (key, value) in data["extras_require"].items():
if isinstance(value, str):
data["extras_require"][key] = value.splitlines()
data["extras_require"][key] = _requirements_sanitize(
data["extras_require"][key])
data["extras_require"][key] = \
py2pack.requires._requirements_sanitize(data["extras_require"][key])

if "data_files" in data:
# data_files may be a sequence of files without a target directory:
Expand All @@ -196,14 +137,10 @@ def _canonicalize_setup_data(data):


def _augment_data_from_tarball(args, filename, data):
setup_filename = "{0}-{1}/setup.py".format(args.name, args.version)
docs_re = re.compile("{0}-{1}\/((?:AUTHOR|ChangeLog|CHANGES|COPYING|LICENSE|NEWS|README).*)".format(args.name, args.version), re.IGNORECASE)
shell_metachars_re = re.compile("[|&;()<>\s]")

if args.run:
names = _run_setup_py(filename, data)
else:
names = _parse_setup_py(filename, setup_filename, data)
names = _run_setup_py(filename, data)

_canonicalize_setup_data(data)

Expand Down Expand Up @@ -315,14 +252,22 @@ def main():
parser_generate.add_argument('version', nargs='?', help='package version (optional)')
parser_generate.add_argument('-t', '--template', choices=file_template_list(), default='opensuse.spec', help='file template')
parser_generate.add_argument('-f', '--filename', help='spec filename (optional)')
parser_generate.add_argument('-r', '--run', action='store_true', help='run setup.py (optional, risky!)')
# TODO (toabctl): remove this is a later release
parser_generate.add_argument(
'-r', '--run', action='store_true',
help='DEPRECATED and noop. will be removed in future releases!')
parser_generate.set_defaults(func=generate)

parser_help = subparsers.add_parser('help', help='show this help')
parser_help.set_defaults(func=lambda args: parser.print_help())

args = parser.parse_args()

# TODO (toabctl): remove this is a later release
if args.run:
warnings.warn("the '--run' switch is deprecated and a noop",
DeprecationWarning)

# set HTTP proxy if one is provided
if args.proxy:
try:
Expand Down
108 changes: 43 additions & 65 deletions py2pack/requires.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,73 +20,11 @@

import json
import os
import re
import six
import pkg_resources
from six.moves import filter
from six.moves import map
import subprocess
import sys
import tempfile
import textwrap


def _safe_eval(descr, code, fallback):
try:
return eval(code)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
code = re.sub("(?m)^", " ", code)
sys.stderr.write(
textwrap.dedent("""\
WARNING: Exception encountered:
%s: %s
whilst eval'ing code for '%s' parameter in setup.py:
%s
Try regenerating with the --run option.
""") % (exc_type.__name__, exc_value, descr, code))
return fallback


def _requires_from_setup_py(file):
"""read requirements from the setup.py file"""
data = {}
contents = six.u(file.read())
fixme = "FIXME: failed to extract %s from setup.py"
match = re.search("ext_modules", contents)
if match:
data["is_extension"] = True
match = re.search("[(,]\s*scripts\s*=\s*(\[.*?\])", contents, flags=re.DOTALL)
if match:
data["scripts"] = \
_safe_eval("scripts", match.group(1), [fixme % "scripts"])
match = re.search("test_suite\s*=\s*(.*)", contents)
if match:
data["test_suite"] = \
_safe_eval("test_suite", match.group(1), [fixme % "test_suite"])
match = re.search("install_requires\s*=\s*(\[.*?\])", contents, flags=re.DOTALL)
if match:
data["install_requires"] = \
_safe_eval("install_requires", match.group(1), [fixme % "install_requires"])
match = re.search("extras_require\s*=\s*(\{.*?\})", contents, flags=re.DOTALL)
if match:
data["extras_require"] = \
_safe_eval("extras_require", match.group(1), [fixme % "extras_require"])
match = re.search("tests_require\s*=\s*(\{.*?\})", contents, flags=re.DOTALL)
if match:
data["tests_require"] = \
_safe_eval("tests_require", match.group(1), [fixme % "tests_require"])
match = re.search("data_files\s*=\s*(\[.*?\])", contents, flags=re.DOTALL)
if match:
data["data_files"] = \
_safe_eval("data_files", match.group(1), [fixme % "data_files"])
match = re.search('entry_points\s*=\s*(\{.*?\}|""".*?"""|".*?")', contents, flags=re.DOTALL)
if match:
data["entry_points"] = \
_safe_eval("entry_points", match.group(1), [fixme % "entry_points"])
return data


def _requires_from_setup_py_run(tmp_dir):
Expand All @@ -112,3 +50,43 @@ def _requires_from_setup_py_run(tmp_dir):
finally:
os.chdir(current_cwd)
return data


def _requirement_filter_by_marker(req):
"""check if the requirement is satisfied by the marker"""
if hasattr(req, 'marker') and req.marker:
# TODO (toabctl): currently we hardcode python 2.7 and linux2
# see https://www.python.org/dev/peps/pep-0508/#environment-markers
marker_env = {'python_version': '2.7', 'sys_platform': 'linux'}
if not req.marker.evaluate(environment=marker_env):
return False
return True


def _requirement_find_lowest_possible(req):
""" find lowest required version"""
version_dep = None
version_comp = None
for dep in req.specs:
version = pkg_resources.parse_version(dep[1])
# we don't want to have a not supported version as minimal version
if dep[0] == '!=':
continue
# try to use the lowest version available
# i.e. for ">=0.8.4,>=0.9.7", select "0.8.4"
if (not version_dep or
version < pkg_resources.parse_version(version_dep)):
version_dep = dep[1]
version_comp = dep[0]
return filter(lambda x: x is not None,
[req.unsafe_name, version_comp, version_dep])


def _requirements_sanitize(req_list):
filtered_req_list = map(
_requirement_find_lowest_possible, filter(
_requirement_filter_by_marker,
map(lambda x: pkg_resources.Requirement.parse(x), req_list)
)
)
return [" ".join(req) for req in filtered_req_list]
32 changes: 0 additions & 32 deletions test/test_py2pack.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import pkg_resources
import unittest
from ddt import ddt, data, unpack

Expand Down Expand Up @@ -57,37 +56,6 @@ def test_normalize_license(self, value, expected_result):
py2pack._normalize_license(d)
self.assertEqual(d['license'], expected_result)

@data(
("pywin32>=1.0;sys_platform=='win32' # PSF", False),
("foobar", True),
("foobar;python_version=='2.7'", True),
("foobar;python_version=='3.5'", False),
)
@unpack
def test__requirement_filter_by_marker(self, req, expected):
pkg = pkg_resources.Requirement.parse(req)
self.assertEqual(py2pack._requirement_filter_by_marker(pkg), expected)

@data(
("foobar>=1.0", ["foobar", ">=", "1.0"]),
("foobar>=1.0,>2", ["foobar", ">=", "1.0"]),
("foobar>=2,>1.0,<=3", ["foobar", ">", "1.0"]),
("foobar>=2,>1.0,!=0.5", ["foobar", ">", "1.0"]),
("foobar!=0.5", ["foobar"]),
)
@unpack
def test__requirement_find_lowest_possible(self, req, expected):
pkg = pkg_resources.Requirement.parse(req)
self.assertEqual(list(py2pack._requirement_find_lowest_possible(pkg)), expected)

@data(
(['six', 'monotonic>=0.1'], ['six', 'monotonic >= 0.1']),
(['monotonic>=1.0,>0.1'], ['monotonic > 0.1']),
)
@unpack
def test__requirements_sanitize(self, req_list, expected):
self.assertEqual(py2pack._requirements_sanitize(req_list), expected)

@data(
(
{'install_requires': ["pywin32>=1.0;sys_platform=='win32'", 'monotonic>=0.1 #comment']},
Expand Down

0 comments on commit 93f3017

Please sign in to comment.