-
Notifications
You must be signed in to change notification settings - Fork 17
/
detect.py
271 lines (212 loc) · 8.79 KB
/
detect.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
import re
import os
import sys
from astroid.builder import AstroidBuilder
from astroid import MANAGER, CallFunc, Name, Assign, Keyword, List, Tuple, Const, AssName
from requirements_detector.requirement import DetectedRequirement
__all__ = ['find_requirements',
'RequirementsNotFound',
'CouldNotParseRequirements']
# PEP263, see http://legacy.python.org/dev/peps/pep-0263/
_ENCODING_REGEXP = re.compile(r'coding[:=]\s*([-\w.]+)')
_PY3K = sys.version_info >= (3, 0)
_PIP_OPTIONS = (
'-i', '--index-url',
'--extra-index-url',
'--no-index',
'-f', '--find-links',
'-r'
)
class RequirementsNotFound(Exception):
pass
class CouldNotParseRequirements(Exception):
pass
def _load_file_contents(filepath):
# This function is a bit of a tedious workaround (AKA 'hack').
# Astroid calls 'compile' under the hood, which refuses to accept a Unicode
# object which contains a PEP-263 encoding definition. However if we give
# Astroid raw bytes, it'll assume ASCII. Therefore we need to detect the encoding
# here, convert the file contents to a Unicode object, *and also strip the encoding
# declaration* to avoid the compile step breaking.
with open(filepath) as f:
if _PY3K:
return f.read()
contents = f.readlines()
result = []
encoding_lines = contents[0:2]
encoding = 'utf-8'
for line in encoding_lines:
match = _ENCODING_REGEXP.search(line)
if match is None:
result.append(line.strip())
else:
encoding = match.group(1)
result += [line.strip() for line in contents[2:]]
result = '\n'.join(result)
return result.decode(encoding)
def find_requirements(path):
"""
This method tries to determine the requirements of a particular project
by inspecting the possible places that they could be defined.
It will attempt, in order:
1) to parse setup.py in the root for an install_requires value
2) to read a requirements.txt file or a requirements.pip in the root
3) to read all .txt files in a folder called 'requirements' in the root
4) to read files matching "*requirements*.txt" and "*reqs*.txt" in the root,
excluding any starting or ending with 'test'
If one of these succeeds, then a list of pkg_resources.Requirement's
will be returned. If none can be found, then a RequirementsNotFound
will be raised
"""
requirements = []
setup_py = os.path.join(path, 'setup.py')
if os.path.exists(setup_py) and os.path.isfile(setup_py):
try:
requirements = from_setup_py(setup_py)
requirements.sort()
return requirements
except CouldNotParseRequirements:
pass
for reqfile_name in ('requirements.txt', 'requirements.pip'):
reqfile_path = os.path.join(path, reqfile_name)
if os.path.exists(reqfile_path) and os.path.isfile(reqfile_path):
try:
requirements += from_requirements_txt(reqfile_path)
except CouldNotParseRequirements as e:
pass
requirements_dir = os.path.join(path, 'requirements')
if os.path.exists(requirements_dir) and os.path.isdir(requirements_dir):
from_dir = from_requirements_dir(requirements_dir)
if from_dir is not None:
requirements += from_dir
from_blob = from_requirements_blob(path)
if from_blob is not None:
requirements += from_blob
requirements = list(set(requirements))
if len(requirements) > 0:
requirements.sort()
return requirements
raise RequirementsNotFound
class SetupWalker(object):
def __init__(self, ast):
self._ast = ast
self._setup_call = None
self._top_level_assigns = {}
self.walk()
def walk(self, node=None):
top = node is None
node = node or self._ast
# test to see if this is a call to setup()
if isinstance(node, CallFunc):
for child_node in node.get_children():
if isinstance(child_node, Name) and child_node.name == 'setup':
# TODO: what if this isn't actually the distutils setup?
self._setup_call = node
for child_node in node.get_children():
if top and isinstance(child_node, Assign):
for target in child_node.targets:
if isinstance(target, AssName):
self._top_level_assigns[target.name] = child_node.value
self.walk(child_node)
def _get_list_value(self, list_node):
values = []
for child_node in list_node.get_children():
if not isinstance(child_node, Const):
# we can't handle anything fancy, only constant values
raise CouldNotParseRequirements
values.append(child_node.value)
return values
def get_requires(self):
# first, if we have a call to setup, then we can see what its "install_requires" argument is
if not self._setup_call:
raise CouldNotParseRequirements
found_requirements = []
for child_node in self._setup_call.get_children():
if not isinstance(child_node, Keyword):
# do we want to try to handle positional arguments?
continue
if child_node.arg not in ('install_requires', 'requires'):
continue
if isinstance(child_node.value, (List, Tuple)):
# joy! this is a simple list or tuple of requirements
# this is a Keyword -> List or Keyword -> Tuple
found_requirements += self._get_list_value(child_node.value)
continue
if isinstance(child_node.value, Name):
# otherwise, it's referencing a value defined elsewhere
# this will be a Keyword -> Name
try:
reqs = self._top_level_assigns[child_node.value.name]
except KeyError:
raise CouldNotParseRequirements
else:
if isinstance(reqs, (List, Tuple)):
found_requirements += self._get_list_value(reqs)
continue
# otherwise it's something funky and we can't handle it
raise CouldNotParseRequirements
# if we've fallen off the bottom with nothing in our list of requirements,
# we simply didn't find anything useful
if len(found_requirements) > 0:
return found_requirements
raise CouldNotParseRequirements
def from_setup_py(setup_file):
try:
from astroid import AstroidBuildingException
except ImportError:
syntax_exceptions = (SyntaxError,)
else:
syntax_exceptions = (SyntaxError, AstroidBuildingException)
try:
contents = _load_file_contents(setup_file)
ast = AstroidBuilder(MANAGER).string_build(contents)
except syntax_exceptions:
# if the setup file is broken, we can't do much about that...
raise CouldNotParseRequirements
walker = SetupWalker(ast)
requirements = []
for req in walker.get_requires():
requirements.append(DetectedRequirement.parse(req, setup_file))
return requirements
def from_requirements_txt(requirements_file):
# see http://www.pip-installer.org/en/latest/logic.html
requirements = []
with open(requirements_file) as f:
for req in f.readlines():
if req.strip() == '':
# empty line
continue
if req.strip().startswith('#'):
# this is a comment
continue
if req.strip().split()[0] in _PIP_OPTIONS:
# this is a pip option
continue
detected = DetectedRequirement.parse(req, requirements_file)
if detected is None:
continue
requirements.append(detected)
return requirements
def from_requirements_dir(path):
requirements = []
for entry in os.listdir(path):
filepath = os.path.join(path, entry)
if not os.path.isfile(filepath):
continue
if entry.endswith('.txt') or entry.endswith('.pip'):
# TODO: deal with duplicates
requirements += from_requirements_txt(filepath)
return requirements
def from_requirements_blob(path):
requirements = []
for entry in os.listdir(path):
filepath = os.path.join(path, entry)
if not os.path.isfile(filepath):
continue
m = re.match(r'^(\w*)req(uirement)?s(\w*)\.txt$', entry)
if m is None:
continue
if m.group(1).startswith('test') or m.group(3).endswith('test'):
continue
requirements += from_requirements_txt(filepath)
return requirements