forked from SRabbelier/Melange
-
Notifications
You must be signed in to change notification settings - Fork 1
/
pavement.py
337 lines (285 loc) · 11.7 KB
/
pavement.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
# -*- coding: utf-8 -*-
"""
Example Usage
=============
The following commands can be run from the root directory of the Mercurial
repo. To run ``paver``, however, you'll need to do ``easy_install Paver``.
Most of the following commands accept other arguments; see ``command --help``
for more information, or ``paver help`` for a list of all the valid commands.
``paver build``
Builds the project. This essentially just runs a bunch of other tasks,
like ``pylint``, ``django_zip`` and ``tinymce_zip``, etc.
``paver pylint``
Runs PyLint on the project.
``paver django_zip``
Builds the Django zip file.
``paver tinymce_zip``
Builds the TinyMCE zip file.
If you specify ``--dry-run`` before a task, then the action of that task will
not actually be carried out, although logging output will be displayed as if
it were. For example, you could run ``paver --dry-run django_zip`` to see what
files would be added to the ``django.zip`` file, etc.
"""
from cStringIO import StringIO
import sys
import zipfile
import paver
import paver.easy
import paver.tasks
from paver.easy import *
from paver.path import path
# Paver comes with Jason Orendorff's 'path' module; this makes path
# manipulation easy and far more readable.
PROJECT_DIR = path(__file__).dirname()
# Set some default options. Having the options at the top of the file cleans
# the whole thing up and makes the behaviour a lot more configurable.
options(
build = Bunch(
project_dir = PROJECT_DIR,
app_build = PROJECT_DIR / 'build',
app_folder = PROJECT_DIR / 'app',
app_files = ['app.yaml', 'cron.yaml', 'index.yaml', 'main.py',
'settings.py', 'shell.py', 'urls.py', 'gae_django.py'],
app_dirs = ['soc', 'ghop', 'gsoc', 'feedparser', 'python25src',
'reflistprop', 'jquery', 'ranklist', 'shell', 'json',
'htmlsanitizer', 'taggable-mixin', 'gviz'],
zip_files = ['tiny_mce.zip'],
skip_pylint = False,
)
)
# The second call to options allows us to re-use some of the constants defined
# in the first call.
options(
clean_build = options.build,
tinymce_zip = options.build,
django_zip = Bunch(
prune_dirs = ['.svn', 'gis', 'admin', 'localflavor', 'mysql',
'mysql_old', 'oracle', 'postgresql',
'postgresql_psycopg2', 'sqlite3', 'test'],
**options.build
),
pylint = Bunch(
check_modules = ['soc', 'reflistprop', 'settings.py', 'urls.py',
'main.py'],
quiet = False,
quiet_args = ['--disable-msg=W0511,R0401', '--reports=no',
'--disable-checker=similarities'],
pylint_args = [],
ignore = False,
**options.build
)
)
# Utility functions
def django_zip_files(django_src_dir):
"""Yields each filename which should go into ``django.zip``."""
for filename in django_src_dir.walkfiles():
# The following seems unnecessarily unreadable, but unfortunately
# it seems it cannot be unobfuscated any more (if someone finds a
# nicer way of writing it, please tell me).
if not (filename.ext in ['.pyc', '.pyo', '.po', '.mo'] or
any(name in filename.splitall()
for name in options.django_zip.prune_dirs)):
# The filename is suitable to be added to the archive. In this
# case, we yield the filename and the name it should be given in
# the Zip archive.
paver.tasks.environment.info(
'%-4sdjango.zip <- %s', '', filename)
arcname = path('django') / django_src_dir.relpathto(filename)
yield filename, arcname
def tinymce_zip_files(tiny_mce_dir):
"""Yields each filename which should go into ``tiny_mce.zip``."""
for filename in tiny_mce_dir.walkfiles():
if '.svn' not in filename.splitall():
# In this case, `tiny_mce/*` is in the root of the zip file, so
# we do not need to prefix `arcname` with 'tinymce/' (like we did
# with `django.zip`).
paver.tasks.environment.info(
'%-4stiny_mce.zip <- %s', '', filename)
arcname = tiny_mce_dir.relpathto(filename)
yield filename, arcname
def write_zip_file(zip_file_handle, files):
if paver.tasks.environment.dry_run:
for args in files:
pass
return
zip_file = zipfile.ZipFile(zip_file_handle, mode='w')
for args in files:
zip_file.write(*args)
zip_file.close()
def symlink(target, link_name):
if hasattr(target, 'symlink'):
target.symlink(link_name)
else:
# If we are on a platform where symlinks are not supported (such as
# Windows), simply copy the files across.
target.copy(link_name)
# Tasks
@task
@cmdopts([
('app-folder=', 'a', 'App folder directory (default /app)'),
('pylint-command=', 'c', 'Specify a custom pylint executable'),
('quiet', 'q', 'Disables a lot of the pylint output'),
('ignore', 'i', 'Ignore PyLint errors')
])
def pylint(options):
"""Check the source code using PyLint."""
from pylint import lint
# Initial command.
arguments = []
if options.quiet:
arguments.extend(options.quiet_args)
if 'pylint_args' in options:
arguments.extend(list(options.pylint_args))
# Add the list of paths containing the modules to check using PyLint.
arguments.extend(
str(options.app_folder / module) for module in options.check_modules)
# By placing run_pylint into its own function, it allows us to do dry runs
# without actually running PyLint.
def run_pylint():
# Add app folder to path.
sys.path.insert(0, options.app_folder.abspath())
# Add google_appengine directory to path.
sys.path.insert(0,
options.project_dir.abspath() /
'thirdparty' / 'google_appengine')
# Specify PyLint RC file.
arguments.append('--rcfile=' +
options.project_dir.abspath() /
'scripts' / 'pylint' / 'pylintrc')
# `lint.Run.__init__` runs the PyLint command.
try:
lint.Run(arguments)
# PyLint will `sys.exit()` when it has finished, so we need to catch
# the exception and process it accordingly.
except SystemExit, exc:
return_code = exc.args[0]
if return_code != 0 and (not options.pylint.ignore):
raise paver.tasks.BuildFailure(
'PyLint finished with a non-zero exit code')
return dry('pylint ' + ' '.join(arguments), run_pylint)
@task
@cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
('app-folder=', 'a', 'App folder directory (default /app)'),
('skip-pylint', 's', 'Skip PyLint checker'),
('ignore-pylint', 'i', 'Ignore results of PyLint (but run it anyway)'),
('quiet-pylint', 'q', 'Make PyLint run quietly'),
])
def build(options):
"""Build the project."""
# If `--skip-pylint` is not provided, run PyLint.
if not options.skip_pylint:
# If `--ignore-pylint` is provided, act as if `paver pylint --ignore`
# was run. Likewise for `--quiet-pylint`.
if options.get('ignore_pylint', False):
options.pylint.ignore = True
if options.get('quiet_pylint', False):
options.pylint.quiet = True
pylint(options)
# Clean old generated zip files from the app folder.
clean_zip(options)
# Clean the App build directory by removing and re-creating it.
clean_build(options)
# Build the django.zip file.
django_zip(options)
# Build the tiny_mce.zip file.
tinymce_zip(options)
# Make the necessary symlinks between the app and build directories.
build_symlinks(options)
@task
@cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def build_symlinks(options):
"""Build symlinks between the app and build folders."""
# Create the symbolic links from the app folder to the build folder.
for filename in options.app_files + options.app_dirs + options.zip_files:
# The `symlink()` function handles discrepancies between platforms.
target = path(options.app_folder) / filename
link = path(options.app_build) / filename
dry(
'%-4s%-20s <- %s' % ('', target, link),
lambda: symlink(target, link))
@task
@cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
])
def clean_build(options):
"""Clean the build folder."""
# Not checking this could cause an error when trying to remove a
# non-existent file.
if path(options.app_build).exists():
path(options.app_build).rmtree()
path(options.app_build).makedirs()
@task
@cmdopts([
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def clean_zip(options):
"""Remove all the generated zip files from the app folder."""
for zip_file in options.zip_files + ['django.zip']:
zip_path = path(options.app_folder) / zip_file
if zip_path.exists():
zip_path.remove()
@task
@cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def django_zip(options):
"""Create the zip file containing Django (minus unwanted stuff)."""
# Write the `django.zip` file. This requires finding all of the necessary
# files and writing them to a `zipfile.ZipFile` instance. Python's
# stdlib `zipfile` module is written in C, so it's fast and doesn't incur
# much overhead.
django_src_dir = path(options.app_folder) / 'django'
django_zip_filename = path(options.app_build) / 'django.zip'
if paver.tasks.environment.dry_run:
django_zip_fp = StringIO()
else:
# Ensure the parent directories exist.
django_zip_filename.dirname().makedirs()
django_zip_fp = open(django_zip_filename, mode='w')
# Write the zip file to disk; this uses the `write_zip_file()` function
# defined above. The try/except/finally makes sure the `django.zip` file
# handle is properly closed no matter what happens.
try:
write_zip_file(django_zip_fp, django_zip_files(django_src_dir))
except Exception, exc:
# Close and delete the (possibly corrupted) `django.zip` file.
django_zip_fp.close()
django_zip_filename.remove()
# Raise the error, causing Paver to exit with a non-zero exit code.
raise paver.tasks.BuildFailure(
'Error occurred creating django.zip: %r' % (exc,))
finally:
# Close the file handle if it isn't already.
if not django_zip_fp.closed:
django_zip_fp.close()
@task
@cmdopts([
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def tinymce_zip(options):
"""Create the zip file containing TinyMCE."""
# This is very similar to django_zip; see the comments there for
# explanations.
tinymce_dir = path(options.app_folder) / 'tiny_mce'
tinymce_zip_filename = path(options.app_folder) / 'tiny_mce.zip'
if paver.tasks.environment.dry_run:
tinymce_zip_fp = StringIO()
else:
# Ensure the parent directories exist.
tinymce_zip_filename.dirname().makedirs()
tinymce_zip_fp = open(tinymce_zip_filename, mode='w')
try:
write_zip_file(tinymce_zip_fp, tinymce_zip_files(tinymce_dir))
except Exception, exc:
tinymce_zip_fp.close()
tinymce_zip_filename.remove()
raise paver.tasks.BuildFailure(
'Error occurred creating tinymce.zip: %r' % (exc,))
finally:
if not tinymce_zip_fp.closed:
tinymce_zip_fp.close()