/
compileapp.py
548 lines (491 loc) · 19 KB
/
compileapp.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This file is part of the web2py Web Framework
Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
Functions required to execute app components
============================================
FOR INTERNAL USE ONLY
"""
import re
import fnmatch
import os
import copy
import random
from storage import Storage, List
from template import parse_template
from restricted import restricted, compile2
from fileutils import mktree, listdir
from myregex import regex_expose
from languages import translator
from dal import BaseAdapter, SQLDB, SQLField, DAL, Field
from sqlhtml import SQLFORM, SQLTABLE
from cache import Cache
from globals import current
import settings
from cfs import getcfs
import html
import validators
from http import HTTP, redirect
import marshal
import shutil
import imp
import logging
logger = logging.getLogger("web2py")
import rewrite
try:
import py_compile
except:
logger.warning('unable to import py_compile')
is_gae = settings.global_settings.web2py_runtime_gae
TEST_CODE = \
r"""
def _TEST():
import doctest, sys, cStringIO, types, cgi, gluon.fileutils
if not gluon.fileutils.check_credentials(request):
raise HTTP(401, web2py_error='invalid credentials')
stdout = sys.stdout
html = '<h2>Testing controller "%s.py" ... done.</h2><br/>\n' \
% request.controller
for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]):
eval_key = eval(key)
if type(eval_key) == types.FunctionType:
number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)])
if number_doctests>0:
sys.stdout = cStringIO.StringIO()
name = '%s/controllers/%s.py in %s.__doc__' \
% (request.folder, request.controller, key)
doctest.run_docstring_examples(eval_key,
globals(), False, name=name)
report = sys.stdout.getvalue().strip()
if report:
pf = 'failed'
else:
pf = 'passed'
html += '<h3 class="%s">Function %s [%s]</h3>\n' \
% (pf, key, pf)
if report:
html += CODE(report, language='web2py', \
link='/examples/global/vars/').xml()
html += '<br/>\n'
else:
html += \
'<h3 class="nodoctests">Function %s [no doctests]</h3><br/>\n' \
% (key)
response._vars = html
sys.stdout = stdout
_TEST()
"""
class LoadFactory(object):
"""
Attention: this helper is new and experimental
"""
def __init__(self,environment):
self.environment = environment
def __call__(self, c=None, f='index', args=[], vars={},
extension=None, target=None,ajax=False,ajax_trap=False,
url=None,user_signature=False, content='loading...',**attr):
import globals
target = target or 'c'+str(random.random())[2:]
attr['_id']=target
request = self.environment['request']
if '.' in f:
f, extension = f.split('.',1)
if url or ajax:
url = url or html.URL(request.application, c, f, r=request,
args=args, vars=vars, extension=extension,
user_signature=user_signature)
script = html.SCRIPT('web2py_component("%s","%s")' % (url, target),
_type="text/javascript")
return html.TAG[''](script, html.DIV(content,**attr))
else:
if not isinstance(args,(list,tuple)):
args = [args]
c = c or request.controller
other_request = globals.Request()
other_request.application = request.application
other_request.controller = c
other_request.function = f
other_request.extension = extension or request.extension
other_request.args = List(args)
other_request.folder = request.folder
other_request.env = request.env
other_request.vars = request.vars
other_request.get_vars = request.get_vars
other_request.post_vars = request.post_vars
other_response = globals.Response()
other_request.env.http_web2py_component_location = \
request.env.path_info
other_request.env.http_web2py_component_element = target
other_response.view = '%s/%s.%s' % (c,f, other_request.extension)
other_environment = copy.copy(self.environment)
other_response._view_environment = other_environment
other_environment['request'] = other_request
other_environment['response'] = other_response
page = run_controller_in(c, f, other_environment)
if isinstance(page, dict):
other_response._vars = page
for key in page:
other_response._view_environment[key] = page[key]
run_view_in(other_response._view_environment)
page = other_response.body.getvalue()
js = None
if ajax_trap:
link = html.URL(request.application, c, f, r=request,
args=args, vars=vars, extension=extension,
user_signature=user_signature)
js = "web2py_trap_form('%s','%s');" % (link, target)
# not sure about this
# for (name,value) in other_response.headers:
# if name == 'web2py-component-command':
# js += value
script = js and html.SCRIPT(js,_type="text/javascript") or ''
return html.TAG[''](html.DIV(html.XML(page),**attr),script)
def local_import_aux(name, force=False, app='welcome'):
"""
In apps, instead of importing a local module
(in applications/app/modules) with::
import a.b.c as d
you should do::
d = local_import('a.b.c')
or (to force a reload):
d = local_import('a.b.c', reload=True)
This prevents conflict between applications and un-necessary execs.
It can be used to import any module, including regular Python modules.
"""
items = name.replace('/','.')
name = "applications.%s.modules.%s" % (app, items)
module = __import__(name)
for item in name.split(".")[1:]:
module = getattr(module, item)
if force:
reload(module)
return module
"""
OLD IMPLEMENTATION:
items = name.replace('/','.').split('.')
filename, modulepath = items[-1], os.path.join(apath,'modules',*items[:-1])
imp.acquire_lock()
try:
file=None
(file,path,desc) = imp.find_module(filename,[modulepath]+sys.path)
if not path in sys.modules or reload:
if is_gae:
module={}
execfile(path,{},module)
module=Storage(module)
else:
module = imp.load_module(path,file,path,desc)
sys.modules[path] = module
else:
module = sys.modules[path]
except Exception, e:
module = None
if file:
file.close()
imp.release_lock()
if not module:
raise ImportError, "cannot find module %s in %s" % (filename, modulepath)
return module
"""
def build_environment(request, response, session):
"""
Build the environment dictionary into which web2py files are executed.
"""
environment = {}
for key in html.__all__:
environment[key] = getattr(html, key)
for key in validators.__all__:
environment[key] = getattr(validators, key)
if not request.env:
request.env = Storage()
current.request = request
current.response = response
current.session = session
current.T = environment['T'] = translator(request)
current.cache = environment['cache'] = Cache(request)
environment['HTTP'] = HTTP
environment['redirect'] = redirect
environment['request'] = request
environment['response'] = response
environment['session'] = session
environment['DAL'] = DAL
environment['Field'] = Field
environment['SQLDB'] = SQLDB # for backward compatibility
environment['SQLField'] = SQLField # for backward compatibility
environment['SQLFORM'] = SQLFORM
environment['SQLTABLE'] = SQLTABLE
environment['LOAD'] = LoadFactory(environment)
environment['local_import'] = \
lambda name, reload=False, app=request.application:\
local_import_aux(name,reload,app)
BaseAdapter.set_folder(os.path.join(request.folder, 'databases'))
response._view_environment = copy.copy(environment)
return environment
def save_pyc(filename):
"""
Bytecode compiles the file `filename`
"""
py_compile.compile(filename)
def read_pyc(filename):
"""
Read the code inside a bytecode compiled file if the MAGIC number is
compatible
:returns: a code object
"""
fp = open(filename, 'rb')
data = fp.read()
fp.close()
if not is_gae and data[:4] != imp.get_magic():
raise SystemError, 'compiled code is incompatible'
return marshal.loads(data[8:])
def compile_views(folder):
"""
Compiles all the views in the application specified by `folder`
"""
path = os.path.join(folder, 'views')
for file in listdir(path, '^[\w/]+\.\w+$'):
data = parse_template(file, path)
filename = ('views/%s.py' % file).replace('/', '_').replace('\\', '_')
filename = os.path.join(folder, 'compiled', filename)
fp = open(filename, 'w')
fp.write(data)
fp.close()
save_pyc(filename)
os.unlink(filename)
def compile_models(folder):
"""
Compiles all the models in the application specified by `folder`
"""
path = os.path.join(folder, 'models')
for file in listdir(path, '.+\.py$'):
fp = open(os.path.join(path, file), 'r')
data = fp.read()
fp.close()
filename = os.path.join(folder, 'compiled','models',file)
mktree(filename)
fp = open(filename, 'w')
fp.write(data)
fp.close()
save_pyc(filename)
os.unlink(filename)
def compile_controllers(folder):
"""
Compiles all the controllers in the application specified by `folder`
"""
path = os.path.join(folder, 'controllers')
for file in listdir(path, '.+\.py$'):
### why is this here? save_pyc(os.path.join(path, file))
fp = open(os.path.join(path,file), 'r')
data = fp.read()
fp.close()
exposed = regex_expose.findall(data)
for function in exposed:
command = data + "\nresponse._vars=response._caller(%s)\n" % \
function
filename = os.path.join(folder, 'compiled', ('controllers/'
+ file[:-3]).replace('/', '_')
+ '_' + function + '.py')
fp = open(filename, 'w')
fp.write(command)
fp.close()
save_pyc(filename)
os.unlink(filename)
def run_models_in(environment):
"""
Runs all models (in the app specified by the current folder)
It tries pre-compiled models first before compiling them.
"""
folder = environment['request'].folder
c = environment['request'].controller
f = environment['request'].function
cpath = os.path.join(folder, 'compiled')
if os.path.exists(cpath):
for model in listdir(cpath, '^models_\w+\.pyc$', 0):
restricted(read_pyc(model), environment, layer=model)
path = os.path.join(cpath, 'models')
models = listdir(path, '^\w+\.pyc$',0,sort=False)
compiled=True
else:
path = os.path.join(folder, 'models')
models = listdir(path, '^\w+\.py$',0,sort=False)
compiled=False
paths = (path, os.path.join(path,c), os.path.join(path,c,f))
for model in models:
if not os.path.split(model)[0] in paths:
continue
elif compiled:
code = read_pyc(model)
elif is_gae:
code = getcfs(model, model,
lambda: compile2(open(model, 'r').read(),model))
else:
code = getcfs(model, model, None)
restricted(code, environment, layer=model)
def run_controller_in(controller, function, environment):
"""
Runs the controller.function() (for the app specified by
the current folder).
It tries pre-compiled controller_function.pyc first before compiling it.
"""
# if compiled should run compiled!
folder = environment['request'].folder
path = os.path.join(folder, 'compiled')
badc = 'invalid controller (%s/%s)' % (controller, function)
badf = 'invalid function (%s/%s)' % (controller, function)
if os.path.exists(path):
filename = os.path.join(path, 'controllers_%s_%s.pyc'
% (controller, function))
if not os.path.exists(filename):
raise HTTP(404,
rewrite.thread.routes.error_message % badf,
web2py_error=badf)
restricted(read_pyc(filename), environment, layer=filename)
elif function == '_TEST':
# TESTING: adjust the path to include site packages
from settings import global_settings
from admin import abspath, add_path_first
paths = (global_settings.gluon_parent, abspath('site-packages', gluon=True), abspath('gluon', gluon=True), '')
[add_path_first(path) for path in paths]
# TESTING END
filename = os.path.join(folder, 'controllers/%s.py'
% controller)
if not os.path.exists(filename):
raise HTTP(404,
rewrite.thread.routes.error_message % badc,
web2py_error=badc)
environment['__symbols__'] = environment.keys()
fp = open(filename, 'r')
code = fp.read()
fp.close()
code += TEST_CODE
restricted(code, environment, layer=filename)
else:
filename = os.path.join(folder, 'controllers/%s.py'
% controller)
if not os.path.exists(filename):
raise HTTP(404,
rewrite.thread.routes.error_message % badc,
web2py_error=badc)
fp = open(filename, 'r')
code = fp.read()
fp.close()
exposed = regex_expose.findall(code)
if not function in exposed:
raise HTTP(404,
rewrite.thread.routes.error_message % badf,
web2py_error=badf)
code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
if is_gae:
layer = filename + ':' + function
code = getcfs(layer, filename, lambda: compile2(code,layer))
restricted(code, environment, filename)
response = environment['response']
vars=response._vars
if response.postprocessing:
for p in response.postprocessing:
vars = p(vars)
if isinstance(vars,unicode):
vars = vars.encode('utf8')
if hasattr(vars,'xml'):
vars = vars.xml()
return vars
def run_view_in(environment):
"""
Executes the view for the requested action.
The view is the one specified in `response.view` or determined by the url
or `view/generic.extension`
It tries the pre-compiled views_controller_function.pyc before compiling it.
"""
request = environment['request']
response = environment['response']
folder = request.folder
path = os.path.join(folder, 'compiled')
badv = 'invalid view (%s)' % response.view
patterns = response.generic_patterns or []
regex = re.compile('|'.join(fnmatch.translate(r) for r in patterns))
allow_generic = regex.search('%s/%s.%s' % (request.controller,
request.function,
request.extension))
if not isinstance(response.view, str):
ccode = parse_template(response.view, os.path.join(folder, 'views'),
context=environment)
restricted(ccode, environment, 'file stream')
elif os.path.exists(path):
x = response.view.replace('/', '_')
files = ['views_%s.pyc' % x]
if allow_generic:
files.append('views_generic.%s.pyc' % request.extension)
# for backward compatibility
if request.extension == 'html':
files.append('views_%s.pyc' % x[:-5])
if allow_generic:
files.append('views_generic.pyc')
# end backward compatibility code
for f in files:
filename = os.path.join(path,f)
if os.path.exists(filename):
code = read_pyc(filename)
restricted(code, environment, layer=filename)
return
raise HTTP(404,
rewrite.thread.routes.error_message % badv,
web2py_error=badv)
else:
filename = os.path.join(folder, 'views', response.view)
if not os.path.exists(filename) and allow_generic:
response.view = 'generic.' + request.extension
filename = os.path.join(folder, 'views', response.view)
if not os.path.exists(filename):
raise HTTP(404,
rewrite.thread.routes.error_message % badv,
web2py_error=badv)
layer = filename
if is_gae:
ccode = getcfs(layer, filename,
lambda: compile2(parse_template(response.view,
os.path.join(folder, 'views'),
context=environment),layer))
else:
ccode = parse_template(response.view,
os.path.join(folder, 'views'),
context=environment)
restricted(ccode, environment, layer)
def remove_compiled_application(folder):
"""
Deletes the folder `compiled` containing the compiled application.
"""
try:
shutil.rmtree(os.path.join(folder, 'compiled'))
path = os.path.join(folder, 'controllers')
for file in listdir(path,'.*\.pyc$',drop=False):
os.unlink(file)
except OSError:
pass
def compile_application(folder):
"""
Compiles all models, views, controller for the application in `folder`.
"""
remove_compiled_application(folder)
os.mkdir(os.path.join(folder, 'compiled'))
compile_models(folder)
compile_controllers(folder)
compile_views(folder)
def test():
"""
Example::
>>> import traceback, types
>>> environment={'x':1}
>>> open('a.py', 'w').write('print 1/x')
>>> save_pyc('a.py')
>>> os.unlink('a.py')
>>> if type(read_pyc('a.pyc'))==types.CodeType: print 'code'
code
>>> exec read_pyc('a.pyc') in environment
1
"""
return
if __name__ == '__main__':
import doctest
doctest.testmod()