Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 417 lines (367 sloc) 16.895 kb
c4678dfa »
2011-09-02 initial commit
1 """
2 tornado_static is a module for displaying static resources in a Tornado web
3 application.
4
5 It can take care of merging, compressing and giving URLs ideal renamings
6 suitable for aggressive HTTP caching.
7
8 (c) mail@peterbe.com
9 """
10
4f3f345b »
2012-10-09 fixed bug to do with replacing URLs within CSS
11 __version__ = '1.9'
c4678dfa »
2011-09-02 initial commit
12
13 import os
14 import cPickle
15 import re
16 import stat
17 import marshal
42b84059 »
2012-02-08 better warnings if you haven't configured any of the optimization too…
18 import warnings
c4678dfa »
2011-09-02 initial commit
19 from time import time
20 from tempfile import gettempdir
21 from base64 import encodestring
22 from subprocess import Popen, PIPE
23 import tornado.web
24
25 try:
26 import cssmin
27 except ImportError:
28 cssmin = None
3f1ffd4f »
2012-06-18 don't bother minifying files called something like foo-min-1.0.js
29
f3aa7aed »
2012-03-23 fix mkdir
30 from .utils import mkdir
c4678dfa »
2011-09-02 initial commit
31
32 ################################################################################
33 # Global variable where we store the conversions so we don't have to do them
34 # again every time the UI module is rendered with the same input
35
36 out_file = os.path.join(os.path.abspath(os.curdir), '.static_name_conversion')
37 def _delete_old_static_name_conversion():
38 """In this app we marshal all static file conversion into a file called
39 '.static_name_conversion' located here in the working directory.
40 The reason we're doing this is so that when you start multiple Python
41 interpreters of the app (e.g. production environment) you only need to
42 work out which name conversions have been done once.
43
44 When you do a new deployment it's perfectly natural that this name
45 conversion should be invalidated since there are now potentially new static
46 resources so it needs to have different static names.
47
48 So delete the file if it's older than a small amount of time in a human
49 sense.
50 """
51 if os.path.isfile(out_file):
52 mtime = os.stat(out_file)[stat.ST_MTIME]
53 age = time() - mtime
54 if age >= 60:
55 os.remove(out_file)
56
57 def load_name_conversion():
58 try:
42b84059 »
2012-02-08 better warnings if you haven't configured any of the optimization too…
59 return marshal.load(open(out_file))
c4678dfa »
2011-09-02 initial commit
60 except IOError:
61 return dict()
62
63 _delete_old_static_name_conversion()
64 _name_conversion = load_name_conversion()
65
66 def save_name_conversion():
42b84059 »
2012-02-08 better warnings if you haven't configured any of the optimization too…
67 marshal.dump(_name_conversion, open(out_file, 'w'))
c4678dfa »
2011-09-02 initial commit
68
69 class StaticURL(tornado.web.UIModule):
70
e0e84bb5 »
2012-03-20 made it possible to render StaticURL UIModule with dont_optimize=True
71 def render(self, *static_urls, **options):
c4678dfa »
2011-09-02 initial commit
72 # the following 4 lines will have to be run for every request. Since
73 # it's just a basic lookup on a dict it's going to be uber fast.
74 basic_name = ''.join(static_urls)
75 already = _name_conversion.get(basic_name)
76 if already:
77 cdn_prefix = self.handler.get_cdn_prefix()
78 if cdn_prefix:
79 already = cdn_prefix + already
80 return already
81
82 new_name = self._combine_filename(static_urls)
83 # If you run multiple tornados (on different ports) it's possible
84 # that another process has already dealt with this static URL.
85 # Therefore we now first of all need to figure out what the final name
86 # is going to be
87 youngest = 0
88 full_paths = []
89 old_paths = {} # maintain a map of what the filenames where before
90 for path in static_urls:
91 full_path = os.path.join(
92 self.handler.settings['static_path'], path)
93 #f = open(full_path)
94 mtime = os.stat(full_path)[stat.ST_MTIME]
95 if mtime > youngest:
96 youngest = mtime
97 full_paths.append(full_path)
98 old_paths[full_path] = path
99
100 n, ext = os.path.splitext(new_name)
101 new_name = "%s.%s%s" % (n, youngest, ext)
c2b99ab6 »
2012-02-16 add cdn_prefix only if None or no optimization was done
102 optimization_done = False
c4678dfa »
2011-09-02 initial commit
103 if os.path.isfile(new_name):
104 # conversion and preparation has already been done!
105 # No point doing it again, so just exit here
106 pass
107 else:
108 destination = file(new_name, 'w')
e0e84bb5 »
2012-03-20 made it possible to render StaticURL UIModule with dont_optimize=True
109 if options.get('dont_optimize'):
110 do_optimize_static_content = False
111 else:
112 do_optimize_static_content = self.handler.settings\
113 .get('optimize_static_content', True)
c4678dfa »
2011-09-02 initial commit
114
115 if do_optimize_static_content:
116 uglifyjs_location = self.handler\
117 .settings.get('UGLIFYJS_LOCATION')
118 closure_location = self.handler\
119 .settings.get('CLOSURE_LOCATION')
120 yui_location = self.handler\
121 .settings.get('YUI_LOCATION')
122
123 for full_path in full_paths:
124 f = open(full_path)
125 code = f.read()
126 if full_path.endswith('.js'):
127 if len(full_paths) > 1:
128 destination.write('/* %s */\n' % os.path.basename(full_path))
e0e84bb5 »
2012-03-20 made it possible to render StaticURL UIModule with dont_optimize=True
129 if (do_optimize_static_content and
130 not self._already_optimized_filename(full_path)):
c2b99ab6 »
2012-02-16 add cdn_prefix only if None or no optimization was done
131 optimization_done = True
c4678dfa »
2011-09-02 initial commit
132 if uglifyjs_location:
133 code = run_uglify_js_compiler(code, uglifyjs_location,
134 verbose=self.handler.settings.get('debug', False))
135 elif closure_location:
136 orig_code = code
137 code = run_closure_compiler(code, closure_location,
138 verbose=self.handler.settings.get('debug', False))
139 elif yui_location:
140 code = run_yui_compressor(code, 'js', yui_location,
42b84059 »
2012-02-08 better warnings if you haven't configured any of the optimization too…
141 verbose=self.handler.settings.get('debug', False))
142 else:
c2b99ab6 »
2012-02-16 add cdn_prefix only if None or no optimization was done
143 optimization_done = False
42b84059 »
2012-02-08 better warnings if you haven't configured any of the optimization too…
144 warnings.warn('No external program configured '
145 'for optimizing .js')
146
c4678dfa »
2011-09-02 initial commit
147 elif full_path.endswith('.css'):
148 if len(full_paths) > 1:
42b84059 »
2012-02-08 better warnings if you haven't configured any of the optimization too…
149 (destination.write('/* %s */\n' %
150 os.path.basename(full_path)))
151 if (do_optimize_static_content and
152 not self._already_optimized_filename(full_path)):
c2b99ab6 »
2012-02-16 add cdn_prefix only if None or no optimization was done
153 optimization_done = True
c4678dfa »
2011-09-02 initial commit
154 if cssmin is not None:
155 code = cssmin.cssmin(code)
156 elif yui_location:
157 code = run_yui_compressor(code, 'css', yui_location,
42b84059 »
2012-02-08 better warnings if you haven't configured any of the optimization too…
158 verbose=self.handler.settings.get('debug', False))
159 else:
c2b99ab6 »
2012-02-16 add cdn_prefix only if None or no optimization was done
160 optimization_done = False
42b84059 »
2012-02-08 better warnings if you haven't configured any of the optimization too…
161 warnings.warn('No external program configured for '
162 'optimizing .css')
c4678dfa »
2011-09-02 initial commit
163 # do run this after the run_yui_compressor() has been used so that
164 # code that is commented out doesn't affect
165 code = self._replace_css_images_with_static_urls(
166 code,
167 os.path.dirname(old_paths[full_path])
168 )
169 else:
170 # this just copies the file
171 pass
172 destination.write(code)
173 destination.write("\n")
174
175 destination.close()
176 prefix = self.handler.settings.get('combined_static_url_prefix', '/combined/')
177 new_name = os.path.join(prefix, os.path.basename(new_name))
178 _name_conversion[basic_name] = new_name
179 save_name_conversion()
180
181 ## Commented out, because I don't want to use CDN when it might take 5 seconds
182 # to generate the new file.
c2b99ab6 »
2012-02-16 add cdn_prefix only if None or no optimization was done
183 # only bother with the cdn_prefix addition if the file wasn't optimized
184 if not optimization_done:
185 cdn_prefix = self.handler.get_cdn_prefix()
186 if cdn_prefix:
187 new_name = cdn_prefix + new_name
c4678dfa »
2011-09-02 initial commit
188 return new_name
189
190
191 def _combine_filename(self, names, max_length=60):
192 # expect the parameter 'names' be something like this:
193 # ['css/foo.css', 'css/jquery/datepicker.css']
194 # The combined filename is then going to be
195 # "/tmp/foo.datepicker.css"
196 first_ext = os.path.splitext(names[0])[-1]
197 save_dir = self.handler.application.settings.get('combined_static_dir')
198 if save_dir is None:
199 save_dir = os.environ.get('TMP_DIR')
200 if not save_dir:
201 save_dir = gettempdir()
202 save_dir = os.path.join(save_dir, 'combined')
203 mkdir(save_dir)
204 combined_name = []
205 _previous_parent_name = None
206 for name in names:
207 parent_name = os.path.split(os.path.dirname(name))[-1]
208 name, ext = os.path.splitext(os.path.basename(name))
209 if parent_name and parent_name != _previous_parent_name:
210 name = '%s.%s' % (parent_name, name)
211 if ext != first_ext:
212 raise ValueError("Mixed file extensions (%s, %s)" %\
213 (first_ext, ext))
214 combined_name.append(name)
215 _previous_parent_name = parent_name
216 if sum(len(x) for x in combined_name) > max_length:
217 combined_name = [x.replace('.min','.m').replace('.pack','.p')
218 for x in combined_name]
219 combined_name = [re.sub(r'-[\d\.]+', '', x) for x in combined_name]
220 while sum(len(x) for x in combined_name) > max_length:
221 try:
222 combined_name = [x[-2] == '.' and x[:-2] or x[:-1]
223 for x in combined_name]
224 except IndexError:
225 break
226
227 combined_name.append(first_ext[1:])
228 return os.path.join(save_dir, '.'.join(combined_name))
229
230 def _replace_css_images_with_static_urls(self, css_code, rel_dir):
231 def replacer(match):
232 filename = match.groups()[0]
233 if (filename.startswith('"') and filename.endswith('"')) or \
234 (filename.startswith("'") and filename.endswith("'")):
235 filename = filename[1:-1]
236 if 'data:image' in filename or filename.startswith('http://'):
4f3f345b »
2012-10-09 fixed bug to do with replacing URLs within CSS
237 return 'url("%s")' % filename
c4678dfa »
2011-09-02 initial commit
238 if filename == '.':
239 # this is a known IE hack in CSS
4f3f345b »
2012-10-09 fixed bug to do with replacing URLs within CSS
240 return 'url(".")'
c4678dfa »
2011-09-02 initial commit
241 # It's really quite common that the CSS file refers to the file
242 # that doesn't exist because if you refer to an image in CSS for
243 # a selector you never use you simply don't suffer.
244 # That's why we say not to warn on nonexisting files
245 new_filename = self.handler.static_url(os.path.join(rel_dir, filename))
246 return match.group().replace(filename, new_filename)
247 _regex = re.compile('url\(([^\)]+)\)')
248 css_code = _regex.sub(replacer, css_code)
249
250 return css_code
251
252 def _already_optimized_filename(self, file_path):
253 file_name = os.path.basename(file_path)
3f1ffd4f »
2012-06-18 don't bother minifying files called something like foo-min-1.0.js
254 for part in ('-min-', '-min.', '.min.', '.minified.', '.pack.', '-jsmin.'):
c4678dfa »
2011-09-02 initial commit
255 if part in file_name:
256 return True
257 return False
258
259
260 class Static(StaticURL):
261 """given a list of static resources, return the whole HTML tag"""
262 def render(self, *static_urls, **options):
263 extension = static_urls[0].split('.')[-1]
264 if extension == 'css':
265 template = '<link rel="stylesheet" type="text/css" href="%(url)s">'
266 elif extension == 'js':
37f19a02 »
2012-07-04 more careful handling of script and link tag attributes
267 template = '<script type="text/javascript" '
268 if options.get('defer'):
c4678dfa »
2011-09-02 initial commit
269 template += 'defer '
37f19a02 »
2012-07-04 more careful handling of script and link tag attributes
270 elif options.get('async'):
c4678dfa »
2011-09-02 initial commit
271 template += 'async '
272 template += 'src="%(url)s"></script>'
273 else:
274 raise NotImplementedError
275 url = super(Static, self).render(*static_urls)
276 return template % dict(url=url)
277
278
279 def run_closure_compiler(code, jar_location, verbose=False): # pragma: no cover
280 if verbose:
281 t0 = time()
282 r = _run_closure_compiler(code, jar_location)
283 if verbose:
284 t1 = time()
285 a, b = len(code), len(r)
286 c = round(100 * float(b) / a, 1)
287 print "Closure took", round(t1 - t0, 4),
288 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
289 return r
290
291 def _run_closure_compiler(jscode, jar_location, advanced_optmization=False): # pragma: no cover
292 cmd = "java -jar %s " % jar_location
293 if advanced_optmization:
294 cmd += " --compilation_level ADVANCED_OPTIMIZATIONS "
295 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
296 try:
297 (stdoutdata, stderrdata) = proc.communicate(jscode)
298 except OSError, msg:
299 # see comment on OSErrors inside _run_yui_compressor()
300 stderrdata = \
301 "OSError: %s. Try again by making a small change and reload" % msg
302 if stderrdata:
303 return "/* ERRORS WHEN RUNNING CLOSURE COMPILER\n" + stderrdata + '\n*/\n' + jscode
304 return stdoutdata
305
306 def run_uglify_js_compiler(code, location, verbose=False): # pragma: no cover
307 if verbose:
308 t0 = time()
309 r = _run_uglify_js_compiler(code, location)
310 if verbose:
311 t1 = time()
312 a, b = len(code), len(r)
313 c = round(100 * float(b) / a, 1)
314 print "UglifyJS took", round(t1 - t0, 4),
315 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
316 return r
317
318 def _run_uglify_js_compiler(jscode, location, options=''): # pragma: no cover
319 cmd = "%s %s" % (location, options)
320 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
321 try:
322 (stdoutdata, stderrdata) = proc.communicate(jscode)
323 except OSError, msg:
324 # see comment on OSErrors inside _run_yui_compressor()
325 stderrdata = \
326 "OSError: %s. Try again by making a small change and reload" % msg
327 if stderrdata:
328 return "/* ERRORS WHEN RUNNING UGLIFYJS COMPILER\n" + stderrdata + '\n*/\n' + jscode
329 return stdoutdata
330
331 def run_yui_compressor(code, type_, jar_location, verbose=False): # pragma: no cover
332 if verbose:
333 t0 = time()
334 r = _run_yui_compressor(code, type_, jar_location)
335 if verbose:
336 t1 = time()
337 a, b = len(code), len(r)
338 c = round(100 * float(b) / a, 1)
339 print "YUI took", round(t1 - t0, 4),
340 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
341 return r
342
343 def _run_yui_compressor(code, type_, jar_location):
344 cmd = "java -jar %s --type=%s" % (jar_location, type_)
345 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
346 try:
347 (stdoutdata, stderrdata) = proc.communicate(code)
348 except OSError, msg:
349 # Sometimes, for unexplicable reasons, you get a Broken pipe when
350 # running the popen instance. It's always non-deterministic problem
351 # so it probably has something to do with concurrency or something
352 # really low level.
353 stderrdata = \
354 "OSError: %s. Try again by making a small change and reload" % msg
355
356 if stderrdata:
357 return "/* ERRORS WHEN RUNNING YUI COMPRESSOR\n" + stderrdata + '\n*/\n' + code
358
359 return stdoutdata
360
361
362 class PlainStaticURL(tornado.web.UIModule):
363 def render(self, url):
364 return self.handler.static_url(url)
365
366 class PlainStatic(tornado.web.UIModule):
367 """Render the HTML that displays a static resource without any optimization
368 or combing.
369 """
370
371 def render(self, *static_urls, **options):
372 extension = static_urls[0].split('.')[-1]
373 if extension == 'css':
374 template = '<link rel="stylesheet" type="text/css" href="%(url)s">'
375 elif extension == 'js':
37f19a02 »
2012-07-04 more careful handling of script and link tag attributes
376 template = '<script type="text/javascript" '
377 if options.get('defer'):
c4678dfa »
2011-09-02 initial commit
378 template += 'defer '
37f19a02 »
2012-07-04 more careful handling of script and link tag attributes
379 elif options.get('async'):
c4678dfa »
2011-09-02 initial commit
380 template += 'async '
381 template += 'src="%(url)s"></script>'
382 else:
383 raise NotImplementedError
384
385 html = []
386 for each in static_urls:
387 url = self.handler.static_url(each)
388 html.append(template % dict(url=url))
389 return "\n".join(html)
390
391
392 _base64_conversion_file = '.base64-image-conversions.pickle'
393 try:
394 _base64_conversions = cPickle.load(file(_base64_conversion_file))
395 #raise IOError
396 except IOError:
397 _base64_conversions = {}
398
399 class Static64(tornado.web.UIModule):
400 def render(self, image_path):
401 already = _base64_conversions.get(image_path)
402 if already:
403 return already
404
405 template = 'data:image/%s;base64,%s'
406 extension = os.path.splitext(os.path.basename(image_path))
407 extension = extension[-1][1:]
408 assert extension in ('gif','png'), extension
409 full_path = os.path.join(
410 self.handler.settings['static_path'], image_path)
411 data = encodestring(file(full_path,'rb').read()).replace('\n','')#.replace('\n','\\n')
412 result = template % (extension, data)
413
414 _base64_conversions[image_path] = result
415 cPickle.dump(_base64_conversions, file(_base64_conversion_file, 'wb'))
416 return result
Something went wrong with that request. Please try again.