Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 433 lines (383 sloc) 17.389 kb
c4678df @peterbe initial commit
authored
1 """
2 tornado_static is a module for displaying static resources in a Tornado web
3 application.
4
5 It can take care of merging, compressing and giving URLs ideal renamings
6 suitable for aggressive HTTP caching.
7
8 (c) mail@peterbe.com
9 """
10
e0e84bb @peterbe made it possible to render StaticURL UIModule with dont_optimize=True
authored
11 __version__ = '1.7'
c4678df @peterbe initial commit
authored
12
13 import os
14 import cPickle
15 import re
16 import stat
17 import marshal
42b8405 @peterbe better warnings if you haven't configured any of the optimization tool o...
authored
18 import warnings
c4678df @peterbe initial commit
authored
19 from time import time
20 from tempfile import gettempdir
21 from base64 import encodestring
22 from subprocess import Popen, PIPE
23 import tornado.web
24
25 try:
26 import cssmin
27 except ImportError:
28 cssmin = None
29
30 def mkdir(newdir):
31 """works the way a good mkdir should :)
32 - already exists, silently complete
33 - regular file in the way, raise an exception
34 - parent directory(ies) does not exist, make them as well
35 """
36 if os.path.isdir(newdir):
37 pass
38 elif os.path.isfile(newdir):
39 raise OSError("a file with the same name as the desired " \
40 "dir, '%s', already exists." % newdir)
41 else:
42 head, tail = os.path.split(newdir)
43 if head and not os.path.isdir(head):
44 _mkdir(head)
45 if tail:
46 os.mkdir(newdir)
47
48 ################################################################################
49 # Global variable where we store the conversions so we don't have to do them
50 # again every time the UI module is rendered with the same input
51
52 out_file = os.path.join(os.path.abspath(os.curdir), '.static_name_conversion')
53 def _delete_old_static_name_conversion():
54 """In this app we marshal all static file conversion into a file called
55 '.static_name_conversion' located here in the working directory.
56 The reason we're doing this is so that when you start multiple Python
57 interpreters of the app (e.g. production environment) you only need to
58 work out which name conversions have been done once.
59
60 When you do a new deployment it's perfectly natural that this name
61 conversion should be invalidated since there are now potentially new static
62 resources so it needs to have different static names.
63
64 So delete the file if it's older than a small amount of time in a human
65 sense.
66 """
67 if os.path.isfile(out_file):
68 mtime = os.stat(out_file)[stat.ST_MTIME]
69 age = time() - mtime
70 if age >= 60:
71 os.remove(out_file)
72
73 def load_name_conversion():
74 try:
42b8405 @peterbe better warnings if you haven't configured any of the optimization tool o...
authored
75 return marshal.load(open(out_file))
c4678df @peterbe initial commit
authored
76 except IOError:
77 return dict()
78
79 _delete_old_static_name_conversion()
80 _name_conversion = load_name_conversion()
81
82 def save_name_conversion():
42b8405 @peterbe better warnings if you haven't configured any of the optimization tool o...
authored
83 marshal.dump(_name_conversion, open(out_file, 'w'))
c4678df @peterbe initial commit
authored
84
85 class StaticURL(tornado.web.UIModule):
86
e0e84bb @peterbe made it possible to render StaticURL UIModule with dont_optimize=True
authored
87 def render(self, *static_urls, **options):
c4678df @peterbe initial commit
authored
88 # the following 4 lines will have to be run for every request. Since
89 # it's just a basic lookup on a dict it's going to be uber fast.
90 basic_name = ''.join(static_urls)
91 already = _name_conversion.get(basic_name)
92 if already:
93 cdn_prefix = self.handler.get_cdn_prefix()
94 if cdn_prefix:
95 already = cdn_prefix + already
96 return already
97
98 new_name = self._combine_filename(static_urls)
99 # If you run multiple tornados (on different ports) it's possible
100 # that another process has already dealt with this static URL.
101 # Therefore we now first of all need to figure out what the final name
102 # is going to be
103 youngest = 0
104 full_paths = []
105 old_paths = {} # maintain a map of what the filenames where before
106 for path in static_urls:
107 full_path = os.path.join(
108 self.handler.settings['static_path'], path)
109 #f = open(full_path)
110 mtime = os.stat(full_path)[stat.ST_MTIME]
111 if mtime > youngest:
112 youngest = mtime
113 full_paths.append(full_path)
114 old_paths[full_path] = path
115
116 n, ext = os.path.splitext(new_name)
117 new_name = "%s.%s%s" % (n, youngest, ext)
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
118 optimization_done = False
c4678df @peterbe initial commit
authored
119 if os.path.isfile(new_name):
120 # conversion and preparation has already been done!
121 # No point doing it again, so just exit here
122 pass
123 else:
124 destination = file(new_name, 'w')
e0e84bb @peterbe made it possible to render StaticURL UIModule with dont_optimize=True
authored
125 if options.get('dont_optimize'):
126 do_optimize_static_content = False
127 else:
128 do_optimize_static_content = self.handler.settings\
129 .get('optimize_static_content', True)
c4678df @peterbe initial commit
authored
130
131 if do_optimize_static_content:
132 uglifyjs_location = self.handler\
133 .settings.get('UGLIFYJS_LOCATION')
134 closure_location = self.handler\
135 .settings.get('CLOSURE_LOCATION')
136 yui_location = self.handler\
137 .settings.get('YUI_LOCATION')
138
139 for full_path in full_paths:
140 f = open(full_path)
141 code = f.read()
142 if full_path.endswith('.js'):
143 if len(full_paths) > 1:
144 destination.write('/* %s */\n' % os.path.basename(full_path))
e0e84bb @peterbe made it possible to render StaticURL UIModule with dont_optimize=True
authored
145 if (do_optimize_static_content and
146 not self._already_optimized_filename(full_path)):
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
147 optimization_done = True
c4678df @peterbe initial commit
authored
148 if uglifyjs_location:
149 code = run_uglify_js_compiler(code, uglifyjs_location,
150 verbose=self.handler.settings.get('debug', False))
151 elif closure_location:
152 orig_code = code
153 code = run_closure_compiler(code, closure_location,
154 verbose=self.handler.settings.get('debug', False))
155 elif yui_location:
156 code = run_yui_compressor(code, 'js', yui_location,
42b8405 @peterbe better warnings if you haven't configured any of the optimization tool o...
authored
157 verbose=self.handler.settings.get('debug', False))
158 else:
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
159 optimization_done = False
42b8405 @peterbe better warnings if you haven't configured any of the optimization tool o...
authored
160 warnings.warn('No external program configured '
161 'for optimizing .js')
162
c4678df @peterbe initial commit
authored
163 elif full_path.endswith('.css'):
164 if len(full_paths) > 1:
42b8405 @peterbe better warnings if you haven't configured any of the optimization tool o...
authored
165 (destination.write('/* %s */\n' %
166 os.path.basename(full_path)))
167 if (do_optimize_static_content and
168 not self._already_optimized_filename(full_path)):
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
169 optimization_done = True
c4678df @peterbe initial commit
authored
170 if cssmin is not None:
171 code = cssmin.cssmin(code)
172 elif yui_location:
173 code = run_yui_compressor(code, 'css', yui_location,
42b8405 @peterbe better warnings if you haven't configured any of the optimization tool o...
authored
174 verbose=self.handler.settings.get('debug', False))
175 else:
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
176 optimization_done = False
42b8405 @peterbe better warnings if you haven't configured any of the optimization tool o...
authored
177 warnings.warn('No external program configured for '
178 'optimizing .css')
c4678df @peterbe initial commit
authored
179 # do run this after the run_yui_compressor() has been used so that
180 # code that is commented out doesn't affect
181 code = self._replace_css_images_with_static_urls(
182 code,
183 os.path.dirname(old_paths[full_path])
184 )
185 else:
186 # this just copies the file
187 pass
188 destination.write(code)
189 destination.write("\n")
190
191 destination.close()
192 prefix = self.handler.settings.get('combined_static_url_prefix', '/combined/')
193 new_name = os.path.join(prefix, os.path.basename(new_name))
194 _name_conversion[basic_name] = new_name
195 save_name_conversion()
196
197 ## Commented out, because I don't want to use CDN when it might take 5 seconds
198 # to generate the new file.
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
199 # only bother with the cdn_prefix addition if the file wasn't optimized
200 if not optimization_done:
201 cdn_prefix = self.handler.get_cdn_prefix()
202 if cdn_prefix:
203 new_name = cdn_prefix + new_name
c4678df @peterbe initial commit
authored
204 return new_name
205
206
207 def _combine_filename(self, names, max_length=60):
208 # expect the parameter 'names' be something like this:
209 # ['css/foo.css', 'css/jquery/datepicker.css']
210 # The combined filename is then going to be
211 # "/tmp/foo.datepicker.css"
212 first_ext = os.path.splitext(names[0])[-1]
213 save_dir = self.handler.application.settings.get('combined_static_dir')
214 if save_dir is None:
215 save_dir = os.environ.get('TMP_DIR')
216 if not save_dir:
217 save_dir = gettempdir()
218 save_dir = os.path.join(save_dir, 'combined')
219 mkdir(save_dir)
220 combined_name = []
221 _previous_parent_name = None
222 for name in names:
223 parent_name = os.path.split(os.path.dirname(name))[-1]
224 name, ext = os.path.splitext(os.path.basename(name))
225 if parent_name and parent_name != _previous_parent_name:
226 name = '%s.%s' % (parent_name, name)
227 if ext != first_ext:
228 raise ValueError("Mixed file extensions (%s, %s)" %\
229 (first_ext, ext))
230 combined_name.append(name)
231 _previous_parent_name = parent_name
232 if sum(len(x) for x in combined_name) > max_length:
233 combined_name = [x.replace('.min','.m').replace('.pack','.p')
234 for x in combined_name]
235 combined_name = [re.sub(r'-[\d\.]+', '', x) for x in combined_name]
236 while sum(len(x) for x in combined_name) > max_length:
237 try:
238 combined_name = [x[-2] == '.' and x[:-2] or x[:-1]
239 for x in combined_name]
240 except IndexError:
241 break
242
243 combined_name.append(first_ext[1:])
244 return os.path.join(save_dir, '.'.join(combined_name))
245
246 def _replace_css_images_with_static_urls(self, css_code, rel_dir):
247 def replacer(match):
248 filename = match.groups()[0]
249 if (filename.startswith('"') and filename.endswith('"')) or \
250 (filename.startswith("'") and filename.endswith("'")):
251 filename = filename[1:-1]
252 if 'data:image' in filename or filename.startswith('http://'):
253 return filename
254 if filename == '.':
255 # this is a known IE hack in CSS
256 return filename
257 # It's really quite common that the CSS file refers to the file
258 # that doesn't exist because if you refer to an image in CSS for
259 # a selector you never use you simply don't suffer.
260 # That's why we say not to warn on nonexisting files
261 new_filename = self.handler.static_url(os.path.join(rel_dir, filename))
262 return match.group().replace(filename, new_filename)
263 _regex = re.compile('url\(([^\)]+)\)')
264 css_code = _regex.sub(replacer, css_code)
265
266 return css_code
267
268 def _already_optimized_filename(self, file_path):
269 file_name = os.path.basename(file_path)
270 for part in ('.min.', '.minified.', '.pack.', '-jsmin.'):
271 if part in file_name:
272 return True
273 return False
274
275
276 class Static(StaticURL):
277 """given a list of static resources, return the whole HTML tag"""
278 def render(self, *static_urls, **options):
279 extension = static_urls[0].split('.')[-1]
280 if extension == 'css':
281 template = '<link rel="stylesheet" type="text/css" href="%(url)s">'
282 elif extension == 'js':
283 template = '<script '
284 if 'defer' in options:
285 template += 'defer '
286 elif 'async' in options:
287 template += 'async '
288 template += 'src="%(url)s"></script>'
289 else:
290 raise NotImplementedError
291 url = super(Static, self).render(*static_urls)
292 return template % dict(url=url)
293
294
295 def run_closure_compiler(code, jar_location, verbose=False): # pragma: no cover
296 if verbose:
297 t0 = time()
298 r = _run_closure_compiler(code, jar_location)
299 if verbose:
300 t1 = time()
301 a, b = len(code), len(r)
302 c = round(100 * float(b) / a, 1)
303 print "Closure took", round(t1 - t0, 4),
304 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
305 return r
306
307 def _run_closure_compiler(jscode, jar_location, advanced_optmization=False): # pragma: no cover
308 cmd = "java -jar %s " % jar_location
309 if advanced_optmization:
310 cmd += " --compilation_level ADVANCED_OPTIMIZATIONS "
311 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
312 try:
313 (stdoutdata, stderrdata) = proc.communicate(jscode)
314 except OSError, msg:
315 # see comment on OSErrors inside _run_yui_compressor()
316 stderrdata = \
317 "OSError: %s. Try again by making a small change and reload" % msg
318 if stderrdata:
319 return "/* ERRORS WHEN RUNNING CLOSURE COMPILER\n" + stderrdata + '\n*/\n' + jscode
320 return stdoutdata
321
322 def run_uglify_js_compiler(code, location, verbose=False): # pragma: no cover
323 if verbose:
324 t0 = time()
325 r = _run_uglify_js_compiler(code, location)
326 if verbose:
327 t1 = time()
328 a, b = len(code), len(r)
329 c = round(100 * float(b) / a, 1)
330 print "UglifyJS took", round(t1 - t0, 4),
331 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
332 return r
333
334 def _run_uglify_js_compiler(jscode, location, options=''): # pragma: no cover
335 cmd = "%s %s" % (location, options)
336 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
337 try:
338 (stdoutdata, stderrdata) = proc.communicate(jscode)
339 except OSError, msg:
340 # see comment on OSErrors inside _run_yui_compressor()
341 stderrdata = \
342 "OSError: %s. Try again by making a small change and reload" % msg
343 if stderrdata:
344 return "/* ERRORS WHEN RUNNING UGLIFYJS COMPILER\n" + stderrdata + '\n*/\n' + jscode
345 return stdoutdata
346
347 def run_yui_compressor(code, type_, jar_location, verbose=False): # pragma: no cover
348 if verbose:
349 t0 = time()
350 r = _run_yui_compressor(code, type_, jar_location)
351 if verbose:
352 t1 = time()
353 a, b = len(code), len(r)
354 c = round(100 * float(b) / a, 1)
355 print "YUI took", round(t1 - t0, 4),
356 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
357 return r
358
359 def _run_yui_compressor(code, type_, jar_location):
360 cmd = "java -jar %s --type=%s" % (jar_location, type_)
361 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
362 try:
363 (stdoutdata, stderrdata) = proc.communicate(code)
364 except OSError, msg:
365 # Sometimes, for unexplicable reasons, you get a Broken pipe when
366 # running the popen instance. It's always non-deterministic problem
367 # so it probably has something to do with concurrency or something
368 # really low level.
369 stderrdata = \
370 "OSError: %s. Try again by making a small change and reload" % msg
371
372 if stderrdata:
373 return "/* ERRORS WHEN RUNNING YUI COMPRESSOR\n" + stderrdata + '\n*/\n' + code
374
375 return stdoutdata
376
377
378 class PlainStaticURL(tornado.web.UIModule):
379 def render(self, url):
380 return self.handler.static_url(url)
381
382 class PlainStatic(tornado.web.UIModule):
383 """Render the HTML that displays a static resource without any optimization
384 or combing.
385 """
386
387 def render(self, *static_urls, **options):
388 extension = static_urls[0].split('.')[-1]
389 if extension == 'css':
390 template = '<link rel="stylesheet" type="text/css" href="%(url)s">'
391 elif extension == 'js':
392 template = '<script '
393 if 'defer' in options:
394 template += 'defer '
395 elif 'async' in options:
396 template += 'async '
397 template += 'src="%(url)s"></script>'
398 else:
399 raise NotImplementedError
400
401 html = []
402 for each in static_urls:
403 url = self.handler.static_url(each)
404 html.append(template % dict(url=url))
405 return "\n".join(html)
406
407
408 _base64_conversion_file = '.base64-image-conversions.pickle'
409 try:
410 _base64_conversions = cPickle.load(file(_base64_conversion_file))
411 #raise IOError
412 except IOError:
413 _base64_conversions = {}
414
415 class Static64(tornado.web.UIModule):
416 def render(self, image_path):
417 already = _base64_conversions.get(image_path)
418 if already:
419 return already
420
421 template = 'data:image/%s;base64,%s'
422 extension = os.path.splitext(os.path.basename(image_path))
423 extension = extension[-1][1:]
424 assert extension in ('gif','png'), extension
425 full_path = os.path.join(
426 self.handler.settings['static_path'], image_path)
427 data = encodestring(file(full_path,'rb').read()).replace('\n','')#.replace('\n','\\n')
428 result = template % (extension, data)
429
430 _base64_conversions[image_path] = result
431 cPickle.dump(_base64_conversions, file(_base64_conversion_file, 'wb'))
432 return result
Something went wrong with that request. Please try again.