Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 429 lines (379 sloc) 17.23 kB
c4678df @peterbe initial commit
authored
1 """
2 tornado_static is a module for displaying static resources in a Tornado web
3 application.
4
5 It can take care of merging, compressing and giving URLs ideal renamings
6 suitable for aggressive HTTP caching.
7
8 (c) mail@peterbe.com
9 """
10
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
11 __version__ = '1.6'
c4678df @peterbe initial commit
authored
12
13 import os
14 import cPickle
15 import re
16 import stat
17 import marshal
42b8405 @peterbe better warnings if you haven't configured any of the optimization too…
authored
18 import warnings
c4678df @peterbe initial commit
authored
19 from time import time
20 from tempfile import gettempdir
21 from base64 import encodestring
22 from subprocess import Popen, PIPE
23 import tornado.web
24
25 try:
26 import cssmin
27 except ImportError:
28 cssmin = None
29
30 def mkdir(newdir):
31 """works the way a good mkdir should :)
32 - already exists, silently complete
33 - regular file in the way, raise an exception
34 - parent directory(ies) does not exist, make them as well
35 """
36 if os.path.isdir(newdir):
37 pass
38 elif os.path.isfile(newdir):
39 raise OSError("a file with the same name as the desired " \
40 "dir, '%s', already exists." % newdir)
41 else:
42 head, tail = os.path.split(newdir)
43 if head and not os.path.isdir(head):
44 _mkdir(head)
45 if tail:
46 os.mkdir(newdir)
47
48 ################################################################################
49 # Global variable where we store the conversions so we don't have to do them
50 # again every time the UI module is rendered with the same input
51
52 out_file = os.path.join(os.path.abspath(os.curdir), '.static_name_conversion')
53 def _delete_old_static_name_conversion():
54 """In this app we marshal all static file conversion into a file called
55 '.static_name_conversion' located here in the working directory.
56 The reason we're doing this is so that when you start multiple Python
57 interpreters of the app (e.g. production environment) you only need to
58 work out which name conversions have been done once.
59
60 When you do a new deployment it's perfectly natural that this name
61 conversion should be invalidated since there are now potentially new static
62 resources so it needs to have different static names.
63
64 So delete the file if it's older than a small amount of time in a human
65 sense.
66 """
67 if os.path.isfile(out_file):
68 mtime = os.stat(out_file)[stat.ST_MTIME]
69 age = time() - mtime
70 if age >= 60:
71 os.remove(out_file)
72
73 def load_name_conversion():
74 try:
42b8405 @peterbe better warnings if you haven't configured any of the optimization too…
authored
75 return marshal.load(open(out_file))
c4678df @peterbe initial commit
authored
76 except IOError:
77 return dict()
78
79 _delete_old_static_name_conversion()
80 _name_conversion = load_name_conversion()
81
82 def save_name_conversion():
42b8405 @peterbe better warnings if you haven't configured any of the optimization too…
authored
83 marshal.dump(_name_conversion, open(out_file, 'w'))
c4678df @peterbe initial commit
authored
84
85 class StaticURL(tornado.web.UIModule):
86
87 def render(self, *static_urls):
88 # the following 4 lines will have to be run for every request. Since
89 # it's just a basic lookup on a dict it's going to be uber fast.
90 basic_name = ''.join(static_urls)
91 already = _name_conversion.get(basic_name)
92 if already:
93 cdn_prefix = self.handler.get_cdn_prefix()
94 if cdn_prefix:
95 already = cdn_prefix + already
96 return already
97
98 new_name = self._combine_filename(static_urls)
99 # If you run multiple tornados (on different ports) it's possible
100 # that another process has already dealt with this static URL.
101 # Therefore we now first of all need to figure out what the final name
102 # is going to be
103 youngest = 0
104 full_paths = []
105 old_paths = {} # maintain a map of what the filenames where before
106 for path in static_urls:
107 full_path = os.path.join(
108 self.handler.settings['static_path'], path)
109 #f = open(full_path)
110 mtime = os.stat(full_path)[stat.ST_MTIME]
111 if mtime > youngest:
112 youngest = mtime
113 full_paths.append(full_path)
114 old_paths[full_path] = path
115
116 n, ext = os.path.splitext(new_name)
117 new_name = "%s.%s%s" % (n, youngest, ext)
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
118 optimization_done = False
c4678df @peterbe initial commit
authored
119 if os.path.isfile(new_name):
120 # conversion and preparation has already been done!
121 # No point doing it again, so just exit here
122 pass
123 else:
124 destination = file(new_name, 'w')
125 do_optimize_static_content = self.handler.settings\
126 .get('optimize_static_content', True)
127
128 if do_optimize_static_content:
129 uglifyjs_location = self.handler\
130 .settings.get('UGLIFYJS_LOCATION')
131 closure_location = self.handler\
132 .settings.get('CLOSURE_LOCATION')
133 yui_location = self.handler\
134 .settings.get('YUI_LOCATION')
135
136 for full_path in full_paths:
137 f = open(full_path)
138 code = f.read()
139 if full_path.endswith('.js'):
140 if len(full_paths) > 1:
141 destination.write('/* %s */\n' % os.path.basename(full_path))
142 if do_optimize_static_content and not self._already_optimized_filename(full_path):
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
143 optimization_done = True
c4678df @peterbe initial commit
authored
144 if uglifyjs_location:
145 code = run_uglify_js_compiler(code, uglifyjs_location,
146 verbose=self.handler.settings.get('debug', False))
147 elif closure_location:
148 orig_code = code
149 code = run_closure_compiler(code, closure_location,
150 verbose=self.handler.settings.get('debug', False))
151 elif yui_location:
152 code = run_yui_compressor(code, 'js', yui_location,
42b8405 @peterbe better warnings if you haven't configured any of the optimization too…
authored
153 verbose=self.handler.settings.get('debug', False))
154 else:
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
155 optimization_done = False
42b8405 @peterbe better warnings if you haven't configured any of the optimization too…
authored
156 warnings.warn('No external program configured '
157 'for optimizing .js')
158
c4678df @peterbe initial commit
authored
159 elif full_path.endswith('.css'):
160 if len(full_paths) > 1:
42b8405 @peterbe better warnings if you haven't configured any of the optimization too…
authored
161 (destination.write('/* %s */\n' %
162 os.path.basename(full_path)))
163 if (do_optimize_static_content and
164 not self._already_optimized_filename(full_path)):
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
165 optimization_done = True
c4678df @peterbe initial commit
authored
166 if cssmin is not None:
167 code = cssmin.cssmin(code)
168 elif yui_location:
169 code = run_yui_compressor(code, 'css', yui_location,
42b8405 @peterbe better warnings if you haven't configured any of the optimization too…
authored
170 verbose=self.handler.settings.get('debug', False))
171 else:
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
172 optimization_done = False
42b8405 @peterbe better warnings if you haven't configured any of the optimization too…
authored
173 warnings.warn('No external program configured for '
174 'optimizing .css')
c4678df @peterbe initial commit
authored
175 # do run this after the run_yui_compressor() has been used so that
176 # code that is commented out doesn't affect
177 code = self._replace_css_images_with_static_urls(
178 code,
179 os.path.dirname(old_paths[full_path])
180 )
181 else:
182 # this just copies the file
183 pass
184 destination.write(code)
185 destination.write("\n")
186
187 destination.close()
188 prefix = self.handler.settings.get('combined_static_url_prefix', '/combined/')
189 new_name = os.path.join(prefix, os.path.basename(new_name))
190 _name_conversion[basic_name] = new_name
191 save_name_conversion()
192
193 ## Commented out, because I don't want to use CDN when it might take 5 seconds
194 # to generate the new file.
c2b99ab @peterbe add cdn_prefix only if None or no optimization was done
authored
195 # only bother with the cdn_prefix addition if the file wasn't optimized
196 if not optimization_done:
197 cdn_prefix = self.handler.get_cdn_prefix()
198 if cdn_prefix:
199 new_name = cdn_prefix + new_name
c4678df @peterbe initial commit
authored
200 return new_name
201
202
203 def _combine_filename(self, names, max_length=60):
204 # expect the parameter 'names' be something like this:
205 # ['css/foo.css', 'css/jquery/datepicker.css']
206 # The combined filename is then going to be
207 # "/tmp/foo.datepicker.css"
208 first_ext = os.path.splitext(names[0])[-1]
209 save_dir = self.handler.application.settings.get('combined_static_dir')
210 if save_dir is None:
211 save_dir = os.environ.get('TMP_DIR')
212 if not save_dir:
213 save_dir = gettempdir()
214 save_dir = os.path.join(save_dir, 'combined')
215 mkdir(save_dir)
216 combined_name = []
217 _previous_parent_name = None
218 for name in names:
219 parent_name = os.path.split(os.path.dirname(name))[-1]
220 name, ext = os.path.splitext(os.path.basename(name))
221 if parent_name and parent_name != _previous_parent_name:
222 name = '%s.%s' % (parent_name, name)
223 if ext != first_ext:
224 raise ValueError("Mixed file extensions (%s, %s)" %\
225 (first_ext, ext))
226 combined_name.append(name)
227 _previous_parent_name = parent_name
228 if sum(len(x) for x in combined_name) > max_length:
229 combined_name = [x.replace('.min','.m').replace('.pack','.p')
230 for x in combined_name]
231 combined_name = [re.sub(r'-[\d\.]+', '', x) for x in combined_name]
232 while sum(len(x) for x in combined_name) > max_length:
233 try:
234 combined_name = [x[-2] == '.' and x[:-2] or x[:-1]
235 for x in combined_name]
236 except IndexError:
237 break
238
239 combined_name.append(first_ext[1:])
240 return os.path.join(save_dir, '.'.join(combined_name))
241
242 def _replace_css_images_with_static_urls(self, css_code, rel_dir):
243 def replacer(match):
244 filename = match.groups()[0]
245 if (filename.startswith('"') and filename.endswith('"')) or \
246 (filename.startswith("'") and filename.endswith("'")):
247 filename = filename[1:-1]
248 if 'data:image' in filename or filename.startswith('http://'):
249 return filename
250 if filename == '.':
251 # this is a known IE hack in CSS
252 return filename
253 # It's really quite common that the CSS file refers to the file
254 # that doesn't exist because if you refer to an image in CSS for
255 # a selector you never use you simply don't suffer.
256 # That's why we say not to warn on nonexisting files
257 new_filename = self.handler.static_url(os.path.join(rel_dir, filename))
258 return match.group().replace(filename, new_filename)
259 _regex = re.compile('url\(([^\)]+)\)')
260 css_code = _regex.sub(replacer, css_code)
261
262 return css_code
263
264 def _already_optimized_filename(self, file_path):
265 file_name = os.path.basename(file_path)
266 for part in ('.min.', '.minified.', '.pack.', '-jsmin.'):
267 if part in file_name:
268 return True
269 return False
270
271
272 class Static(StaticURL):
273 """given a list of static resources, return the whole HTML tag"""
274 def render(self, *static_urls, **options):
275 extension = static_urls[0].split('.')[-1]
276 if extension == 'css':
277 template = '<link rel="stylesheet" type="text/css" href="%(url)s">'
278 elif extension == 'js':
279 template = '<script '
280 if 'defer' in options:
281 template += 'defer '
282 elif 'async' in options:
283 template += 'async '
284 template += 'src="%(url)s"></script>'
285 else:
286 raise NotImplementedError
287 url = super(Static, self).render(*static_urls)
288 return template % dict(url=url)
289
290
291 def run_closure_compiler(code, jar_location, verbose=False): # pragma: no cover
292 if verbose:
293 t0 = time()
294 r = _run_closure_compiler(code, jar_location)
295 if verbose:
296 t1 = time()
297 a, b = len(code), len(r)
298 c = round(100 * float(b) / a, 1)
299 print "Closure took", round(t1 - t0, 4),
300 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
301 return r
302
303 def _run_closure_compiler(jscode, jar_location, advanced_optmization=False): # pragma: no cover
304 cmd = "java -jar %s " % jar_location
305 if advanced_optmization:
306 cmd += " --compilation_level ADVANCED_OPTIMIZATIONS "
307 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
308 try:
309 (stdoutdata, stderrdata) = proc.communicate(jscode)
310 except OSError, msg:
311 # see comment on OSErrors inside _run_yui_compressor()
312 stderrdata = \
313 "OSError: %s. Try again by making a small change and reload" % msg
314 if stderrdata:
315 return "/* ERRORS WHEN RUNNING CLOSURE COMPILER\n" + stderrdata + '\n*/\n' + jscode
316 return stdoutdata
317
318 def run_uglify_js_compiler(code, location, verbose=False): # pragma: no cover
319 if verbose:
320 t0 = time()
321 r = _run_uglify_js_compiler(code, location)
322 if verbose:
323 t1 = time()
324 a, b = len(code), len(r)
325 c = round(100 * float(b) / a, 1)
326 print "UglifyJS took", round(t1 - t0, 4),
327 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
328 return r
329
330 def _run_uglify_js_compiler(jscode, location, options=''): # pragma: no cover
331 cmd = "%s %s" % (location, options)
332 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
333 try:
334 (stdoutdata, stderrdata) = proc.communicate(jscode)
335 except OSError, msg:
336 # see comment on OSErrors inside _run_yui_compressor()
337 stderrdata = \
338 "OSError: %s. Try again by making a small change and reload" % msg
339 if stderrdata:
340 return "/* ERRORS WHEN RUNNING UGLIFYJS COMPILER\n" + stderrdata + '\n*/\n' + jscode
341 return stdoutdata
342
343 def run_yui_compressor(code, type_, jar_location, verbose=False): # pragma: no cover
344 if verbose:
345 t0 = time()
346 r = _run_yui_compressor(code, type_, jar_location)
347 if verbose:
348 t1 = time()
349 a, b = len(code), len(r)
350 c = round(100 * float(b) / a, 1)
351 print "YUI took", round(t1 - t0, 4),
352 print "seconds to compress %d bytes into %d (%s%%)" % (a, b, c)
353 return r
354
355 def _run_yui_compressor(code, type_, jar_location):
356 cmd = "java -jar %s --type=%s" % (jar_location, type_)
357 proc = Popen(cmd, shell=True, stdout=PIPE, stdin=PIPE, stderr=PIPE)
358 try:
359 (stdoutdata, stderrdata) = proc.communicate(code)
360 except OSError, msg:
361 # Sometimes, for unexplicable reasons, you get a Broken pipe when
362 # running the popen instance. It's always non-deterministic problem
363 # so it probably has something to do with concurrency or something
364 # really low level.
365 stderrdata = \
366 "OSError: %s. Try again by making a small change and reload" % msg
367
368 if stderrdata:
369 return "/* ERRORS WHEN RUNNING YUI COMPRESSOR\n" + stderrdata + '\n*/\n' + code
370
371 return stdoutdata
372
373
374 class PlainStaticURL(tornado.web.UIModule):
375 def render(self, url):
376 return self.handler.static_url(url)
377
378 class PlainStatic(tornado.web.UIModule):
379 """Render the HTML that displays a static resource without any optimization
380 or combing.
381 """
382
383 def render(self, *static_urls, **options):
384 extension = static_urls[0].split('.')[-1]
385 if extension == 'css':
386 template = '<link rel="stylesheet" type="text/css" href="%(url)s">'
387 elif extension == 'js':
388 template = '<script '
389 if 'defer' in options:
390 template += 'defer '
391 elif 'async' in options:
392 template += 'async '
393 template += 'src="%(url)s"></script>'
394 else:
395 raise NotImplementedError
396
397 html = []
398 for each in static_urls:
399 url = self.handler.static_url(each)
400 html.append(template % dict(url=url))
401 return "\n".join(html)
402
403
404 _base64_conversion_file = '.base64-image-conversions.pickle'
405 try:
406 _base64_conversions = cPickle.load(file(_base64_conversion_file))
407 #raise IOError
408 except IOError:
409 _base64_conversions = {}
410
411 class Static64(tornado.web.UIModule):
412 def render(self, image_path):
413 already = _base64_conversions.get(image_path)
414 if already:
415 return already
416
417 template = 'data:image/%s;base64,%s'
418 extension = os.path.splitext(os.path.basename(image_path))
419 extension = extension[-1][1:]
420 assert extension in ('gif','png'), extension
421 full_path = os.path.join(
422 self.handler.settings['static_path'], image_path)
423 data = encodestring(file(full_path,'rb').read()).replace('\n','')#.replace('\n','\\n')
424 result = template % (extension, data)
425
426 _base64_conversions[image_path] = result
427 cPickle.dump(_base64_conversions, file(_base64_conversion_file, 'wb'))
428 return result
Something went wrong with that request. Please try again.