Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
762 lines (641 sloc) 29.8 KB
#-*- coding: iso-8859-1 -*
##
## Fry-IT Zope Bases
## Peter Bengtsson, peter@fry-it.com, (c) 2005-2007
##
import re
import os
import gzip
import stat
from tempfile import gettempdir
from time import time
try:
from slimmer import js_slimmer, css_slimmer
except ImportError:
css_slimmer = js_slimmer = None
# Zope
import App
from Globals import package_home, DevelopmentMode
from ZPublisher.Iterators import filestream_iterator
from Acquisition import aq_inner, aq_parent
try:
from zope.app.content_types import guess_content_type
except ImportError:
# hmm, must be zope < 2.10
from OFS.content_types import guess_content_type
# For GzippedImageFile
from App.Common import rfc1123_date
from Utils import anyTrue, base_hasattr, getEnvBool
FILESTREAM_ITERATOR_THRESHOLD = 2 << 16 # 128 Kb (from LocalFS StreamingFile.py)
EXPIRY_INFINITY = 60*60*24*365*5 # 5 years
DATA64_REGEX = re.compile('(DATA64\(([\w/\.]+\.(gif|png|jpg))\))')
referred_css_images_regex = re.compile('url\(([^\)]+)\)')
from base64 import encodestring
def _find_related_context(context, path):
"""Return a new context relative to this path.
For example if context os <MyApp at 0x1235> and path is something like
../images/foo then the new path will be:
aq_parent(aq_inner(context)).images.foo
If the path starts with / then we have to hope that acquisition will do its
magic.
"""
for bit in [x for x in path.split('/') if x]:
if bit == '..':
context = aq_parent(aq_inner(context))
else:
context = getattr(context, bit)
return context
def my_guess_content_type(path, data):
content_type, enc = guess_content_type(path, data)
if content_type in ('text/plain', 'text/html','text/x-unknown-content-type'):
if os.path.basename(path).endswith('.js-slimmed.js'):
content_type = 'application/x-javascript'
elif os.path.basename(path).find('.css-slimmed.css') > -1:
# the find() covers both 'foo.css-slimmed' and
# 'foo.css-slimmed-data64expanded'
content_type = 'text/css'
elif os.path.basename(path).find('.css-aliased.css') > -1:
content_type = 'text/css'
return content_type, enc
def _getAutogeneratedFilepath(filepath, directoryname='.autogenerated'):
""" given a filepath, return (and make sure the directory exists) the filepath
as landed in the filpath.
Eg. If filepath='/home/peterbe/css/screen.css' then return
'/home/peterbe/css/.autogenerated/screen.css'
"""
dirname = os.path.dirname(filepath)
basename = os.path.basename(filepath)
if directoryname in dirname.split(os.path.sep):
return filepath
dirname = os.path.join(dirname, directoryname)
if not os.path.isdir(dirname):
os.mkdir(dirname)
return os.path.join(dirname, basename)
class BetterImageFile(App.ImageFile.ImageFile): # that name needs to improve
def __init__(self, path, _prefix=None, max_age_development=60, max_age_production=3600,
content_type=None, set_expiry_header=True):
if _prefix is None:
_prefix = getConfiguration().softwarehome
elif type(_prefix) is not type(''):
_prefix = package_home(_prefix)
path = os.path.join(_prefix, path)
self.path = self.original_path = path
self.set_expiry_header = set_expiry_header
if DevelopmentMode:
# In development mode, a shorter time is handy
max_age = max_age_development
else:
# A longer time reduces latency in production mode
max_age = max_age_production
self.max_age = max_age
self.cch = 'public,max-age=%d' % max_age
data = open(path, 'rb').read()
if content_type is None:
content_type, __ = my_guess_content_type(path, data)
if content_type:
self.content_type=content_type
else:
raise ValueError, "content_type not set or couldn't be guessed"
#self.content_type='text/plain'
self.__name__=path[path.rfind('/')+1:]
self.lmt=float(os.stat(path)[8]) or time.time()
self.lmh=rfc1123_date(self.lmt)
self.content_size = os.stat(path)[stat.ST_SIZE]
def index_html(self, REQUEST, RESPONSE):
"""Default document"""
# HTTP If-Modified-Since header handling. This is duplicated
# from OFS.Image.Image - it really should be consolidated
# somewhere...
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Last-Modified', self.lmh)
RESPONSE.setHeader('Cache-Control', self.cch)
RESPONSE.setHeader('Content-Length', self.content_size)
if self.set_expiry_header:
RESPONSE.setHeader('Expires', self._expires())
header=REQUEST.get_header('If-Modified-Since', None)
if header is not None:
header=header.split(';')[0]
# Some proxies seem to send invalid date strings for this
# header. If the date string is not valid, we ignore it
# rather than raise an error to be generally consistent
# with common servers such as Apache (which can usually
# understand the screwy date string as a lucky side effect
# of the way they parse it).
try: mod_since=long(DateTime(header).timeTime())
except: mod_since=None
if mod_since is not None:
if getattr(self, 'lmt', None):
last_mod = long(self.lmt)
else:
last_mod = long(0)
if last_mod > 0 and last_mod <= mod_since:
RESPONSE.setStatus(304)
return ''
if self.content_size > FILESTREAM_ITERATOR_THRESHOLD:
return filestream_iterator(self.path, 'rb')
else:
return open(self.path,'rb').read()
HEAD__roles__=None
def HEAD(self, REQUEST, RESPONSE):
""" """
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Last-Modified', self.lmh)
RESPONSE.setHeader('Content-Length', self.content_size)
return ''
def _expires(self):
return rfc1123_date(time()+self.max_age)
def __str__(self):
""" return the content of the file """
return open(self.path,'rb').read()
class GzippedFile(BetterImageFile):
def __init__(self, path, _prefix=None, max_age_development=60, max_age_production=3600,
set_expiry_header=False):
BetterImageFile.__init__(self, path, _prefix=_prefix,
set_expiry_header=set_expiry_header,
max_age_development=max_age_development,
max_age_production=max_age_production)
self._prepareGZippedContent()
def _prepareGZippedContent(self):
# Make a gzip copy and
if not hasattr(self, 'original_path'):
self.original_path = self.path
if hasattr(self, 'original_path') and not hasattr(self, 'original_content_size'):
self.original_content_size = os.stat(self.original_path)[stat.ST_SIZE]
data = open(self.original_path, 'rb').read()
gz_path = _getAutogeneratedFilepath(self.original_path + '.gz')
gzip.open(gz_path, 'wb').write(data)
self.path = gz_path
self.content_size = os.stat(self.path)[stat.ST_SIZE]
def index_html(self, REQUEST, RESPONSE):
"""Default document"""
# HTTP If-Modified-Since header handling. This is duplicated
# from OFS.Image.Image - it really should be consolidated
# somewhere...
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Last-Modified', self.lmh)
RESPONSE.setHeader('Cache-Control', self.cch)
if self.set_expiry_header:
RESPONSE.setHeader('Expires', self._expires())
header=REQUEST.get_header('If-Modified-Since', None)
if header is not None:
header=header.split(';')[0]
# Some proxies seem to send invalid date strings for this
# header. If the date string is not valid, we ignore it
# rather than raise an error to be generally consistent
# with common servers such as Apache (which can usually
# understand the screwy date string as a lucky side effect
# of the way they parse it).
try: mod_since=long(DateTime(header).timeTime())
except: mod_since=None
if mod_since is not None:
if getattr(self, 'lmt', None):
last_mod = long(self.lmt)
else:
last_mod = long(0)
if last_mod > 0 and last_mod <= mod_since:
RESPONSE.setStatus(304)
return ''
if not os.path.isfile(self.path):
self._prepareGZippedContent()
if REQUEST['HTTP_ACCEPT_ENCODING'].find('gzip')>-1:
# HTTP/1.1 'gzip', HTTP/1.0 'x-gzip'
RESPONSE.setHeader("Content-encoding", "gzip")
RESPONSE.setHeader("Content-Length", self.content_size)
if self.content_size > FILESTREAM_ITERATOR_THRESHOLD:
return filestream_iterator(self.path,'rb')
else:
return open(self.path,'rb').read()
else:
RESPONSE.setHeader("Content-Length", self.original_content_size)
if self.original_content_size > FILESTREAM_ITERATOR_THRESHOLD:
return filestream_iterator(self.original_path, 'rb')
else:
return open(self.original_path,'rb').read()
def __str__(self):
return open(self.original_path,'rb').read()
def attachImages(class_, files, globals_):
""" do attachImage() multiple times """
for file in files:
attachImage(class_, file, globals_)
def attachImage(class_, file, globals_):
""" attach an BetterImageFile instance into a class.
The file parameter must be a path to an image like
'images/foo.jpg' and then any instance of class_
(eg. localhost:8080/Bar), the image will be available as
localhost:8080/Bar/foo.jpg.
"""
import warnings
warnings.warn("Don't use this. Use registerImage() or registerImages() instead",
DeprecationWarning, 2)
if not os.path.isfile(file) and os.path.isfile(os.path.join('images',file)):
file = os.path.join('images', file)
filename = os.path.basename(file)
setattr(class_, filename, App.ImageFile.ImageFile(file, globals_))
def registerJSFiles(product, files, Globals=globals(), rel_path='js',
slim_if_possible=True, gzip_if_possible=False,
max_age_development=60, max_age_production=3600,
set_expiry_header=True):
""" register all js files """
result = _registerFiles(registerJSFile, product, files, Globals,
rel_path=rel_path,
slim_if_possible=slim_if_possible,
gzip_if_possible=gzip_if_possible,
max_age_development=max_age_development,
max_age_production=max_age_production,
set_expiry_header=set_expiry_header)
return result
def registerJSFile(product, filename, epath=None, Globals=globals(),
rel_path='js',
slim_if_possible=True,
gzip_if_possible=False,
set_expiry_header=False,
max_age_development=60, max_age_production=3600):
p_home = package_home(Globals) # product home
if filename.count('-slimmed.js'):
raise SystemError, "Again!??!"
objectid = filename
path = "%s/" % rel_path
if epath:
path = "%s/%s/" % (rel_path, epath)
filepath = '%s%s' % (path, filename)
if len(filename.split(',')) > 1:
# it's a combo name!!
real_filepath = _getAutogeneratedFilepath(os.path.join(p_home, filepath))
out = open(real_filepath, 'w')
mtimes = []
for filepath_ in [os.path.join(p_home, '%s%s' % (path, x))
for x in filename.split(',')]:
content = open(filepath_).read()
if slim_if_possible and js_slimmer and not dont_slim_file(filepath_):
content = js_slimmer(content)
out.write(content + '\n')
mtimes.append(os.stat(filepath_)[stat.ST_MTIME])
out.close()
filepath = real_filepath
mtime = max(mtimes)
# since we've taken the slimming thought into account already
# here in the loop there is no need to consider slimming the
# content further down.
slim_if_possible = False
else:
mtime = os.stat(os.path.join(p_home, filepath))[stat.ST_MTIME]
setattr(product,
objectid,
BetterImageFile(filepath, Globals,
max_age_development=max_age_development,
max_age_production=max_age_production,
set_expiry_header=set_expiry_header)
)
obj = getattr(product, objectid)
if slim_if_possible and dont_slim_file(os.path.join(p_home, filepath)):
slim_if_possible = False
if slim_if_possible and \
os.path.isfile(os.path.join(p_home, filepath)):
if os.path.isfile(os.path.join(p_home, filepath+'.nogzip')):
gzip_if_possible = False
if js_slimmer is not None and slim_if_possible:
obj = getattr(product, objectid)
slimmed = js_slimmer(open(obj.path,'rb').read(), hardcore=False)
filepath = _getAutogeneratedFilepath(obj.path + '-slimmed.js')
open(filepath, 'wb').write(slimmed)
setattr(obj, 'path', filepath)
# set up an alias too with near infinite max_age
a, b = os.path.splitext(filename)
objectid_alias = a + '.%s' % mtime + b
setattr(product,
objectid_alias,
BetterImageFile(obj.path, Globals,
set_expiry_header=set_expiry_header,
max_age_development=EXPIRY_INFINITY, # 5 years
max_age_production=EXPIRY_INFINITY # 5 years
)
)
# make a note of the alias
if hasattr(product, 'misc_infinite_aliases'):
aliases = product.misc_infinite_aliases
else:
aliases = {}
aliases[objectid] = objectid_alias
setattr(product, 'misc_infinite_aliases', aliases)
if gzip_if_possible:
setattr(product, objectid,
GzippedFile(filepath, Globals,
max_age_development=max_age_development,
max_age_production=max_age_production,
set_expiry_header=set_expiry_header)
)
# then set up an alias (overwrite the previous alias)
# for the gzipped version too.
setattr(product, objectid_alias,
GzippedFile(filepath, Globals,
max_age_development=EXPIRY_INFINITY,
max_age_production=EXPIRY_INFINITY,
set_expiry_header=set_expiry_header)
)
def registerCSSFiles(product, files, Globals=globals(),
rel_path='css', slim_if_possible=True, gzip_if_possible=False,
max_age_development=60, max_age_production=3600,
set_expiry_header=False,
expand_data64=False,
replace_images_with_aliases=False):
""" register all js files """
result = _registerFiles(registerCSSFile, product, files, Globals,
rel_path=rel_path,
slim_if_possible=slim_if_possible,
gzip_if_possible=gzip_if_possible,
max_age_development=max_age_development,
max_age_production=max_age_production,
expand_data64=expand_data64,
set_expiry_header=set_expiry_header,
replace_images_with_aliases=replace_images_with_aliases)
return result
def registerCSSFile(product, filename, epath=None, Globals=globals(),
rel_path='css',
slim_if_possible=True, gzip_if_possible=False,
set_expiry_header=False,
max_age_development=60, max_age_production=3600,
expand_data64=False,
replace_images_with_aliases=False):
p_home = package_home(Globals) # product home
if filename.count('-slimmed.css'):
raise SystemError, "Again!??!"
objectid = filename
path = "%s/" % rel_path
if epath:
path = "%s/%s/" % (rel_path, epath)
filepath = '%s%s' % (path, filename)
if len(filename.split(',')) > 1:
# it's a combo name!!
real_filepath = _getAutogeneratedFilepath(os.path.join(p_home, filepath))
out = open(real_filepath, 'w')
mtimes = []
for filepath_ in [os.path.join(p_home, '%s%s' % (path, x))
for x in filename.split(',')]:
content = open(filepath_).read()
if slim_if_possible and css_slimmer and not dont_slim_file(filepath_):
content = css_slimmer(content)
out.write(content+'\n')
mtimes.append(os.stat(filepath_)[stat.ST_MTIME])
out.close()
filepath = real_filepath
mtime = max(mtimes)
# since we've taken the slimming thought into account already
# here in the loop there is no need to consider slimming the
# content further down.
slim_if_possible = False
else:
mtime = os.stat(os.path.join(p_home, filepath))[stat.ST_MTIME]
setattr(product,
objectid,
BetterImageFile(filepath, Globals,
set_expiry_header=set_expiry_header,
max_age_development=max_age_development,
max_age_production=max_age_production)
)
obj = getattr(product, objectid)
if slim_if_possible and dont_slim_file(os.path.join(p_home, filepath)):
slim_if_possible = False
if slim_if_possible and \
os.path.isfile(os.path.join(p_home, filepath)):
if os.path.isfile(os.path.join(p_home, filepath+'.nogzip')):
gzip_if_possible = False
if css_slimmer is not None and slim_if_possible:
slimmed = css_slimmer(open(obj.path,'rb').read())
filepath = _getAutogeneratedFilepath(obj.path + '-slimmed.css')
open(filepath, 'wb').write(slimmed)
setattr(obj, 'path', filepath)
if expand_data64:
# If this is true, then try to convert things like
# 'DATA64(www/img.gif)' into
# 'data:image/gif;base64,ASIDJADIJAIW00100.... ...\n'
# This feature is highly experimental and has proved problematic
# since it sometimes just doesn't work and it's really hard to
# debug why certain images aren't working. Use carefully.
#
# The spec is here: http://tools.ietf.org/html/rfc2397
# The inspiration came from here:
# http://developer.yahoo.com/performance/rules.html#num_http
new_content = False
content = file(obj.path).read()
for whole_tag, path, extension in DATA64_REGEX.findall(content):
fp = os.path.join(p_home, path)
content = content.replace(whole_tag,
'data:image/%s;base64,%s' % \
(extension,
encodestring(file(fp,'rb').read()).replace('\n','\\n')))
new_content = content
if new_content:
filepath = _getAutogeneratedFilepath(obj.path + '-data64expanded')
open(filepath, 'wb').write(new_content)
setattr(obj, 'path', filepath)
if replace_images_with_aliases:
# This feature opens the CSS content and looks for things like
# url(/misc_/MyProduct/close.png) and replaces that with
# url(/misc_/MyProduct/close.1223555.png) if possible.
new_content = False
if getattr(product, 'misc_infinite_aliases', {}):
content = file(obj.path).read()
images = []
if content.count('/misc_/'):
filenames = []
for org, alias in product.misc_infinite_aliases.items():
if anyTrue(org.endswith, ('.gif','.jpg','.png')):
filenames.append(org)
regex = 'url\(["\']*/misc_/%s/(' % product + \
'|'.join([re.escape(x) for x in filenames]) + \
')["\']*\)'
regex = re.compile(regex)
def replacer(g):
whole = g.group()
better_filename = product.misc_infinite_aliases.get(g.groups()[0], g.groups()[0])
whole = whole.replace(g.groups()[0], better_filename)
return whole
new_content = regex.sub(replacer, content)
else:
def replacer(match):
filepath, filename = match.groups()[0].rsplit('/', 1)
try:
image_product = _find_related_context(product, filepath)
except AttributeError:
#import warnings
import logging
logging.warn("Unable to find image product of %s from %r" % \
(product, filepath))
return match.group()
aliased = getattr(image_product, 'misc_infinite_aliases',
{}).get(filename)
if aliased:
return match.group().replace(filename, aliased)
else:
return match.group()
#new_content = content
new_content = referred_css_images_regex.sub(replacer, content)
if new_content:
filepath = _getAutogeneratedFilepath(obj.path + '-aliased.css')
open(filepath, 'wb').write(new_content)
setattr(obj, 'path', filepath)
# set up an alias too with near infinit max_age
a, b = os.path.splitext(filename)
objectid_alias = a + '.%s' % mtime + b
setattr(product,
objectid_alias,
BetterImageFile(obj.path, Globals,
set_expiry_header=set_expiry_header,
max_age_development=EXPIRY_INFINITY, # 5 years
max_age_production=EXPIRY_INFINITY # 5 years
)
)
# make a note of the alias
if hasattr(product, 'misc_infinite_aliases'):
aliases = product.misc_infinite_aliases
else:
aliases = {}
if objectid in aliases:
# this is not supposed to happen in the same sense that you can't have
# two files by the same name in the same directory
if getEnvBool('PREVENT_DUPLICATE_STATIC_STORAGE', False):
# by default we don't worry about that much
raise ValueError, "%r is already defined in %s.misc_infinite_aliases" %\
(objectid, product)
aliases[objectid] = objectid_alias
setattr(product, 'misc_infinite_aliases', aliases)
if gzip_if_possible:
setattr(product, objectid,
GzippedFile(filepath, Globals,
set_expiry_header=set_expiry_header,
max_age_development=max_age_development,
max_age_production=max_age_production)
)
# also set up the alias which overwrites the previously
# set up alias.
setattr(product, objectid_alias,
GzippedFile(filepath, Globals,
set_expiry_header=set_expiry_header,
max_age_development=EXPIRY_INFINITY,
max_age_production=EXPIRY_INFINITY)
)
def registerImages(product, images, Globals=globals(), rel_path='www',
max_age_development=60, max_age_production=3600,
set_expiry_header=False,
use_rel_path_in_alias=False):
""" register all images """
result = _registerFiles(registerImage, product, images, Globals, rel_path=rel_path,
max_age_development=max_age_development,
max_age_production=max_age_production,
set_expiry_header=set_expiry_header,
use_rel_path_in_alias=use_rel_path_in_alias)
return result
def registerIcons(product, images, Globals=globals()):
""" legacy name of function """
import warnings
warnings.warn("Please use registerImages() instead",
DeprecationWarning, 2)
registerImages(product, images, Globals)
def registerImage(product, filename, idreplacer={}, epath=None,
Globals=globals(), rel_path='www',
set_expiry_header=False,
max_age_development=60, max_age_production=3600,
use_rel_path_in_alias=False):
# A helper function that takes an image filename (assumed
# to live in a 'www' subdirectory of this package). It
# creates an ImageFile instance and adds it as an attribute
# of misc_.MyPackage of the zope application object (note
# that misc_.MyPackage has already been created by the product
# initialization machinery by the time registerIcon is called).
p_home = package_home(Globals) # product home
if use_rel_path_in_alias:
objectid = filename
else:
objectid = os.path.basename(filename)
if epath:
path = "%s/%s/" % (rel_path, epath)
else:
path = "%s/" % (rel_path)
#fullpath = '%s%s' % (path, filename)
if os.path.isfile(os.path.join(path, filename)):
fullpath = os.path.join(path, filename)
else:
# let's try to manually guess it
fullpath = os.path.join(p_home, path, filename)
fullpath = fullpath.replace('//','/')
for k,v in idreplacer.items():
objectid = objectid.replace(k,v)
mtime = os.stat(fullpath)[stat.ST_MTIME]
a, b = os.path.splitext(objectid)
objectid_alias = a + '.%s' % mtime + b
setattr(product,
objectid,
BetterImageFile(fullpath, Globals,
set_expiry_header=set_expiry_header,
max_age_development=max_age_development,
max_age_production=max_age_production)
)
# set up an alias with near infinite max_age
setattr(product,
objectid_alias,
BetterImageFile(fullpath, Globals,
set_expiry_header=set_expiry_header,
max_age_development=EXPIRY_INFINITY,
max_age_production=EXPIRY_INFINITY)
)
# make a non-persistent note of this added available alias
if base_hasattr(product, 'misc_infinite_aliases'):
aliases = product.misc_infinite_aliases
else:
aliases = {}
aliases[objectid] = objectid_alias
setattr(product, 'misc_infinite_aliases', aliases)
def registerIcon(product, filename, idreplacer={}, epath=None,
Globals=globals()):
import warnings
warnings.warn("Please use registerImage() instead",
DeprecationWarning, 2)
return registerImage(product, filename,
idreplacer=idreplacer,
epath=epath,
Globals=Globals)
def _registerFiles(handler, product, files, Globals, rel_path='www', **kw):
""" I really shold right a comment here """
all = {}
number = 0
for each in files:
if isinstance(each, basestring):
names = each
dirname = ''
elif isinstance(each, (list, tuple)):
dirname = ''
names = ','.join(each)
else:
names = each['n']
dirname = each.get('d','')
if isinstance(names, str) and len(names.split(':')) > 1:
names = names.split(':')
names = [x.strip() for x in names]
elif type(names)==type('s'):
names = [names]
prev = all.get(dirname, [])
prev.extend(names)
all[dirname] = prev
for dirname, names in all.items():
for name in names:
handler(product, name, epath=dirname, Globals=Globals,
rel_path=rel_path, **kw)
number += 1
return number
def dont_slim_file(filepath):
""" return true if there's enough evidence that the file shouldn't
be slimmed. Note that this has nothing to do with manual configuration
just other factors such as filename and presence of marker files.
The evidence for NOT slimming is:
1. Is there a <filename>.noslim ?
2. Does the filename contain .packed. or something similar
"""
assert os.path.isfile(filepath), \
"Can't test for reason not to slim because %s doesn't exist" % filepath
if os.path.isfile(filepath+'.noslim'):
return True
for part in ('-min.', '.min.', '-packed.','.packed'):
if os.path.basename(filepath).find(part) > -1:
return True
# default is False, i.e. there's NO reason not to slim
return False