This repository has been archived by the owner. It is now read-only.
Permalink
Browse files

Split thumbnails and sr images across multiple buckets.

This allows for HTTP parallelization.
Also, allow users to select JPEG or PNG for SR images.
  • Loading branch information...
andre-d authored and spladug committed Oct 25, 2011
1 parent a8e2d55 commit 4653c16a72d301c4c0ac1639e529aa3499968116
View
@@ -319,13 +319,31 @@ words_file = /usr/dict/words
# -- media stuff --
# user agent for the scraper
useragent = Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)
+
+# The storage method to use for media
+# Options:
+# s3 - Uses Amazon S3 to upload media to buckets
+media_store = s3
+
# your s3 credentials
S3KEY_ID =
S3SECRET_KEY =
-# s3 bucket
-s3_thumb_bucket = test.your.domain.here
+
+# This bucket is used for old media, new installs may leave this blank
+s3_old_thumb_bucket =
+
+# May be one bucket, or many buckets seperated by commas
+s3_media_buckets =
+
+# Store direct urls for images, rather than buckets
+# For the bucket mybucket with the image helloworld.jpg the stored url would be:
+# true: http://s3.amazonaws.com/mybucket/helloworld.jpg
+# false: http://mybucket/helloworld.jpg
+s3_media_direct = true
+
default_thumb = /static/noimage.png
self_thumb = /static/self_default2.png
+
media_domain = localhost
# Embedly API Key
embedly_api_key =
@@ -410,3 +428,4 @@ beaker.session_secret = somesecret
# execute malicious code after an exception is raised.
#set debug = false
+
View
@@ -1158,10 +1158,11 @@ def GET_upload_sr_img(self, *a, **kw):
VModhash(),
file = VLength('file', max_length=1024*500),
name = VCssName("name"),
+ img_type = VImageType('img_type'),
form_id = VLength('formid', max_length = 100),
header = VInt('header', max=1, min=0),
sponsor = VInt('sponsor', max=1, min=0))
- def POST_upload_sr_img(self, file, header, sponsor, name, form_id):
+ def POST_upload_sr_img(self, file, header, sponsor, name, form_id, img_type):
"""
Called on /about/stylesheet when an image needs to be replaced
or uploaded, as well as on /about/edit for updating the
@@ -1181,43 +1182,42 @@ def POST_upload_sr_img(self, file, header, sponsor, name, form_id):
# default error list (default values will reset the errors in
# the response if no error is raised)
errors = dict(BAD_CSS_NAME = "", IMAGE_ERROR = "")
- try:
- cleaned = cssfilter.clean_image(file,'PNG')
- if header:
- # there is one and only header, and it is unnumbered
- resource = None
- elif sponsor and c.user_is_admin:
- resource = "sponsor"
- elif not name:
- # error if the name wasn't specified or didn't satisfy
- # the validator
+ add_image_to_sr = False
+ if sponsor and not c.user_is_admin:
+ return self.abort(403, 'forbidden')
+
+ if not sponsor and not header:
+ add_image_to_sr = True
+ if not name:
+ # error if the name wasn't specified and the image was not for a sponsored link or header
+ # this may also fail if a sponsored image was added and the user is not an admin
errors['BAD_CSS_NAME'] = _("bad image name")
- else:
- resource = c.site.add_image(name, max_num = g.max_sr_images)
- c.site._commit()
-
- except cssfilter.BadImage:
- # if the image doesn't clean up nicely, abort
- errors["IMAGE_ERROR"] = _("bad image")
- except ValueError:
- # the add_image method will raise only on too many images
- errors['IMAGE_ERROR'] = (
- _("too many images (you only get %d)") % g.max_sr_images)
+
+ if c.site.images:
+ if c.site.images.has_key(name):
+ errors['IMAGE_ERROR'] = (_("An image with that name already exists"))
+ elif c.site.get_num_images() >= g.max_sr_images:
+ errors['IMAGE_ERROR'] = (_("too many images (you only get %d)") % g.max_sr_images)
if any(errors.values()):
- return UploadedImage("", "", "", errors = errors).render()
+ return UploadedImage("", "", "", errors = errors).render()
else:
# with the image num, save the image an upload to s3. the
# header image will be of the form "${c.site._fullname}.png"
# while any other image will be ${c.site._fullname}_${resource}.png
- new_url = cssfilter.save_sr_image(c.site, cleaned,
- resource = resource)
+ try:
+ new_url = cssfilter.save_sr_image(c.site, file, suffix = '.' + img_type)
+ except cssfilter.BadImage:
+ errors['IMAGE_ERROR'] = _("Invalid image or general image error")
+ return UploadedImage("", "", "", errors = errors).render()
+
if header:
c.site.header = new_url
elif sponsor and c.user_is_admin:
c.site.sponsorship_img = new_url
- c.site._commit()
+ c.site.add_image(name, url = new_url)
+ c.site._commit()
return UploadedImage(_('saved'), new_url, name,
errors = errors, form_id = form_id).render()
@@ -482,15 +482,14 @@ def POST_link_thumb(self, link=None, file=None):
errors = dict(BAD_CSS_NAME = "", IMAGE_ERROR = "")
try:
# thumnails for promoted links can change and therefore expire
- force_thumbnail(link, file, fileType=".jpg")
+ force_thumbnail(link, file, file_type=".jpg")
except cssfilter.BadImage:
# if the image doesn't clean up nicely, abort
errors["IMAGE_ERROR"] = _("bad image")
if any(errors.values()):
return UploadedImage("", "", "upload", errors = errors,
form_id = "image-upload").render()
else:
- link.thumbnail_version = sha.new(file).hexdigest()
link._commit()
return UploadedImage(_('saved'), thumbnail_url(link), "",
errors = errors,
@@ -1236,6 +1236,21 @@ def run(self, val):
else:
return val
+class VImageType(Validator):
+ def run(self, img_type):
+ if not img_type in ('png', 'jpg'):
+ return 'png'
+ return img_type
+
+class VSubredditSponsorship(VInt):
+ max = 1
+ min = 0
+ def run(self, val):
+ s = super(VSubredditSponsorship, self).run(val)
+ if s and not c.user_is_admin:
+ abort(403, "forbidden")
+ return s
+
class ValidEmails(Validator):
"""Validates a list of email addresses passed in as a string and
delineated by whitespace, ',' or ';'. Also validates quantity of
View
@@ -92,6 +92,7 @@ class Globals(object):
'frontpage_dart',
'allow_wiki_editing',
'heavy_load_mode',
+ 's3_media_direct',
'disable_captcha',
'disable_ads',
]
@@ -111,6 +112,7 @@ class Globals(object):
'authorized_cnames',
'hardcache_categories',
'proxy_addr',
+ 's3_media_buckets',
'allowed_pay_countries',
'case_sensitive_domains']
View
@@ -36,6 +36,10 @@
from md5 import md5
from r2.lib.contrib.nymph import optimize_png
+from r2.lib.media import upload_media
+
+from r2.lib.template_helpers import s3_https_if_secure
+
import re
from urlparse import urlparse
@@ -207,10 +211,16 @@ def valid_url(prop,value,report):
name = custom_img_urls.match(url).group(1)
# the label -> image number lookup is stored on the subreddit
if c.site.images.has_key(name):
- num = c.site.images[name]
- value._setCssText("url(http://%s/%s_%d.png?v=%s)"
- % (g.s3_thumb_bucket, c.site._fullname[::-1], num,
- randstr(36)))
+ url = c.site.images[name]
+ if isinstance(url, int): # legacy url, needs to be generated
+ bucket = g.s3_old_thumb_bucket
+ baseurl = "http://%s" % (bucket)
+ if g.s3_media_direct:
+ baseurl = "http://%s/%s" % (s3_direct_url, bucket)
+ url = "%s/%s_%d.png"\
+ % (baseurl, c.site._fullname, url)
+ url = s3_https_if_secure(url)
+ value._setCssText("url(%s)"%url)
else:
# unknown image label -> error
report.append(ValidationError(msgs['broken_url']
@@ -390,60 +400,13 @@ def rendered_link(links, media, compress):
def rendered_comment(comments):
return wrap_links(comments, num = 1).render(style = "html")
-class BadImage(Exception): pass
-
-def clean_image(data,format):
- import Image
- from StringIO import StringIO
+class BadImage(Exception):
+ def __init__(self, error = None):
+ self.error = error
+def save_sr_image(sr, data, suffix = '.png'):
try:
- in_file = StringIO(data)
- out_file = StringIO()
-
- im = Image.open(in_file)
- im = im.resize(im.size)
-
- im.save(out_file,format)
- ret = out_file.getvalue()
- except IOError,e:
+ return upload_media(data, file_type = suffix)
+ except Exception as e:
raise BadImage(e)
- finally:
- out_file.close()
- in_file.close()
-
- return ret
-
-def save_sr_image(sr, data, resource = None):
- """
- uploades image data to s3 as a PNG and returns its new url. Urls
- will be of the form:
- http://${g.s3_thumb_bucket}/${sr._fullname}[_${num}].png?v=${md5hash}
- [Note: g.s3_thumb_bucket begins with a "/" so the above url is valid.]
- """
- hash = md5(data).hexdigest()
-
- f = tempfile.NamedTemporaryFile(suffix = '.png',delete=False)
- try:
- f.write(data)
- f.close()
-
- optimize_png(f.name, g.png_optimizer)
- contents = open(f.name).read()
-
- if resource is not None:
- resource = "_%s" % resource
- else:
- resource = ""
- fname = resource = sr._fullname[::-1] + resource + ".png"
-
- s3cp.send_file(g.s3_thumb_bucket, fname, contents, 'image/png')
-
- finally:
- os.unlink(f.name)
-
- return 'http://%s/%s?v=%s' % (g.s3_thumb_bucket,
- resource.split('/')[-1], hash)
-
-
-
Oops, something went wrong.

0 comments on commit 4653c16

Please sign in to comment.