Permalink
Browse files

Added first attempt at rendering support

  • Loading branch information...
1 parent 33d4234 commit 737f74fbfb767db675639cdaed53a33dbf394031 Nick Johnson committed Oct 20, 2011
Showing with 175 additions and 89 deletions.
  1. +2 −1 backends.yaml
  2. +150 −87 main.py
  3. +5 −0 models.py
  4. +1 −1 ndb/tasklets.py
  5. +17 −0 templates/index.html
View
@@ -1,4 +1,5 @@
backends:
- name: renderer
class: B4
- instances: 5
+ instances: 6
+ options: dynamic
View
237 main.py
@@ -1,6 +1,7 @@
import cStringIO
import datetime
import logging
+import math
import time
import urllib
import urlparse
@@ -9,6 +10,7 @@
from google.appengine.api import backends
from google.appengine.api import files
from google.appengine.api import urlfetch
+from google.appengine.ext import blobstore
from google.appengine.runtime import apiproxy_errors
from ndb import context, model, tasklets
from webapp2_extras import jinja2
@@ -23,6 +25,7 @@
NUM_STRIPES = 16
PARALLELISM = 4
+NUM_BACKENDS = 6
class BaseHandler(webapp2.RequestHandler):
@@ -72,104 +75,164 @@ def mapper_task():
raise tasklets.Return(results)
+@tasklets.tasklet
+def render_image(x, y, width, height, px_width):
+ """Renders an image of part of the mandelbrot set.
+
+ The coordinate system used has the top left corner of the whole mandelbrot
+ image at 0,0 and the bottom right at 1,1.
+
+ Args:
+ x, y: Coordinates of the top left corner of the rendered image.
+ width, height: Width and height of the section to render.
+ px_width, px_height: Width and height of the generated image.
+ Returns:
+ A PIL Image object containing the rendered image.
+ """
+ total_pixel_width = px_width / width
+ level = int(math.ceil(math.log(total_pixel_width, 2)))
+ tiles_per_side = (2 ** max(1, level - mandelbrot.TILE_SIZE_BITS))
+ top_left_tile = (int(x * tiles_per_side),
+ int(y * tiles_per_side))
+ bottom_right_tile = (int(math.ceil((x + width) * tiles_per_side)),
+ int(math.ceil((y + height) * tiles_per_side)))
+
+ tile_args = [(level, tx, ty)
+ for tx in range(top_left_tile[0], bottom_right_tile[0] + 1)
+ for ty in range(top_left_tile[1], bottom_right_tile[1] + 1)]
+ tiles = yield [fetch_or_render_tile(*x) for x in tile_args]
+
+ real_width = mandelbrot.TILE_SIZE * (bottom_right_tile[0] - top_left_tile[0])
+ real_height = mandelbrot.TILE_SIZE * (bottom_right_tile[1] - top_left_tile[1])
+ image = Image.new('RGB', (real_width, real_height))
+
+ for tile, tile_img in tiles:
+ if not tile_img:
+ tile_img = Image.open(blobstore.BlobReader(tile.tile))
+ _, tile_x, tile_y = tile.position
+ image.paste(tile_img, (tile_x * mandelbrot.TILE_SIZE,
+ tile_y * mandelbrot.TILE_SIZE))
+
+ raise tasklets.Return(image)
+
+
+@tasklets.tasklet
+def fetch_or_render_tile(level, x, y):
+ tile_key = models.CachedTile.key_for_tile('exabrot', level, x, y)
+ tile = tile_key.get()
+ img = None
+ if not tile:
+ tile, img = yield render_tile(level, x, y)
+ raise tasklets.Return(tile, img)
+
+
+@tasklets.tasklet
+def render_tile(level, x, y):
+ # Compute the bounds of this tile
+ xmin, ymin, xsize, ysize, tilesize = mandelbrot.calculate_bounds(
+ level, x, y)
+ # Divide the tile up into vertical stripes
+ stripe_size = ysize / NUM_STRIPES
+ stripe_height = tilesize / NUM_STRIPES
+ stripes = [
+ (xmin, ymin + stripe_size * i, xsize, stripe_size, tilesize,
+ stripe_height) for i in range(NUM_STRIPES)]
+
+ # Construct the image that will hold the final tile
+ img = Image.new('RGB', (tilesize, tilesize))
+ operation_cost = 0
+ start_time = time.time()
+
+ map_result = yield ndb_map(get_image, stripes, PARALLELISM)
+ for stripe_num, (stripe, opcost) in enumerate(map_result):
+ # Paste the result into the final image
+ operation_cost += opcost
+ stripe_img = Image.open(cStringIO.StringIO(stripe))
+ img.paste(stripe_img, (0, stripe_num * stripe_height))
+ elapsed = time.time() - start_time
+
+ # Save the image to the datastore and return it
+ logging.info("Rendered tile %s/%s/%s in %.2f seconds with %d operations.",
+ level, x, y, elapsed, operation_cost)
+
+ tile = write_tile(level, x, y, operation_cost, elapsed, img)
+ yield tile.put_async()
+ raise tasklets.Return(tile, img)
+
+def write_tile(level, x, y, operation_cost, elapsed, img):
+ """Writes a tile to the blobstore and returns the datastore object."""
+ tiledata = cStringIO.StringIO()
+ img.save(tiledata, 'PNG')
+
+ write_start = time.time()
+ tile_filename = files.blobstore.create(mime_type='image/png')
+ with files.open(tile_filename, 'a') as f:
+ f.write(tiledata.getvalue())
+ files.finalize(tile_filename)
+ logging.info("Blobstore write took %.2f seconds", time.time() - write_start)
+
+ return models.CachedTile(
+ key=models.CachedTile.key_for_tile('exabrot', level, x, y),
+ tile=files.blobstore.get_blob_key(tile_filename),
+ rendered=datetime.datetime.utcnow(),
+ operation_cost=operation_cost,
+ render_time=elapsed,
+ level=level)
+
+@tasklets.tasklet
+def get_image(xmin, ymin, xsize, ysize, width, height):
+ params = urllib.urlencode({
+ 'xmin': xmin,
+ 'ymin': ymin,
+ 'xsize': xsize,
+ 'ysize': ysize,
+ 'width': width,
+ 'height': height,
+ })
+ for i in range(3): # Retries
+ instance_id = hash(params) % NUM_BACKENDS
+ url = urlparse.urljoin(backends.get_url('renderer', instance=instance_id),
+ '/backend/render_tile?%s' % params)
+ rpc = urlfetch.create_rpc(deadline=10.0)
+ urlfetch.make_fetch_call(rpc, url)
+ try:
+ response = yield rpc
+ if response.status_code not in (500, 0):
+ break
+ except (apiproxy_errors.DeadlineExceededError,
+ urlfetch.DeadlineExceededError):
+ pass
+ logging.warn("Backend failed to render tile; retrying")
+ # Wait a little before retrying
+ time.sleep(0.2)
+ assert response.status_code == 200, \
+ "Expected status 200, got %s" % response.status_code
+ raise tasklets.Return(
+ response.content,
+ int(response.headers['X-Operation-Cost']))
+
class TileHandler(BaseHandler):
@context.toplevel
def get(self, level, x, y):
self.response.headers['Content-Type'] = 'image/png'
-
- tile_key = models.CachedTile.key_for_tile('exabrot', level, x, y)
- tile = tile_key.get()
- if not tile:
- tile = yield self.render_tile(int(level), int(x), int(y))
+ tile, img = yield fetch_or_render_tile(int(level), int(x), int(y))
self.response.headers['X-AppEngine-BlobKey'] = str(tile.tile)
- @tasklets.tasklet
- def render_tile(self, level, x, y):
- # Compute the bounds of this tile
- xmin, ymin, xsize, ysize, tilesize = mandelbrot.calculate_bounds(
- level, x, y)
- # Divide the tile up into vertical stripes
- stripe_size = ysize / NUM_STRIPES
- stripe_height = tilesize / NUM_STRIPES
- stripes = [
- (xmin, ymin + stripe_size * i, xsize, stripe_size, tilesize,
- stripe_height) for i in range(NUM_STRIPES)]
-
- # Construct the image that will hold the final tile
- img = Image.new('RGB', (tilesize, tilesize))
- operation_cost = 0
- start_time = time.time()
-
- map_result = yield ndb_map(self.get_image, stripes, PARALLELISM)
- for stripe_num, (stripe, opcost) in enumerate(map_result):
- # Paste the result into the final image
- operation_cost += opcost
- stripe_img = Image.open(cStringIO.StringIO(stripe))
- img.paste(stripe_img, (0, stripe_num * stripe_height))
- elapsed = time.time() - start_time
-
- # Save the image to the datastore and return it
- logging.info("Rendered tile %s/%s/%s in %.2f seconds with %d operations.",
- level, x, y, elapsed, operation_cost)
-
- tile = self.write_tile(level, x, y, operation_cost, elapsed, img)
- tile.put()
- raise tasklets.Return(tile)
-
- def write_tile(self, level, x, y, operation_cost, elapsed, img):
- """Writes a tile to the blobstore and returns the datastore object."""
- tiledata = cStringIO.StringIO()
- img.save(tiledata, 'PNG')
-
- write_start = time.time()
- tile_filename = files.blobstore.create(mime_type='image/png')
- with files.open(tile_filename, 'a') as f:
- f.write(tiledata.getvalue())
- files.finalize(tile_filename)
- logging.info("Blobstore write took %.2f seconds", time.time() - write_start)
-
- return models.CachedTile(
- key=models.CachedTile.key_for_tile('exabrot', level, x, y),
- tile=files.blobstore.get_blob_key(tile_filename),
- rendered=datetime.datetime.utcnow(),
- operation_cost=operation_cost,
- render_time=elapsed,
- level=level)
- @tasklets.tasklet
- def get_image(self, xmin, ymin, xsize, ysize, width, height):
- params = urllib.urlencode({
- 'xmin': xmin,
- 'ymin': ymin,
- 'xsize': xsize,
- 'ysize': ysize,
- 'width': width,
- 'height': height,
- })
- for i in range(3): # Retries
- url = urlparse.urljoin(backends.get_url('renderer'),
- '/backend/render_tile?%s' % params)
- rpc = urlfetch.create_rpc(deadline=10.0)
- urlfetch.make_fetch_call(rpc, url)
- try:
- response = yield rpc
- if response.status_code not in (500, 0):
- break
- except (apiproxy_errors.DeadlineExceededError,
- urlfetch.DeadlineExceededError):
- pass
- logging.warn("Backend failed to render tile; retrying")
- # Wait a little before retrying
- time.sleep(0.2)
- assert response.status_code == 200, \
- "Expected status 200, got %s" % response.status_code
- raise tasklets.Return(
- response.content,
- int(response.headers['X-Operation-Cost']))
+class RenderHandler(BaseHandler):
+ @context.toplevel
+ def get(self, x, y, width, height):
+ x, y, width, height = [float(z) for z in (x, y, width, height)]
+ image = yield render_image(x, y, width, height, 512)
+ image_data = cStringIO.StringIO()
+ image.save(image_data, 'PNG')
+ self.response.headers['Content-Type'] = 'image/png'
+ self.response.write(image_data.getvalue())
application = webapp2.WSGIApplication([
('/', IndexHandler),
+ ('/render/([0-9.e-]+)/([0-9.e-]+)/([0-9.e-]+)/([0-9.e-]+)\.png', RenderHandler),
('/exabrot_files/(\d+)/(\d+)_(\d+).png', TileHandler),
], debug=True)
View
@@ -10,6 +10,11 @@ class CachedTile(model.Model):
#_use_datastore = False
_use_memcache = False
+ @property
+ def position(self):
+ """Returns the level/x/y tuple for this tile."""
+ return tuple(int(x) for x in self.key.id.split('/')[1:])
+
@classmethod
def key_for_tile(cls, type, level, x, y):
return model.Key(cls, '%s/%s/%s/%s' % (type, level, x, y))
View
@@ -205,7 +205,7 @@ def add_callback(self, callback, *args, **kwds):
self._callbacks.append((callback, args, kwds))
def set_result(self, result):
- assert not self._done
+ assert not self._done, (self._result, result)
self._result = result
self._done = True
_state.remove_pending(self)
View
@@ -20,6 +20,23 @@
document.location.hash = get_permalink();
});
+ var linkbutton = new Seadragon.Button(
+ "Permalink",
+ "images/link_rest.png",
+ "images/link_rest.png",
+ "images/link_hover.png",
+ "images/link_click.png",
+ null,
+ function() {
+ var bounds = viewer.viewport.getBounds();
+ var newpath = '/render/' + [bounds.x, bounds.y, bounds.width, bounds.height].join('/')
+ document.location.pathname = newpath;
+ },
+ null,
+ null,
+ null);
+ viewer.addControl(linkbutton.elmt, Seadragon.ControlAnchor.BOTTOM_LEFT);
+
var fragment = document.location.hash;
if(fragment != '#' && fragment != '') {
viewer.addEventListener("open", function(viewer) {

0 comments on commit 737f74f

Please sign in to comment.