Permalink
Browse files

Fixed is_ready issue (I think) by removing update_user_info call from…

… the IndexHander - it's likely that it was undoing the is_ready=True by putting an old copy of the UserInfo back in the datastore. Re-fixed static map generation, since I was accidentally truncating my latlng floats and somehow lost that change before. Split task queues into two, based on type of work.
  • Loading branch information...
1 parent 1d15b72 commit bda438c53ffa9048e3da9bb71d0d9733b94e00a9 @lehrblogger committed Jan 14, 2012
Showing with 21 additions and 72 deletions.
  1. +1 −4 gheatae/tile.py
  2. +2 −13 handlers.py
  3. +0 −1 models.py
  4. +8 −3 queue.yaml
  5. +5 −6 scripts/fetch_foursquare_data.py
  6. +5 −24 scripts/update_user_maps.py
  7. +0 −21 templates/private_forbidden.html
View
5 gheatae/tile.py
@@ -26,11 +26,9 @@ def __init__(self, user, lat_north, lng_west, range_lat, range_lng):
else:
self.level_max = int(constants.level_const)
self.color_scheme = color_scheme.color_schemes[constants.default_color]
-
self.cache_levels = []
for i in range(self.level_max - 1, -1, -1):
self.cache_levels.append(int(((-(pow(float(i) - self.level_max, 2))/self.level_max) + self.level_max) / self.level_max * 255))
-
if not constants.provider:
constants.provider = provider.DBProvider()
uservenues = constants.provider.get_user_data(user, lat_north, lng_west, range_lat, range_lng)
@@ -45,7 +43,7 @@ def plot_image(self, points):
start = datetime.now()
for i, point in enumerate(points):
self.__merge_point_in_space(space_level, point, rad)
- logging.debug(' point %d of %d, start at %s, done at %s' % (i, len(points), start, datetime.now()))
+ # logging.debug(' point %d of %d, start at %s, done at %s' % (i, len(points), start, datetime.now()))
return self.convert_image(space_level)
def __merge_point_in_space(self, space_level, point, rad):
@@ -105,7 +103,6 @@ def image_out(self):
class CustomTile(BasicTile):
def __init__(self, user, zoom, lat_north, lng_west, offset_x_px, offset_y_px):
- logging.info("drawing new custom tile")
self.zoom = zoom
self.decay = 0.5
#dot_radius = int(math.ceil(len(dot[self.zoom]) / 2))
View
15 handlers.py
@@ -46,21 +46,12 @@ def get(self):
'static_url': 'http://maps.google.com/maps/api/staticmap?center=40.7427050566%2C-73.9888000488&format=png&zoom=13&key=ABQIAAAAwA6oEsCLgzz6I150wm3ELBQO7aMTgd18mR6eRdj9blrVCeGU7BS14EnkGH_2LpNpZ8DJW0u7G5ocLQ&sensor=false&size=640x640',
'mapimage_url': 'map/%s.png' % 'ag93aGVyZS1kby15b3UtZ29yEAsSCE1hcEltYWdlGM2LRgw',
}
- foursquare_is_happy = True
user = users.get_current_user()
if user:
welcome_data['user'] = user
welcome_data['url'] = users.create_logout_url(self.request.uri)
userinfo = UserInfo.all().filter('user =', user).get()
if userinfo:
- if userinfo.is_authorized:
- try:
- fetch_foursquare_data.update_user_info(userinfo)
- except foursquare.FoursquareRemoteException, err:
- if str(err).find('403 Forbidden') >= 0:
- foursquare_is_happy = False
- else:
- raise err
welcome_data['userinfo'] = userinfo
welcome_data['real_name'] = userinfo.real_name
welcome_data['photo_url'] = userinfo.photo_url
@@ -71,9 +62,7 @@ def get(self):
os_path = os.path.dirname(__file__)
self.response.out.write(template.render(os.path.join(os_path, 'templates/header.html'), {'key': constants.get_google_maps_apikey()}))
self.response.out.write(template.render(os.path.join(os_path, 'templates/private_welcome.html'), welcome_data))
- if not foursquare_is_happy:
- self.response.out.write(template.render(os.path.join(os_path, 'templates/private_forbidden.html'), None))
- elif user and userinfo:
+ if user and userinfo:
if userinfo.is_authorized:
self.response.out.write(template.render(os.path.join(os_path, 'templates/private_sidebar.html'), sidebar_data))
else:
@@ -114,7 +103,7 @@ def get(self):
raise err
try:
fetch_foursquare_data.update_user_info(userinfo)
- fetch_foursquare_data.fetch_and_store_checkins_next(userinfo, limit=2)
+ fetch_foursquare_data.fetch_and_store_checkins_next(userinfo, limit=50)
except foursquare.FoursquareRemoteException, err:
if str(err).find('403 Forbidden') >= 0:
pass # if a user tries to sign up while my app is blocked, then it currently just redirects to the signup page
View
1 models.py
@@ -44,7 +44,6 @@ class MapImage(db.Model):
zoom = db.IntegerProperty()
width = db.IntegerProperty()
height = db.IntegerProperty()
- input_imgs = db.ListProperty(db.Blob, default=[])
img = db.BlobProperty()
static_url = db.StringProperty()
update_count = db.IntegerProperty(default=0)
View
11 queue.yaml
@@ -1,4 +1,9 @@
queue:
-- name: default
- rate: 6/m
- bucket_size: 20
+
+- name: checkins
+ rate: 60/m
+ bucket_size: 30
+
+- name: tiles
+ rate: 120/m
+ bucket_size: 60
View
11 scripts/fetch_foursquare_data.py
@@ -85,13 +85,12 @@ def fetch_and_store_checkins_next(userinfo, limit=100):
num_added, num_received = fetch_and_store_checkins(userinfo, limit)
logging.info("num_added = %d, num_received = %d" % (num_added, num_received))
if num_added == 0:
- userinfo.is_ready = True
- userinfo.put()
+ def put_ready_userinfo(userinfo_param):
+ userinfo_param.is_ready = True
+ userinfo_param.put()
+ db.run_in_transaction(put_ready_userinfo, userinfo)
else:
- taskqueue.add(url='/fetch_foursquare_data/next_for_user/%s' % userinfo.key())
- # def put_ready_userinfo(userinfo_param):
- # userinfo_param.put()
- # db.run_in_transaction(put_ready_userinfo, userinfo)
+ taskqueue.add(queue_name='checkins', url='/fetch_foursquare_data/next_for_user/%s' % userinfo.key())
def update_user_info(userinfo):
fs = get_new_fs_for_userinfo(userinfo)
View
29 scripts/update_user_maps.py
@@ -16,29 +16,16 @@
def draw_static_tile(user, mapimage_key, zoom, northlat, westlng, offset_x_px, offset_y_px):
new_tile = tile.CustomTile(user, zoom, northlat, westlng, offset_x_px, offset_y_px)
- def compose_and_save(key, tile, x, y): # this has to be done in a transaction - otherwise the different threads will overwrite each other's progress on the share mapimage
+ def compose_and_save(key, tile, x, y): # this has to be done in a transaction - otherwise the different threads will overwrite each other's progress on the shared mapimage
mapimage = db.get(key)
- input_tuples = [(new_tile.image_out(), x, y, 1.0, images.TOP_LEFT)] # http://code.google.com/appengine/docs/python/images/functions.html
- logging.warning('in the transaction')
+ input_tuples = [(tile.image_out(), x, y, 1.0, images.TOP_LEFT)]
if mapimage.img:
input_tuples.append((mapimage.img, 0, 0, 1.0, images.TOP_LEFT))
img = images.composite(inputs=input_tuples, width=mapimage.width, height=mapimage.height, color=0, output_encoding=images.PNG) # redraw main image every time to show progress
mapimage.img = db.Blob(img)
mapimage.last_updated = datetime.now()
mapimage.put()
- try:
- db.run_in_transaction(compose_and_save, mapimage_key, new_tile, offset_x_px, offset_y_px)
- except err:
- logging.error(err)
-
-# def update_map_image(user, zoom, width, height, northlat, westlng):
-# input_tuples = []
-# for offset_x_px in range (0, width, 256):
-# for offset_y_px in range (0, height, 256):
-# new_tile = tile.CustomTile(user, int(zoom), northlat, westlng, offset_x_px, offset_y_px)
-# input_tuples.append() # http://code.google.com/appengine/docs/python/images/functions.html
-# img = images.composite(inputs=input_tuples, width=width, height=height, color=0, output_encoding=images.PNG)
-# return img
+ db.run_in_transaction(compose_and_save, mapimage_key, new_tile, offset_x_px, offset_y_px)
def generate_static_map(user, widthxheight, zoom, centerpoint, northwest):
try:
@@ -74,36 +61,30 @@ def generate_static_map(user, widthxheight, zoom, centerpoint, northwest):
mapimage.height = int(height)
mapimage.width = int(width)
mapimage.static_url = "http://maps.google.com/maps/api/staticmap?" + urllib.urlencode(google_data)
- mapimage.input_tuples = [];
mapimage.put()
for offset_x_px in range (0, mapimage.width, 256):
for offset_y_px in range (0, mapimage.height, 256):
- taskqueue.add(url='/draw_static_tile/%s/%d/%d/%d/%d/%d' % (mapimage.key(), mapimage.zoom, mapimage.northlat, mapimage.westlng, offset_x_px, offset_y_px))
+ taskqueue.add(queue_name='tiles', url='/draw_static_tile/%s/%d/%f/%f/%d/%d' % (mapimage.key(), mapimage.zoom, mapimage.northlat, mapimage.westlng, offset_x_px, offset_y_px))
except DeadlineExceededError, err:
logging.error("Ran out of time before creating a map! %s" % err)
if __name__ == '__main__':
fragments = environ['PATH_INFO'].split('/')
- logging.warning(fragments)
fragments.pop(0)
func = fragments.pop(0)
user = users.get_current_user()
try:
if user and func == 'generate_static_map':
- logging.warning('generate_static_map')
- logging.warning(fragments)
assert len(fragments) == 4, "fragments should have 4 elements %s" % str(fragments)
widthxheight, zoom, centerpoint, northwest = fragments
generate_static_map(user, widthxheight, zoom, centerpoint, northwest)
elif func == 'draw_static_tile':
- logging.warning('draw_static_tile')
- logging.warning(fragments)
mapimage_key = fragments.pop(0)
mapimage = db.get(mapimage_key)
if mapimage:
assert len(fragments) == 5, "fragments should have 5 elements %s" % str(fragments)
zoom, northlat, westlng, offset_x_px, offset_y_px = fragments
- draw_static_tile(mapimage, int(zoom), float(northlat), float(westlng), int(offset_x_px), int(offset_y_px))
+ draw_static_tile(mapimage.user, mapimage_key, int(zoom), float(northlat), float(westlng), int(offset_x_px), int(offset_y_px))
else:
logging.warning('No mapimage found for key %s' % mapimage_key)
except AssertionError, err:
View
21 templates/private_forbidden.html
@@ -1,21 +0,0 @@
-<div id="left-lower" class="span-8 last">
- <h2>uh-oh!</h2>
- <p>
- It looks like Foursquare is having some issues with their API...
- </p>
- <p>
- Please come back and try again in a few hours!
- </p>
-</div>
-<div id="regenerate" class="span-8 last">
- <h2>take a snapshot anyway:</h2>
- <p>
- <input type="submit" name="submit" class="button" id="regenerate_button" value="create your map with the current view" />
- <span id="regenerate_status">(re)generating map... <img src='static/spinner-small.gif'/></span>
- <label for="regenerate_button"><br/>check-ins are only stored for a week, so come<br/>back here to update your map with new check-ins</label
- </p>
- <p>
- <input type="submit" name="submit" class="button" id="delete_map_button" value="delete your public map and personalized page" />
- <span id="delete_map_status">deleting map... <img src='static/spinner-small.gif'/></span>
- </p>
-</div>

0 comments on commit bda438c

Please sign in to comment.