Changed Backend-Update to work with a task. Therefore slow SC API cal…

…ls will be repeated and no tracks are lost

app.yaml

@@ -14,6 +14,10 @@ handlers:
   script: backend_update_track.py
   login: admin
 
+- url: /backend-update-task
+  script: backend_update_task.py
+  login: admin
+
 - url: /backend-update
   script: backend_update.py
   login: admin

backend_update.py

@@ -22,47 +22,36 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 from google.appengine.runtime import DeadlineExceededError
-from google.appengine.api.labs import taskqueue 
-from google.appengine.api import memcache
+from google.appengine.api.labs import taskqueue
 import logging
-import time
 import os
+import datetime
 
-import models
 import backend_utils 
-import settings
 
 def main():
-	"""
-	This method queries the SoundCloud API, fetches the latest tracks having been uploaded
-	since the last backend update, and adds them to the task queue for further processing.
-	It is intended to be called by a cronjob on a short basis, like every 3 minutes.
-	"""
-	try:
-		logging.info("Backend update started")
-
-		logging.info("Fetching latest tracks from SoundCloud")
-		tracks = backend_utils.get_latest_tracks_from_soundcloud()
-		logging.info("Fetched %i tracks from Soundcloud" % len(tracks)) 
-		if len(tracks) > 0:
-			counter = 0    
-			for track in tracks:
-				track['id'] = unicode(track['id'])
-				if memcache.add(track['id'], track, time=settings.TRACK_BACKEND_UPDATE_LIFETIME*60, namespace="backend_update_track"): 
-					taskqueue.add(url='/backend-update/track', params={'track_id': track['id'], 'time_track_added_to_queue': str(int(time.time()))})
-					logging.info("Added track_id %s to memcache and task queue." % track['id'])
-					counter += 1					
-				else:
-					logging.error("Setting Memcache failed for track \"%s\" by \"%s\" (id: %s, created at: %s)." % \
-											(track['title'], track['user']['username'], track['id'], track['created_at']))
-			logging.info("Added %i tracks to the taskqueue" % counter)
-		else:
-			logging.info("Backend update finished without new tracks") 
-
-	except DeadlineExceededError:
-		logging.warning("Backend Update has been canceled due to Deadline Exceeded")
-		for name in os.environ.keys():
-			logging.info("%s = %s" % (name, os.environ[name]))
-
+  """
+  This method adds a task to the task queue intended to querie the SoundCloud API, 
+  fetch the latest tracks having been uploaded since the last backend update,
+  and adds them to the task queue for further processing.
+  It is intended to be called by a cronjob on a short basis, like every 3 minutes.
+  """
+  try:
+    logging.info("Backend update started")
+
+    time_from = backend_utils.calculate_time_from()
+    time_to = datetime.datetime.now().isoformat()
+
+    taskqueue.add(url='/backend-update-task',
+                  params={'time_from': time_from, 'time_to': time_to})
+
+    logging.info("Added backend update task to task queue. time_from: %s time_to: %s" % \
+                (time_from, time_to))               
+
+  except DeadlineExceededError:
+    logging.warning("Backend Update has been canceled due to Deadline Exceeded")
+    for name in os.environ.keys():
+      logging.info("%s = %s" % (name, os.environ[name]))
+
 if __name__ == '__main__':
   main()

backend_update_task.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2009 Johan Uhle
+# 
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from google.appengine.runtime import DeadlineExceededError
+from google.appengine.api.labs import taskqueue 
+from google.appengine.api import memcache
+from google.appengine.ext import webapp 
+
+import wsgiref.handlers     
+import logging
+import time
+import os
+
+import models
+import backend_utils 
+import settings
+
+class BackendUpdate(webapp.RequestHandler):
+
+  def post(self):
+	"""
+	This method queries the SoundCloud API, fetches the tracks having been uploaded
+	in the specified time and adds them to the task queue for further processing.
+	"""
+	try:
+		logging.info("Backend update task started")
+
+		logging.info("Fetching the tracks from SoundCloud")
+		time_from = self.request.get('time_from')
+		time_to = self.request.get('time_to')		
+		tracks = backend_utils.get_latest_tracks_from_soundcloud()
+		logging.info("Fetched %i tracks from Soundcloud" % len(tracks)) 
+		if len(tracks) > 0:
+			counter = 0    
+			for track in tracks:
+				track['id'] = unicode(track['id'])
+				memcache_add = memcache.add(track['id'], 
+				                            track,
+				                            time=settings.TRACK_BACKEND_UPDATE_LIFETIME*60, 
+				                            namespace="backend_update_track")
+				if memcache_add: 
+					taskqueue.add(url='/backend-update/track', 
+					              params={'track_id': track['id'], 
+					                      'time_track_added_to_queue': str(int(time.time()))})
+					logging.info("Added track_id %s to memcache and task queue." % track['id'])
+					counter += 1					
+				else:
+					logging.error("Setting Memcache failed for track \"%s\" by \"%s\" (id: %s, created at: %s)." % \
+											(track['title'], track['user']['username'], track['id'], track['created_at']))
+			logging.info("Added %i tracks to the taskqueue" % counter)
+		else:
+			logging.info("Backend update finished without new tracks") 
+
+	except DeadlineExceededError:
+		logging.warning("Backend Update has been canceled due to Deadline Exceeded")
+		for name in os.environ.keys():
+			logging.info("%s = %s" % (name, os.environ[name]))
+
+def main():
+  wsgiref.handlers.CGIHandler().run(webapp.WSGIApplication([
+    ('/backend-update-task', BackendUpdate),
+  ]))            
+
+if __name__ == '__main__':
+  main()

backend_utils.py

@@ -67,14 +67,24 @@ def open_remote_api(query, api):
   logging.info("Result for %s Query: %s" % (api_name, result.content))
   return json.loads(result.content)
 
-def get_latest_tracks_from_soundcloud():
+def calculate_time_from():
+  time_from = datetime.datetime.now()
+  time_from -= datetime.timedelta(hours=settings.SOUNDCLOUD_TIMEZONE_ADJUSTMENT)
+  time_from -= datetime.timedelta(minutes=settings.API_QUERY_INTERVAL)
+  time_from = time_from.isoformat()
+  return time_from
+
+def get_latest_tracks_from_soundcloud(time_from=None, time_to=None):
   """
+  time_form and time_to must be datetime.isoformat()
   Get Latest Tracks from Soundcloud
   """
-  created_at_time = datetime.datetime.now()
-  created_at_time -= datetime.timedelta(hours=settings.SOUNDCLOUD_TIMEZONE_ADJUSTMENT)
-  created_at_time -= datetime.timedelta(minutes=settings.API_QUERY_INTERVAL)
-  query = "/tracks.json?created_at[from]=%s&duration[to]=%s" % (created_at_time.isoformat(), settings.DURATION_LIMIT)
+  if not time_from: time_from = calculate_time_from()    
+  if not time_to: time_to = datetime.datetime.now().isoformat()
+  query = "/tracks.json?"
+  query += "created_at[from]=" + time_from
+  query += "&created_at[to]=" + time_to
+  query += "&duration[to]=" + settings.DURATION_LIMIT
   tracks = open_remote_api(query, "soundcloud") 
   return tracks