Using a new send_datastore which prevents endless loop and removes ig…

…nores objects taht could not be saved.

measure.py

@@ -61,15 +61,22 @@ def post(self, key, path, format):
     self.response.set_status(error and 304 or 201)
 
 def send_to_datastore(models):
-  lm = len(models)    
-  if lm:
-    try:
-      db.put(models);
-      logging.info('::STATS:: db.put(%s)' % lm)
-    except:
-      logging.warning('Could not save whole set')
-      send_to_datastore(models[:lm / 2])
-      send_to_datastore(models[lm / 2:])
+  payload = [models]
+
+  while payload:
+    models = payload.pop(0)
+    l = len(models)
+    if l:
+      try:
+        db.put(models)
+        logging.info('::STATS:: db.put(%s)', l)
+      except:
+        logging.info('::STATS:: !db.put(%s)', l)
+        if l == 1:
+          logging.warning('Could not save: %s', models)
+          continue
+        payload.append(models[:l / 2])
+        payload.append(models[l / 2:])
 
 def create_datum(campaign, ns, obj = {}):
   ns = ns.strip('/').replace('/', '.')    
@@ -80,7 +87,8 @@ def create_datum(campaign, ns, obj = {}):
 
   key = '%s.%s' % (campaign, ns)
   if (not _Stats.has_key(key)):
-    datum.stats = _Stats[key] = Statistics.get_by_key_name_or_insert(key, campaign = campaign, namespace = ns, type = kind)
+    datum.stats = _Stats[key] = Statistics.get_by_key_name_or_insert(key, campaign = campaign, namespace = ns)
+    datum.stats.type = kind
   else:
     datum.stats = _Stats[key]
 

myapp/models.py

@@ -39,6 +39,10 @@ class Campaign(db.Model):
   homepage = db.StringProperty()
   organizer = db.ReferenceProperty(collection_name = 'campaigns')
   created_on = db.DateTimeProperty(auto_now_add = 1)
+
+  @classmethod
+  def kind(cls):
+    return 'Campaign'
 
 class Storage(SerializableExpando):
   json_does_not_include = ['campaign', 'namespace', 'type', 'prev', 'stats']
@@ -48,6 +52,10 @@ class Storage(SerializableExpando):
   type = db.StringProperty(required = True)
   created_on = db.DateTimeProperty(auto_now_add = 1)
   stats = db.ReferenceProperty(collection_name = 'statistics')
+
+  @classmethod
+  def kind(cls):
+    return 'Storage'
 
 class Statistics (SerializableExpando):
   json_does_not_include = ['campaign', 'namespace', 'histograms']
@@ -56,7 +64,11 @@ class Statistics (SerializableExpando):
   namespace = db.StringProperty(required = True)
   count = db.IntegerProperty(default = 0)
   histograms = db.StringListProperty()
-  type = db.StringProperty(required = True)
+  type = db.StringProperty()
+
+  @classmethod
+  def kind(cls):
+    return 'Statistics'
 
   @staticmethod
   def get_by_campaign_and_namespace(campaign, namespace):
@@ -83,11 +95,19 @@ class Histogram(SerializableExpando):
 
   statistic = db.ReferenceProperty(Statistics, collection_name = 'statistic')
   name = db.StringProperty(required = True)
+
+  @classmethod
+  def kind(cls):
+    return 'Histogram'
 
 class TaskModel(db.Expando):
   object = db.ReferenceProperty(required = True)
   task = db.StringProperty(required = True)
 
+  @classmethod
+  def kind(cls):
+    return 'TaskModel'
+
   def execute(self):
      obj = self.properties()['object'].get_value_for_datastore(self)
      import tasks