initial stack overflow harvesting

appengine/services/so_tasks.py

@@ -17,12 +17,13 @@
 
 import webapp2
 import logging
-import re
-from datetime import datetime
-from datetime import date
+from datetime import datetime, date, timedelta
 
+import json
 import urllib2
+from urllib2 import Request, urlopen, URLError
 
+from google.appengine.ext import ndb
 from google.appengine.api import taskqueue
 from google.appengine.api import mail
 from google.appengine.api import app_identity
@@ -33,10 +34,15 @@
 from models import ActivityRecord
 from models import ActivityMetaData
 from models import Account
+from models import ProductGroup
 from models import activity_record as ar
 
 from .utils import get_server_api_key
 
+SO_ROOT = "https://api.stackexchange.com/2.2"
+
+# https://api.stackexchange.com/2.2/users/2748916/tags/[google-drive-sdk]/top-answers?((&site=stackoverflow&order=desc&sort=activity&filter=default
+
 
 class CronHarvestSO(webapp2.RequestHandler):
 
@@ -75,5 +81,82 @@ def post(self):
 
         gplus_id = self.request.get('gplus_id')
         logging.info(gplus_id)
-        key = self.request.get('key')
-        logging.info(key)
+        safe_key = self.request.get('key')
+        logging.info(safe_key)
+
+        gde = ndb.Key(urlsafe=safe_key).get()
+        logging.info(gde.gplus_id)
+
+        today = date.today().strftime('%s')
+        yesterday = (date.today() - timedelta(1000)).strftime('%s')
+        logging.info(today)
+        logging.info(yesterday)
+
+        q = "/top-answers?((&site=stackoverflow&order=desc&sort=activity&filter=default"
+        q = q + "&fromdate={}&todate={}".format(yesterday, today)
+
+        # product_group is a repeated property : some GDE's have multiple hats
+        for pg in gde.product_group:
+            product_group = ndb.Key('ProductGroup', pg).get()
+            logging.info(product_group)
+            so_tags = product_group.so_tags
+
+            logging.info(so_tags)
+            # so_tags = str([x.encode('UTF8') for x in so_tags])
+            # logging.info(so_tags)
+
+            # this routine is predicated on the daily rate of answer for a tag
+            # by a GDE is less than 30
+
+            daily_answers = 0
+            daily_score = 0
+            daily_accepted = 0
+
+            for so_tag in so_tags:
+
+                #q = "/top-answers?((&site=stackoverflow&order=desc&sort=activity&filter=default"
+                # fromdate=1383264000&todate=1427760000
+                url = SO_ROOT + \
+                    "/users/{}/tags/[{}]".format(gde.so_id, so_tag) + q
+                logging.info(url)
+
+                req = Request(url)
+                try:
+                    response = urlopen(req)
+                except URLError as e:
+                    if hasattr(e, 'reason'):
+                        logging.error('We failed to reach a server.')
+                        logging.error('Reason: ', e.reason)
+                    elif hasattr(e, 'code'):
+                        logging.error(
+                            'The server couldn\'t fulfill the request.')
+                        logging.error('Error code: ', e.code)
+                else:
+                    r = response.read()
+                    logging.info(r)
+                    answers = json.loads(r)["items"]
+                    count = len(answers)
+                    score = 0
+                    accepted = 0
+                    for answer in answers:
+                        score += answer["score"]
+                        if answer["is_accepted"]:
+                            accepted += 1
+
+                    logging.info(count)
+                    logging.info(score)
+                    logging.info(accepted)
+
+            daily_answers += count
+            daily_score += score
+            daily_accepted += accepted
+
+            logging.info(daily_answers)
+            logging.info(daily_score)
+            logging.info(daily_accepted)
+
+            # create(get) activity record
+            # polymer #forumpost #stackOverflow
+
+            activities = ActivityRecord.query(ActivityRecord.gplus_id == gplus_id,
+                                              ActivityRecord.metadata.type == '#stackOverflow')

appengine/services/web_endpoints.py

@@ -263,7 +263,7 @@ def ag_list(self, query):
 class ProductGroupService(remote.Service):
 
     @ProductGroup.method(path='/productGroup/{id}', http_method='POST', name='insert',
-                         request_fields=('id', 'tag', 'description', 'url', 'image', 'api_key'))
+                         request_fields=('id', 'tag', 'description', 'url', 'image', 'api_key', 'so_tags'))
     def pg_insert(self, product_group):
         if not check_auth(None, product_group.api_key):
             raise endpoints.UnauthorizedException(