Browse files

- coded shy hnsearch andalchemy api services

  • Loading branch information...
1 parent 8351016 commit 65eb792b1b90deb88b3affd32f3141b877f73a71 @zemanel committed Jun 10, 2011
Showing with 166 additions and 25 deletions.
  1. +9 −0 app/app.yaml
  2. +74 −11 app/mood/handlers.py
  3. +17 −9 app/mood/models.py
  4. +49 −4 app/notes.txt
  5. +7 −1 app/urls.py
  6. +10 −0 docs/alchemy_response.js
View
9 app/app.yaml
@@ -24,6 +24,15 @@ error_handlers:
file: templates/error_handlers/timeout.html
handlers:
+
+- url: /jobs/.*
+ script: main.py
+ login: admin
+
+- url: /tasks/.*
+ script: main.py
+ login: admin
+
- url: /favicon\.ico
static_files: static/favicon.ico
upload: static/favicon.ico
View
85 app/mood/handlers.py
@@ -1,26 +1,89 @@
# -*- coding: utf-8 -*-
import logging
+import urllib
+import urllib2
+from django.utils import simplejson as json
+from google.appengine.ext import db
+
from tipfy.app import Response
from tipfy.handler import RequestHandler
from tipfyext.jinja2 import Jinja2Mixin
+from models import NewsItem
+
logger = logging.getLogger(__name__)
-class HomeHandler(RequestHandler, Jinja2Mixin):
+class HomePage(RequestHandler, Jinja2Mixin):
def get(self):
context = {
}
- key = self.app.config['alchemyapi']['API_KEY']
return self.render_response('home.html', **context)
-class PollHNSearchHandler(RequestHandler):
- '''
- Poll HNSearch API for news comments and create tasks for sentiment analysis
- '''
- pass
-
-class PollAlchemySAHandler(RequestHandler):
+class PollHNSearchJob(RequestHandler):
+ def get(self):
+ '''Poll HNSearch API for news comments and create tasks for sentiment analysis
+ '''
+ #url = "http://api.thriftdb.com/api.hnsearch.com/items/_search?filter[fields][type]=comment&filter[fields][create_ts]=[NOW-30MINUTES%20TO%20NOW]&sortby=create_ts%20desc&limit=100&pretty_print=true"
+ url = "http://api.thriftdb.com/api.hnsearch.com/items/_search?" + urllib.urlencode({
+ 'filter[fields][type]' : 'comment',
+ 'filter[fields][create_ts]' : '[NOW-30MINUTES TO NOW]',
+ 'sortby' : 'create_ts desc',
+ 'limit' : 100,
+ 'pretty_print': True
+ })
+ try:
+ result = urllib2.urlopen(url)
+ content = json.loads(result.read())
+ self._parse_results(content['results'])
+ except Exception, e:
+ logger.exception(e)
+ return Response('Application error', status=500)
+ return Response('OK', status=200)
+
+ def _parse_results(self, results):
+ '''stores news items in the datastore
+ '''
+ for item in results:
+ #logger.debug(item)
+ newsitem = NewsItem.get_or_insert(key_name=str(item['item']['id']), text=item['item']['text'])
+ #TODO: add task
+
+ #logger.info(newsitem)
+# newsitem.sentiment_type = 'positive'
+# newsitem.sentiment_score = float('-0.100542')
+
+class PollAlchemyTask(RequestHandler):
'''
- Poll Alchemy API sentimental analisys for a news comment
+ Poll Alchemy API Sentimental Analisys for processing a news comment
'''
- pass
+ def get(self):
+ api_key = self.app.config['alchemyapi']['API_KEY']
+ itemid = self.request.args.get('itemid', None)
+ newsitem = NewsItem.get_by_key_name(itemid)
+ url = "http://access.alchemyapi.com/calls/text/TextGetTextSentiment"
+ try:
+ result = urllib2.urlopen(url, urllib.urlencode({
+ 'apikey' : api_key,
+ 'outputMode' : 'json',
+ 'text' : newsitem.text
+ }))
+ content = json.loads(result.read())
+ logger.debug(content)
+ if content['status']=='OK':
+ newsitem.sentiment_type = content['docSentiment']['type']
+ newsitem.sentiment_score = float(content['docSentiment']['score'])
+ newsitem.put()
+ else:
+ raise Exception(content['statusInfo'])
+ except Exception, e:
+ logger.exception(e)
+ return Response('Application error', status=500)
+ return Response('OK', status=200)
+
+
+
+
+
+
+
+
View
26 app/mood/models.py
@@ -20,13 +20,21 @@
cache_ts - When the item was last cached
"""
+class NewsItem(db.Model):
+ text = db.TextProperty()
+
+ #sentimental analisys
+ is_sentiment_processed = db.BooleanProperty(default=False)
+ sentiment_type = db.StringProperty()
+ sentiment_score = db.FloatProperty()
-class News(db.Model):
- title = db.StringProperty(required=True)
- description = db.TextProperty()
- time = db.DateTimeProperty()
- location = db.TextProperty()
- creator = db.UserProperty()
- edit_link = db.TextProperty()
- gcal_event_link = db.TextProperty()
- gcal_event_xml = db.TextProperty()
+ #id: will use key of db model
+ #points = db.IntegerProperty()
+ #parent_id: will use parent of db model instance
+ #username = db.StringProperty()
+ #type = db.StringProperty()
+ #url = db.LinkProperty()
+ #domain = db.StringProperty()
+ #title = db.StringProperty()
+ #num_comments = db.IntegerProperty()
+ #created_on = db.DateTimeProperty()
View
53 app/notes.txt
@@ -1,5 +1,35 @@
+Items - http://api.thriftdb.com/api.hnsearch.com/items
+id - The item's unique integer id (not searchable)
+parent_id - The parent comment's id
+parent_sigid - The parent comment's signed id
+points - The number of points
+username - The submitter's username
+type - Item type (submission|comment)
+url - A submission url
+domain - A submission url's domain name
+title - A submission title
+num_comments - Number of submission comments
+text - The submission/comment content
+discussion{} - A comment's parent discussion
+id - The discussion's item id (not searchable)
+sigid - The discussion's signed id
+title - The discussion's item title (not searchable)
+create_ts - When the item was created
+cache_ts - When the item was last cached
+AlchemyAPI
+--------
+
+
+
+Appengine queues:
+-----------------
+http://www.youtube.com/watch?v=o3TuRs9ANhs&feature=player_embedded
+
+
+http://docs.python.org/release/2.6/howto/urllib2.html
+
javascript:(function(){
_s=document.createElement('SCRIPT');
_s.type='text/javascript';
@@ -28,17 +58,32 @@ else:
http://www.thriftdb.com/documentation/rest-api/search-api
-facebook&weights[title]=1.1&weights[text]=0.7&weights[domain]=2.0&weights[username]=0.1&weights[type]=0.0&boosts[fields][points]=0.15&boosts[fields][num_comments]=0.15&boosts[functions][pow(2,div(div(ms(create_ts,NOW),3600000),72))]=200.0&pretty_print=true
+http://api.thriftdb.com/api.hnsearch.com/items/_search?q=facebook&weights[title]=1.1&weights[text]=0.7&weights[domain]=2.0&weights[username]=0.1&weights[type]=0.0&boosts[fields][points]=0.15&boosts[fields][num_comments]=0.15&boosts[functions][pow(2,div(div(ms(create_ts,NOW),3600000),72))]=200.0&pretty_print=true
+
+
+* Items from last day
+http://api.thriftdb.com/api.hnsearch.com/items/_search?filter[fields][create_ts]=[NOW-24HOURS%20TO%20NOW]&sortby=create_ts%20desc
+
+* Items from last hour
+http://api.thriftdb.com/api.hnsearch.com/items/_search?filter[fields][create_ts]=[NOW-1HOUR%20TO%20NOW]&sortby=create_ts%20desc&pretty_print=true
+
+
+* Items from 30 minutes
+http://api.thriftdb.com/api.hnsearch.com/items/_search?filter[fields][create_ts]=[NOW-30MINUTES%20TO%20NOW]&sortby=create_ts%20desc&limit=100&pretty_print=true
-http://api.thriftdb.com/test_bucket/cars/_search?filter[queries][]=color%3Ablack+OR+color%3Ared&filter[queries][]=year%3A%5B1970+TO+1980%5D&pretty_print=true
"cache_ts": "2011-06-08T14:50:32Z",
year:[1970+TO+1980]
+http://lucene.apache.org/java/2_9_1/queryparsersyntax.html#Range Searches
+mod_date:[20020101 TO 20030101]
+
+
+"type": "",
+
http://api.thriftdb.com/api.hnsearch.com/items/_search?
q=
+sortby=
filter[queries][]=cache_ts:[*+TO+]
-http://lucene.apache.org/java/2_9_1/queryparsersyntax.html#Range Searches
-mod_date:[20020101 TO 20030101]
View
8 app/urls.py
@@ -3,6 +3,12 @@
from tipfy.routing import Rule
rules = [
- Rule('/', name='home', handler='mood.handlers.HomeHandler'),
+ Rule('/', name='home', handler='mood.handlers.HomePage'),
#Rule('/pretty', name='hello-world-pretty', handler='hello_world.handlers.PrettyHelloWorldHandler'),
+
+ # cron jobs
+ Rule('/jobs/poll_hnsearch', name='job-poll-hnsearch', handler='mood.handlers.PollHNSearchJob'),
+
+ # tasks
+ Rule('/tasks/poll_alchemyapi', name='task-poll-hnsearch', handler='mood.handlers.PollAlchemyTask'),
]
View
10 docs/alchemy_response.js
@@ -0,0 +1,10 @@
+{
+ "status": "OK",
+ "usage": "By accessing AlchemyAPI or using information generated by AlchemyAPI, you are agreeing to be bound by the AlchemyAPI Terms of Use: http://www.alchemyapi.com/company/terms.html",
+ "url": "http://www.cnn.com/2009/CRIME/01/13/missing.pilot/index.html",
+ "language": "english",
+ "docSentiment": {
+ "type": "negative",
+ "score": "-0.100542"
+ }
+}

0 comments on commit 65eb792

Please sign in to comment.