Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

TopQuestionsRetriever

  • Loading branch information...
commit 63c05b508318f05f21a20db9261ebeae4864a883 1 parent b0967f1
@systempuntoout authored
View
2  app/config/constant.py
@@ -1,5 +1,5 @@
UNICORN_MESSAGE_ERROR = "..for the love it bears to fair maidens forgets its ferocity and wildness.. "
-GENERIC_ERROR = "Ooooops!"
+GENERIC_ERROR = "Ooooops, It works on my machine. Please try again later."
NOT_FOUND_ERROR = "Not found!"
SERVER_ERROR = "Server problem"
UNSUPPORTED_SERVICE_ERROR = "Unsupported service"
View
1  app/config/urls.py
@@ -15,5 +15,6 @@
'/quicklook','app.controllers.ajax.Quicklook',
'/admin','app.controllers.admin.Admin',
'/admin/authtokenrenewal','app.controllers.admin.AuthTokenRenewal',
+ '/admin/topquestionsretriever','app.controllers.admin.TopQuestionsRetriever',
'/_ah/warmup','app.controllers.admin.Warmup',
)
View
46 app/controllers/admin.py
@@ -2,12 +2,15 @@
from google.appengine.api import urlfetch
from app.lib.key import api_key
from google.appengine.api import memcache
-from app.core.stackprinterdownloader import StackAuthDownloader
+from app.core.stackprinterdownloader import StackAuthDownloader,StackExchangeDownloader
from app.utility.utils import TokenManager
from google.appengine.ext import deferred
import app.utility.worker as worker
+from google.appengine.api.taskqueue import taskqueue
import app.db.question as dbquestion
+VOTES_ENTRY_LEVEL = 15
+
render = web.render
class Admin:
@@ -32,7 +35,7 @@ def GET(self):
elif action =='memcacheflush':
result['result'] = memcache.flush_all()
elif action =='normalize':
- deferred.defer(worker.normalize_printed_question)
+ deferred.defer(worker.deferred_normalize_printed_question)
result['result'] = True
elif action =='delete':
service = web.input(service = None)['service']
@@ -51,6 +54,45 @@ def GET(self):
result = {}
result['result'] = StackAuthDownloader.renew_auth_token()
return render.admin(result)
+
+class TopQuestionsRetriever:
+ """
+ Retrieve the hottest questions (week by default)
+ """
+ def GET(self):
+
+ result = {}
+ service_parameter = web.input(service = None)['service']
+ question_id_parameter = web.input(question_id = None)['question_id']
+ sort_parameter = web.input(sort = 'week')['sort']
+ pagesize_parameter = web.input(pagesize = 30)['pagesize']
+
+ if service_parameter:
+ se_downloader = StackExchangeDownloader(service_parameter)
+ if question_id_parameter:
+ se_downloader.get_post(question_id_parameter)
+ else:
+ questions = se_downloader.get_questions_by_hotness(pagesize = pagesize_parameter,
+ sort = sort_parameter)
+ for question in questions:
+ question_id = int(question['question_id'])
+ score = int(question['score'])
+ if score > VOTES_ENTRY_LEVEL:
+ taskqueue.add(url='/admin/topquestionsretriever?service=%s&question_id=%s&sort=%s' % \
+ (service_parameter, question_id, sort_parameter) ,
+ method = 'GET',
+ queue_name = 'retriever')
+ else:
+ supported_services = StackAuthDownloader.get_supported_services()
+ for service in supported_services.keys:
+ if not service.startswith('meta.'):
+ taskqueue.add(url='/admin/topquestionsretriever?service=%s&sort=%s' % \
+ (service, sort_parameter),
+ method = 'GET',
+ queue_name = 'retriever')
+ result['result'] = True
+ return render.admin(result)
+
class Warmup:
"""
View
23 app/core/stackprinterdownloader.py
@@ -2,7 +2,6 @@
from google.appengine.api import urlfetch
from google.appengine.ext import deferred
from app.config.constant import UNSUPPORTED_SERVICE_ERROR
-from app.config.constant import CODE_API_ERROR_THROTTLING
import app.lib.sepy as sepy
import app.lib.deliciousapi as deliciousapi
import app.utility.utils as utils
@@ -87,6 +86,12 @@ def get_answer_quicklook(self, answer_id):
return answer[0]
else:
return None
+
+ def get_questions_by_hotness(self, page = 1 , pagesize = 30, sort = 'week'):
+ questions_by_hotness = []
+ results = self.retriever.get_questions(self.api_endpoint, page, pagesize, sort)
+ questions = results["questions"]
+ return questions
def get_questions_by_tags(self, tagged, page):
questions_by_tags = []
@@ -235,12 +240,16 @@ def get_post(self, question_id):
raise
if post.is_printable():
- deferred.defer(worker.deferred_store_print_statistics,
- post.question['question_id'],
- self.service,
- post.question['title'],
- post.question['tags'],
- post.deleted)
+ try:
+ deferred.defer(worker.deferred_store_print_statistics,
+ post.question['question_id'],
+ self.service,
+ post.question['title'],
+ post.question['tags'],
+ post.deleted)
+ except:
+ logging.info("%s - defer error trying to store print statistics : %s" % (self.service, question_id))
+
return post
else:
return None
View
4 app/static/docs/changelog.txt
@@ -1,3 +1,7 @@
+Version 0.9.9.3
+2011-01-13
+TopQuestions retriever
+
Version 0.9.9.2
2011-01-08
Top Printed promoted
View
5 app/static/docs/todo.txt
@@ -1,7 +1,6 @@
Last update: 2011-01-08
-Retrieve the hottest weekly questions
-Fix printing with TeX
-Memcache to avoid Api calls
+Some better caching when bots hammer
+Store votes/answers on top printed
View
22 app/tests/test_db.py
@@ -12,25 +12,31 @@ class QuestionTestCase(unittest.TestCase):
def test_store_questions(self):
- dbquestion.store_printed_question(question_id = 1, service = 'fooservice', title = 'footitle', tags = ['tag1', 'tag2'])
- questions_list = dbquestion.get_top_printed_questions()
+ dbquestion.store_printed_question(question_id = 1, service = 'fooservice', title = 'footitle', tags = ['tag1', 'tag2'],deleted = False)
+ questions_list = dbquestion.get_top_printed_questions(1)
self.assertEquals(len(questions_list), 1)
self.assertEquals(questions_list[0].question_id, 1)
- dbquestion.store_printed_question(question_id = 1, service = 'fooservice', title = 'footitle', tags = ['tag1', 'tag2'])
+ dbquestion.store_printed_question(question_id = 1, service = 'fooservice', title = 'footitle', tags = ['tag1', 'tag2'],deleted = False)
self.assertEquals(len(questions_list), 1)
self.assertEquals(questions_list[0].question_id, 1)
- dbquestion.store_printed_question(question_id = 2, service = 'fooservice', title = 'footitle2', tags = ['tag1', 'tag2'])
- questions_list = dbquestion.get_top_printed_questions()
+ dbquestion.store_printed_question(question_id = 2, service = 'fooservice', title = 'footitle2', tags = ['tag1', 'tag2'],deleted = False)
+ questions_list = dbquestion.get_top_printed_questions(1)
self.assertEquals(len(questions_list), 2)
self.assertEquals(questions_list[0].question_id, 1)
self.assertEquals(questions_list[1].question_id, 2)
- dbquestion.store_printed_question(question_id = 2, service = 'fooservice', title = 'footitle2', tags = ['tag1', 'tag2'])
- dbquestion.store_printed_question(question_id = 2, service = 'fooservice', title = 'footitle2', tags = ['tag1', 'tag2'])
- questions_list = dbquestion.get_top_printed_questions()
+ dbquestion.store_printed_question(question_id = 2, service = 'fooservice', title = 'footitle2', tags = ['tag1', 'tag2'],deleted = False)
+ dbquestion.store_printed_question(question_id = 2, service = 'fooservice', title = 'footitle2', tags = ['tag1', 'tag2'],deleted = False)
+ questions_list = dbquestion.get_top_printed_questions(1)
self.assertEquals(len(questions_list), 2)
self.assertEquals(questions_list[0].question_id, 2)
self.assertEquals(questions_list[1].question_id, 1)
self.assertEquals(questions_list[1].tags, ['tag1', 'tag2'])
+ dbquestion.store_printed_question(question_id = 3, service = 'fooservice', title = 'footitle2', tags = ['tag1', 'tag2'],deleted = True)
+ questions_list = dbquestion.get_top_printed_questions(1)
+ self.assertEquals(len(questions_list), 3)
+ questions_list = dbquestion.get_deleted_questions()
+ self.assertEquals(len(questions_list), 1)
+
if __name__ == '__main__':
unittest.main()
View
5 app/utility/worker.py
@@ -5,6 +5,7 @@
CHUNK_SIZE = 150
+
def deferred_store_question_to_cache(question_id, service, question_data):
try:
CachedQuestionModel(key_name = '%s_%s' % (question_id, service ), data = question_data).put()
@@ -38,6 +39,6 @@ def deferred_store_print_statistics(question_id, service, title, tags, deleted):
except Exception, exception:
logging.error(exception)
-def normalize_printed_question():
+def deferred_normalize_printed_question():
for printed_question in dbquestion.PrintedQuestionModel.all():
- pass
+ pass
View
1  app/views/__init__.py
@@ -93,6 +93,7 @@ def __template__ (result):
yield '', join_(' <input type="submit" value="Delete printed and cached Q/A"/>\n')
yield '', join_(' </form>\n')
yield '', join_(' </li>\n')
+ yield '', join_(' <li><a href="/admin/topquestionsretriever">Top Question Retriever</a></li>\n')
yield '', join_(' </ul> \n')
yield '', join_(' \n')
yield '', join_(' <p><b>Result:</b></p>\n')
View
1  app/views/admin.html
@@ -23,6 +23,7 @@
<input type="submit" value="Delete printed and cached Q/A"/>
</form>
</li>
+ <li><a href="/admin/topquestionsretriever">Top Question Retriever</a></li>
</ul>
<p><b>Result:</b></p>
View
7 cron.yaml
@@ -6,4 +6,9 @@ cron:
- description: Auth Token Renewal
url: /admin/authtokenrenewal
schedule: every 6 hours
- timezone: Europe/Rome
+ timezone: Europe/Rome
+- description: Sunday morning top questions retriever
+ url: /admin/topquestionsretriever
+ schedule: every sunday 12:00
+ timezone: Europe/Rome
+
View
4 queue.yaml
@@ -0,0 +1,4 @@
+queue:
+- name: retriever
+ rate: 1/m
+ bucket_size: 1
Please sign in to comment.
Something went wrong with that request. Please try again.