Permalink
Browse files

Merge branch 'master' of git://github.com/pkra/mathblogging

  • Loading branch information...
2 parents 5b46396 + 1a0e6de commit 390fcc982f60f372dc708c36f831a89b02259c0e @fbreuer fbreuer committed Dec 19, 2011
Showing with 220 additions and 21 deletions.
  1. +3 −0 app.yaml
  2. +36 −0 bulkloader.yaml
  3. +1 −1 categoryview.py
  4. +1 −1 cron.yaml
  5. +1 −1 dataexport.py
  6. +1 −1 dateview.py
  7. +1 −1 dateviewhisartvis.py
  8. +1 −1 dateviewresearch.py
  9. +1 −1 dateviewteacher.py
  10. +1 −1 feedhandler.py
  11. +61 −1 main.py
  12. +3 −3 planetmo.py
  13. +103 −3 planettag.py
  14. +5 −5 statsview.py
  15. +1 −1 weeklypicks.py
View
@@ -43,6 +43,9 @@ handlers:
- url: /globaltaglistworker
script: main.py
login: admin
+- url: /prefeed
+ script: main.py
+ login: admin
- url: .*
script: main.py
View
@@ -98,3 +98,39 @@ transformers:
- property: taglist
external_name: taglist
import_transform: transform.none_if_empty(list)
+
+- kind: PreFeed
+ connector: csv
+ connector_options:
+ # TODO: Add connector options here--these are specific to each connector.
+ property_map:
+ - property: __key__
+ external_name: key
+ export_transform: transform.key_id_or_name_as_string
+
+ - property: comments_url
+ external_name: comments_url
+
+ - property: person
+ external_name: person
+ # Type: String Stats: 128 properties of this type in this kind.
+
+ - property: category
+ external_name: category
+ # Type: String Stats: 128 properties of this type in this kind.
+
+ - property: language
+ external_name: language
+ # Type: String Stats: 128 properties of this type in this kind.
+
+ - property: posts_url
+ external_name: posts_url
+ # Type: String Stats: 128 properties of this type in this kind.
+
+ - property: homepage
+ external_name: homepage
+
+ - property: title
+ external_name: title
+ # Type: String Stats: 128 properties of this type in this kind.
+
View
@@ -21,7 +21,7 @@ def generateSection(self,pair):
for entry in Post.gql("WHERE homepage = :1 ORDER BY timestamp_created DESC LIMIT 7", feed.homepage):
output.append("""
<li><a href="%(link)s" title="%(title)s">%(title)s</a></li>
- """ % {'link': entry.link, 'title': html_escape(entry.title)})
+ """ % {'link': html_escape(entry.link), 'title': html_escape(entry.title)})
output.append("""
</ul> </div> </li>
""")
View
@@ -1,7 +1,7 @@
cron:
- description: check for feed updates
url: /allworker
- schedule: every 6 hours
+ schedule: every 8 hours
- description: clean up datastore
url: /cleanupdatastore
schedule: every monday 02:00
View
@@ -76,7 +76,7 @@ class WeeklyPicksJSONPHandler(JSONPHandler):
def generatePage(self):
picks = [ {
"url": "http://rjlipton.wordpress.com/2011/12/03/the-meaning-of-omega/",
- "caption": "If you haven't followed the debate on TCS breakthrough in matrix multiplication, you can read up on it at Gödel's Lost Letter and P=NP (and you might also check out a short comment at Yet Another Math Programmer)." } ]
+ "caption": "If you haven't followed the debate on TCS breakthrough in matrix multiplication, you can read up on it at Godel's Lost Letter and P=NP (and you might also check out a short comment at Yet Another Math Programmer)." } ]
output = picks
return simplejson.dumps(output)
View
@@ -48,5 +48,5 @@ def generateContent(self):
<a href="%(link)s" title="%(title)s">%(title)s</a>
</div>
</td>
- </tr>""" % {'time': post.printShortTime_created(), 'homepage': post.homepage, 'service': html_escape(post.service), 'title': post.title, 'link': post.link }
+ </tr>""" % {'time': post.printShortTime_created(), 'homepage': post.homepage, 'service': html_escape(post.service), 'title': html_escape(post.title), 'link': html_escape(post.link) }
return s + "</tbody></table>"
View
@@ -48,5 +48,5 @@ def generateContent(self):
<a href="%(link)s" title="%(title)s">%(title)s</a>
</div>
</td>
- </tr>""" % {'time': post.printShortTime_created(), 'homepage': post.homepage, 'service': html_escape(post.service), 'title': post.title, 'link': post.link }
+ </tr>""" % {'time': post.printShortTime_created(), 'homepage': post.homepage, 'service': html_escape(post.service), 'title': html_escape(post.title), 'link': html_escape(post.link) }
return s + "</tbody></table>"
View
@@ -48,5 +48,5 @@ def generateContent(self):
<a href="%(link)s" title="%(title)s">%(title)s</a>
</div>
</td>
- </tr>""" % {'time': post.printShortTime_created(), 'homepage': post.homepage, 'service': html_escape(post.service), 'title': post.title, 'link': post.link }
+ </tr>""" % {'time': post.printShortTime_created(), 'homepage': post.homepage, 'service': html_escape(post.service), 'title': html_escape(post.title), 'link': html_escape(post.link) }
return s + "</tbody></table>"
View
@@ -48,5 +48,5 @@ def generateContent(self):
<a href="%(link)s" title="%(title)s">%(title)s</a>
</div>
</td>
- </tr>""" % {'time': post.printShortTime_created(), 'homepage': post.homepage, 'service': html_escape(post.service), 'title': post.title, 'link': post.link }
+ </tr>""" % {'time': post.printShortTime_created(), 'homepage': post.homepage, 'service': html_escape(post.service), 'title': html_escape(post.title), 'link': html_escape(post.link) }
return s + "</tbody></table>"
View
@@ -30,7 +30,7 @@ def generatePage(self):
<updated>%(tupdated)s</updated>
<id>%(link)s</id>
<content type="html"> %(content_trunc)s [Length: approx %(length)d words.] </content>
- </entry>""" % {'service': html_escape(entry.service),'title': entry.title, 'homepage': entry.homepage , 'link':entry.link, 'tcreated': entry.printTime_created_rfc3339(), 'tupdated':entry.printTime_updated_rfc3339(), 'length': entry.length/5, 'content_trunc': truncate_html_words(entry.content, 40)} )
+ </entry>""" % {'service': html_escape(entry.service),'title': html_escape(entry.title), 'homepage': html_escape(entry.homepage) , 'link':html_escape(entry.link), 'tcreated': entry.printTime_created_rfc3339(), 'tupdated':entry.printTime_updated_rfc3339(), 'length': entry.length/5, 'content_trunc': truncate_html_words(entry.content, 40)} )
output.append("</feed>")
return "".join(output)
View
62 main.py
@@ -377,6 +377,48 @@ class Comment(Entry):
#######################################
+### PreFeed: for extending the database by generating most of the information for the feedobjects
+#######################################
+
+
+class PreFeed(db.Model):
+ posts_url = db.LinkProperty()
+ comments_url = db.StringProperty()
+ category = db.StringProperty()
+ person = db.StringProperty()
+ language = db.StringProperty()
+ homepage = db.StringProperty()
+ title = db.StringProperty()
+
+ def generate_feedobject(self):
+ x = Feed()
+ try:
+ x.posts_url = self.posts_url
+ x.homepage = self.homepage
+ x.title = self.title
+ x.listtitle = self.title.lower()
+ x.person = self.person
+ x.category = self.category
+ x.taglist = []
+ x.language = self.language
+ x.priority = 1
+ x.favicon = ""
+ x.comments_url = self.comments_url
+ x.comments_day = 0
+ x.comments_week = 0
+ x.posts_week = 0
+ x.posts_month = 0
+ x.checksum_posts = '' # checksum of original rss-file
+ x.checksum_comments = '' # checksum of original rss-file
+ x.last_successful_posts_fetch_date = datetime.datetime(1970,1,1)
+ x.last_successful_comments_fetch_date = datetime.datetime(1970,1,1)
+ x.put()
+ except Exception, e: # TODO more exception catching: 'NoneType' error when feed is malformed not enough for bug tracking.
+ logging.warning("There was an error processing the Feedobject :" + str(e))
+
+
+
+#######################################
### Storing for caching
#######################################
@@ -526,7 +568,7 @@ def get(self):
if feed.category != 'community': ### TODO GET YOUR ACT TOGETHER AND RE-ADD THEM
taskqueue.add(url="/fetch", params={'url': feed.posts_url})
- pages_to_cache_list = ["/", "/feeds","/bytype","/weekly-picks","/bydate","/byresearchdate","/byartvishisdate","/byteacherdate","/bystats","/planetmo", "/planetmo-feed","/feed_pure","/feed_applied","/feed_history","/feed_art","/feed_fun","/feed_general","/feed_journals","/feed_teachers","/feed_visual","/feed_journalism","/feed_institutions","/feed_communities","/feed_commercial","/feed_newssite","/feed_carnival","/feed_all","/feed_researchers"]
+ pages_to_cache_list = ["/", "/feeds","/bytype","/weekly-picks","/bydate","/byresearchdate","/byartvishisdate","/byteacherdate","/bystats","/planetmo", "/planetmo-feed","/feed_pure","/feed_applied","/feed_history","/feed_art","/feed_fun","/feed_general","/feed_journals","/feed_teachers","/feed_visual","/feed_journalism","/feed_institutions","/feed_communities","/feed_commercial","/feed_newssite","/feed_carnival","/feed_all","/feed_researchers","/bystats-researchers","bystats-educators","bystats-artvis"]
for page in pages_to_cache_list:
taskqueue.add(url=page, method="GET")
self.response.set_status(200)
@@ -576,6 +618,19 @@ def get(self):
x.put()
self.response.set_status(200)
+#################################
+### GENERATING FEEDOBJECTS FROM PREFEEDs
+#################################
+
+
+class PreFeedWorker(webapp.RequestHandler): ### runs at \prefeed with admin privileges -- make sure to delete prefeeds afterwards!
+ def get(self):
+ logging.info("PreFeedWorker")
+ for prefeed in PreFeed.all():
+ prefeed.generate_feedobject()
+ self.response.set_status(200)
+
+
#################################
### CLEANING UP THE DATASTORE // DELETING OLD ENTRIES
@@ -663,6 +718,9 @@ def main():
('/byartvishisdate', DateViewHisArtVis),
('/byteacherdate', DateViewTeacher),
('/bystats', StatsView),
+ ('/bystats-researchers', StatsViewResearchers),
+ ('/bystats-educators', StatsViewEducators),
+ ('/bystats-artvis', StatsViewArtVis),
('/planetmo', PlanetMO),
('/planetmo-feed', PlanetMOfeed),
('/database.csv', CsvView),
@@ -675,6 +733,7 @@ def main():
('/feedtaglistfetch', FeedTagListFetchWorker),
('/feedtaglistworker', FeedTagListWorker),
('/globaltaglistworker', GlobalTagListWorker),
+ ('/prefeed', PreFeedWorker),
('/reboot', RebootCommand),
('/feed_pure', FeedHandlerPure),
('/feed_applied', FeedHandlerApplied),
@@ -701,6 +760,7 @@ def main():
('/feed_academics', FeedHandlerResearchers), # left for transition
('/feed_institution', FeedHandlerInstitutions),
('/planettag', PlanetTag),
+ ('/planettag2', PlanetTagBlogs),
# Testing
('/gridview', GridView)
],
View
@@ -16,14 +16,14 @@ def generateContent(self):
<div class="planetsubtitle">Posted on <a href="%(homepage)s" title="%(service)s">%(service)s</a> at %(tcreated)s.
</div>
<div class="planettags">
- <i>Tags:</i> """ % {"link":entry.link, "title":entry.title, "homepage":entry.homepage, "service":html_escape(entry.service), "tcreated":entry.printShortTime_created()} )
+ <i>Tags:</i> """ % {"link":html_escape(entry.link), "title":html_escape(entry.title), "homepage":html_escape(entry.homepage), "service":html_escape(entry.service), "tcreated":entry.printShortTime_created()} )
for tag in entry.tags:
- output.append(tag)
+ output.append(html_escape(tag))
output.append(", ")
output.append( """ </div>
</div>
<div class="planetbody">
""" )
- output.append( entry.content )
+ output.append( truncate_html_words(entry.content,100) )
output.append( """ </div> </div> """ )
return "".join(output)
Oops, something went wrong.

0 comments on commit 390fcc9

Please sign in to comment.