Permalink
Browse files

cleaning up code to reduce unnecessary warning messages; clean up rem…

…oving all cheetah used
  • Loading branch information...
pkra
pkra committed Oct 17, 2011
1 parent d803cfb commit 3534948bf113fbeb4a6f53191d05b3af8706e435
Showing with 56 additions and 222 deletions.
  1. +1 −1 bulkloader.yaml
  2. +1 −1 cron.yaml
  3. +0 −10 cse-config.tmpl
  4. +15 −0 dataexport.py
  5. +0 −44 feeds.tmpl
  6. +37 −32 main.py
  7. +0 −18 opml.tmpl
  8. +0 −116 start.tmpl
  9. +2 −0 temp_global.py
View
@@ -85,4 +85,4 @@ transformers:
- property: taglist
external_name: taglist
- import_transform: eval
+ import_transform: transform.none_if_empty(list)
View
@@ -1,7 +1,7 @@
cron:
- description: check for feed updates
url: /allworker
- schedule: every 2 hours
+ schedule: every 3 hours
- description: clean up datastore
url: /cleanupdatastore
schedule: every monday 02:00
View
@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<Annotations>
- #for $feed in $qf.get()
- #if feed.homepage != ""
- <Annotation about="<%=feed.cse_homepage()%>*">
- <Label name="_cse_et7bffbfveg"/>
- </Annotation>
- #end if
- #end for
-</Annotations>
View
@@ -26,3 +26,18 @@ def generatePage(self):
xmlUrl="%(url)s" htmlUrl="%(homepage)s"/>""" % {'title': feed.title,'homepage': feed.homepage, 'person': feed.person , 'category':feed.category, 'url': feed.posts_url,'comments': feed.comments_url, 'favicon': feed.favicon, 'listtitle': feed.listtitle, 'language' : feed.language } )
output.append("""</outline></body></opml>""")
return "".join(output)
+
+class CSEConfig(CachedPage):
+ cacheName = "CSEConfig"
+ def generatePage(self):
+ output = []
+ output.append( """ <?xml version="1.0" encoding="UTF-8" ?>
+ <Annotations>""")
+ for feed in Feed.all():
+ output.append( """
+ <Annotation about="%(homepage)s*">
+ <Label name="_cse_et7bffbfveg"/>
+ </Annotation>
+ """ % {'homepage': add_slash(strip_http(feed.homepage)) } )
+ output.append("""</Annotations>""")
+ return "".join(output)
View
@@ -1,44 +0,0 @@
-$header
-
-$menu
-
-<div class="content">
-
- <h2> The Feeds </h2>
- <p>We offer some very simplistic feeds since the primary purpose of mathblogging.org is to help you find blogs, not to bundle them. <br/> There are much better tools around if you want to generate a metafeed of your favorite feeds; e.g., google reader, friendfeed, facebook etc.</p>
- <p> You can also <a href="/database-opml.xml">download our OPML-file</a> to import our database into your feed reader or <a href="/database.csv"> get our database as a csv</a>. </p>
-
- <ul>
- <li> <b> The feeds by category</b> </li>
- <li> </li>
- <li> <a href="/feed_pure">The pure feed</a> </li>
- <li> <a href="/feed_applied">The applied feed</a> </li>
- <li> <a href="/feed_teachers">The teachers feed</a> </li>
- <li> <a href="/feed_visual">The visual feed</a> </li>
- <li> <a href="/feed_history">The history feed</a> </li>
- <li> <a href="/feed_art">The art feed</a> </li>
- <li> <a href="/feed_fun">The fun feed</a> </li>
- <li> <a href="/feed_general">The general feed</a> </li>
- <li> <a href="/feed_journalism">The journalism feed</a> </li>
- <li> <a href="/feed_journals">The journals feed</a> </li>
- <li> <a href="/feed_commercial">The commercial feed</a> </li>
- <li> <a href="/feed_institutions">The institutions feed</a> </li>
- <li> <a href="/feed_communities">The communities feed</a> </li>
- <li> <a href="/feed_newssite">The news-sites feed</a> </li>
- <li> <a href="/feed_carnival">The carnival feed</a> </li>
- </ul>
- <ul>
- <li> <b>Some mixed feeds</b> </li>
- <li> </li>
- <li> <a href="/feed_researchers">The researchers feed</a> (pure, applied and history)</li>
- <li> <a href="/feed_people">The people feed (everything but institutions, communities, journals, commercial)</a> </li>
- <li> <a href="/feed_all">The full feed</a> </li>
- </ul>
-
-$disqus
-
-</div>
-
-$footer
- </body>
-</html>
View
69 main.py
@@ -14,8 +14,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from Cheetah.Template import Template
-
import wsgiref.handlers
import os
import xml.dom.minidom
@@ -40,6 +38,13 @@
import cgi
from google.appengine.ext.webapp.util import run_wsgi_app
+# from http://code.google.com/appengine/docs/python/tools/libraries.html#Django
+# import os #done earlier
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+
+from google.appengine.dist import use_library
+use_library('django', '0.96')
+
#import pickle # for listoflist in datastore
from django.utils import simplejson
@@ -103,7 +108,7 @@ def truncate_html_words(s, num):
if words <= length:
# Don't try to close tags if we don't need to truncate
return s
- out = s[:ellipsis_pos] + ' ...'
+ out = s[:ellipsis_pos] + ' [...]'
# Close any tags still open
for tag in open_tags:
out += '</%s>' % tag
@@ -232,8 +237,8 @@ def fetch_feed_and_generate_entries(self,url,_type):
_type.generate_entry(entry,feed,self)
except LookupError, e:
logging.warning("There was an error parsing the feed " + url + ":" + str(e))
- def cse_homepage(self): # REMINDER: for CSE = google custome search engine = search for startpage
- return add_slash(strip_http(self.homepage))
+ #def cse_homepage(self): # REMINDER: for CSE = google custome search engine = search for startpage
+ # return add_slash(strip_http(self.homepage))
@@ -292,7 +297,7 @@ def generate_entry(cls, entry, feedparser_feed, database_feed):
x.timestamp_created = timestamp_updated
x.guid = guid
x.put()
- except Exception, e:
+ except Exception, e: # TODO more exception catching: 'NoneType' error when feed is malformed not enough for bug tracking.
logging.warning("There was an error processing an Entry of the Feed :" + str(e))
def printTime_created_rfc3339(self):
@@ -400,11 +405,11 @@ def get(self):
if not Stored_Page.gql("WHERE name = :1", self.cacheName).get():
logging.info("Writing to datastore: " + self.cacheName)
self.write_page_to_datastore()
- try:
- content = Stored_Page.gql("WHERE name = :1", self.cacheName).get().html_content
- memcache.set(self.cacheName,content,self.cacheTime)
- except Exception, e:
- logging.warning("Error setting memcache from Stored_Page object :" + str(e))
+ content = Stored_Page.gql("WHERE name = :1", self.cacheName).get().html_content
+ if not content == 'NoneType':
+ memcache.set(self.cacheName,content,self.cacheTime) ### TODO NoneType error messages on the App Engine but everything works
+ #except Exception, e:
+ # logging.warning("Error setting memcache from Stored_Page object :" + str(e))
self.response.out.write(memcache.get(self.cacheName))
### Adding header, footer, menu
@@ -416,12 +421,12 @@ def generatePage(self):
### TODO remove leftover static cheetah pages???
-class SimpleCheetahPage(CachedPage):
- templateName = "default.tmpl"
- def generatePage(self):
- template_values = { 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
- path = os.path.join(os.path.dirname(__file__), self.templateName)
- return str(Template( file = path, searchList = (template_values,) ))
+#class SimpleCheetahPage(CachedPage):
+# templateName = "default.tmpl"
+# def generatePage(self):
+# template_values = { 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
+# path = os.path.join(os.path.dirname(__file__), self.templateName)
+# return str(Template( file = path, searchList = (template_values,) ))
@@ -430,13 +435,13 @@ def generatePage(self):
#################################
-class StartPage(SimpleCheetahPage):
- cacheName = "StartPage"
- templateName = "start.tmpl"
+#class StartPage(SimpleCheetahPage):
+# cacheName = "StartPage"
+# templateName = "start.tmpl"
-class FeedsPage(SimpleCheetahPage):
- cacheName = "FeedsPage"
- templateName = "feeds.tmpl"
+#class FeedsPage(SimpleCheetahPage):
+# cacheName = "FeedsPage"
+# templateName = "feeds.tmpl"
#################################
@@ -445,11 +450,11 @@ class FeedsPage(SimpleCheetahPage):
### TODO is completely outdated!!!!!
-class CSEConfig(webapp.RequestHandler):
- def get(self):
- template_values = { 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
- path = os.path.join(os.path.dirname(__file__), 'cse-config.tmpl')
- self.response.out.write(Template( file = path, searchList = (template_values,) ))
+#class CSEConfig(webapp.RequestHandler):
+# def get(self):
+# template_values = { 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
+# path = os.path.join(os.path.dirname(__file__), 'cse-config.tmpl')
+# self.response.out.write(Template( file = path, searchList = (template_values,) ))
#################################
@@ -481,7 +486,7 @@ def get(self):
#logging.info("Adding fetch task for feed " + feed.title + " with url: " + feed.posts_url)
taskqueue.add(url="/fetch", params={'url': feed.posts_url})
- pages_to_cache_list = ["/", "/feeds","/bytype","/weekly-picks","/bydate","/byresearchdate","/byartvishisdate","/byteacherdate","/bystats","/planetmo", "/planettag", "/planetmo-feed","/feed_pure","/feed_applied","/feed_history","/feed_art","/feed_fun","/feed_general","/feed_journals","/feed_teachers","/feed_visual","/feed_journalism","/feed_institutions","/feed_communities","/feed_commercial","/feed_newssite","/feed_carnival","/feed_all","/feed_researchers"]
+ pages_to_cache_list = ["/", "/feeds","/bytype","/weekly-picks","/bydate","/byresearchdate","/byartvishisdate","/byteacherdate","/bystats","/planetmo", "/planetmo-feed","/feed_pure","/feed_applied","/feed_history","/feed_art","/feed_fun","/feed_general","/feed_journals","/feed_teachers","/feed_visual","/feed_journalism","/feed_institutions","/feed_communities","/feed_commercial","/feed_newssite","/feed_carnival","/feed_all","/feed_researchers"]
for page in pages_to_cache_list:
taskqueue.add(url=page, method="GET")
self.response.set_status(200)
@@ -509,9 +514,8 @@ def post(self):
class FeedTagListWorker(webapp.RequestHandler):
def get(self):
logging.info("FeedTagListWorker")
- for feed in Feed.all():
+ for feed in Feed.gql("WHERE category IN :1", ['history','fun','general','commercial','art','visual','pure','applied','teacher','journalism']):
taskqueue.add(url="/feedtaglistfetch", params={'url': feed.posts_url})
- taskqueue.add(url="/globaltaglistworker", method="GET")
self.response.set_status(200)
class GlobalTagListWorker(webapp.RequestHandler):
@@ -584,6 +588,7 @@ def get(self):
### Dynamically generated web pages // the main content of the site
#################################
+from startpage import *
from dateview import DateView ### TODO make abstract and call with "all, research, teacher, hisartvis"
from dateviewresearch import DateViewResearch
from dateviewteacher import DateViewTeacher
@@ -596,7 +601,7 @@ def get(self):
from grid import *
from weeklypicks import *
from statsview import * ### TODO make like dateview all, research, teacher, hisartivs
-
+from feedspage import *
View
@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<opml version="1.0">
- <head>
- <title>Mathblogging.org Database export to OPML</title>
- </head>
- <body>
-
- <outline title="Mathblogging.org" text="Mathblogging.org">
-
-#for $feed in $qf.get().order("category")
-
- <outline text="<%=feed.title%>" title="<%=feed.title%>" type="rss"
- xmlUrl="<%=feed.url%>" htmlUrl="<%=feed.homepage%>"/>
-#end for
- </outline>
-
- </body>
-</opml>
View
@@ -1,116 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"><head>
- <meta http-equiv="content-type" content="application/xhtml+xml; charset=UTF-8"/>
- <link rel="stylesheet" type="text/css" href="/content/site.css"/>
- <link rel="icon" href="/favicon.ico" type="image/x-icon" />
- <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon" />
- <style type="text/css">
-body {
- height: 600px;
- background: #DDDDDD;
- background-repeat: repeat-x;
- background-image: url("/content/background-frontpage.png");
- background-image: -webkit-gradient(linear,left top,left bottom, color-stop(0, #444444), color-stop(0.33, #AAAAAA), color-stop(1, #DDDDDD));
- background-image: -moz-linear-gradient(center top , #444444 0%, #AAAAAA 200px, #DDDDDD 100%);
-}
-h1 {
- margin-bottom: 20px;
-}
-a {
- padding: 10pt;
-}
-.footer {
- padding-top: 100px;
-}
- </style>
- <title>Mathblogging.org</title>
- </head><body>
-<h1 style="text-align:center">Mathblogging.org <span style="font-size: 10pt; color: rgb(204, 204, 204);">your one-stop shop for mathematical blogs</span></h1>
-
-
-<table style="margin: auto; width: 592px;" border="0" cellpadding="0" cellspacing="0">
-<tbody align="center">
-<tr align="center">
- <td align="center"><a href="/bydate" ><img src="/content/date-128.png" alt="View the latest posts"/></a><br/><b>View the Latest Posts</b></td>
- <td align="center"><a href="/bytype"><img src="/content/type-128.png" alt="View by type"/></a><br/><b>View by Category</b></td>
- <td align="center"><a href="/bystats"><img src="/content/ranking-128.png" alt="View by stats"/></a><br/><b>View by Stats</b></td>
-</tr>
-<tr align="center">
- <td align="center" style="padding-top:30px;"><a href="/weekly-picks"><img src="/content/favorites-128.png" alt="View our favorites"/></a><br/><b>Our weekly picks</b></td>
- <td align="center" style="padding-top:30px;"><a href="/planettag"><img src="/content/tag-128.png" alt="View Posts by Tag"/></a><br/><b>View by Tag</b></td>
- <td align="center" style="padding-top:30px;"><a href="/planetmo"><img src="/content/planet-128.png" alt="PlanetMO"/></a><br/><b>PlanetMO</b></td>
-</tr>
-</tbody></table>
-
-
-<div style="margin:20px;">
-
-<div id="cse-search-form" style="width: 100%;">Loading custom search. If you don't have JavaScript, you can use <a href="http://www.google.com/cse/home?cx=011837388000883284523:et7bffbfveg">this link</a> to search.</div>
-<script src="http://www.google.com/jsapi" type="text/javascript"></script>
-<script type="text/javascript">
- google.load('search', '1', {language : 'en', style : google.loader.themes.SHINY});
- google.setOnLoadCallback(function() {
- var customSearchControl = new google.search.CustomSearchControl('011837388000883284523:et7bffbfveg');
- customSearchControl.setResultSetSize(google.search.Search.FILTERED_CSE_RESULTSET);
- var options = new google.search.DrawOptions();
- options.setSearchFormRoot('cse-search-form');
- customSearchControl.draw('cse', options);
- }, true);
-</script>
-<div id="cse-search-form2" style="width: 100%;">
-</div>
-
-<div id="cse" style="width:100%;"></div>
-
-<link rel="stylesheet" href="/content/cse-frontpage-2.css" type="text/css" />
-</div>
-
-<table style="margin:auto; width: 450px; padding-top:10px;" border="0" cellpadding="0" cellspacing="0">
-<tbody align="center">
-<tr align="center">
- <td align="center"><a href="https://mathblogging.wordpress.com/contact-us/"><img src="/content/mail-64.png" alt="A blog's not listed?"/></a><br/><span style="font-size:90%">A blog's not listed? <br/> Tell us!</span></td>
- <td align="center"><a href="feeds"><img src="/content/rss-64.png" alt="Grab a feed"/></a><br/><span style="font-size:90%"> Visiting too often? <br/> Grab a feed! </span></td>
- <td align="center"><a href="http://mathblogging.wordpress.com/about"><img src="/content/about-64.png" alt="More about us"/></a><br/><span style="font-size:90%;">More about us? <br/> Visit our blog.</span></td>
-</tr>
-</tbody>
-</table>
-
-<div class="twitter">
-<script type="text/javascript" src="http://widgets.twimg.com/j/2/widget.js"></script>
-<script>
-new TWTR.Widget({
- version: 2,
- type: 'list',
- rpp: 30,
- interval: 6000,
- title: '@mathblogging',
- subject: 'math bloggers on twitter',
- width: 400,
- height: 400,
- theme: {
- shell: {
- background: '#ffffff',
- color: '#000000'
- },
- tweets: {
- background: '#ffffff',
- color: '#000000',
- links: '#878787'
- }
- },
- features: {
- scrollbar: true,
- loop: false,
- live: true,
- hashtags: true,
- timestamp: true,
- avatars: true,
- behavior: 'all'
- }
-}).render().setList('mathblogging', 'math-bloggers').start();
-</script>
-</div>
-
-$footer
-
- </body></html>
View
@@ -88,3 +88,5 @@
</div>
<!-- end copyright footer -->
"""
+
+

0 comments on commit 3534948

Please sign in to comment.