Skip to content
This repository
Browse code

attempt of writing statsview.py, doesn't work for unknown reasons

  • Loading branch information...
commit 289d3ceaf702c97647d6b3f9d25f3902bcdca295 1 parent ce8a765
authored August 09, 2011
9  app.yaml
@@ -6,7 +6,7 @@ api_version: 1
6 6
 handlers:
7 7
 - url: /remote_api
8 8
   script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py
9  
-  login: admin
  9
+  login: admin  
10 10
 - url: /content/(.*)
11 11
   static_files: content/\1
12 12
   upload: content/(.*)
@@ -16,10 +16,10 @@ handlers:
16 16
 - url: /fetch
17 17
   script: main.py
18 18
   login: admin
19  
-- url: /fetchall
  19
+- url: /allworker
20 20
   script: main.py
21 21
   login: admin
22  
-- url: /fetchsync
  22
+- url: /taglistworker
23 23
   script: main.py
24 24
   login: admin
25 25
 - url: /clearpagecache
@@ -30,3 +30,6 @@ handlers:
30 30
   login: admin
31 31
 - url: .*
32 32
   script: main.py
  33
+  
  34
+builtins:
  35
+- datastore_admin: on
20  bulkloader.yaml
@@ -36,8 +36,8 @@ transformers:
36 36
     - property: homepage
37 37
       external_name: homepage
38 38
 
39  
-    - property: comments
40  
-      external_name: comments
  39
+    - property: comments_url
  40
+      external_name: comments_url
41 41
 
42 42
     - property: favicon
43 43
       external_name: favicon
@@ -67,7 +67,19 @@ transformers:
67 67
       external_name: language
68 68
       # Type: String Stats: 128 properties of this type in this kind.
69 69
 
70  
-    - property: url
71  
-      external_name: url
  70
+    - property: posts_url
  71
+      external_name: posts_url
72 72
       # Type: String Stats: 128 properties of this type in this kind.
73 73
 
  74
+    - property: comments_day
  75
+      external_name: comments_day
  76
+
  77
+    - property: comments_week
  78
+      external_name: comments_week
  79
+
  80
+    - property: posts_week
  81
+      external_name: posts_week
  82
+
  83
+    - property: posts_month
  84
+      external_name: posts_month
  85
+
6  bydate.tmpl
... ...
@@ -1,3 +1,5 @@
  1
+#from main import *
  2
+
1 3
 $header
2 4
 
3 5
 $menu
@@ -22,18 +24,20 @@ $menu
22 24
   </tr>
23 25
   </thead>
24 26
   <tbody>
25  
-  #for $entry in $allentries
  27
+  #for $entry in $Post.gql("WHERE category IN ['pure','applied'] ORDER BY timestamp_created LIMIT 150")
26 28
   <tr>
27 29
     <td valign="bottom" class="datecolumn">
28 30
       <div>
29 31
 	<%= entry.printShortTime_created() %>
30 32
       </div>
31 33
     </td>
  34
+  #raw
32 35
     <td valign="bottom" class="blogcolumn">
33 36
       <div>
34 37
 	<a href="$entry.homepage" title="<%=entry.service%>"><%=entry.service%></a>
35 38
       </div>
36 39
     </td>
  40
+  #end raw
37 41
     <td valign="bottom" class="postcolumn">
38 42
       <div>
39 43
 	<a href="$entry.link" title="<%=entry.title%>"><%=entry.title%></a>
11  bystats.tmpl
@@ -4,10 +4,7 @@ $menu
4 4
 
5 5
 <div class="content">
6 6
 
7  
-<p> Warning! We calculate the numbers based on the current feeds. </p>
8  
-<p>If a feed only contains 10 entries, the stats will be at most 10 -- beware of multiples of 5!</p>
9  
-
10  
-<table class="rankingcolumn" id="postsbydate">
  7
+<table class="bydate" id="commentsperday">
11 8
   <thead>
12 9
   <tr>
13 10
     <th align="left" class="datecolumn">
@@ -35,7 +32,7 @@ $menu
35 32
   </tbody>
36 33
 </table>
37 34
 
38  
-<table class="rankingcolumn" id="postsbydate">
  35
+<table class="bydate" id="commentsperweek">
39 36
   <thead>
40 37
   <tr>
41 38
     <th align="left" class="datecolumn">
@@ -64,7 +61,7 @@ $menu
64 61
 </table>
65 62
 
66 63
 
67  
-<table class="rankingcolumn" id="postsbydate">
  64
+<table class="bydate" id="postsperweek">
68 65
   <thead>
69 66
   <tr>
70 67
     <th align="left" class="datecolumn">
@@ -96,7 +93,7 @@ $menu
96 93
   </tbody>
97 94
 </table>
98 95
 
99  
-<table class="rankingcolumn" id="postsbydate">
  96
+<table class="bydate" id="postspermonth">
100 97
   <thead>
101 98
   <tr>
102 99
     <th align="left" class="datecolumn">
23  content/site.css
@@ -263,6 +263,7 @@ width:500px;
263 263
 {
264 264
 float:left;
265 265
 display:block;
  266
+width: 75px;
266 267
 font-size:small;
267 268
 font-weight:bold;
268 269
 color:#FFFFFF;
@@ -281,7 +282,9 @@ background-color:black;
281 282
 
282 283
 /* ------------ Begin Date View Table --------------- */
283 284
 
284  
-#postsbydate {
  285
+.bydate {
  286
+    float: left;
  287
+    margin: 1ex !important;
285 288
     table-layout: fixed;
286 289
     margin: 5ex;
287 290
     padding: 5px;
@@ -290,37 +293,37 @@ background-color:black;
290 293
     -webkit-box-shadow: 4px 4px 8px black;
291 294
 }
292 295
 
293  
-#postsbydate td {
  296
+.bydate td {
294 297
     border-bottom: 1px dashed gray;
295 298
 }
296 299
 
297  
-#postsbydate tbody .datecolumn {
  300
+.bydate tbody .datecolumn {
298 301
     width: 7ex;
299 302
     font-size: 80%;
300 303
     text-align: left;
301 304
 }
302 305
 
303  
-#postsbydate tbody div {
  306
+.bydate tbody div {
304 307
     height: 1.2em;
305 308
     overflow: hidden;
306 309
 }
307 310
 
308  
-#postsbydate tbody .blogcolumn {
  311
+.bydate tbody .blogcolumn {
309 312
     width: 30ex;
310 313
     font-size: 80%;
311 314
     text-align: left;
312 315
 }
313 316
 
314  
-#postbydate tbody .postcolumn {
  317
+.bydate tbody .postcolumn {
315 318
     text-align: left;
316 319
     width: 50ex;
317 320
 }
318 321
 
319  
-#postsbydate a {
  322
+.bydate a {
320 323
    text-decoration: none;
321 324
 }
322 325
 
323  
-#postsbydate .postcolumn a {
  326
+.bydate .postcolumn a {
324 327
     font-size: 90%;
325 328
     color: black;
326 329
 }
@@ -361,7 +364,11 @@ background-color:black;
361 364
     -webkit-box-shadow: 4px 4px 7px black;
362 365
     border: 1px solid gray;
363 366
     padding: 18px 18px 18px 18px;
  367
+<<<<<<< HEAD
  368
+    margin: 10px 10px 30px 10px;
  369
+=======
364 370
     margin: 10px 10px 50px 10px;
  371
+>>>>>>> 1bb0d5aa9e26b8ac4f90bbc4ede5fe9b3bc86ce2
365 372
 }
366 373
 
367 374
 .planetbody, .planetbody h1, .planetbody h2, .planetbody h3, .planetbody h4 {
4  cron.yaml
... ...
@@ -1,4 +1,4 @@
1 1
 cron:
2 2
 - description: check for feed updates
3  
-  url: /fetchall
4  
-  schedule: every 3 hours
  3
+  url: /allworker
  4
+  schedule: every 2 hours
2  database.tmpl
... ...
@@ -1,4 +1,4 @@
1 1
 title, homepage, person, category, url, comments, priority, subject, favicon
2  
-#for $feed in $qf.get().order("categoyr")
  2
+#for $feed in $qf.get().order("category")
3 3
 "<%=feed.title%>",<%=feed.homepage%>,<%=feed.person%>,<%=feed.category%>,<%=feed.url%>,<%=feed.comments%>,<%=feed.priority%>,<%=feed.subject%>,<%=feed.favicon%>
4 4
 #end for
8  faq.html
... ...
@@ -1,8 +0,0 @@
1  
-<html>
2  
-<head>
3  
-  <title>FAQ</title>
4  
-</head>
5  
-<body>
6  
-A FAQ.
7  
-</body>
8  
-</html>
2,103  feedparser.py
1577 additions, 526 deletions not shown
3  feeds.tmpl
@@ -6,6 +6,7 @@ $menu
6 6
 
7 7
       <h2> The Feeds </h2>
8 8
     <p>We offer some very simplistic feeds since the primary purpose of mathblogging.org is to help you find blogs, not to bundle them. <br/> There are much better tools around if you want to generate a metafeed of your favorite feeds; e.g., google reader, friendfeed, facebook etc.</p>
  9
+    <p> You can also <a href="/database-opml.xml">download our OPML-file</a> to import our database into your feed reader or <a href="/database.csv"> get our database as a csv</a>. </p>
9 10
 
10 11
       <ul>
11 12
 	    <li> <b> The feeds by category</b> </li>
@@ -23,6 +24,8 @@ $menu
23 24
 	    <li> <a href="/feed_commercial">The commercial feed</a> </li>
24 25
 	    <li> <a href="/feed_institutions">The institutions feed</a> </li>
25 26
 	    <li> <a href="/feed_communities">The communities feed</a> </li>
  27
+	    <li> <a href="/feed_newssite">The news-sites feed</a> </li>
  28
+	    <li> <a href="/feed_carnival">The carnival feed</a> </li>
26 29
       </ul>
27 30
       <ul>
28 31
 	    <li> <b>Some mixed feeds</b> </li> 
79  htmlcutstring.py
... ...
@@ -1,79 +0,0 @@
1  
-# This package is used to cut the string which is having html tags. 
2  
-# It does not count the html tags, it just count the string inside tags and keeps
3  
-# the tags as it is.
4  
-
5  
-# ex: If the string is "welcome to <b>Python World</b> <br> Python is bla". and If we want to cut the string of 16 charaters then output will be "welcome to <b>Python</b>". 
6  
-
7  
-# Here while cutting the string it keeps the tags for the cutting string and skip the rest and without distorbing the div structure.
8  
-
9  
-# USAGE1:
10  
-#  obj = HtmlCutString("welcome to <b>Python World</b> <br> Python is",16)
11  
-#  newCutString = obj.cut()
12  
-
13  
-# USAGE2:
14  
-#  newCutString = cutHtmlString("welcome to <b>Python World</b> <br> Python is",16)
15  
-
16  
-
17  
-from xml.dom.minidom import getDOMImplementation
18  
-from xml.dom.minidom import parseString
19  
-
20  
-class HtmlCutString():
21  
-
22  
-    def __init__(self,string, limit):
23  
-        # temparary node to parse the html tags in the string
24  
-        self.tempDiv = parseString('<div>'+string+'</div>');
25  
-        # while parsing text no of characters parsed
26  
-        self.charCount = 0
27  
-        self.limit = limit
28  
-
29  
-
30  
-    def cut(self):
31  
-        impl = getDOMImplementation()
32  
-        newdoc = impl.createDocument(None, "some_tag", None)
33  
-        newDiv = newdoc.documentElement
34  
-
35  
-        self.searchEnd(self.tempDiv, newDiv)
36  
-        # removeng some_tag that we added above
37  
-        newContent = newDiv.firstChild.toxml('utf-8')
38  
-        # removing div tag that we added in the __init__
39  
-        return newContent[5:-6]
40  
-
41  
-    def deleteChildren(self,node):
42  
-        while node.firstChild:
43  
-            self.deleteChildren(node.firstChild)
44  
-            node.removeChild(node.firstChild)
45  
-             
46  
-  
47  
-    def searchEnd(self,parseDiv, newParent):
48  
-        for ele in parseDiv.childNodes:
49  
-            # not text node
50  
-            if ele.nodeType != 3:
51  
-                newEle = ele.cloneNode(True)
52  
-                newParent.appendChild(newEle)
53  
-                if len(ele.childNodes) == 0:
54  
-                    continue
55  
-                self.deleteChildren(newEle)
56  
-                res = self.searchEnd(ele,newEle)
57  
-                if res :
58  
-                    return res
59  
-                else:
60  
-                    continue;
61  
-
62  
-
63  
-            # the limit of the char count reached
64  
-            if (len(ele.nodeValue) + self.charCount) >= self.limit:
65  
-                newEle = ele.cloneNode(True)
66  
-                newEle.nodeValue = ele.nodeValue[0:(self.limit - self.charCount)]
67  
-                newParent.appendChild(newEle)
68  
-                return True
69  
-
70  
-            newEle = ele.cloneNode(True)
71  
-            newParent.appendChild(newEle)
72  
-            self.charCount += len(ele.nodeValue)
73  
-
74  
-        return False
75  
-    
76  
-def cutHtmlString(string, limit):
77  
-    output = HtmlCutString(string,limit)
78  
-    return output.cut()
79  
-
774  main.py
... ...
@@ -1,8 +1,3 @@
1  
-#TODO: 
2  
-#main.py : feeds auf neue categories umstellen
3  
-#templates auf categories checken
4  
-#JOURNALS!!
5  
-
6 1
 # Mathblogging is a simple blog aggregator.
7 2
 # Copyright (C) 2010 Felix Breuer, Frederik von Heymann, Peter Krautzberger
8 3
 #
@@ -28,7 +23,7 @@
28 23
 import datetime
29 24
 import time
30 25
 import logging
31  
-import counter
  26
+import string
32 27
 
33 28
 from operator import attrgetter
34 29
 from time import strftime, strptime, gmtime
@@ -37,6 +32,7 @@
37 32
 from google.appengine.api import users
38 33
 from google.appengine.ext import webapp
39 34
 from google.appengine.ext import db
  35
+from google.appengine.ext.db import polymodel
40 36
 from google.appengine.api import urlfetch
41 37
 from google.appengine.api import memcache
42 38
 from google.appengine.api.labs import taskqueue
@@ -44,11 +40,16 @@
44 40
 import cgi
45 41
 from google.appengine.ext.webapp.util import run_wsgi_app
46 42
 
47  
-# Escape HTML entities.
  43
+
  44
+### some variables like header, footer etc.
  45
+from temp_global import *
  46
+
  47
+
  48
+## Escape HTML entities.
48 49
 html_escape_table = {
49 50
     "&": "&amp;",
50 51
     '"': "&quot;",
51  
-#    "'": "&apos;",
  52
+    "'": "&apos;",
52 53
     ">": "&gt;",
53 54
     "<": "&lt;",
54 55
     }
@@ -56,99 +57,9 @@
56 57
 def html_escape(text):
57 58
     """Produce entities within text."""
58 59
     return "".join(html_escape_table.get(c,c) for c in text)
59  
-# end
60  
-
61  
-header = """
62  
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
63  
-<html xmlns="http://www.w3.org/1999/xhtml">
64  
-  <head>
65  
-    <meta http-equiv="content-type" content="application/xhtml+xml; charset=UTF-8"/>
66  
-    <link rel="stylesheet" type="text/css" href="/content/site.css"/>
67  
-    <link rel="icon" href="/favicon.ico" type="image/x-icon" />
68  
-    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon" />
69  
-    <title>Mathblogging.org</title>
70  
-    <script type="text/javascript" src="/content/jquery-1.5.2.min.js"></script>         
71  
-    <link rel="stylesheet" type="text/css" href="/content/jqcloud.css" />
72  
-    <script type="text/javascript" src="/content/jqcloud-0.1.8.js"></script>
73  
-  </head>
74  
-  <body>
75  
-    <h1> <a style="text-decoration:none;color:white;" href="/">Mathblogging.org <small style="color: #CCC">beta</small></a></h1>
76  
-"""
77  
-
78  
-menu = """
79  
-<!-- Top Navigation -->
80  
-<div id="menu">
81  
-<ul>
82  
-  <li><h2><a href="/bydate" title="Recent posts">Posts</a></h2>
83  
-  <ul>
84  
-    <li><h2><a href="/byresearchdate" title="Recent posts in Research">Researchers</a></h2>
85  
-    </li>
86  
-    <li><h2><a href="/byartvishisdate" title="Recent posts in Art,Visual,History">Art/Visual/History</a></h2>
87  
-    </li>
88  
-    <li><h2><a href="/byteacherdate" title="Recent posts from Teachers">Teachers</a></h2>
89  
-    </li>
90  
-  </ul>
91  
-  </li>
92  
-  <li><h2><a href="/bytype" title="Blogs by Category">Blogs</a></h2>
93  
-  </li>
94  
-  <li><h2><a href="/bystats" title="Recent statistics">Stats</a></h2>
95  
-  </li>
96  
-  <li><h2><a href="/weekly-picks" title="Our weekly picks">Weekly Picks</a></h2>
97  
-  </li>     
98  
-  <li><h2><a href="/planettag" title="PlanetTAG">PlanetTAG</a></h2>
99  
-  </li>
100  
-  <li><h2><a href="/planetmo" title="PlanetMO">PlanetMO</a></h2>
101  
-  </li>     
102  
-  <li><h2><a href="/feeds" title="Feeds">Feeds</a></h2>
103  
-  </li>
104  
-  <li><h2><a href="https://mathblogging.wordpress.com/" title="About us">About us</a></h2>
105  
-  </li>
106  
-  <li><h2><a href="/" title="Search">Search</a></h2>
107  
-  </li>
108  
-</ul>						
109  
-</div>
110  
-<!-- end Top Navigation -->
111  
-"""
112  
-
113  
-disqus = """
114  
-<!-- disqus code-->
115  
-<div class="disqus">
116  
-<hr/>
117  
-<div id="disqus_thread"></div>
118  
-<script type="text/javascript">
119  
-    /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */
120  
-    var disqus_shortname = 'mathblogging'; // required: replace example with your forum shortname
121  
-
122  
-    // The following are highly recommended additional parameters. Remove the slashes in front to use.
123  
-    // var disqus_identifier = 'unique_dynamic_id_1234';
124  
-    // var disqus_url = 'http://example.com/permalink-to-page.html';
125  
-
126  
-    /* * * DON'T EDIT BELOW THIS LINE * * */
127  
-    (function() {
128  
-        var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
129  
-        dsq.src = 'http://' + disqus_shortname + '.disqus.com/embed.js';
130  
-        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
131  
-    })();
132  
-</script>
133  
-<noscript><p>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript">comments powered by Disqus.</a></p></noscript>
134  
-<a href="http://disqus.com" class="dsq-brlink">blog comments powered by <span class="logo-disqus">Disqus</span></a>
135  
-</div>
136  
-<!-- end disqus code-->
137  
-"""
138  
-
139  
-footer = """
140  
-<!-- copyright footer -->
141  
-<div class="footer">
142  
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/3.0/">
143  
-  <img alt="Creative Commons License" src="http://i.creativecommons.org/l/by-nc-sa/3.0/80x15.png"/>
144  
-</a>
145  
-<p>
146  
-mathblogging.org is licensed under a <br/> <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/3.0/">Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License</a>.
147  
-</p>
148  
-</div>
149  
-<!-- end copyright footer -->
150  
-"""
  60
+## end
151 61
 
  62
+### strip_http, add_slash: for Google Custom Search
152 63
 def strip_http(str):
153 64
     if str[0:7] == "http://":
154 65
       return str[7:]
@@ -160,190 +71,185 @@ def add_slash(str):
160 71
       return str+"/"
161 72
     else:
162 73
       return str
163  
-
  74
+### preliminary definitions for processing with feedparser.py
164 75
 def get_feedparser_entry_content(entry):
165 76
     try:
166  
-        return " ".join([content.value for content in entry.content])            
  77
+        return " ".join([content.value for content in entry.content])
167 78
     except AttributeError:
168 79
         try:
169 80
             return entry['summary']
170 81
         except AttributeError:
171 82
             return ""
172  
-            
173  
-
  83
+### creating a reliable guid for our own feeds
  84
+def feedparser_entry_to_guid(entry):
  85
+    theid = None
  86
+    try:
  87
+        theid = html_escape(entry.id)
  88
+    except AttributeError:
  89
+        t = time.gmtime(0)
  90
+        try:
  91
+            t = entry.updated_parsed
  92
+        except AttributeError:
  93
+            pass
  94
+        dt = datetime.datetime(t[0],t[1],t[2],t[3],t[4],t[5])
  95
+        try:
  96
+            theid = html_escape((entry['link'] + str(dt) + entry.title)[0:500])
  97
+        except AttributeError:
  98
+            pass
  99
+    return theid
  100
+### The first major class: Feed
174 101
 class Feed(db.Model):
175  
-    url = db.LinkProperty()
  102
+    posts_url = db.LinkProperty()
176 103
     homepage = db.StringProperty()
177 104
     title = db.StringProperty()
178 105
     listtitle = db.StringProperty()
179 106
     person = db.StringProperty()
180  
-    category = db.StringProperty() # history fun general commercial art visual pure applied teacher journalism community institution  
181  
-    # was: 'groups', 'research', 'educator', 'journalism', 'institution', 'community', ('commercial')
  107
+    category = db.StringProperty() # history fun general commercial art visual pure applied teacher journalism community institution journal news carnival
182 108
     language = db.StringProperty()
183 109
     priority = db.IntegerProperty()
184 110
     favicon = db.StringProperty()
185  
-    comments = db.StringProperty()
  111
+    comments_url = db.StringProperty()
186 112
     comments_day = db.IntegerProperty()
187 113
     comments_week = db.IntegerProperty()
188 114
     posts_week = db.IntegerProperty()
189 115
     posts_month = db.IntegerProperty()
190  
-    def restore_cache(self):
191  
-        logging.info("Restoring Cache of Feed " + self.title)
192  
-        #try: 
193  
-        updates = self.fetch_entries()
194  
-        memcache.set(self.url,updates,86400) # 1 day 
195  
-        # memcache element for comment feeds. fetch_comments_entries accumulates the list, self.comments is the database object to call up later
196  
-        comments_updates = self.fetch_comments_entries()
197  
-        memcache.set(self.comments,comments_updates,86400) # 1 day
198  
-        logging.info("Memcache updated successfully.")
199  
-        self.comments_day = len([item for item in self.comments_entries() if time.mktime(time.localtime()) - time.mktime(item.gettime()) <= 86400 ])
200  
-        self.comments_week = len([item for item in self.comments_entries() if time.mktime(time.localtime()) - time.mktime(item.gettime()) <= 604800 ])
201  
-        self.posts_month = len([item for item in self.entries() if time.mktime(time.localtime()) - time.mktime(item.gettime()) <= 2592000 ])
202  
-        self.posts_week = len([item for item in self.entries() if time.mktime(time.localtime()) - time.mktime(item.gettime()) <= 604800 ])
  116
+    def update_database(self):
  117
+        logging.info("Update Database for Feed: " + self.title)
  118
+        self.fetch_feed_and_generate_entries(self.posts_url,Post)
  119
+        self.fetch_feed_and_generate_entries(self.comments_url,Comment)
  120
+        self.update_stats()
  121
+    def update_stats(self):
  122
+        # calculate stats
  123
+        self.comments_day = Comment.gql("WHERE service = :1 AND timestamp_created > :2", self.title, datetime.datetime.now() - datetime.timedelta(1)).count()
  124
+        self.comments_week = Comment.gql("WHERE service = :1 AND timestamp_created > :2", self.title, datetime.datetime.now() - datetime.timedelta(7)).count()
  125
+        self.posts_week = Post.gql("WHERE service = :1 AND timestamp_created > :2", self.title, datetime.datetime.now() - datetime.timedelta(7)).count()
  126
+        self.posts_month = Post.gql("WHERE service = :1 AND timestamp_created > :2", self.title, datetime.datetime.now() - datetime.timedelta(30)).count()
  127
+        #self.comments_day = Comment.all().filter("service =",self.title).filter("timestamp_created > ", datetime.datetime.now() - datetime.timedelta(1)).count()
  128
+        #self.comments_week = Comment.all().filter("service =",self.title).filter("timestamp_created > ", datetime.datetime.now() - datetime.timedelta(7)).count()
  129
+        #self.posts_week = Post.all().filter("service =",self.title).filter("timestamp_created > ", datetime.datetime.now() - datetime.timedelta(7)).count()
  130
+        #self.posts_month = Post.all().filter("service =",self.title).filter("timestamp_created > ", datetime.datetime.now() - datetime.timedelta(30)).count()
203 131
         logging.info("Feed " + self.title + " has stats " + str(self.comments_day) + " " + str(self.comments_week) + " " + str(self.posts_month) + " " + str(self.posts_week))
204 132
         self.put()
205  
-    def entries(self,num=None):
206  
-        if not memcache.get(self.url):
207  
-            return [] # TODO: schedule a fetch-task !
208  
-        if num == None:
209  
-            return memcache.get(self.url)
210  
-        result = memcache.get(self.url)
211  
-        return result[0:num]
212  
-    def fetch_entries(self):
  133
+    #def entries(self,num=None):
  134
+    #    result = [entry for entry in Post.all().filter("service=",self.title)]
  135
+    #    return result[0:num]
  136
+    def fetch_feed_and_generate_entries(self,url,_type):
  137
+        if url == "":
  138
+            return
213 139
         try:
214  
-            result = urlfetch.fetch(self.url,deadline=10) # 10 is max deadline
  140
+            result = urlfetch.fetch(url,deadline=10) # 10 is max deadline
215 141
         except urlfetch.DownloadError:
216  
-            logging.warning("Downloading URL " + self.url + "failed: timeout.")
217  
-            return []
  142
+            logging.warning("Downloading URL " + url + "failed: timeout.")
  143
+            return
218 144
         except urlfetch.ResponseTooLargeError:
219  
-            logging.warning("Downloading URL " + self.url + "failed: response tooo large.")
220  
-            return []
221  
-        updates = []
  145
+            logging.warning("Downloading URL " + url + "failed: response tooo large.")
  146
+            return
  147
+        except urlfetch.InvalidURLError:
  148
+            logging.warning("Downloading URL " + url + "failed: invalid url.")
  149
+            return
222 150
         if result.status_code == 200:
223  
-            logging.info("Successfully fetched URL " + self.url)
  151
+            logging.info("Successfully fetched URL " + url)
224 152
             try:
225 153
                 feed = feedparser.parse(result.content)
226 154
                 for entry in feed['entries']:
227  
-                    try:
228  
-                        x = Entry()
229  
-                        x.service = self.title
230  
-                        x.title = entry['title']
231  
-                        x.link = html_escape(entry['link'])
232  
-                        x.length = len( get_feedparser_entry_content(entry) )
233  
-                        x.content = get_feedparser_entry_content(entry)
234  
-                        #x.cleancontent = ' '.join(BeautifulSoup(x.content).findAll(text=True))
235  
-                        #x.sanitizedcontent = x.content
236  
-                        x.homepage = self.homepage
237  
-                        try:
238  
-                            x.tags = entry.tags
239  
-                        except AttributeError:
240  
-                            x.tags = [ ]
241  
-                        try:
242  
-                            x.timestamp_updated = entry.updated_parsed
243  
-                        except AttributeError:
244  
-                            #x.timestamp = time.strptime("01.01.1970","%d.%m.%Y")
245  
-                            x.timestamp_updated = time.gmtime(0)
246  
-                        try:
247  
-                            x.timestamp_created = entry.published_parsed
248  
-                        except AttributeError:
249  
-                            try:
250  
-                                x.timestamp_created = entry.updated_parsed
251  
-                            except AttributeError:
252  
-                                #x.timestamp = time.strptime("01.01.1970","%d.%m.%Y")
253  
-                                x.timestamp_created = time.gmtime(0)
254  
-                        updates.append(x)
255  
-                    except Exception, e:
256  
-                        logging.warning("There was an error processing an Entry of the Feed " + self.title + ":" + str(e))        
  155
+                    _type.generate_entry(entry,feed,self) 
257 156
             except LookupError, e:
258  
-                logging.warning("There was an error parsing the feed " + self.title + ":" + str(e))
259  
-                    
260  
-        return updates
  157
+                logging.warning("There was an error parsing the feed " + url + ":" + str(e))
261 158
     def cse_homepage(self): # REMINDER: for CSE = google custome search engine = search for startpage
262 159
         return add_slash(strip_http(self.homepage))
263  
-    def top_entries(self):
264  
-        return self.entries()[0:10]
265  
-    def template_top(self):
266  
-        return {'title': self.title, 'entries': self.top_entries() }
  160
+    #def top_entries(self):
  161
+    #    return self.entries()[0:10]
  162
+    #def template_top(self):
  163
+    #    return {'title': self.title, 'entries': self.top_entries() }
267 164
     # comments_entries the abstract construct
268  
-    def comments_entries(self,num=None):
269  
-        if not memcache.get(self.comments):
270  
-            return [] # TODO: schedule a fetch-task !
271  
-        if num == None:
272  
-            return memcache.get(self.comments)
273  
-        result = memcache.get(self.comments)
274  
-        return result[0:num]
275  
-    # fetching entries from comment feeds (just like regular feed)
276  
-    def fetch_comments_entries(self):
277  
-        if self.comments == "":
278  
-            return []
  165
+    #def comments_entries(self,num=None):
  166
+    #    if not memcache.get(self.comments):
  167
+    #        return [] # TODO: schedule a fetch-task !
  168
+    #    if num == None:
  169
+    #        return memcache.get(self.comments)
  170
+    #    result = memcache.get(self.comments)
  171
+    #    return result[0:num]
  172
+
  173
+
  174
+
  175
+
  176
+
  177
+
  178
+
  179
+
  180
+#### The second major class: Entry
  181
+
  182
+
  183
+
  184
+class Entry(polymodel.PolyModel):
  185
+    title = db.TextProperty()
  186
+    link = db.StringProperty()
  187
+    homepage = db.StringProperty()
  188
+    service = db.StringProperty()
  189
+    timestamp_created = db.DateTimeProperty()
  190
+    timestamp_updated = db.DateTimeProperty()
  191
+    length = db.IntegerProperty()
  192
+    content = db.TextProperty()
  193
+    tags = db.StringListProperty()
  194
+    category = db.StringProperty()
  195
+    guid = db.StringProperty()
  196
+        
  197
+    # cls has to be one of the classes Post or Comment
  198
+    # entry is a feedparser entry
  199
+    # feedparser_feed is a feedparser feed
  200
+    # database_feed is corresponding Feed object
  201
+    @classmethod
  202
+    def generate_entry(cls, entry, feedparser_feed, database_feed):
279 203
         try:
280  
-            result = urlfetch.fetch(self.comments,deadline=10) # 10 is max deadline
281  
-        except urlfetch.DownloadError:
282  
-            logging.warning("Downloading URL " + self.comments + "failed: timeout.")
283  
-            return []
284  
-        except urlfetch.ResponseTooLargeError:
285  
-            logging.warning("Downloading URL " + self.comments + "failed: response tooo large.")
286  
-            return []
287  
-        except urlfetch.InvalidURLError:
288  
-            logging.warning("Downloading URL " + self.comments + "failed: invalid url.")
289  
-            return []
290  
-        comments_updates = []
291  
-        if result.status_code == 200:
292  
-            logging.info("Successfully fetched URL " + self.comments)
293  
-            try:
294  
-                feed = feedparser.parse(result.content)
295  
-                for entry in feed['entries']:
296  
-                    try:
297  
-                        x = Entry()
298  
-                        x.service = self.title
299  
-                        x.title = entry['title']
300  
-                        x.link = html_escape(entry['link'])
301  
-                        x.length = len( get_feedparser_entry_content(entry) )
302  
-                        x.homepage = self.homepage
303  
-                        try:
304  
-                            x.timestamp_updated = entry.updated_parsed
305  
-                        except AttributeError:
306  
-                            #x.timestamp = time.strptime("01.01.1970","%d.%m.%Y")
307  
-                            x.timestamp_updated = time.gmtime(0)
308  
-                        try:
309  
-                            x.timestamp_created = entry.published_parsed
310  
-                        except AttributeError:
311  
-                            try:
312  
-                                x.timestamp_created = entry.updated_parsed
313  
-                            except AttributeError:
314  
-                                #x.timestamp = time.strptime("01.01.1970","%d.%m.%Y")
315  
-                                x.timestamp_created = time.gmtime(0)
316  
-                        comments_updates.append(x)
317  
-                    except Exception, e:
318  
-                        logging.warning("There was an error processing an Entry of the Feed " + self.title + ":" + str(e))        
319  
-            except LookupError, e:
320  
-                logging.warning("There was an error parsing the feed " + self.title + ":" + str(e))
321  
-
322  
-        return comments_updates
323  
-
324  
-class Entry:
325  
-    def __init__(self=None, title=None, link=None, timestamp_created=None, timestamp_updated=None, service=None, homepage=None, length=0, content="", cleancontent="", sanitizedcontent=""):
326  
-        self.title = title
327  
-        self.link = link
328  
-        self.homepage = homepage
329  
-        self.service = service
330  
-        self.timestamp_created = timestamp_created
331  
-        self.timestamp_updated = timestamp_updated
332  
-        self.length = length
333  
-        self.content = content
334  
-        self.cleancontent = cleancontent
335  
-        self.sanitizedcontent = sanitizedcontent
  204
+            # if entry is in database, i.e.,
  205
+### PETER TRYING TO OPTIMIZE CODE
  206
+#ORIGINAL            result = cls.all().filter("service=",database_feed.title).filter("guid=",feedparser_entry_to_guid(entry)).get()
  207
+#FIRST ATTEMPT            result = cls.gql("WHERE guid = :1", feedparser_entry_to_guid(entry))[0]
  208
+            if cls.gql("WHERE guid = :1", feedparser_entry_to_guid(entry)).count() != 0:
  209
+               logging.info("guid exists: " + entry['title'])
  210
+#ORIGINAL            if result != None:
  211
+#ORIGINAL                #   update entry attributes if changed?
  212
+#ORIGINAL               # pass
  213
+            else:
  214
+                #   add entry to database!
  215
+                x = cls()
  216
+                x.service = html_escape(database_feed.title)
  217
+                x.title = html_escape(entry['title'])
  218
+                x.link = html_escape(entry['link'])
  219
+                x.length = len( get_feedparser_entry_content(entry) )
  220
+                x.content = get_feedparser_entry_content(entry)
  221
+                x.category = database_feed.category
  222
+                x.homepage = html_escape(database_feed.homepage)
  223
+                try:
  224
+                    x.tags = [ string.capwords(tag.term) for tag in entry.tags ]
  225
+                except AttributeError:
  226
+                    x.tags = [ ]
  227
+                t = time.gmtime(0)
  228
+                try:
  229
+                    t = entry.updated_parsed
  230
+                except AttributeError:
  231
+                    pass
  232
+                x.timestamp_updated = datetime.datetime(t[0],t[1],t[2],t[3],t[4],t[5])
  233
+                try:
  234
+                    t = entry.published_parsed
  235
+                except AttributeError:
  236
+                    pass
  237
+                x.timestamp_created = datetime.datetime(t[0],t[1],t[2],t[3],t[4],t[5])
  238
+                x.guid = feedparser_entry_to_guid(entry)
  239
+                x.put()
  240
+        except Exception, e:
  241
+            logging.warning("There was an error processing an Entry of the Feed :" + str(e))
336 242
 
337 243
     def printTime_created_rfc3339(self):
338 244
         try:
339  
-            res = strftime('%Y-%m-%dT%H:%M:%SZ',self.timestamp_created)
  245
+            res = self.timestamp_created.strftime('%Y-%m-%dT%H:%M:%SZ')
340 246
         except TypeError:
341 247
             res = ""
342 248
         return res
343 249
 
344 250
     def printTime_updated_rfc3339(self):
345 251
         try:
346  
-            res = strftime('%Y-%m-%dT%H:%M:%SZ',self.timestamp_updated)
  252
+            res = self.timestamp_updated.strftime('%Y-%m-%dT%H:%M:%SZ')
347 253
         except TypeError:
348 254
             res = ""
349 255
         return res
@@ -360,6 +266,7 @@ def printTime_updated(self):
360 266
         except TypeError:
361 267
             res = ""
362 268
         return res
  269
+### Is gettime used anywhere???
363 270
     def gettime(self): #REMINDER for future code reading: change name to gettime_created -- after Felix fixes/improves statsview to fix the bug
364 271
         if self.timestamp_created == None:
365 272
             return time.gmtime(0)
@@ -367,16 +274,25 @@ def gettime(self): #REMINDER for future code reading: change name to gettime_cre
367 274
             return self.timestamp_created
368 275
     def printShortTime_created(self):
369 276
         try:
370  
-            today = time.localtime()
371  
-            if today[0] == self.timestamp_created[0] and today[1] <= self.timestamp_created[1] and today[2] <= self.timestamp_created[2]:
  277
+            today = datetime.datetime.now()
  278
+            if today.year == self.timestamp_created.year and today.month <= self.timestamp_created.month and today.day <= self.timestamp_created.day:
372 279
                 return "today"
373  
-            #if today[0] == self.timestamp[0] and today[1] <= self.timestamp[1] and today[2] - 1 <= self.timestamp[2]:
374  
-            #    return "yesterday"
375  
-            res = strftime('%b %d',self.timestamp_created)
376  
-        except TypeError:
  280
+            res = self.timestamp_created.strftime('%b %d')
  281
+        except TypeError, e:
  282
+            logging.warning("problem: " + str(e))
377 283
             res = ""
378 284
         return res
379 285
 
  286
+class Post(Entry):
  287
+    iamapost = db.StringProperty()
  288
+
  289
+class Comment(Entry):
  290
+    iamacomment = db.StringProperty()
  291
+
  292
+
  293
+### The smaller classes:  preliminary work for generating actual webpages
  294
+
  295
+
380 296
 class MainPage(webapp.RequestHandler):
381 297
   def get(self):
382 298
       self.redirect("/content/start.html")
@@ -390,13 +306,21 @@ def get(self, string):
390 306
       return db.GqlQuery(string)
391 307
 
392 308
 class CachedPage(webapp.RequestHandler):
  309
+    # NOTE: the empty string as cacheName turns off caching
393 310
     cacheName = "default"
394 311
     cacheTime = 2700
395 312
     def get(self):
396  
-        if not memcache.get(self.cacheName):
397  
-            memcache.set(self.cacheName,self.generatePage(),self.cacheTime)
398  
-        #self.response.headers['Cache-Control'] = 'public; max-age=2700;'
399  
-        self.response.out.write(memcache.get(self.cacheName))
  313
+        if self.cacheName == "":
  314
+            self.response.out.write(self.generatePage())
  315
+        else:
  316
+            if not memcache.get(self.cacheName):
  317
+                memcache.set(self.cacheName,self.generatePage(),self.cacheTime)
  318
+            #self.response.headers['Cache-Control'] = 'public; max-age=2700;'
  319
+            self.response.out.write(memcache.get(self.cacheName))
  320
+
  321
+class TemplatePage(CachedPage):
  322
+    def generatePage(self):
  323
+        return header + menu + "<div class='content'>" + self.generateContent() + disqus + "</div>" + footer + "</body></html>"
400 324
 
401 325
 class SimpleCheetahPage(CachedPage):
402 326
     templateName = "default.tmpl"
@@ -405,6 +329,8 @@ def generatePage(self):
405 329
         path = os.path.join(os.path.dirname(__file__), self.templateName)
406 330
         return str(Template( file = path, searchList = (template_values,) ))
407 331
 
  332
+### static pages
  333
+
408 334
 class StartPage(SimpleCheetahPage):
409 335
     cacheName = "StartPage"
410 336
     templateName = "start.tmpl"
@@ -417,17 +343,22 @@ class FeedsPage(SimpleCheetahPage):
417 343
     cacheName = "FeedsPage"
418 344
     templateName = "feeds.tmpl"
419 345
 
420  
-class CategoryView(SimpleCheetahPage):
421  
-    cacheName = "CategoryView"
422  
-    templateName = "bycategory.tmpl"
  346
+### the old Dynamic pages -- OBSOLETE?
  347
+
  348
+#class CategoryView(SimpleCheetahPage):
  349
+#    cacheName = "CategoryView"
  350
+#    templateName = "bycategory.tmpl"
423 351
 
424 352
 class WeeklyPicks(SimpleCheetahPage):
425  
-       cacheName = "WeeklyPicks"
426  
-       def generatePage(self):
427  
-        entries = [ entry for feed in Feed.all().filter("person =","Mathblogging.org") for entry in feed.entries() ]
428  
-        has_tag = lambda entry: len(filter(lambda tag: tag.term.lower() == "weekly picks", entry.tags)) > 0
429  
-        picks = filter(has_tag, entries)
430  
-        picks.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
  353
+    cacheName = "WeeklyPicks"
  354
+    # TODO: do everything in template???
  355
+    def generatePage(self):
  356
+        # ############ PROBLEM SOLVED ############
  357
+        # This is how to test list membership (even though it is counter-intuitive), see:
  358
+        #  http://wenku.baidu.com/view/7861b3f9aef8941ea76e0562.html
  359
+        #  http://code.google.com/p/typhoonae/issues/detail?id=40
  360
+        # 
  361
+        picks = Post.gql("WHERE service = 'Mathblogging.org - the Blog' AND tags = 'weekly picks' ORDER BY timestamp_created LIMIT 15")
431 362
         template_values = { 'qf': QueryFactory(), 'picks_entries': picks, 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header}
432 363
         
433 364
         path = os.path.join(os.path.dirname(__file__), 'weekly_picks.tmpl')
@@ -436,6 +367,7 @@ def generatePage(self):
436 367
         
437 368
 class StatsView(CachedPage):
438 369
     cacheName = "StatsView"
  370
+    # TODO: do everything in template???
439 371
     def generatePage(self):
440 372
         feeds_w_comments_day = db.GqlQuery("SELECT * FROM Feed WHERE comments_day != 0 ORDER BY comments_day DESC").fetch(1000)
441 373
         feeds_w_comments_week = db.GqlQuery("SELECT * FROM Feed WHERE comments_week != 0 ORDER BY comments_week DESC").fetch(1000)
@@ -447,91 +379,47 @@ def generatePage(self):
447 379
         renderedString = str(Template( file = path, searchList = (template_values,) ))
448 380
         return renderedString
449 381
 
450  
-class DateView(CachedPage):
451  
-    cacheName = "DateView"
452  
-    def generatePage(self):
453  
-        all_entries = [ entry for feed in Feed.all().filter("category !=","institution").filter("category !=","community") for entry in feed.entries() ]
454  
-        all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
455  
-        template_values = { 'qf':  QueryFactory(), 'allentries': all_entries[0:150], 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
456  
-        path = os.path.join(os.path.dirname(__file__), 'bydate.tmpl')
457  
-        return str(Template( file = path, searchList = (template_values,) ))
458  
-
459 382
         
460 383
 class DateResearchView(CachedPage):
  384
+    cacheName = "DateResearchView"
  385
+    # TODO: do everything in template???
461 386
     def get(self):
462  
-        all_entries = [ entry for feed in Feed.all().filter("category =","pure") for entry in feed.entries() ]
463  
-        applied_entries = [ entry for feed in Feed.all().filter("category =","applied") for entry in feed.entries() ]
464  
-        all_entries.extend(applied_entries)
465  
-        all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
466  
-        template_values = { 'qf':  QueryFactory(), 'allentries': all_entries[0:150], 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
  387
+        template_values = { 'qf':  QueryFactory(), 
  388
+                            'allentries': Post.gql("WHERE category IN ['pure','applied'] ORDER BY timestamp_created LIMIT 150"), 
  389
+                            'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
467 390
 
468 391
         path = os.path.join(os.path.dirname(__file__), 'bydate.tmpl')
469 392
         self.response.out.write(Template( file = path, searchList = (template_values,) ))
470 393
 
471 394
 class DateHisArtVisView(CachedPage):
  395
+    cacheName = "DateHisArtVisView"
472 396
     def get(self):
473  
-        all_entries = [ entry for feed in Feed.all().filter("category =","visual") for entry in feed.entries() ]
474  
-        history_entries = [ entry for feed in Feed.all().filter("category =","history") for entry in feed.entries() ]
475  
-        visual_entries = [ entry for feed in Feed.all().filter("category =","art") for entry in feed.entries() ]
476  
-        all_entries.extend(visual_entries)
477  
-        all_entries.extend(history_entries)
478  
-        all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
479  
-        template_values = { 'qf':  QueryFactory(), 'allentries': all_entries[0:150], 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
  397
+        template_values = { 'qf':  QueryFactory(), 
  398
+                            'allentries': Post.gql("WHERE category IN ['visual','history','art'] ORDER BY timestamp_created LIMIT 150"),
  399
+                            'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
480 400
 
481 401
         path = os.path.join(os.path.dirname(__file__), 'bydate.tmpl')
482 402
         self.response.out.write(Template( file = path, searchList = (template_values,) ))
483 403
 
484 404
 class DateTeacherView(CachedPage):
  405
+    cacheName = "DateTeacherView"
485 406
     def get(self):
486  
-        all_entries = [ entry for feed in Feed.all().filter("category =","teacher") for entry in feed.entries() ]
487  
-        all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
488  
-        template_values = { 'qf':  QueryFactory(), 'allentries': all_entries[0:150], 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
  407
+        template_values = { 'qf':  QueryFactory(), 
  408
+                            'allentries': Post.gql("WHERE category = 'teacher' ORDER BY timestamp_created LIMIT 150"),
  409
+                            'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
489 410
 
490 411
         path = os.path.join(os.path.dirname(__file__), 'bydate.tmpl')
491 412
         self.response.out.write(Template( file = path, searchList = (template_values,) ))
492  
-# outdated
493  
-#class TagsView(webapp.RequestHandler):
494  
-    #def get(self):
495  
-        #all_entries = [ entry for feed in Feed.all().filter("category !=","micro").filter("category !=","community") for entry in feed.entries() ]
496  
-        #all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
497  
-        #template_values = { 'qf':  QueryFactory(), 'allentries': all_entries, 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
498  
-    
499  
-        #path = os.path.join(os.path.dirname(__file__), 'bytags.tmpl')
500  
-        #self.response.out.write(Template( file = path, searchList = (template_values,) ))
501  
-
502  
-class PlanetMath(webapp.RequestHandler):
503  
-    def get(self):
504  
-        all_entries = [ entry for feed in Feed.all() for entry in feed.entries() ]
505  
-        has_tag_math = lambda entry: len(filter(lambda tag: tag.term.lower().find("math") == 0, entry.tags)) > 0
506  
-        entries_tagged_math = filter(has_tag_math, all_entries)
507  
-        entries_tagged_math.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
508  
-        template_values = { 'qf':  QueryFactory(), 'mathentries': entries_tagged_math[0:20], 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
509  
-    
510  
-        path = os.path.join(os.path.dirname(__file__), 'planetmath.tmpl')
511  
-        self.response.out.write(Template( file = path, searchList = (template_values,) ))
512 413
 
513 414
 class PlanetMO(webapp.RequestHandler):
514 415
     def get(self):
515  
-        all_entries = [ entry for feed in Feed.all() for entry in feed.entries() ]
516  
-        has_tag_math = lambda entry: len(filter(lambda tag: tag.term.lower() == "mathoverflow" or tag.term.lower() == "mo" or tag.term.lower() == "planetmo", entry.tags)) > 0
517  
-        entries_tagged_math = filter(has_tag_math, all_entries)
518  
-        entries_tagged_math.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
519  
-        template_values = { 'qf':  QueryFactory(), 'moentries': entries_tagged_math[0:50], 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header}
  416
+        template_values = { 'qf':  QueryFactory(), 
  417
+                            'moentries': Post.gql("WHERE tags IN ['mathoverflow','mo','planetmo'] ORDER BY timestamp_created LIMIT 50"), 
  418
+                            'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header}
520 419
     
521 420
         path = os.path.join(os.path.dirname(__file__), 'planetmo.tmpl')
522 421
         self.response.out.write(Template( file = path, searchList = (template_values,) ))
523 422
 
524  
-class PlanetMOfeed(webapp.RequestHandler):
525  
-    def get(self):
526  
-        all_entries = [ entry for feed in Feed.all() for entry in feed.entries() ]
527  
-        has_tag_math = lambda entry: len(filter(lambda tag: tag.term.lower() == "mathoverflow" or tag.term.lower() == "mo" or tag.term.lower() == "planetmo", entry.tags)) > 0
528  
-        entries_tagged_math = filter(has_tag_math, all_entries)
529  
-        entries_tagged_math.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
530  
-        template_values = { 'qf':  QueryFactory(), 'allentries': entries_tagged_math, 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
531  
-    
532  
-        path = os.path.join(os.path.dirname(__file__), 'atom.tmpl')
533  
-        self.response.out.write(Template( file = path, searchList = (template_values,) ))
534  
-
535 423
 # Database output
536 424
 class CsvView(webapp.RequestHandler):
537 425
     def get(self):
@@ -541,246 +429,106 @@ def get(self):
541 429
         self.response.headers['Content-Type'] = 'text/csv'
542 430
         self.response.out.write(Template( file = path, searchList = (template_values,) ))
543 431
 
544  
-# deprecated
545  
-#class SearchView(webapp.RequestHandler):
546  
-    #def get(self):
547  
-        #all_entries = [ entry for feed in Feed.all().filter("category !=","micro").filter("category !=","community") for entry in feed.entries() ]
548  
-        #all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
549  
-        #template_values = { 'qf':  QueryFactory(), 'allentries': all_entries[0:150], 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
550  
-    
551  
-        #path = os.path.join(os.path.dirname(__file__), 'search.tmpl')
552  
-        #self.response.out.write(Template( file = path, searchList = (template_values,) ))
  432
+## Database OPML output
  433
+#class OPMLView(webapp.RequestHandler):
  434
+#    def get(self):
  435
+#        template_values = { 'qf':  QueryFactory(), 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header}
  436
+#    
  437
+#        path = os.path.join(os.path.dirname(__file__), 'opml.tmpl')
  438
+#        self.response.headers['Content-Type'] = 'text/xml'
  439
+#        self.response.out.write(Template( file = path, searchList = (template_values,) ))
553 440
 
554  
-# google custom search engine
  441
+#### CSE = google custom search engine
555 442
 class CSEConfig(webapp.RequestHandler):
556 443
     def get(self):
557  
-        all_entries = [ entry for feed in Feed.all().filter("category !=","community") for entry in feed.entries() ]
558  
-        all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
559  
-        template_values = { 'qf':  QueryFactory(), 'allentries': all_entries[0:150], 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
  444
+        template_values = { 'qf':  QueryFactory(), 'menu': menu, 'footer': footer, 'disqus': disqus, 'header': header }
560 445
     
561 446
         path = os.path.join(os.path.dirname(__file__), 'cse-config.tmpl')
562 447
         self.response.out.write(Template( file = path, searchList = (template_values,) ))
563 448
 
564 449
 
565  
-class FeedHandlerBase(CachedPage):
566  
-    def generatePage(self):
567  
-        all_entries = [ entry for feed in self.feeds() for entry in feed.entries() ]
568  
-        all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
569  
-        template_values = { 'qf':  QueryFactory(), 'allentries': all_entries[0:150], 'menu': menu, 'disqus': disqus, 'header': header }
570  
-    
571  
-        path = os.path.join(os.path.dirname(__file__), 'atom.tmpl')
572  
-        return str(Template( file = path, searchList = (template_values,) ))
573  
-        
574  
-class FeedHandlerAll(FeedHandlerBase):
575  
-    cacheName = "FeedAll"
576  
-    def feeds(self):
577  
-        return Feed.all()
578  
-
579  
-class FeedHandlerResearchers(CachedPage):
580  
-    cacheName = "FeedResearchers"
581  
-    def get(self):
582  
-        all_entries = [ entry for feed in Feed.all().filter("category =","pure") for entry in feed.entries() ]
583  
-        applied_entries = [ entry for feed in Feed.all().filter("category =","applied") for entry in feed.entries() ]
584  
-        history_entries = [ entry for feed in Feed.all().filter("category =","history") for entry in feed.entries() ]
585  
-        all_entries.extend(applied_entries)
586  
-        all_entries.extend(history_entries)
587  
-        all_entries.sort( lambda a,b: - cmp(a.timestamp_created,b.timestamp_created) )
588  
-        template_values = { 'qf':  QueryFactory(), 'allentries': all_entries[0:150], 'menu': menu, 'footer': footer, 'disqus':  disqus, 'header': header }
589  
-
590  
-        path = os.path.join(os.path.dirname(__file__), 'atom.tmpl')
591  
-        self.response.out.write(Template( file = path, searchList = (template_values,) ))    
592  
-
593  
-class FeedHandlerPure(FeedHandlerBase):
594  
-    cacheName = "FeedPure"
595  
-    def feeds(self):
596  
-        return Feed.all().filter("category =","pure")
597  
-
598  
-class FeedHandlerApplied(FeedHandlerBase):
599  
-    cacheName = "FeedApplied"
600  
-    def feeds(self):
601  
-        return Feed.all().filter("category =","applied")
602  
-
603  
-class FeedHandlerHistory(FeedHandlerBase):
604  
-    cacheName = "FeedHistory"
605  
-    def feeds(self):
606  
-        return Feed.all().filter("category =","history")
607  
-
608  
-class FeedHandlerVisual(FeedHandlerBase):
609  
-    cacheName = "FeedVisual"
610  
-    def feeds(self):
611  
-        return Feed.all().filter("category =","visual")
612  
-
613  
-class FeedHandlerArt(FeedHandlerBase):
614  
-    cacheName = "FeedArt"
615  
-    def feeds(self):
616  
-        return Feed.all().filter("category =","art")
617  
-
618  
-class FeedHandlerTeachers(FeedHandlerBase):
619  
-    cacheName = "FeedTeachers"
620  
-    def feeds(self):
621  
-        return Feed.all().filter("category =","teacher")
622  
-
623  
-class FeedHandlerFun(FeedHandlerBase):
624  
-    cacheName = "FeedFun"
625  
-    def feeds(self):
626  
-        return Feed.all().filter("category =","fun")
627  
-
628  
-class FeedHandlerGeneral(FeedHandlerBase):
629  
-    cacheName = "FeedGeneral"
630  
-    def feeds(self):
631  
-        return Feed.all().filter("category =","general")
632  
-
633  
-class FeedHandlerJournals(FeedHandlerBase):
634  
-    cacheName = "FeedJournals"
635  
-    def feeds(self):
636  
-        return Feed.all().filter("category =","journal")
637  
-
638  
-class FeedHandlerJournalism(FeedHandlerBase):
639  
-    cacheName = "FeedJournalism"
640  
-    def feeds(self):
641  
-        return Feed.all().filter("category =","journalism")
642  
-    
643  
-class FeedHandlerInstitutions(FeedHandlerBase):
644  
-    cacheName = "FeedInstitutions"
645  
-    def feeds(self):
646  
-        return Feed.all().filter("category =","institution")
647  
-
648  
-class FeedHandlerCommunities(FeedHandlerBase):
649  
-    cacheName = "FeedCommunities"
650  
-    def feeds(self):
651  
-        return Feed.all().filter("category =","community")
652  
-
653  
-class FeedHandlerCommercial(FeedHandlerBase):
654  
-    cacheName = "FeedCommercial"
655  
-    def feeds(self):
656  
-        return Feed.all().filter("category =","commercial")
657  
-
658  
-
659  
-class FeedHandlerPeople(FeedHandlerBase):
660  
-    cacheName = "FeedPeople"
661  
-    def feeds(self):
662  
-        return Feed.all().filter("category !=","community").filter("category !=","institution").filter("category !=","journals").filter("category !=","commercial")
663  
-
664  
-# replaced by researcher
665  
-#class FeedHandlerAcademics(FeedHandlerBase):
666  
-    #cacheName = "FeedAcademics"
667  
-    #def feeds(self):
668  
-        #return Feed.all().filter("category !=","community").filter("category !=","educator").filter("category !=","journalism")       
669  
-    
670  
-    
671  
-
  450
+### Worker classes: downloading the feeds
672 451
 class FetchWorker(webapp.RequestHandler):
673 452
     def post(self):
674 453
         try:
675 454
             url = self.request.get('url')
676 455
             logging.info("FetchWorker: " + url)
677 456
             if url:
678  
-                feed = Feed.all().filter("url =", url).get()
  457
+#ORIGINAL                feed = Feed.all().filter("posts_url =", url).get()
  458
+                feed = Feed.gql("WHERE posts_url = :1", url).get()
679 459
                 if feed:
680  
-                    feed.restore_cache()
  460
+                    feed.update_database()
681 461
             self.response.set_status(200)
682