use Ajax to get news / packages info & delete bad cache

The page gets rendered instantly, since we can fetch the rss data in the backgournd using jQuery and ajax. Bad feed cache data will be deleted and refetched when necessary. Some versions of python 2.7 bsddb module is broke.
pyther · Dec 10, 2011 · 6bf18ab · 6bf18ab
1 parent 4bbc812
commit 6bf18ab
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 62 deletions.
diff --git a/index.py b/index.py
@@ -24,13 +24,14 @@
 import sys
 import shelve
 
-import os.path
+import os
 import datetime
 
 from arch import Arch
 
 urls = (
-  '/', 'index'
+  '/', 'index',
+  '/fetch', 'fetchFeeds'
 )
 
 web.config.debug = True
@@ -40,50 +41,24 @@
 app = web.application(urls, globals())
 web.template.Template.globals['render'] = render
 
-#Downloads RSS for news feed items
-def getFeeds():
-
-    #Time until cached feeds expire
-    timeToLiveSeconds=3600   #60 Minutes
-
-    #Stores file as .nfeed_cache in CWD
-    newsPath = shelve.open('./cache/news_cache')
-    i686Path = shelve.open('./cache/i686_cache')
-    x86_64Path = shelve.open('./cache/x86_64_cache')
-
-    #Fetches the feed from cache or from the website
-    news = feedcache.Cache(newsPath,timeToLiveSeconds).fetch('http://www.archlinux.org/feeds/news/')
-    i686 = feedcache.Cache(i686Path,timeToLiveSeconds).fetch('http://www.archlinux.org/feeds/packages/i686/')
-    x86_64 = feedcache.Cache(x86_64Path,timeToLiveSeconds).fetch('http://www.archlinux.org/feeds/packages/x86_64/')
-
-    #Closes feed
-    newsPath.close()
-    i686Path.close()
-    x86_64Path.close()
-
-    return news, i686, x86_64
+#Time until cached feeds expire
+timeToLiveSeconds=3600   #60 Minutes
 
+# Feeds
+newsRss='http://www.archlinux.org/feeds/news/'
+x86Rss='http://www.archlinux.org/feeds/packages/i686/'
+x64Rss='http://www.archlinux.org/feeds/packages/x86_64/'
 
+newsFile='./cache/news_cache'
+x86File='./cache/i686_cache'
+x64File='./cache/x86_64_cache'
 
 class index:
     def GET(self):
 
         #web.header('Content-Type','application/xhtml+xml; charset=utf-8')
 
-        # Store title and url for news together, only store 4 entries
-        newsFeed, i686Feed, x86_64Feed = getFeeds();
-
-        news = [(x.title, x.link) for x in newsFeed.entries][:maxNEWS]
-        i686PKGs = [(x.title, x.category, x.link, x.summary) for x in i686Feed.entries][:maxPKGS]
-        x86_64PKGs = [(x.title, x.category, x.link, x.summary) for x in x86_64Feed.entries][:maxPKGS]
-
-        i686=Arch()
-        x86_64=Arch()
-
-        i686.add_packages(i686PKGs)
-        x86_64.add_packages(x86_64PKGs)
-
-        return render.index(news, i686, x86_64)
+        return render.index()
 
     #This function will get the search query and process it...
     def POST(self):
@@ -111,6 +86,78 @@ def POST(self):
         else:
             return web.badrequest()
 
+class fetchFeeds:
+    def GET(self):
+        # Corrupt Cache
+        #  * Importing bsddb fails on many 2.7 instances,
+        #    python tries to import corrupt cached using bsddb, this fails
+        #  * DBPageNotFoundError, since python tried to import corrupt cache
+        #    as a bsddb file, and it is not, it will error out
+
+        i = web.input()
+        feed = web.websafe(i.feed)
+
+        if feed == 'news':
+            try:
+                newsCache = shelve.open(newsFile)
+            except ImportError:
+                os.remove(newsFile)
+                newsCache = shelve.open(newsFile)
+
+            try:
+                newsFeed = feedcache.Cache(newsCache,timeToLiveSeconds).fetch(newsRss)
+            except:
+                newsCache.close()
+                newsCache = shelve.open(newsFile)
+                os.remove(newsFile)
+                newsFeed = feedcache.Cache(newsCache,timeToLiveSeconds).fetch(newsRss)
+            newsCache.close()
+
+            news = [(x.title, x.link) for x in newsFeed.entries][:maxNEWS]
+            return render.news(news)
+        elif feed == 'i686':
+            try:
+                x86Cache = shelve.open(x86File)
+            except:
+                os.remove(x86File)
+                x86Cache = shelve.open(x86File)
+
+            try:
+                x86Feed = feedcache.Cache(x86Cache,timeToLiveSeconds).fetch(x86Rss)
+            except:
+                x86Cache.close()
+                os.remove(x86File)
+                x86Cache = shelve.open(x86File)
+                x86Feed = feedcache.Cache(x86Cache,timeToLiveSeconds).fetch(x86Rss)
+            x86Cache.close()
+
+            x86Pkgs = [(x.title, x.category, x.link, x.summary) for x in x86Feed.entries][:maxPKGS]
+            x86=Arch()
+            x86.add_packages(x86Pkgs)
+
+            return render.packages(x86)
+        elif feed == 'x86_64':
+            try:
+                x64Cache = shelve.open(x64File)
+            except ImportError:
+                os.remove(x64File)
+                x64Cache = shelve.open(x64File)
+
+            try:
+                x64Feed = feedcache.Cache(x64Cache,timeToLiveSeconds).fetch(x64Rss)
+            except:
+                x64Cache.close()
+                os.remove(x64File)
+                x64Cache = shelve.open(x64File)
+                x64Feed = feedcache.Cache(x64Cache,timeToLiveSeconds).fetch(x64Rss)
+            x64Cache.close()
+
+            x64Pkgs = [(x.title, x.category, x.link, x.summary) for x in x64Feed.entries][:maxPKGS]
+            x64=Arch()
+            x64.add_packages(x64Pkgs)
+        return render.packages(x64)
+
+
 if __name__ == "__main__":
     if SERVER is "webpy" or SERVER is "lighttpd":
         app.run()

diff --git a/static/jquery.js b/static/jquery.js
diff --git a/templates/index.html b/templates/index.html
@@ -1,9 +1,17 @@
-$def with (news, i686, x86_64)
+$def with ()
 
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> 
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
     <head>
+        <script type="text/javascript" src="./static/jquery.js"></script>
+        <script type="text/javascript">
+                \$(document).ready(function () {
+                    \$('#top').load('./fetch?feed=news');
+                    \$('#i686pkgs').load('./fetch?feed=i686');
+                    \$('#x86_64pkgs').load('./fetch?feed=x86_64');
+                });
+        </script>
         <title>Home Page</title>
         <style type="text/css">
             #top{
@@ -12,6 +20,7 @@
                 font-family: "DejaVu Sans", "Bitstreams Vera Sans", helvetica, arial;
                 padding: 0.18em;
                 margin: 0px;
+                height: auto;
                 position: absolute;
                 top: 0;
                 left: 0;
@@ -102,15 +111,9 @@
         </style>
     </head>
     <body onload="document.forms['f'].elements['q'].focus();">
-        $#Links
-        $if news:
             <div id="top">
-                News:
-                <a href="$:news[0][1]">$:news[0][0]</a>&nbsp;-&nbsp;
-                <a href="$:news[1][1]">$:news[1][0]</a>&nbsp;-&nbsp; 
-                <a href="$:news[2][1]">$:news[2][0]</a>&nbsp;-&nbsp;
-                <a href="$:news[3][1]">$:news[3][0]</a>
-            </div> 
+                News: Loading...
+            </div>
         <div id="cont">
                 <a href="http://archlinux.org"><img src="static/logo.png" alt="Logo" /></a><br />
                 <form name="f" method="post" action="./">
@@ -132,23 +135,19 @@
             <h2>Recent Updates</h2>
                 <div id="i686">
                     <h3>i686</h3>
-                    <ul>
-                        $for x in i686.packages:
-                            $if x['pkgrepo']=="Testing" or x['pkgrepo']=="Community-Testing":
-                                <li style="font-style: italic;" title="$x['pkgname'] - $x['pkgdesc']"><a href="$x['pkgurl']">$x['short_pkgname']</a></li>
-                            $else:
-                                <li title="$x['pkgname'] - $x['pkgdesc']"><a href="$x['pkgurl']">$x['short_pkgname']</a></li>
-                    </ul>
+                    <div id="i686pkgs">
+                        <ul>
+                            <li>Loading...</li>
+                        </ul>
+                    </div>
                 </div>
                 <div id="x86_64">
                     <h3>x86_64</h3>
-                    <ul>
-                        $for x in x86_64.packages:
-                            $if x['pkgrepo']=="Testing" or x['pkgrepo']=="Community-Testing":
-                                <li style="font-style: italic;" title="$x['pkgname'] - $x['pkgdesc']"><a href="$x['pkgurl']">$x['short_pkgname']</a></li>
-                            $else:
-                                <li title="$x['pkgname'] - $x['pkgdesc']"><a href="$x['pkgurl']">$x['short_pkgname']</a></li>
-                    </ul>
+                    <div id="x86_64pkgs">
+                        <ul>
+                            <li>Loading...</li>
+                        </ul>
+                    </div>
                 </div>
         </div>
         <div id="footer">

diff --git a/templates/news.html b/templates/news.html
@@ -0,0 +1,7 @@
+$def with (news)
+
+News:
+<a href="$:news[0][1]">$:news[0][0]</a>&nbsp;-&nbsp;
+<a href="$:news[1][1]">$:news[1][0]</a>&nbsp;-&nbsp;
+<a href="$:news[2][1]">$:news[2][0]</a>&nbsp;-&nbsp;
+<a href="$:news[3][1]">$:news[3][0]</a>
diff --git a/templates/packages.html b/templates/packages.html
@@ -0,0 +1,9 @@
+$def with (arch)
+
+<ul>
+    $for x in arch.packages:
+        $if x['pkgrepo']=="Testing" or x['pkgrepo']=="Community-Testing":
+            <li style="font-style: italic;" title="$x['pkgname'] - $x['pkgdesc']"><a href="$x['pkgurl']">$x['short_pkgname']</a></li>
+        $else:
+            <li title="$x['pkgname'] - $x['pkgdesc']"><a href="$x['pkgurl']">$x['short_pkgname']</a></li>
+</ul>