caching, robots

app/config/settings.py

@@ -25,7 +25,8 @@
     - [web-frameworks+python](http://www.gaecupboard.com/tag/web-frameworks/python?category=Libraries) in the _Libraries_ category  
     - [reviews](http://www.gaecupboard.com/tag/reviews?category=Articles) in the _Articles_ category  
     - [rants](http://www.gaecupboard.com/tag/rants?category=Articles)  in the _Articles_ category  
-    - [java](http://www.gaecupboard.com/tag/java?category=Books)  in the _Books_ category
+    - [java](http://www.gaecupboard.com/tag/java?category=Books)  in the _Books_ category  
+    - [pricing](http://www.gaecupboard.com/tag/pricing?category=Articles)  in the _Articles_ category  
 
 This tool is a work in progress, so expect errors, dragons and other nasty things.    
 
@@ -66,6 +67,7 @@
 GENERIC_ERROR = "Ooooops, it works on my machine. Please try again later."
 NOT_FOUND_ERROR = "Not found!"
 SERVER_ERROR = "Server problem"
+BOTS=['Googlebot','Slurp','Twiceler','msnbot','KaloogaBot','YodaoBot','Baiduspider','googlebot','Speedy Spider','DotBot']
 
 #Mapping Discovery
 AUTO_CONTENT_BY_LINK = {

app/config/urls.py

@@ -12,7 +12,7 @@
   '/post/([\w_-]+)(?:/([\w-]+))?/?', 'app.controllers.main.Post',
   '/about', 'app.controllers.main.About',
   '/search', 'app.controllers.main.Search',
-  '/img/([\w_-]+)', 'app.controllers.main.Image',
+  '/img/([\w_-]+)(?:/.*)?', 'app.controllers.main.Image',
   '/ajax/tags','app.controllers.ajax.Tags',
   '/ajax/links','app.controllers.ajax.Links',
   '/admin','app.controllers.admin.Admin',

app/controllers/main.py

@@ -1,16 +1,21 @@
-from app.config.settings import *
-from app.config.urls import sitemap_urls
-import app.db.models as models
-import app.utility.utils as utils
-from app.utility.utils import cachepage
+import os
+import logging
+import re
+
 from google.appengine.api import memcache
-import logging, web, re
 from google.appengine.ext import ereporter
 from google.appengine.ext import db
 from google.appengine.api import users
 from google.appengine.api import mail
 from google.appengine.ext.db import BadKeyError
 
+import web
+from app.config.settings import *
+from app.config.urls import sitemap_urls
+import app.db.models as models
+import app.utility.utils as utils
+from app.utility.utils import cachepage
+
 ereporter.register_logger()
 
 render = web.render 
@@ -79,7 +84,11 @@ def GET(self, post_id = None, slug = None):
                 #Return to canonical if the slug is truncated
                 if slug and slug.strip() != post.slug:
                     return web.seeother('/post/%s/%s' % (post_id, post.slug))
-                prev_post, next_post = models.Post.get_prev_next(post)
+                #ALERT:Trying to save some reads
+                if not utils.check_useragent_for_bots(os.environ.get('HTTP_USER_AGENT')):
+                    prev_post, next_post = models.Post.get_prev_next(post)
+                else:
+                    prev_post = next_post = None
             else:
                 raise web.notfound()
 
@@ -131,12 +140,15 @@ def GET(self, tags = None):
         posts_count = models.Post.get_posts_count(tags_filter = tags, category_filter= category)
         posts = models.Post.get_posts(page, limit = POSTS_PER_PAGE, offset = POSTS_PER_PAGE * (page - 1), tags_filter = tags, category_filter= category)
 
+        if not posts:
+            raise web.notfound(render.layout(render.index([]), title ='Home', navbar = True, is_user_admin = users.is_current_user_admin()))
+
         return render_template(render.index(posts, 
                                             tags, 
                                             category, 
                                             pagination = utils.Pagination(posts_count, page, POSTS_PER_PAGE),
                                             is_user_admin = users.is_current_user_admin()),
-                                            title = "%s %s" % (category if category else 'Home',' '.join(tags)))
+                                            title = "%s %s" % (category if category else '',' '.join(tags)))
 
 
 class TagCloud:
@@ -219,20 +231,27 @@ class Robots:
        Robots
        """
        def GET(self):
-           return render.robots()          
+           disallow_all = False
+           if REDIRECT_FROM_APPENGINE_HOST_TO_HOST and os.environ.get('HTTP_HOST').endswith(APPENGINE_HOST):
+                disallow_all = True
+           return render.robots(disallow_all)          
 
 
 class Image:
      """
      Image
      """
      def GET(self, post_id):
-         post = models.Post.get_post(post_id)
+         try:
+             post = models.Post.get_post(post_id)
 
-         if post and post.thumbnail:
-             web.header('Content-type', 'image/png')
-             return post.thumbnail
-         else:
+             if post and post.thumbnail:
+                 web.header('Content-type', 'image/png')
+                 web.header('Cache-Control','public, max-age=300')
+                 return post.thumbnail
+             else:
+                 raise web.notfound()
+         except:
              raise web.notfound()
 
 

app/db/models.py

@@ -95,7 +95,7 @@ def get_post(post_id):
 
     def get_image_path(self):
         if self.thumbnail is not None:
-            return "/img/%s" % self.key()
+            return "/img/%s/%s.png" % (self.key(), self.slug)
         else:
             return utils.get_predefined_image_link(self.link, self.category)
 

app/utility/utils.py

@@ -146,6 +146,12 @@ def check_link_weight(link):
        return (question['score']) >= 3
    return True    
 
+def check_useragent_for_bots(useragent):
+    for botname in BOTS:
+        if botname in useragent:
+            return True
+    return False
+
 def get_metadescription(post):
 
     return ("%s - %s" % ((CMS_NAME + ' ' + post.category if post.category else '').strip(),

app/views/__init__.py

@@ -444,6 +444,7 @@ def layout (content, title = None , tag_cloud = [], categories = [], navbar = Tr
     extend_([u'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n'])
     extend_([u'<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US">\n'])
     extend_([u'<head>\n'])
+    extend_([u'    <meta name="google-site-verification" content="m05zkgXk41nY4dXLheGpLdTiTnx-JduC2XOamAra_6Q" />\n'])
     extend_([u'    <META name="y_key" content="88698ce39daf8603" />\n'])
     extend_([u'    <link rel="alternate" type="application/atom+xml" href="/index.xml" />\n'])
     extend_([u'    <meta http-equiv="content-type" content="', escape_(settings.HTML_MIME_TYPE, True), u'"/>\n'])
@@ -723,14 +724,17 @@ def post (post, prev_post = None, next_post = None, content_discovered = '', is_
 join_ = post._join; escape_ = post._escape
 
 # coding: utf-8
-def robots():
-    __lineoffset__ = -5
+def robots (disallow_all = False):
+    __lineoffset__ = -4
     loop = ForLoop()
     self = TemplateResult(); extend_ = self.extend
     extend_([u'User-Agent: *\n'])
-    extend_([u'Disallow: /admin\n'])
-    extend_([u'Sitemap: http://', escape_((settings.HOST), True), u'/sitemap_index.xml\n'])
-    extend_([u'Sitemap: http://', escape_((settings.HOST), True), u'/sitemap.xml\n'])
+    if disallow_all:
+        extend_([u'Disallow: /\n'])
+    else:
+        extend_([u'Disallow: /admin\n'])
+        extend_([u'Sitemap: http://', escape_((settings.HOST), True), u'/sitemap_index.xml\n'])
+        extend_([u'Sitemap: http://', escape_((settings.HOST), True), u'/sitemap.xml\n'])
 
     return self
 

app/views/layout.html

@@ -3,6 +3,7 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US">
 <head>
+    <meta name="google-site-verification" content="m05zkgXk41nY4dXLheGpLdTiTnx-JduC2XOamAra_6Q" />
     <META name="y_key" content="88698ce39daf8603" />
     <link rel="alternate" type="application/atom+xml" href="/index.xml" />
     <meta http-equiv="content-type" content="$settings.HTML_MIME_TYPE"/>

app/views/robots.txt

@@ -1,4 +1,8 @@
+$def with (disallow_all = False)
 User-Agent: *
-Disallow: /admin
-Sitemap: http://$(settings.HOST)/sitemap_index.xml
-Sitemap: http://$(settings.HOST)/sitemap.xml
+$if disallow_all:
+    Disallow: /
+$else:
+    Disallow: /admin
+    Sitemap: http://$(settings.HOST)/sitemap_index.xml
+    Sitemap: http://$(settings.HOST)/sitemap.xml

application.py

@@ -46,7 +46,7 @@ def redirect_if_needed(env, start_response):
             request = webob.Request(env)
             scheme, netloc, path, query, fragment = urlparse.urlsplit(request.url)
             url = urlparse.urlunsplit([scheme, to_server, path, query, fragment])
-            if not path.startswith('/admin'):
+            if not path.startswith('/admin') and not path.startswith('/robots.txt'):
                 start_response("301 Moved Permanently", [("Location", url)])
                 return ["301 Moved Peramanently", "Click Here %s" % url]
         return wsgi_app(env, start_response)