Merge branch 'purge-requests' into develop

.gitignore

@@ -18,4 +18,5 @@ TAGS
 /public/download
 /public/*theme
 /vendor/bundle
-.bundle
+.bundle
+bin/

Gemfile

@@ -16,6 +16,7 @@ gem 'json', '~> 1.5.1'
 gem 'mahoro'
 gem 'memcache-client', :require => 'memcache'
 gem 'locale', '>= 2.0.5'
+gem 'net-purge'
 gem 'rack', '~> 1.1.0'
 gem 'rdoc', '~> 2.4.3'
 gem 'recaptcha', '~> 0.3.1', :require => 'recaptcha/rails'
@@ -37,3 +38,7 @@ group :test do
   gem 'fakeweb'
   gem 'rspec-rails', '~> 1.3.4'
 end
+
+group :develop do
+  gem 'ruby-debug'
+end

Gemfile.lock

@@ -17,14 +17,18 @@ GEM
     activeresource (2.3.14)
       activesupport (= 2.3.14)
     activesupport (2.3.14)
+    columnize (0.3.6)
     fakeweb (1.3.0)
     fast_gettext (0.6.1)
     gettext (2.1.0)
       locale (>= 2.0.5)
     json (1.5.4)
+    linecache (0.46)
+      rbx-require-relative (> 0.0.4)
     locale (2.0.5)
     mahoro (0.3)
     memcache-client (1.8.5)
+    net-purge (0.1.0)
     pg (0.11.0)
     rack (1.1.0)
     rails (2.3.14)
@@ -35,6 +39,7 @@ GEM
       activesupport (= 2.3.14)
       rake (>= 0.8.3)
     rake (0.9.2)
+    rbx-require-relative (0.0.9)
     rdoc (2.4.3)
     recaptcha (0.3.1)
     rmagick (2.13.1)
@@ -44,6 +49,11 @@ GEM
     rspec-rails (1.3.4)
       rack (>= 1.0.0)
       rspec (~> 1.3.1)
+    ruby-debug (0.10.4)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.4.0)
+    ruby-debug-base (0.10.4)
+      linecache (>= 0.3)
     ruby-msg (1.5.0)
       ruby-ole (>= 1.2.8)
       vpim (>= 0.360)
@@ -64,6 +74,7 @@ DEPENDENCIES
   locale (>= 2.0.5)
   mahoro
   memcache-client
+  net-purge
   pg
   rack (~> 1.1.0)
   rails (= 2.3.14)
@@ -73,6 +84,7 @@ DEPENDENCIES
   routing-filter (~> 0.2.4)
   rspec (~> 1.3.2)
   rspec-rails (~> 1.3.4)
+  ruby-debug
   ruby-msg (~> 1.5.0)
   vpim
   will_paginate (~> 2.3.11)

app/controllers/admin_controller.rb

@@ -36,6 +36,8 @@ def expire_for_request(info_request)
 
         # also force a search reindexing (so changed text reflected in search)
         info_request.reindex_request_events
+        # and remove from varnsi
+        info_request.purge_in_cache
     end
 
     # Expire cached attachment files for a user

app/controllers/application_controller.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # controllers/application.rb:
 # Parent class of all controllers in FOI site. Filters added to this controller
 # apply to all controllers in the application. Likewise, all the methods added
@@ -543,16 +544,6 @@ def country_from_ip
         return country
     end
 
-    def quietly_try_to_open(url)
-        begin 
-            result = open(url).read.strip
-        rescue OpenURI::HTTPError, SocketError, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH
-            logger.warn("Unable to open third-party URL #{url}")
-            result = ""
-        end
-        return result
-    end
-
     # URL generating functions are needed by all controllers (for redirects),
     # views (for links) and mailers (for use in emails), so include them into
     # all of all.

app/models/incoming_message.rb

@@ -816,7 +816,6 @@ def extract_attachments!
                                          :filename => _get_part_file_name(leaf),
                                          :charset => leaf.charset,
                                          :within_rfc822_subject => within_rfc822_subject,
-                                         :display_size => "0K",
                                          :body => body)
             attachment.save!
             attachments << attachment.id

app/models/info_request.rb

@@ -23,6 +23,9 @@
 require 'digest/sha1'
 
 class InfoRequest < ActiveRecord::Base
+    include ActionView::Helpers::UrlHelper
+    include ActionController::UrlWriter
+
     strip_attributes!
 
     validates_presence_of :title, :message => N_("Please enter a summary of your request")
@@ -453,7 +456,6 @@ def receive(email, raw_email_data, override_stop_new_responses = false, rejected
     # An annotation (comment) is made
     def add_comment(body, user)
         comment = Comment.new
-
         ActiveRecord::Base.transaction do
             comment.body = body
             comment.user = user
@@ -1042,6 +1044,18 @@ def json_for_api(deep)
         end
         return ret
     end
+
+    before_save :purge_in_cache
+    def purge_in_cache
+        if !MySociety::Config.get('VARNISH_HOST').nil? && !self.id.nil?
+            # we only do this for existing info_requests (new ones have a nil id)
+            path = url_for(:controller => 'request', :action => 'show', :url_title => self.url_title, :only_path => true, :locale => :none)
+            req = PurgeRequest.new(:url => path,
+                                   :model => self.class.base_class.to_s,
+                                   :model_id => self.id)
+            req.save()
+        end
+    end
 end
 
 

app/models/outgoing_message.rb

@@ -267,7 +267,10 @@ def fully_destroy
         end
     end
 
-
+    after_save(:purge_in_cache)
+    def purge_in_cache
+        self.info_request.purge_in_cache
+    end
 end
 
 

app/models/public_body.rb

@@ -240,7 +240,7 @@ def update_url_name
     # Return the short name if present, or else long name
     def short_or_long_name
         if self.short_name.nil? || self.short_name.empty?   # 'nil' can happen during construction
-            self.name
+            self.name.nil? ? "" : self.name
         else
             self.short_name
         end
@@ -547,6 +547,11 @@ def json_for_api
         }
     end
 
+    after_save(:purge_in_cache)
+    def purge_in_cache
+        self.info_requests.each {|x| x.purge_in_cache}
+    end
+
 end
 
 

app/models/purge_request.rb

@@ -0,0 +1,41 @@
+# models/purge_request.rb:
+# A queue of URLs to purge
+#
+# Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
+# Email: francis@mysociety.org; WWW: http://www.mysociety.org/
+#
+
+class PurgeRequest < ActiveRecord::Base
+    def self.purge_all
+        done_something = false
+        for item in PurgeRequest.all()
+            item.purge
+            done_something = true
+        end
+        return done_something
+    end
+
+    def self.purge_all_loop
+        # Run purge_all in an endless loop, sleeping when there is nothing to do
+        while true
+            sleep_seconds = 1
+            while !purge_all
+                sleep sleep_seconds
+                sleep_seconds *= 2
+                sleep_seconds = 30 if sleep_seconds > 30
+            end
+        end
+    end
+
+    def purge
+        config = MySociety::Config.load_default()
+        varnish_url = config['VARNISH_HOST']
+        result = quietly_try_to_purge(varnish_url, self.url)
+        if result == "200"
+            self.delete()
+        end
+    end
+end
+
+
+

app/models/track_thing.rb

@@ -108,15 +108,15 @@ def track_query_description
         end
         descriptions = []
         if varieties.include? _("requests")
-            descriptions << _("requests which are {{list_of_statuses}}", :list_of_statuses => Array(statuses).join(_(' or ')))
+            descriptions << _("requests which are {{list_of_statuses}}", :list_of_statuses => Array(statuses).sort.join(_(' or ')))
             varieties -= [_("requests")]
         end
         if descriptions.empty? and varieties.empty?
             varieties << _("anything")
         end
         descriptions += Array(varieties)
         parsed_text = parsed_text.strip
-        descriptions = descriptions.join(_(" or "))
+        descriptions = descriptions.sort.join(_(" or "))
         if !parsed_text.empty?
             descriptions += _("{{list_of_things}} matching text '{{search_query}}'", :list_of_things => "", :search_query => parsed_text)
         end

app/models/user.rb

@@ -422,5 +422,12 @@ def User.record_bounce_for_email(email, message)
         end
         return true
     end
+
+    after_save(:purge_in_cache)
+    def purge_in_cache
+        # XXX should only be if specific attributes have changed
+        self.info_requests.each {|x| x.purge_in_cache}
+    end
+
 end
 

config/crontab.ugly

@@ -13,6 +13,7 @@ MAILTO=cron-!!(*= $site *)!!@mysociety.org
 */5 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/change-xapian-database.lock "/data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> /data/vhost/!!(*= $vhost *)!!/logs/update-xapian-index.log || echo "stalled?"
 # Every 10 minutes
 5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-alert-tracks check
+5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/purge-varnish check
 
 # Once an hour
 39 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-overdue-requests.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?"

config/environment.rb

@@ -135,6 +135,7 @@
 require 'rack_quote_monkeypatch.rb'
 require 'world_foi_websites.rb'
 require 'alaveteli_external_command.rb'
+require 'quiet_opener.rb'
 
 ExceptionNotification::Notifier.sender_address = MySociety::Config::get('EXCEPTION_NOTIFICATIONS_FROM')
 ExceptionNotification::Notifier.exception_recipients = MySociety::Config::get('EXCEPTION_NOTIFICATIONS_TO')

config/environments/development.rb

@@ -1,5 +1,7 @@
 # Settings specified here will take precedence over those in config/environment.rb
 
+config.log_level = :info
+
 # In the development environment your application's code is reloaded on
 # every request.  This slows down response time but is perfect for development
 # since you don't have to restart the webserver when you make code changes.

config/general.yml-example

@@ -142,3 +142,7 @@ EXCEPTION_NOTIFICATIONS_TO:
 
 # This rate limiting can be turned off per-user via the admin interface
 MAX_REQUESTS_PER_USER_PER_DAY: 6
+
+# This is used to work out where to send purge requests.  Should be
+# unset if you aren't running behind varnish
+VARNISH_HOST: localhost

config/purge-varnish-debian.ugly

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides: purge-varnish
+# Required-Start: $local_fs $syslog
+# Required-Stop: $local_fs $syslog
+# Default-Start: 2 3 4 5 
+# Default-Stop: 0 1 6
+# Short-Description: purge-varnish is a daemon running the Alaveteli email alerts
+# Description: purge-varnish send Alaveteli email alerts as required
+### END INIT INFO
+#
+# !!(*= $daemon_name *)!! Start the Alaveteli email alert daemon
+
+NAME=!!(*= $daemon_name *)!!
+DAEMON=/data/vhost/!!(*= $vhost *)!!/alaveteli/script/runner
+DAEMON_ARGS="--daemon PurgeRequest.purge_all_loop"
+PIDFILE=/data/vhost/!!(*= $vhost *)!!/purge-varnish.pid
+LOGFILE=/data/vhost/!!(*= $vhost *)!!/logs/purge-varnish.log
+DUSER=!!(*= $user *)!!
+
+trap "" 1
+
+export PIDFILE LOGFILE
+
+quietly_start_daemon() {
+    /sbin/start-stop-daemon --quiet --start --pidfile "$PIDFILE" --chuid "$DUSER" --startas "$DAEMON" -- $DAEMON_ARGS
+}
+
+start_daemon() {
+    /sbin/start-stop-daemon --start --pidfile "$PIDFILE" --chuid "$DUSER" --startas "$DAEMON" -- $DAEMON_ARGS
+}
+
+stop_daemon() {
+    /sbin/start-stop-daemon --stop --oknodo --pidfile "$PIDFILE"
+}
+
+restart() { stop; start; }
+
+case "$1" in
+  check)
+      quietly_start_daemon
+      if [ $? -ne 1 ]
+      then
+          echo "Alaveteli alert daemon was not running; now restarted"
+          exit 1
+      else
+          exit 0
+      fi
+      ;;
+
+  start)
+      echo -n "Starting Alaveteli alert daemon: $NAME"
+      start_daemon
+      ;;
+
+  stop)
+        echo -n "Stopping Alaveteli alert daemon: $NAME"
+        stop_daemon
+        ;;
+
+  restart)
+      echo -n "Restarting Alaveteli alert daemon: $NAME"
+      stop_daemon
+      start_daemon
+      ;;
+
+  *)
+    echo "Usage: /etc/init.d/$NAME {start|stop|restart|check}"
+    exit 1
+    ;;
+esac
+
+if [ $? -eq 0 ]; then
+    echo .
+    exit 0
+else
+    echo " failed"
+    exit 1
+fi
+

config/test.yml

@@ -124,3 +124,4 @@ EXCEPTION_NOTIFICATIONS_TO:
 
 MAX_REQUESTS_PER_USER_PER_DAY: 2
 
+VARNISH_HOST: varnish.localdomain