Merge pull request #14 from freerange/import-all-a-single-users-messages

Import many messages from multiple accounts

Chris, James M and I reviewed Tom's changes and agreed to merge into master.

.travis.yml

@@ -1,2 +1,4 @@
 rvm:
-  - 1.9.3
+  - 1.9.3
+before_script:
+  - "bundle exec rake db:migrate db:test:prepare"

app/controllers/messages_controller.rb

@@ -4,4 +4,8 @@ class MessagesController < ApplicationController
   def index
     @messages = MessageRepository.messages
   end
+
+  def show
+    @message = MessageRepository.find(params[:id])
+  end
 end

app/views/messages/index.html.erb

@@ -4,8 +4,8 @@
   <% @messages.each do |message| %>
     <li class="message">
       <span class="date"><%= message.date.strftime("%Y-%m-%D %H:%M:%S") %></span>
-      <span class="subject"><%= message.subject %></span>
-      <span class="sender"><%= message.from.first %></span>
+      <span class="subject"><%= link_to message.subject, message_path(message) %></span>
+      <span class="sender"><%= message.from %></span>
     </li>
   <% end %>
 </ul>

app/views/messages/show.html.erb

@@ -0,0 +1,8 @@
+<ul>
+<li><%= @message.subject %></li>
+<li><%= @message.date %></li>
+<li><%= @message.from %></li>
+</ul>
+<pre>
+<%= @message.original.to_s %>
+</pre>

config/database.yml

@@ -5,7 +5,7 @@
 #   gem 'sqlite3'
 development:
   adapter: sqlite3
-  database: ":memory:"
+  database: "db/development.sqlite3"
   pool: 5
   timeout: 5000
 
@@ -14,13 +14,13 @@ development:
 # Do not set this db to the same as development or production.
 test: &test
   adapter: sqlite3
-  database: ":memory:"
+  database: "db/test.sqlite3"
   pool: 5
   timeout: 5000
 
 production:
   adapter: sqlite3
-  database: ":memory:"
+  database: "db/production.sqlite3"
   pool: 5
   timeout: 5000
 

config/environments/test.rb

@@ -37,5 +37,5 @@
 end
 
 require Rails.root + 'test' + 'fakes' + 'fake_gmail'
-require 'gmail_imap_client'
-GmailImapClient.connection_class = FakeGmail::Connection
+require 'google_mail/mailbox'
+GoogleMail::Mailbox.connection_class = FakeGmail::Connection

config/routes.rb

@@ -1,5 +1,6 @@
 Sauron::Application.routes.draw do
   root to: "messages#index"
+  resources :messages, only: [:show]
 
   if Rails.env.test?
     # So that we can test arbitrary test controllers but avoid exposing this catch-all route in production

db/migrate/20120327123609_initial_schema.rb

@@ -0,0 +1,11 @@
+class InitialSchema < ActiveRecord::Migration
+  def change
+    create_table :messages do |table|
+      table.string :account
+      table.string :uid
+      table.string :subject
+      table.datetime :date
+      table.string :from
+    end
+  end
+end

db/schema.rb

@@ -11,6 +11,14 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 0) do
+ActiveRecord::Schema.define(:version => 20120327123609) do
+
+  create_table "messages", :force => true do |t|
+    t.string   "account"
+    t.string   "uid"
+    t.string   "subject"
+    t.datetime "date"
+    t.string   "from"
+  end
 
 end

features/step_definitions/walking_skeleton.rb

@@ -9,12 +9,12 @@
     Mail.new("Subject: Message one\nDate: 2012-05-23 12:34:45\nFrom: Dave"),
     Mail.new("Subject: Message two\nDate: 2012-06-22 09:21:31\nFrom: Barry")
   ].each do |message|
-    FakeGmail.server.accounts[account].add_message('INBOX', message)
+    FakeGmail.server.accounts[account].add_message(message)
   end
 end
 
 When /^the messages for account "([^"]*)" are imported$/ do |account|
-  MessageImporter.new(GmailImapClient.connect(account, 'password')).import_into(MessageRepository.instance)
+  AccountMessageImporter.import_for(account, 'password')
 end
 
 Then /^they should be visible on the messages page$/ do

lib/account_message_importer.rb

@@ -1,11 +1,10 @@
-require 'gmail_imap_client'
 require 'message_repository'
 
 class AccountMessageImporter
   class << self
     def import_for(email, password)
-      imap_client = GmailImapClient.connect(email, password)
-      MessageImporter.new(imap_client).import_into(MessageRepository.instance)
+      mailbox = GoogleMail::Mailbox.connect(email, password)
+      MessageImporter.new(mailbox).import_into(MessageRepository)
     end
   end
 end

lib/cache_backed_message_store.rb

@@ -0,0 +1,25 @@
+class CacheBackedMessageStore
+  def initialize(cache = ActiveSupport::Cache::FileStore.new(Rails.root + 'data' + Rails.env + 'messages'))
+    @cache = cache
+  end
+
+  def add(account, uid, message)
+    @cache.write [account, uid], message
+  end
+
+  def find(account, uid)
+    @cache.read [account, uid]
+  end
+
+  class << self
+    delegate :add, :find, to: :instance
+
+    def instance
+      @instance ||= new
+    end
+
+    def configure(*args)
+      @instance = new(*args)
+    end
+  end
+end

lib/file_based_message_store.rb

@@ -1,19 +1,35 @@
 require 'fileutils'
+require 'base64'
 
 class FileBasedMessageStore
-  def initialize(root_path)
+  attr_reader :root_path
+
+  def initialize(root_path = Rails.root + 'data' + Rails.env + 'messages')
     @root_path = root_path
   end
 
+  def include?(key)
+    File.exist? key_path(key)
+  end
+
+  def [](key)
+    if include?(key)
+      Base64.strict_decode64(File.read(key_path(key)))
+    end
+  end
+
   def []=(key, value)
-    full_path = File.expand_path(key, @root_path)
-    FileUtils.mkdir_p File.dirname(full_path)
-    File.write full_path, value
+    FileUtils.mkdir_p File.dirname(key_path(key))
+    File.write key_path(key), Base64.strict_encode64(value)
   end
 
   def values
-    Dir["#{@root_path}/**"].map do |path|
-      File.read(path)
+    Dir["#{root_path}/**"].map do |path|
+      Base64.strict_decode64(File.read(path))
     end
   end
+
+  def key_path(key)
+    File.expand_path(Digest::MD5.hexdigest(key.to_s), root_path)
+  end
 end

lib/google_mail/imap_cache.rb

@@ -0,0 +1,21 @@
+module GoogleMail
+  class ImapCache
+    delegate :read, :write, :fetch, to: :@cache
+
+    def initialize(cache = ActiveSupport::Cache::FileStore.new(Rails.root + 'tmp' + 'cache' + 'imap'))
+      @cache = cache
+    end
+
+    class << self
+      delegate :read, :write, :fetch, to: :instance
+
+      def instance
+        @instance ||= new
+      end
+
+      def configure(*args)
+        @instance = new(*args)
+      end
+    end
+  end
+end

lib/google_mail/mailbox.rb

@@ -0,0 +1,61 @@
+require 'net/imap'
+
+module GoogleMail
+  class Mailbox
+    class AuthenticatedConnection
+      attr_reader :email
+      delegate :examine, :uid_search, :list, :uid_fetch, to: :@imap
+
+      def initialize(email, password)
+        @imap = ::Net::IMAP.new 'imap.gmail.com', 993, true
+        @imap.login email, password
+        @email = email
+      end
+    end
+
+    class CachedConnection
+      delegate :email, :examine, :uid_search, :list, to: :@connection
+
+      def initialize(email, password, cache = GoogleMail::ImapCache)
+        @connection = AuthenticatedConnection.new(email, password)
+        @cache = cache
+      end
+
+      def uid_fetch(uid, command)
+        @cache.fetch [@connection.email, uid, command] do
+          @connection.uid_fetch(uid, command)
+        end
+      end
+    end
+
+    cattr_accessor :connection_class
+    self.connection_class = CachedConnection
+
+    attr_reader :connection
+    delegate :email, to: :connection
+
+    def initialize(connection)
+      @connection = connection
+      mailboxes = @connection.list('', '%').collect(&:name)
+      if mailboxes.include?('[Gmail]')
+        @connection.examine '[Gmail]/All Mail'
+      else
+        @connection.examine '[Google Mail]/All Mail'
+      end
+    end
+
+    def uids
+      connection.uid_search('ALL')
+    end
+
+    def message(uid)
+      connection.uid_fetch(uid, 'BODY.PEEK[]').map {|m| m.attr['BODY[]']}.first
+    end
+
+    class << self
+      def connect(email, password)
+        new connection_class.new(email, password)
+      end
+    end
+  end
+end

lib/message_importer.rb

@@ -1,13 +1,15 @@
 class MessageImporter
-  attr_reader :message_client
+  attr_reader :mailbox
 
-  def initialize(message_client)
-    @message_client = message_client
+  def initialize(mailbox)
+    @mailbox = mailbox
   end
 
   def import_into(repository)
-    message_client.inbox_messages.each do |message|
-      repository.store(message)
+    mailbox.uids.each do |uid|
+      unless repository.exists?(mailbox.email, uid)
+        repository.add mailbox.email, uid, mailbox.message(uid)
+      end
     end
   end
 end