Store mailbox and message state information in a local database to sp…

…eed up future syncs

Rakefile

@@ -48,8 +48,10 @@ Gem::Specification.new do |s|
   s.require_path          = 'lib'
   s.required_ruby_version = '>= 1.8.6'
 
-  s.add_dependency('highline', '~> 1.5.0')
-  s.add_dependency('trollop',  '~> 1.13')
+  s.add_dependency('highline',     '~> 1.5.0')
+  s.add_dependency('sequel',       '~> 3.3.0')
+  s.add_dependency('sqlite3-ruby', '~> 1.2.5')
+  s.add_dependency('trollop',      '~> 1.13')
 
   s.files = [
     'HISTORY',

bin/larch

@@ -22,7 +22,7 @@ EOS
     opt :from, "URI of the source IMAP server.", :short => '-f', :type => :string, :required => true
     opt :to,   "URI of the destination IMAP server.", :short => '-t', :type => :string, :required => true
 
-    text "\nCopy Options:"
+    text "\nSync Options:"
     opt :all,              "Copy all folders recursively", :short => :none
     opt :all_subscribed,   "Copy all subscribed folders recursively", :short => :none
     opt :exclude,          "List of mailbox names/patterns that shouldn't be copied", :short => :none, :type => :strings, :multi => true
@@ -35,7 +35,8 @@ EOS
     opt :to_user,          "Destination server username (default: prompt)", :short => :none, :type => :string
 
     text "\nGeneral Options:"
-    opt :dry_run,          "Don't actually make any changes.", :short => '-n'
+    opt :database,         "Specify a non-default message database to use", :short => :none, :default => File.join('~', '.larch.db')
+    opt :dry_run,          "Don't actually make any changes", :short => '-n'
     opt :fast_scan,        "Use a faster (but less accurate) method to scan mailboxes. This may result in messages being re-copied.", :short => :none
     opt :max_retries,      "Maximum number of times to retry after a recoverable error", :short => :none, :default => 3
     opt :no_create_folder, "Don't create destination folders that don't already exist", :short => :none
@@ -104,10 +105,10 @@ EOS
   uri_to.password   ||= CGI.escape(ask("Destination password (#{uri_to.host}): ") {|q| q.echo = false })
 
   # Go go go!
-  init(
-    options[:verbosity],
-    options[:exclude] ? options[:exclude].flatten : [],
-    options[:exclude_file]
+  init(options[:database],
+    :exclude      => options[:exclude] ? options[:exclude].flatten : [],
+    :exclude_file => options[:exclude_file],
+    :log_level    => options[:verbosity]
   )
 
   Net::IMAP.debug = true if @log.level == :insane

lib/larch.rb

@@ -8,6 +8,9 @@
 require 'time'
 require 'uri'
 
+require 'sequel'
+require 'sequel/extensions/migration'
+
 require 'larch/errors'
 require 'larch/imap'
 require 'larch/imap/mailbox'
@@ -17,24 +20,31 @@
 module Larch
 
   class << self
-    attr_reader :log, :exclude
+    attr_reader :db, :log, :exclude
 
     EXCLUDE_COMMENT = /#.*$/
     EXCLUDE_REGEX   = /^\s*\/(.*)\/\s*/
     GLOB_PATTERNS   = {'*' => '.*', '?' => '.'}
+    LIB_DIR         = File.join(File.dirname(File.expand_path(__FILE__)), 'larch')
+
+    def init(database, config = {})
+      @config = {
+        :exclude   => [],
+        :log_level => :info
+      }.merge(config)
 
-    def init(log_level = :info, exclude = [], exclude_file = nil)
-      @log = Logger.new(log_level)
+      @log = Logger.new(@config[:log_level])
+      @db  = open_db(database)
 
-      @exclude = exclude.map do |e|
+      @exclude = @config[:exclude].map do |e|
         if e =~ EXCLUDE_REGEX
           Regexp.new($1, Regexp::IGNORECASE)
         else
           glob_to_regex(e.strip)
         end
       end
 
-      load_exclude_file(exclude_file) if exclude_file
+      load_exclude_file(@config[:exclude_file]) if @config[:exclude_file]
 
       # Stats
       @copied = 0
@@ -97,6 +107,43 @@ def copy_folder(imap_from, imap_to)
       summary
     end
 
+    # Opens a connection to the Larch message database, creating it if
+    # necessary.
+    def open_db(database)
+      filename = File.expand_path(database)
+      exists   = File.exist?(filename)
+
+      begin
+        db = Sequel.connect("sqlite://#{filename}")
+        db.test_connection
+      rescue => e
+        @log.fatal "Unable to open message database: #{e}"
+        abort
+      end
+
+      # Ensure that the database schema is up to date.
+      migration_dir = File.join(LIB_DIR, 'db', 'migrate')
+
+      unless Sequel::Migrator.get_current_migration_version(db) ==
+          Sequel::Migrator.latest_migration_version(migration_dir)
+        begin
+          Sequel::Migrator.apply(db, migration_dir)
+        rescue => e
+          @log.fatal "Unable to migrate message database: #{e}"
+          abort
+        end
+      end
+
+      # chmod the db file if we just created it
+      File.chmod(0600, filename) unless exists
+
+      require 'larch/db/message'
+      require 'larch/db/mailbox'
+      require 'larch/db/account'
+
+      db
+    end
+
     def summary
       @log.info "#{@copied} message(s) copied, #{@failed} failed, #{@total - @copied - @failed} untouched out of #{@total} total"
     end
@@ -118,11 +165,11 @@ def copy_messages(imap_from, mailbox_from, imap_to, mailbox_to)
 
       @total += mailbox_from.length
 
-      mailbox_from.each do |id|
-        next if mailbox_to.has_message?(id)
+      mailbox_from.each_guid do |guid|
+        next if mailbox_to.has_guid?(guid)
 
         begin
-          msg = mailbox_from.peek(id)
+          next unless msg = mailbox_from.peek(guid)
 
           if msg.envelope.from
             env_from = msg.envelope.from.first

lib/larch/db/account.rb

@@ -0,0 +1,12 @@
+module Larch; module Database
+
+class Account < Sequel::Model
+  plugin :hook_class_methods
+  one_to_many :mailboxes, :class => Larch::Database::Mailbox
+
+  before_destroy do
+    Mailbox.filter(:account_id => id).destroy
+  end
+end
+
+end; end

lib/larch/db/mailbox.rb

@@ -0,0 +1,12 @@
+module Larch; module Database
+
+class Mailbox < Sequel::Model
+  plugin :hook_class_methods
+  one_to_many :messages, :class => Larch::Database::Message
+
+  before_destroy do
+    Message.filter(:mailbox_id => id).destroy
+  end
+end
+
+end; end

lib/larch/db/message.rb

@@ -0,0 +1,6 @@
+module Larch; module Database
+
+class Message < Sequel::Model
+end
+
+end; end

lib/larch/db/migrate/001_create_schema.rb

@@ -0,0 +1,42 @@
+class CreateSchema < Sequel::Migration
+  def down
+    drop_table :accounts, :mailboxes, :messages
+  end
+
+  def up
+    create_table :accounts do
+      primary_key :id
+      text :hostname, :null => false
+      text :username, :null => false
+
+      unique [:hostname, :username]
+    end
+
+    create_table :mailboxes do
+      primary_key :id
+      foreign_key :account_id, :table => :accounts
+      text :name, :null => false
+      text :delim, :null => false
+      text :attr, :null => false, :default => ''
+      integer :subscribed, :null => false, :default => 0
+      integer :uidvalidity
+      integer :uidnext
+
+      unique [:account_id, :name, :uidvalidity]
+    end
+
+    create_table :messages do
+      primary_key :id
+      foreign_key :mailbox_id, :table => :mailboxes
+      integer :uid, :null => false
+      text :guid, :null => false
+      text :message_id
+      integer :rfc822_size, :null => false
+      integer :internaldate, :null => false
+      text :flags, :null => false, :default => ''
+
+      index :guid
+      unique [:mailbox_id, :uid]
+    end
+  end
+end

lib/larch/imap.rb

@@ -6,14 +6,14 @@ module Larch
 # required reading if you're doing anything with IMAP in Ruby:
 # http://sup.rubyforge.org
 class IMAP
-  attr_reader :conn, :options
+  attr_reader :conn, :db_account, :options
 
   # URI format validation regex.
   REGEX_URI = URI.regexp(['imap', 'imaps'])
 
   # Larch::IMAP::Message represents a transferable IMAP message which can be
   # passed between Larch::IMAP instances.
-  Message = Struct.new(:id, :envelope, :rfc822, :flags, :internaldate)
+  Message = Struct.new(:guid, :envelope, :rfc822, :flags, :internaldate)
 
   # Initializes a new Larch::IMAP instance that will connect to the specified
   # IMAP URI.
@@ -62,9 +62,14 @@ def initialize(uri, options = {})
 
     raise ArgumentError, "must provide a username and password" unless @uri.user && @uri.password
 
-    @conn      = nil
-    @mailboxes = {}
-    @mutex     = Mutex.new
+    @conn       = nil
+    @mailboxes  = {}
+    @mutex      = Mutex.new
+
+    @db_account = Database::Account.find_or_create(
+      :hostname => host,
+      :username => username
+    )
 
     # Create private convenience methods (debug, info, warn, etc.) to make
     # logging easier.
@@ -333,6 +338,11 @@ def update_mailboxes
             subscribed.any?{|s| s.name == mb.name}, mb.attr)
       end
     end
+
+    # Remove mailboxes that no longer exist from the database.
+    @db_account.mailboxes.each do |db_mailbox|
+      db_mailbox.destroy unless @mailboxes.has_key?(db_mailbox.name)
+    end
   end
 
 end