lib/larch/config.rb @@ -6,9 +6,16 @@ Version 1.1.0 (git) database, which allows Larch to resync and resume interrupted syncs much more quickly without having to rescan all messages. As a result, SQLite 3 is now a dependency. + * Larch now loads config options from ~/.larch/config.yaml if it exists, or + from the file specified by the --config command-line option. This file may + contain multiple sections. If a section name is specified via the + command-line, Larch will use the options in that section for the session; + otherwise it will use the options in the "default" section. See the README + for more details. * Folders are now copied recursively by default. * Progress information is now displayed regularly while scanning large mailboxes. + * Added short versions of common command-line options. * The --fast-scan option has been removed. Version 1.0.2 (2009-08-05) HISTORY @@ -2,90 +2,99 @@ Larch is a tool to copy messages from one IMAP server to another quickly and safely. It's smart enough not to copy messages that already exist on the -destination and robust enough to deal with ornery or misbehaving servers. +destination and robust enough to deal with interruptions caused by flaky +connections or misbehaving servers. Larch is particularly well-suited for copying email to, from, or between Gmail accounts. *Author*:: Ryan Grove (mailto:ryan@wonko.com) -*Version*:: 1.0.2 (2009-08-05) +*Version*:: 1.1.0 (git) *Copyright*:: Copyright (c) 2009 Ryan Grove. All rights reserved. *License*:: GPL 2.0 (http://opensource.org/licenses/gpl-2.0.php) *Website*:: http://github.com/rgrove/larch == Installation -Install Larch via RubyGems: +Install the latest stable version of Larch via RubyGems: gem install larch -== Usage - - larch --from <uri> --to <uri> [options] +Or you can install the latest development version from GitHub: - Required: - --from, -f <s>: URI of the source IMAP server. - --to, -t <s>: URI of the destination IMAP server. - - Copy Options: - --all: Copy all folders recursively - --all-subscribed: Copy all subscribed folders recursively - --exclude <s+>: List of mailbox names/patterns that shouldn't be - copied - --exclude-file <s>: Filename containing mailbox names/patterns that - shouldn't be copied - --from-folder <s>: Source folder to copy from (default: INBOX) - --from-pass <s>: Source server password (default: prompt) - --from-user <s>: Source server username (default: prompt) - --to-folder <s>: Destination folder to copy to (default: INBOX) - --to-pass <s>: Destination server password (default: prompt) - --to-user <s>: Destination server username (default: prompt) + gem sources -a http://gems.github.com + gem install rgrove-larch - General Options: - --dry-run, -n: Don't actually make any changes. - --fast-scan: Use a faster (but less accurate) method to scan - mailboxes. This may result in messages being - re-copied. - --max-retries : Maximum number of times to retry after a recoverable - error (default: 3) - --no-create-folder: Don't create destination folders that don't already - exist - --ssl-certs <s>: Path to a trusted certificate bundle to use to - verify server SSL certificates - --ssl-verify: Verify server SSL certificates - --verbosity, -V <s>: Output verbosity: debug, info, warn, error, or fatal - (default: info) - --version, -v: Print version and exit - --help, -h: Show this message - -== Examples - -Larch is run from the command line. At a minimum, you must specify a source -server and a destination server in the form of IMAP URIs. You may also specify -one or more options: +== Usage + larch [config section] [options] larch --from <uri> --to <uri> [options] -For an overview of all available command-line options, run: - - larch --help + Server Options: + --from, -f <s>: URI of the source IMAP server. + --from-folder, -F <s>: Source folder to copy from (default: INBOX) + --from-pass, -p <s>: Source server password (default: prompt) + --from-user, -u <s>: Source server username (default: prompt) + --to, -t <s>: URI of the destination IMAP server. + --to-folder, -T <s>: Destination folder to copy to (default: INBOX) + --to-pass, -P <s>: Destination server password (default: prompt) + --to-user, -U <s>: Destination server username (default: prompt) + + Sync Options: + --all, -a: Copy all folders recursively + --all-subscribed, -s: Copy all subscribed folders recursively + --exclude <s+>: List of mailbox names/patterns that shouldn't be + copied + --exclude-file <s>: Filename containing mailbox names/patterns that + shouldn't be copied -Specify a source server and a destination server and Larch will prompt you for -the necessary usernames and passwords, then sync the contents of the source's -+INBOX+ folder to the destination: + General Options: + --config, -c <s>: Specify a non-default config file to use (default: + ~/.larch/config.yaml) + --database <s>: Specify a non-default message database to use + (default: ~/.larch/larch.db) + --dry-run, -n: Don't actually make any changes + --max-retries : Maximum number of times to retry after a recoverable + error (default: 3) + --no-create-folder: Don't create destination folders that don't already + exist + --ssl-certs <s>: Path to a trusted certificate bundle to use to + verify server SSL certificates + --ssl-verify: Verify server SSL certificates + --verbosity, -V <s>: Output verbosity: debug, info, warn, error, or fatal + (default: info) + --version, -v: Print version and exit + --help, -h: Show this message + +== Usage Examples + +Larch is run from the command line. The following examples demonstrate how to +run Larch using only command line arguments, but you may also place these +options in a config file and run Larch without any arguments if you prefer. See +the "Configuration" section below for more details. + +For an overview of all available options, run: + + larch -h + +At a minimum, you must specify a source server and a destination server +in the form of IMAP URIs: larch --from imap://mail.example.com --to imap://imap.gmail.com +Larch will prompt you for the necessary usernames and passwords, then sync the +contents of the source's +INBOX+ folder to the destination's INBOX folder. + To connect using SSL, specify a URI beginning with <tt>imaps://</tt>: larch --from imaps://mail.example.com --to imaps://imap.gmail.com If you'd like to sync a specific folder other than +INBOX+, specify the source and destination folders using <tt>--from-folder</tt> and -<tt>--to-folder</tt>: +<tt>--to-folder</tt>. Folder names containing spaces must be enclosed in quotes: larch --from imaps://mail.example.com --to imaps://imap.gmail.com \ - --from-folder "Sent Mail" --to-folder "Sent Mail" + --from-folder 'Sent Mail' --to-folder 'Sent Mail' To sync all folders, use the <tt>--all</tt> option (or <tt>--all-subscribed</tt> if you only want to sync subscribed folders): @@ -99,7 +108,7 @@ already exist. To prevent this, add the <tt>--no-create-folder</tt> option: --no-create-folder You can prevent Larch from syncing one or more folders by using the -<tt>--exclude</tt> option: +<tt>--exclude</tt> option, which accepts multiple arguments: larch --from imaps://mail.example.com --to imaps://imap.gmail.com --all \ --exclude Spam Trash Drafts "[Gmail]/*" @@ -112,10 +121,83 @@ option: --exclude-file exclude.txt The wildcard characters <tt>*</tt> and <tt>?</tt> are supported in exclusion -lists. You can also use a regular expression by enclosing a pattern in +lists. You may also use a regular expression by enclosing a pattern in forward slashes, so the previous example could be achieved with the pattern <tt>/(Spam|Trash|Drafts|\[Gmail\]\/.*)/</tt> +== Configuration + +While it's possible to control Larch entirely from the command line, this can be +inconvenient if you need to specify a lot of options or if you run Larch +frequently and can't always remember which options to use. Using a configuration +file can simplify things. + +By default, Larch looks for a config file at <tt>~/.larch/config.yaml</tt> and +uses it if found. You may specify a custom config file using the +<tt>--config</tt> command line option. + +The Larch configuration file is a simple YAML[http://yaml.org/] file that may +contain multiple sections, each with a different set of options, as well as a +special +default+ section. The options in the +default+ section will be used +unless they're overridden either in another config section or on the command +line. + +=== Example + +Here's a sample Larch config file: + + default: + all-subscribed: true # Copy all subscribed folders by default + + # Copy mail from Gmail to my server, excluding stuff I don't want. + gmail to my server: + from: imaps://imap.gmail.com + from-user: example + from-pass: secret + + to: imaps://mail.example.com + to-user: example + to-pass: secret + + exclude: + - "[Gmail]/Sent Mail" + - "[Gmail]/Spam" + - "[Gmail]/Trash" + + # Copy mail from my INBOX to Gmail's INBOX + my inbox to gmail inbox: + all-subscribed: false + + from: imaps://mail.example.com + from-folder: INBOX + from-user: example + from-pass: secret + + to: imaps://imap.gmail.com + to-folder: INBOX + to-user: example + to-pass: secret + +This file contains three sections. The options from +default+ will be used in +all other sections as well unless they're overridden. + +To specify which config section you want Larch to use, just pass its name on the +command line (use quotes if the name contains spaces): + + larch 'gmail to my server' + +If you specify additional command line options, they'll override options in the +config file: + + larch 'gmail to my server' --from-user anotheruser + +Running Larch with no command line arguments will cause the +default+ section +to be used. With the example above, this will result in an error since the ++default+ section doesn't contain the required +from+ and +to+ options, but if +you only need to use Larch with a single configuration, you could use the ++default+ section for everything and save yourself some typing on the command +line. + == Server Compatibility Larch should work well with any server that properly supports README.rdoc @@ -45,7 +45,7 @@ Rake::RDocTask.new do |rd| rd.title = 'Larch Documentation' rd.rdoc_dir = 'doc' - rd.rdoc_files.include('README.rdoc', 'lib/**/*.rb') + rd.rdoc_files.include('README.rdoc', 'HISTORY', 'lib/**/*.rb') rd.options << '--line-numbers' << '--inline-source' end Rakefile @@ -11,120 +11,97 @@ module Larch # Parse command-line options. options = Trollop.options do version "Larch #{APP_VERSION}\n" << APP_COPYRIGHT - banner <<EOS + banner <<-EOS Larch syncs messages from one IMAP server to another. Awesomely. Usage: + larch [config section] [options] larch --from <uri> --to <uri> [options] -Required: +Server Options: EOS - opt :from, "URI of the source IMAP server.", :short => '-f', :type => :string, :required => true - opt :to, "URI of the destination IMAP server.", :short => '-t', :type => :string, :required => true + opt :from, "URI of the source IMAP server.", :short => '-f', :type => :string + opt :from_folder, "Source folder to copy from", :short => '-F', :default => Config::DEFAULT['from-folder'] + opt :from_pass, "Source server password (default: prompt)", :short => '-p', :type => :string + opt :from_user, "Source server username (default: prompt)", :short => '-u', :type => :string + opt :to, "URI of the destination IMAP server.", :short => '-t', :type => :string + opt :to_folder, "Destination folder to copy to", :short => '-T', :default => Config::DEFAULT['to-folder'] + opt :to_pass, "Destination server password (default: prompt)", :short => '-P', :type => :string + opt :to_user, "Destination server username (default: prompt)", :short => '-U', :type => :string text "\nSync Options:" - opt :all, "Copy all folders recursively", :short => :none - opt :all_subscribed, "Copy all subscribed folders recursively", :short => :none + opt :all, "Copy all folders recursively", :short => '-a' + opt :all_subscribed, "Copy all subscribed folders recursively", :short => '-s' opt :exclude, "List of mailbox names/patterns that shouldn't be copied", :short => :none, :type => :strings, :multi => true opt :exclude_file, "Filename containing mailbox names/patterns that shouldn't be copied", :short => :none, :type => :string - opt :from_folder, "Source folder to copy from", :short => :none, :default => 'INBOX' - opt :from_pass, "Source server password (default: prompt)", :short => :none, :type => :string - opt :from_user, "Source server username (default: prompt)", :short => :none, :type => :string - opt :to_folder, "Destination folder to copy to", :short => :none, :default => 'INBOX' - opt :to_pass, "Destination server password (default: prompt)", :short => :none, :type => :string - opt :to_user, "Destination server username (default: prompt)", :short => :none, :type => :string text "\nGeneral Options:" - opt :database, "Specify a non-default message database to use", :short => :none, :default => File.join('~', '.larch', 'larch.db') + opt :config, "Specify a non-default config file to use", :short => '-c', :default => Config::DEFAULT['config'] + opt :database, "Specify a non-default message database to use", :short => :none, :default => Config::DEFAULT['database'] opt :dry_run, "Don't actually make any changes", :short => '-n' - opt :max_retries, "Maximum number of times to retry after a recoverable error", :short => :none, :default => 3 + opt :max_retries, "Maximum number of times to retry after a recoverable error", :short => :none, :default => Config::DEFAULT['max-retries'] opt :no_create_folder, "Don't create destination folders that don't already exist", :short => :none opt :ssl_certs, "Path to a trusted certificate bundle to use to verify server SSL certificates", :short => :none, :type => :string opt :ssl_verify, "Verify server SSL certificates", :short => :none - opt :verbosity, "Output verbosity: debug, info, warn, error, or fatal", :short => '-V', :default => 'info' + opt :verbosity, "Output verbosity: debug, info, warn, error, or fatal", :short => '-V', :default => Config::DEFAULT['verbosity'] end - # Validate command-line options. - [:from, :to].each do |sym| - unless options[sym] =~ IMAP::REGEX_URI - Trollop.die sym, "must be a valid IMAP URI (e.g. imap://example.com)" - end - end - - unless Logger::LEVELS.has_key?(options[:verbosity].to_sym) - Trollop.die :verbosity, "must be one of: #{Logger::LEVELS.keys.join(', ')}" - end - - if options[:exclude_file_given] - filename = options[:exclude_file] - - Trollop.die :exclude_file, ": file not found: #{filename}" unless File.file?(filename) - Trollop.die :exclude_file, ": file cannot be read: #{filename}" unless File.readable?(filename) + # Load config. + config = Config.new(ARGV.shift || 'default', options[:config], options) + + if options[:config_given] + Trollop.die :config, ": file not found: #{options[:config]}" unless File.exist?(options[:config]) end - # Prevent conflicting options from being used. - if options[:all_given] - [:all_subscribed, :from_folder, :to_folder].each do |o| - Trollop.die :all, "may not be used with --#{o.to_s.gsub('_', '-')}" if options["#{o}_given".to_sym] - end - end - - if options[:all_subscribed_given] - [:all, :from_folder, :to_folder].each do |o| - Trollop.die :all_subscribed, "may not be used with --#{o.to_s.gsub('_', '-')}" if options["#{o}_given".to_sym] - end + # Validate config. + begin + config.validate + rescue Config::Error => e + abort "Config error: #{e}" end # Create URIs. - uri_from = URI(options[:from]) - uri_to = URI(options[:to]) + uri_from = URI(config.from) + uri_to = URI(config.to) - # --all and --all-subscribed options override folders specified in URIs - if options[:all_given] || options[:all_subscribed_given] + # Use --from-folder and --to-folder unless folders were specified in the URIs. + uri_from.path ||= '/' + CGI.escape(config.from_folder.gsub(/^\//, '')) + uri_to.path ||= '/' + CGI.escape(config.to_folder.gsub(/^\//, '')) + + # --all and --all-subscribed options override folders + if config.all || config.all_subscribed uri_from.path = '' uri_to.path = '' end - # --from-folder and --to-folder options override folders specified in URIs - if options[:from_folder_given] || options[:to_folder_given] - uri_from.path = '/' + CGI.escape(options[:from_folder].gsub(/^\//, '')) - uri_to.path = '/' + CGI.escape(options[:to_folder].gsub(/^\//, '')) - end - # Usernames and passwords specified as arguments override those in the URIs - uri_from.user = CGI.escape(options[:from_user]) if options[:from_user] - uri_from.password = CGI.escape(options[:from_pass]) if options[:from_pass] - uri_to.user = CGI.escape(options[:to_user]) if options[:to_user] - uri_to.password = CGI.escape(options[:to_pass]) if options[:to_pass] + uri_from.user = CGI.escape(config.from_user) if config.from_user + uri_from.password = CGI.escape(config.from_pass) if config.from_pass + uri_to.user = CGI.escape(config.to_user) if config.to_user + uri_to.password = CGI.escape(config.to_pass) if config.to_pass - # If usernames/passwords aren't specified in either URIs or args, then prompt. + # If usernames/passwords aren't specified in either URIs or config, then prompt. uri_from.user ||= CGI.escape(ask("Source username (#{uri_from.host}): ")) uri_from.password ||= CGI.escape(ask("Source password (#{uri_from.host}): ") {|q| q.echo = false }) uri_to.user ||= CGI.escape(ask("Destination username (#{uri_to.host}): ")) uri_to.password ||= CGI.escape(ask("Destination password (#{uri_to.host}): ") {|q| q.echo = false }) # Go go go! - init(options[:database], - :exclude => options[:exclude] ? options[:exclude].flatten : [], - :exclude_file => options[:exclude_file], - :log_level => options[:verbosity] - ) - - Net::IMAP.debug = true if @log.level == :insane + init(config) imap_from = Larch::IMAP.new(uri_from, - :dry_run => options[:dry_run], - :max_retries => options[:max_retries], - :ssl_certs => options[:ssl_certs] || nil, - :ssl_verify => options[:ssl_verify] + :dry_run => config[:dry_run], + :max_retries => config[:max_retries], + :ssl_certs => config[:ssl_certs] || nil, + :ssl_verify => config[:ssl_verify] ) imap_to = Larch::IMAP.new(uri_to, - :create_mailbox => !options[:no_create_folder] && !options[:dry_run], - :dry_run => options[:dry_run], - :max_retries => options[:max_retries], - :ssl_certs => options[:ssl_certs] || nil, - :ssl_verify => options[:ssl_verify] + :create_mailbox => !config[:no_create_folder] && !config[:dry_run], + :dry_run => config[:dry_run], + :max_retries => config[:max_retries], + :ssl_certs => config[:ssl_certs] || nil, + :ssl_verify => config[:ssl_verify] ) unless RUBY_PLATFORM =~ /mswin|mingw|bccwin|wince|java/ @@ -136,9 +113,9 @@ EOS end end - if options[:all_given] + if config.all copy_all(imap_from, imap_to) - elsif options[:all_subscribed_given] + elsif config.all_subscribed copy_all(imap_from, imap_to, true) else copy_folder(imap_from, imap_to) bin/larch @@ -2,15 +2,15 @@ Gem::Specification.new do |s| s.name = %q{larch} - s.version = "1.0.2.2" + s.version = "1.0.2.3" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Ryan Grove"] - s.date = %q{2009-08-19} + s.date = %q{2009-08-22} s.default_executable = %q{larch} s.email = %q{ryan@wonko.com} s.executables = ["larch"] - s.files = ["HISTORY", "LICENSE", "README.rdoc", "bin/larch", "lib/larch/db/account.rb", "lib/larch/db/mailbox.rb", "lib/larch/db/message.rb", "lib/larch/db/migrate/001_create_schema.rb", "lib/larch/errors.rb", "lib/larch/imap/mailbox.rb", "lib/larch/imap.rb", "lib/larch/logger.rb", "lib/larch/version.rb", "lib/larch.rb"] + s.files = ["HISTORY", "LICENSE", "README.rdoc", "bin/larch", "lib/larch/config.rb", "lib/larch/db/account.rb", "lib/larch/db/mailbox.rb", "lib/larch/db/message.rb", "lib/larch/db/migrate/001_create_schema.rb", "lib/larch/errors.rb", "lib/larch/imap/mailbox.rb", "lib/larch/imap.rb", "lib/larch/logger.rb", "lib/larch/version.rb", "lib/larch.rb"] s.homepage = %q{http://github.com/rgrove/larch/} s.require_paths = ["lib"] s.required_ruby_version = Gem::Requirement.new(">= 1.8.6") larch.gemspec @@ -8,10 +8,12 @@ require 'fileutils' require 'net/imap' require 'time' require 'uri' +require 'yaml' require 'sequel' require 'sequel/extensions/migration' +require 'larch/config' require 'larch/errors' require 'larch/imap' require 'larch/imap/mailbox' @@ -21,21 +23,19 @@ require 'larch/version' module Larch class << self - attr_reader :db, :log, :exclude + attr_reader :config, :db, :log, :exclude EXCLUDE_COMMENT = /#.*$/ EXCLUDE_REGEX = /^\s*\/(.*)\/\s*/ GLOB_PATTERNS = {'*' => '.*', '?' => '.'} LIB_DIR = File.join(File.dirname(File.expand_path(__FILE__)), 'larch') - def init(database, config = {}) - @config = { - :exclude => [], - :log_level => :info - }.merge(config) + def init(config) + raise ArgumentError, "config must be a Larch::Config instance" unless config.is_a?(Config) - @log = Logger.new(@config[:log_level]) - @db = open_db(database) + @config = config + @log = Logger.new(@config[:verbosity]) + @db = open_db(@config[:database]) @exclude = @config[:exclude].map do |e| if e =~ EXCLUDE_REGEX @@ -47,6 +47,8 @@ module Larch load_exclude_file(@config[:exclude_file]) if @config[:exclude_file] + Net::IMAP.debug = true if @log.level == :insane + # Stats @copied = 0 @failed = 0 lib/larch.rb @@ -1,6 +1,10 @@ module Larch class Error < StandardError; end + class Config + class Error < Larch::Error; end + end + class IMAP class Error < Larch::Error; end class FatalError < Error; end lib/larch/errors.rb @@ -1,6 +1,6 @@ module Larch APP_NAME = 'Larch' - APP_VERSION = '1.0.2.2' + APP_VERSION = '1.0.2.3' APP_AUTHOR = 'Ryan Grove' APP_EMAIL = 'ryan@wonko.com' APP_URL = 'http://github.com/rgrove/larch/' lib/larch/version.rb b4fc00f0add21a4ed0f0bf35f72a79c4250aed06 Ryan Grove ryan@wonko.com http://github.com/rgrove/larch/commit/b51d341170ab4166019f2208898ee94bec3fd226 b51d341170ab4166019f2208898ee94bec3fd226

2009-08-22T17:35:29-07:00

Add config file support and documentation 27ddc31ed269c3186d21a1b29125ade56a6c0ac9 Ryan Grove ryan@wonko.com