Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 325 lines (246 sloc) 8.913 kb
e86f20e @rgrove Initial commit
authored
1 require 'cgi'
2 require 'digest/md5'
eabe18f @rgrove Use ~/.larch/larch.db as the default database path
authored
3 require 'fileutils'
e86f20e @rgrove Initial commit
authored
4 require 'net/imap'
5 require 'time'
6 require 'uri'
b51d341 @rgrove Add config file support and documentation
authored
7 require 'yaml'
e86f20e @rgrove Initial commit
authored
8
4c786cb @rgrove Store mailbox and message state information in a local database to speed...
authored
9 require 'sequel'
10 require 'sequel/extensions/migration'
11
5ae6f07 @rgrove Monkeypatch Net::IMAP in Ruby <= 1.9.1 to fix broken response handling, ...
authored
12 require 'larch/monkeypatch/net/imap'
13
b51d341 @rgrove Add config file support and documentation
authored
14 require 'larch/config'
e86f20e @rgrove Initial commit
authored
15 require 'larch/errors'
16 require 'larch/imap'
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Net::...
authored
17 require 'larch/imap/mailbox'
e86f20e @rgrove Initial commit
authored
18 require 'larch/logger'
19 require 'larch/version'
20
21 module Larch
22
23 class << self
fe6c811 @rgrove Cleanup
authored
24 attr_reader :config, :db, :exclude, :log
e86f20e @rgrove Initial commit
authored
25
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that should ...
authored
26 EXCLUDE_COMMENT = /#.*$/
27 EXCLUDE_REGEX = /^\s*\/(.*)\/\s*/
28 GLOB_PATTERNS = {'*' => '.*', '?' => '.'}
4c786cb @rgrove Store mailbox and message state information in a local database to speed...
authored
29 LIB_DIR = File.join(File.dirname(File.expand_path(__FILE__)), 'larch')
30
b51d341 @rgrove Add config file support and documentation
authored
31 def init(config)
32 raise ArgumentError, "config must be a Larch::Config instance" unless config.is_a?(Config)
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that should ...
authored
33
b51d341 @rgrove Add config file support and documentation
authored
34 @config = config
35 @log = Logger.new(@config[:verbosity])
36 @db = open_db(@config[:database])
e86f20e @rgrove Initial commit
authored
37
fe6c811 @rgrove Cleanup
authored
38 parse_exclusions
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that should ...
authored
39
b51d341 @rgrove Add config file support and documentation
authored
40 Net::IMAP.debug = true if @log.level == :insane
41
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that should ...
authored
42 # Stats
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
43 @copied = 0
44 @deleted = 0
45 @failed = 0
46 @total = 0
e86f20e @rgrove Initial commit
authored
47 end
48
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
49 # Recursively copies all messages in all folders from the source to the
50 # destination.
19fdcb5 @rgrove Add --all-subscribed option to copy all subscribed folders recursively. ...
authored
51 def copy_all(imap_from, imap_to, subscribed_only = false)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
52 raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
53 raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
54
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
55 @copied = 0
56 @deleted = 0
57 @failed = 0
58 @total = 0
05e169b @rgrove Show cumulative stats rather than per-folder stats
authored
59
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
60 imap_from.each_mailbox do |mailbox_from|
4969b46 @rgrove Ensure that excluded folders aren't created
authored
61 next if excluded?(mailbox_from.name)
19fdcb5 @rgrove Add --all-subscribed option to copy all subscribed folders recursively. ...
authored
62 next if subscribed_only && !mailbox_from.subscribed?
63
d2bfaa3 @rgrove Better handling and resolution of config vs. command line conflicts; --t...
authored
64 if imap_to.uri_mailbox
65 mailbox_to = imap_to.mailbox(imap_to.uri_mailbox)
66 else
67 mailbox_to = imap_to.mailbox(mailbox_from.name, mailbox_from.delim)
68 end
69
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
70 mailbox_to.subscribe if mailbox_from.subscribed?
71
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
72 copy_messages(mailbox_from, mailbox_to)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
73 end
74
75 rescue => e
76 @log.fatal e.message
05e169b @rgrove Show cumulative stats rather than per-folder stats
authored
77
78 ensure
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
79 summary
3eeac4e @rgrove Clean up accounts older than 30 days and vacuum the database on exit.
authored
80 db_maintenance
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
81 end
82
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
83 # Copies the messages in a single IMAP folder and all its subfolders
84 # (recursively) from the source to the destination.
1a5873f @rgrove Specify username and password in the IMAP URI instead of as parameters t...
authored
85 def copy_folder(imap_from, imap_to)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
86 raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
87 raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
e86f20e @rgrove Initial commit
authored
88
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
89 @copied = 0
90 @deleted = 0
91 @failed = 0
92 @total = 0
e86f20e @rgrove Initial commit
authored
93
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
94 mailbox_from = imap_from.mailbox(imap_from.uri_mailbox || 'INBOX')
95 mailbox_to = imap_to.mailbox(imap_to.uri_mailbox || 'INBOX')
4969b46 @rgrove Ensure that excluded folders aren't created
authored
96
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
97 copy_mailbox(mailbox_from, mailbox_to)
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Net::...
authored
98
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
99 imap_from.disconnect
100 imap_to.disconnect
101
102 rescue => e
103 @log.fatal e.message
104
105 ensure
106 summary
3eeac4e @rgrove Clean up accounts older than 30 days and vacuum the database on exit.
authored
107 db_maintenance
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
108 end
109
4c786cb @rgrove Store mailbox and message state information in a local database to speed...
authored
110 # Opens a connection to the Larch message database, creating it if
111 # necessary.
112 def open_db(database)
5af3962 @rgrove Add support for in-memory SQLite databases
authored
113 unless database == ':memory:'
114 filename = File.expand_path(database)
115 directory = File.dirname(filename)
eabe18f @rgrove Use ~/.larch/larch.db as the default database path
authored
116
5af3962 @rgrove Add support for in-memory SQLite databases
authored
117 unless File.exist?(directory)
118 FileUtils.mkdir_p(directory)
119 File.chmod(0700, directory)
120 end
eabe18f @rgrove Use ~/.larch/larch.db as the default database path
authored
121 end
4c786cb @rgrove Store mailbox and message state information in a local database to speed...
authored
122
123 begin
9ce554e @rgrove Use Amalgalite instead of requiring that SQLite 3 already be installed.
authored
124 db = Sequel.amalgalite(:database => filename)
4c786cb @rgrove Store mailbox and message state information in a local database to speed...
authored
125 db.test_connection
126 rescue => e
eabe18f @rgrove Use ~/.larch/larch.db as the default database path
authored
127 @log.fatal "unable to open message database: #{e}"
4c786cb @rgrove Store mailbox and message state information in a local database to speed...
authored
128 abort
129 end
130
131 # Ensure that the database schema is up to date.
132 migration_dir = File.join(LIB_DIR, 'db', 'migrate')
133
84f3f14 @ehames Removed calls to (get_current|latest)_migration_version.
ehames authored
134 begin
135 Sequel::Migrator.apply(db, migration_dir)
136 rescue => e
137 @log.fatal "unable to migrate message database: #{e}"
138 abort
4c786cb @rgrove Store mailbox and message state information in a local database to speed...
authored
139 end
140
141 require 'larch/db/message'
142 require 'larch/db/mailbox'
143 require 'larch/db/account'
144
145 db
146 end
147
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
148 def summary
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
149 @log.info "#{@copied} message(s) copied, #{@failed} failed, #{@deleted} deleted out of #{@total} total"
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
150 end
151
3eeac4e @rgrove Clean up accounts older than 30 days and vacuum the database on exit.
authored
152
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
153 private
154
3eeac4e @rgrove Clean up accounts older than 30 days and vacuum the database on exit.
authored
155
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
156 def copy_mailbox(mailbox_from, mailbox_to)
157 raise ArgumentError, "mailbox_from must be a Larch::IMAP::Mailbox instance" unless mailbox_from.is_a?(Larch::IMAP::Mailbox)
158 raise ArgumentError, "mailbox_to must be a Larch::IMAP::Mailbox instance" unless mailbox_to.is_a?(Larch::IMAP::Mailbox)
159
160 return if excluded?(mailbox_from.name) || excluded?(mailbox_to.name)
161
162 mailbox_to.subscribe if mailbox_from.subscribed?
163 copy_messages(mailbox_from, mailbox_to)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
164
6f8a462 @rgrove Add --no-recurse option.
authored
165 unless @config['no-recurse']
166 mailbox_from.each_mailbox do |child_from|
167 next if excluded?(child_from.name)
168 child_to = mailbox_to.imap.mailbox(child_from.name, child_from.delim)
169 copy_mailbox(child_from, child_to)
170 end
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
171 end
172 end
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that should ...
authored
173
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
174 def copy_messages(mailbox_from, mailbox_to)
175 raise ArgumentError, "mailbox_from must be a Larch::IMAP::Mailbox instance" unless mailbox_from.is_a?(Larch::IMAP::Mailbox)
176 raise ArgumentError, "mailbox_to must be a Larch::IMAP::Mailbox instance" unless mailbox_to.is_a?(Larch::IMAP::Mailbox)
e86f20e @rgrove Initial commit
authored
177
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
178 return if excluded?(mailbox_from.name) || excluded?(mailbox_to.name)
a1bb01f @rgrove Perform the initial mailbox scans asynchronously to speed things up.
authored
179
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
180 imap_from = mailbox_from.imap
181 imap_to = mailbox_to.imap
5f08103 @rgrove Fix a whole crapload of threading issues and work around Net::IMAP deadl...
authored
182
957fd5e @rgrove More concise log messages to reduce visual clutter in the log.
authored
183 @log.info "#{imap_from.host}/#{mailbox_from.name} -> #{imap_to.host}/#{mailbox_to.name}"
1692bb2 @rgrove Copy folders recursively by default. Closes #5
authored
184
185 @total += mailbox_from.length
7b6e93a @rgrove Don't connect to the destination server until a connection is actually n...
authored
186
abc6649 @rgrove Add --sync-flags option to synchronize message flags (like Seen, Flagged...
authored
187 mailbox_from.each_db_message do |from_db_message|
188 guid = from_db_message.guid
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
189 uid = from_db_message.uid
abc6649 @rgrove Add --sync-flags option to synchronize message flags (like Seen, Flagged...
authored
190
191 if mailbox_to.has_guid?(guid)
192 begin
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
193 if @config['sync_flags']
194 to_db_message = mailbox_to.fetch_db_message(guid)
195
196 if to_db_message.flags != from_db_message.flags
197 new_flags = from_db_message.flags_str
198 new_flags = '(none)' if new_flags.empty?
abc6649 @rgrove Add --sync-flags option to synchronize message flags (like Seen, Flagged...
authored
199
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
200 @log.info "[>] syncing flags: uid #{uid}: #{new_flags}"
201 mailbox_to.set_flags(guid, from_db_message.flags)
202 end
203 end
abc6649 @rgrove Add --sync-flags option to synchronize message flags (like Seen, Flagged...
authored
204
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
205 if @config['delete'] && !from_db_message.flags.include?(:Deleted)
206 @log.info "[<] deleting uid #{uid} (already exists at destination)"
2782b52 @rgrove Move deleted Gmail messages to the Trash. Closes #29
authored
207 @deleted += 1 if mailbox_from.delete_message(guid)
abc6649 @rgrove Add --sync-flags option to synchronize message flags (like Seen, Flagged...
authored
208 end
2782b52 @rgrove Move deleted Gmail messages to the Trash. Closes #29
authored
209
abc6649 @rgrove Add --sync-flags option to synchronize message flags (like Seen, Flagged...
authored
210 rescue Larch::IMAP::Error => e
211 @log.error e.message
212 end
213
214 next
215 end
e86f20e @rgrove Initial commit
authored
216
217 begin
4c786cb @rgrove Store mailbox and message state information in a local database to speed...
authored
218 next unless msg = mailbox_from.peek(guid)
a1bb01f @rgrove Perform the initial mailbox scans asynchronously to speed things up.
authored
219
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Net::...
authored
220 if msg.envelope.from
221 env_from = msg.envelope.from.first
222 from = "#{env_from.mailbox}@#{env_from.host}"
223 else
224 from = '?'
225 end
cae47ce @rgrove Add a watchdog thread to work around Net::IMAP deadlock issues resulting...
authored
226
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
227 @log.info "[>] copying uid #{uid}: #{from} - #{msg.envelope.subject}"
e86f20e @rgrove Initial commit
authored
228
1a5873f @rgrove Specify username and password in the IMAP URI instead of as parameters t...
authored
229 mailbox_to << msg
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Net::...
authored
230 @copied += 1
e86f20e @rgrove Initial commit
authored
231
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
232 if @config['delete']
233 @log.info "[<] deleting uid #{uid}"
2782b52 @rgrove Move deleted Gmail messages to the Trash. Closes #29
authored
234 @deleted += 1 if mailbox_from.delete_message(guid)
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
235 end
236
5f08103 @rgrove Fix a whole crapload of threading issues and work around Net::IMAP deadl...
authored
237 rescue Larch::IMAP::Error => e
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Net::...
authored
238 @failed += 1
e86f20e @rgrove Initial commit
authored
239 @log.error e.message
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Net::...
authored
240 next
5f08103 @rgrove Fix a whole crapload of threading issues and work around Net::IMAP deadl...
authored
241 end
242 end
0d47dc8 @rgrove Add --delete option to delete messages from the source after copying the...
authored
243
244 if @config['expunge']
245 begin
246 @log.debug "[<] expunging deleted messages"
247 mailbox_from.expunge
248 rescue Larch::IMAP::Error => e
249 @log.error e.message
250 end
251 end
47dd339 @rgrove Rescue recoverable errors during mailbox scans and continue processing o...
authored
252
253 rescue Larch::IMAP::Error => e
254 @log.error e.message
255
e86f20e @rgrove Initial commit
authored
256 end
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that should ...
authored
257
3eeac4e @rgrove Clean up accounts older than 30 days and vacuum the database on exit.
authored
258 def db_maintenance
259 @log.debug 'performing database maintenance'
260
261 # Remove accounts that haven't been used in over 30 days.
262 Database::Account.filter(:updated_at => nil).destroy
263 Database::Account.filter('? - updated_at >= 2592000', Time.now.to_i).destroy
264
265 # Release unused disk space and defragment the database.
266 @db.run('VACUUM')
267 end
268
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that should ...
authored
269 def excluded?(name)
270 name = name.downcase
271
272 @exclude.each do |e|
273 return true if (e.is_a?(Regexp) ? !!(name =~ e) : File.fnmatch?(e, name))
274 end
275
276 return false
277 end
278
279 def glob_to_regex(str)
280 str.gsub!(/(.)/) {|c| GLOB_PATTERNS[$1] || Regexp.escape(c) }
281 Regexp.new("^#{str}$", Regexp::IGNORECASE)
282 end
283
284 def load_exclude_file(filename)
285 @exclude ||= []
286 lineno = 0
287
288 File.open(filename, 'rb') do |f|
289 f.each do |line|
290 lineno += 1
291
292 # Strip comments.
293 line.sub!(EXCLUDE_COMMENT, '')
294 line.strip!
295
296 # Skip empty lines.
297 next if line.empty?
298
299 if line =~ EXCLUDE_REGEX
300 @exclude << Regexp.new($1, Regexp::IGNORECASE)
301 else
302 @exclude << glob_to_regex(line)
303 end
304 end
305 end
306
307 rescue => e
308 raise Larch::IMAP::FatalError, "error in exclude file at line #{lineno}: #{e}"
309 end
310
fe6c811 @rgrove Cleanup
authored
311 def parse_exclusions
312 @exclude = @config[:exclude].map do |e|
313 if e =~ EXCLUDE_REGEX
314 Regexp.new($1, Regexp::IGNORECASE)
315 else
316 glob_to_regex(e.strip)
317 end
318 end
319
320 load_exclude_file(@config[:exclude_file]) if @config[:exclude_file]
321 end
e86f20e @rgrove Initial commit
authored
322 end
323
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that should ...
authored
324 end
Something went wrong with that request. Please try again.