Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 240 lines (180 sloc) 6.271 kB
b6b2fb0 @rgrove Fix issues when transferring messages between servers that use differ…
authored
1 # Prepend this file's directory to the include path if it's not there already.
e86f20e @rgrove Initial commit
authored
2 $:.unshift(File.dirname(File.expand_path(__FILE__)))
3 $:.uniq!
4
5 require 'cgi'
6 require 'digest/md5'
eabe18f @rgrove Use ~/.larch/larch.db as the default database path
authored
7 require 'fileutils'
e86f20e @rgrove Initial commit
authored
8 require 'net/imap'
9 require 'time'
10 require 'uri'
11
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
12 require 'sequel'
13 require 'sequel/extensions/migration'
14
e86f20e @rgrove Initial commit
authored
15 require 'larch/errors'
16 require 'larch/imap'
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Ne…
authored
17 require 'larch/imap/mailbox'
e86f20e @rgrove Initial commit
authored
18 require 'larch/logger'
19 require 'larch/version'
20
21 module Larch
22
23 class << self
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
24 attr_reader :db, :log, :exclude
e86f20e @rgrove Initial commit
authored
25
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that shou…
authored
26 EXCLUDE_COMMENT = /#.*$/
27 EXCLUDE_REGEX = /^\s*\/(.*)\/\s*/
28 GLOB_PATTERNS = {'*' => '.*', '?' => '.'}
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
29 LIB_DIR = File.join(File.dirname(File.expand_path(__FILE__)), 'larch')
30
31 def init(database, config = {})
32 @config = {
33 :exclude => [],
34 :log_level => :info
35 }.merge(config)
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that shou…
authored
36
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
37 @log = Logger.new(@config[:log_level])
38 @db = open_db(database)
e86f20e @rgrove Initial commit
authored
39
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
40 @exclude = @config[:exclude].map do |e|
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that shou…
authored
41 if e =~ EXCLUDE_REGEX
42 Regexp.new($1, Regexp::IGNORECASE)
43 else
44 glob_to_regex(e.strip)
45 end
46 end
47
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
48 load_exclude_file(@config[:exclude_file]) if @config[:exclude_file]
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that shou…
authored
49
50 # Stats
bd0affb @rgrove Use Monitor instead of Mutex to allow for reentrant methods
authored
51 @copied = 0
52 @failed = 0
53 @total = 0
e86f20e @rgrove Initial commit
authored
54 end
55
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
56 # Recursively copies all messages in all folders from the source to the
57 # destination.
19fdcb5 @rgrove Add --all-subscribed option to copy all subscribed folders recursivel…
authored
58 def copy_all(imap_from, imap_to, subscribed_only = false)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
59 raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
60 raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
61
05e169b @rgrove Show cumulative stats rather than per-folder stats
authored
62 @copied = 0
63 @failed = 0
64 @total = 0
65
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
66 imap_from.each_mailbox do |mailbox_from|
4969b46 @rgrove Ensure that excluded folders aren't created
authored
67 next if excluded?(mailbox_from.name)
19fdcb5 @rgrove Add --all-subscribed option to copy all subscribed folders recursivel…
authored
68 next if subscribed_only && !mailbox_from.subscribed?
69
b6b2fb0 @rgrove Fix issues when transferring messages between servers that use differ…
authored
70 mailbox_to = imap_to.mailbox(mailbox_from.name, mailbox_from.delim)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
71 mailbox_to.subscribe if mailbox_from.subscribed?
72
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
73 copy_messages(imap_from, mailbox_from.name, imap_to, mailbox_to.name)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
74 end
75
76 rescue => e
77 @log.fatal e.message
05e169b @rgrove Show cumulative stats rather than per-folder stats
authored
78
79 ensure
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
80 summary
81 end
82
6386d10 @rgrove Add --dry-run option. Closes #2
authored
83 # Copies the messages in a single IMAP folder (non-recursively) from the
84 # source to the destination.
1a5873f @rgrove Specify username and password in the IMAP URI instead of as parameter…
authored
85 def copy_folder(imap_from, imap_to)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
86 raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
87 raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
e86f20e @rgrove Initial commit
authored
88
bd0affb @rgrove Use Monitor instead of Mutex to allow for reentrant methods
authored
89 @copied = 0
90 @failed = 0
91 @total = 0
e86f20e @rgrove Initial commit
authored
92
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
93 from_mb_name = imap_from.uri_mailbox || 'INBOX'
94 to_mb_name = imap_to.uri_mailbox || 'INBOX'
4969b46 @rgrove Ensure that excluded folders aren't created
authored
95
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
96 return if excluded?(from_mb_name) || excluded?(to_mb_name)
4969b46 @rgrove Ensure that excluded folders aren't created
authored
97
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
98 copy_messages(imap_from, from_mb_name, imap_to, to_mb_name)
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Ne…
authored
99
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
100 imap_from.disconnect
101 imap_to.disconnect
102
103 rescue => e
104 @log.fatal e.message
105
106 ensure
107 summary
108 end
109
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
110 # Opens a connection to the Larch message database, creating it if
111 # necessary.
112 def open_db(database)
eabe18f @rgrove Use ~/.larch/larch.db as the default database path
authored
113 filename = File.expand_path(database)
114 directory = File.dirname(filename)
115
116 unless File.exist?(directory)
117 FileUtils.mkdir_p(directory)
118 File.chmod(0700, directory)
119 end
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
120
121 begin
122 db = Sequel.connect("sqlite://#{filename}")
123 db.test_connection
124 rescue => e
eabe18f @rgrove Use ~/.larch/larch.db as the default database path
authored
125 @log.fatal "unable to open message database: #{e}"
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
126 abort
127 end
128
129 # Ensure that the database schema is up to date.
130 migration_dir = File.join(LIB_DIR, 'db', 'migrate')
131
132 unless Sequel::Migrator.get_current_migration_version(db) ==
133 Sequel::Migrator.latest_migration_version(migration_dir)
134 begin
135 Sequel::Migrator.apply(db, migration_dir)
136 rescue => e
eabe18f @rgrove Use ~/.larch/larch.db as the default database path
authored
137 @log.fatal "unable to migrate message database: #{e}"
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
138 abort
139 end
140 end
141
142 require 'larch/db/message'
143 require 'larch/db/mailbox'
144 require 'larch/db/account'
145
146 db
147 end
148
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
149 def summary
150 @log.info "#{@copied} message(s) copied, #{@failed} failed, #{@total - @copied - @failed} untouched out of #{@total} total"
151 end
152
153 private
154
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
155 def copy_messages(imap_from, mb_name_from, imap_to, mb_name_to)
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
156 raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
157 raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
158
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
159 return if excluded?(mb_name_from) || excluded?(mb_name_to)
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that shou…
authored
160
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
161 @log.info "copying messages from #{imap_from.host}/#{mb_name_from} to #{imap_to.host}/#{mb_name_to}"
e86f20e @rgrove Initial commit
authored
162
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
163 mailbox_from = imap_from.mailbox(mb_name_from)
a1bb01f @rgrove Perform the initial mailbox scans asynchronously to speed things up.
authored
164
9a20439 @rgrove Add --all option to copy all folders recursively. Closes #1
authored
165 @total += mailbox_from.length
5f08103 @rgrove Fix a whole crapload of threading issues and work around Net::IMAP de…
authored
166
7b6e93a @rgrove Don't connect to the destination server until a connection is actuall…
authored
167 mailbox_to = imap_to.mailbox(mb_name_to)
168
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
169 mailbox_from.each_guid do |guid|
170 next if mailbox_to.has_guid?(guid)
e86f20e @rgrove Initial commit
authored
171
172 begin
4c786cb @rgrove Store mailbox and message state information in a local database to sp…
authored
173 next unless msg = mailbox_from.peek(guid)
a1bb01f @rgrove Perform the initial mailbox scans asynchronously to speed things up.
authored
174
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Ne…
authored
175 if msg.envelope.from
176 env_from = msg.envelope.from.first
177 from = "#{env_from.mailbox}@#{env_from.host}"
178 else
179 from = '?'
180 end
cae47ce @rgrove Add a watchdog thread to work around Net::IMAP deadlock issues result…
authored
181
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Ne…
authored
182 @log.info "copying message: #{from} - #{msg.envelope.subject}"
e86f20e @rgrove Initial commit
authored
183
1a5873f @rgrove Specify username and password in the IMAP URI instead of as parameter…
authored
184 mailbox_to << msg
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Ne…
authored
185 @copied += 1
e86f20e @rgrove Initial commit
authored
186
5f08103 @rgrove Fix a whole crapload of threading issues and work around Net::IMAP de…
authored
187 rescue Larch::IMAP::Error => e
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Ne…
authored
188 @failed += 1
e86f20e @rgrove Initial commit
authored
189 @log.error e.message
f16a12e @rgrove Fuck you, threads. You're not worth the trouble. And fuck you too, Ne…
authored
190 next
5f08103 @rgrove Fix a whole crapload of threading issues and work around Net::IMAP de…
authored
191 end
192 end
e86f20e @rgrove Initial commit
authored
193 end
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that shou…
authored
194
195 def excluded?(name)
196 name = name.downcase
197
198 @exclude.each do |e|
199 return true if (e.is_a?(Regexp) ? !!(name =~ e) : File.fnmatch?(e, name))
200 end
201
202 return false
203 end
204
205 def glob_to_regex(str)
206 str.gsub!(/(.)/) {|c| GLOB_PATTERNS[$1] || Regexp.escape(c) }
207 Regexp.new("^#{str}$", Regexp::IGNORECASE)
208 end
209
210 def load_exclude_file(filename)
211 @exclude ||= []
212 lineno = 0
213
214 File.open(filename, 'rb') do |f|
215 f.each do |line|
216 lineno += 1
217
218 # Strip comments.
219 line.sub!(EXCLUDE_COMMENT, '')
220 line.strip!
221
222 # Skip empty lines.
223 next if line.empty?
224
225 if line =~ EXCLUDE_REGEX
226 @exclude << Regexp.new($1, Regexp::IGNORECASE)
227 else
228 @exclude << glob_to_regex(line)
229 end
230 end
231 end
232
233 rescue => e
234 raise Larch::IMAP::FatalError, "error in exclude file at line #{lineno}: #{e}"
235 end
236
e86f20e @rgrove Initial commit
authored
237 end
238
71eb0a3 @rgrove Add --exclude and --exclude-file options to specify folders that shou…
authored
239 end
Something went wrong with that request. Please try again.