-
Notifications
You must be signed in to change notification settings - Fork 285
/
imap_utils.py
executable file
·971 lines (768 loc) · 40.6 KB
/
imap_utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
# -*- coding: utf-8 -*-
'''
Gmvault: a tool to backup and restore your gmail account.
Copyright (C) <since 2011> <guillaume Aubert (guillaume dot aubert at gmail do com)>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Module containing the IMAPFetcher object which is the Wrapper around the modified IMAPClient object
'''
import math
import time
import socket
import re
import functools
import ssl
import imaplib
import gmv.gmvault_const as gmvault_const
import gmv.log_utils as log_utils
import gmv.credential_utils as credential_utils
import gmv.gmvault_utils as gmvault_utils
import gmv.mod_imap as mimap
LOG = log_utils.LoggerFactory.get_logger('imap_utils')
class PushEmailError(Exception):
"""
PushEmail Error
"""
def __init__(self, a_msg, quarantined = False):
"""
Constructor
"""
super(PushEmailError, self).__init__(a_msg)
self._in_quarantine = quarantined
def quarantined(self):
""" Get email to quarantine """
return self._in_quarantine
class LabelError(Exception):
"""
LabelError. Exception send when there is an error when adding labels to message
"""
def __init__(self, a_msg, ignore = False):
"""
Constructor
"""
super(LabelError, self).__init__(a_msg)
self._ignore = ignore
def ignore(self):
""" ignore """
return self._ignore
#retry decorator with nb of tries and sleep_time and backoff
def retry(a_nb_tries=3, a_sleep_time=1, a_backoff=1): #pylint:disable=R0912
"""
Decorator for retrying command when it failed with a imap or socket error.
Should be used exclusively on imap exchanges.
Strategy, always retry on any imaplib or socket error. Wait few seconds before to retry
backoff sets the factor by which the a_sleep_time should lengthen after each failure. backoff must be greater than 1,
or else it isn't really a backoff
"""
if a_backoff < 1:
raise ValueError("a_backoff must be greater or equal to 1")
a_nb_tries = math.floor(a_nb_tries)
if a_nb_tries < 0:
raise ValueError("a_nb_tries must be 0 or greater")
if a_sleep_time <= 0:
raise ValueError("a_sleep_time must be greater than 0")
def reconnect(the_self, rec_nb_tries, total_nb_tries, rec_error, rec_sleep_time = [1]): #pylint: disable=W0102
"""
Reconnect procedure. Sleep and try to reconnect
"""
# go in retry mode if less than a_nb_tries
while rec_nb_tries[0] < total_nb_tries:
LOG.critical("Disconnecting from Gmail Server and sleeping ...")
the_self.disconnect()
# add X sec of wait
time.sleep(rec_sleep_time[0])
rec_sleep_time[0] *= a_backoff #increase sleep time for next time
rec_nb_tries[0] += 1
#increase total nb of reconns
the_self.total_nb_reconns += 1
# go in retry mode: reconnect.
# retry reconnect as long as we have tries left
try:
LOG.critical("Reconnecting to the from Gmail Server.")
#reconnect to the current folder
the_self.connect(go_to_current_folder = True )
return
except Exception, ignored:
# catch all errors and try as long as we have tries left
LOG.exception(ignored)
else:
#cascade error
raise rec_error
def inner_retry(the_func): #pylint:disable=C0111,R0912
def wrapper(*args, **kwargs): #pylint:disable=C0111,R0912
nb_tries = [0] # make it mutable in reconnect
m_sleep_time = [a_sleep_time] #make it mutable in reconnect
while True:
try:
return the_func(*args, **kwargs)
except PushEmailError, p_err:
LOG.debug("error message = %s. traceback:%s" % (p_err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical("Cannot reach the Gmail server. Wait %s second(s) and retrying." % (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
reconnect(args[0], nb_tries, a_nb_tries, p_err, m_sleep_time)
except imaplib.IMAP4.abort, err: #abort is recoverable and error is not
LOG.debug("IMAP (abort) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical("Received an IMAP abort error. Wait %s second(s) and retrying." % (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
# problem with this email, put it in quarantine
reconnect(args[0], nb_tries, a_nb_tries, err, m_sleep_time)
except socket.error, sock_err:
LOG.debug("error message = %s. traceback:%s" % (sock_err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical("Cannot reach the Gmail server. Wait %s second(s) and retrying." % (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
reconnect(args[0], nb_tries, a_nb_tries, sock_err, m_sleep_time)
except ssl.SSLError, ssl_err:
LOG.debug("error message = %s. traceback:%s" % (ssl_err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical("Cannot reach the Gmail server. Wait %s second(s) and retrying." % (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
reconnect(args[0], nb_tries, a_nb_tries, sock_err, m_sleep_time)
except imaplib.IMAP4.error, err:
#just trace it back for the moment
LOG.debug("IMAP (normal) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback()))
if nb_tries[0] < a_nb_tries:
LOG.critical("Error when reaching Gmail server. Wait %s second(s) and retry up to 2 times." \
% (m_sleep_time[0]))
else:
LOG.critical("Stop retrying, tried too many times ...")
#raise err
# retry 2 times before to quit
reconnect(args[0], nb_tries, 2, err, m_sleep_time)
return functools.wraps(the_func)(wrapper)
#return wrapper
return inner_retry
class GIMAPFetcher(object): #pylint:disable=R0902,R0904
'''
IMAP Class reading the information
'''
GMAIL_EXTENSION = 'X-GM-EXT-1' # GMAIL capability
GMAIL_ALL = u'[Gmail]/All Mail' #GMAIL All Mail mailbox
GENERIC_GMAIL_ALL = u'\\AllMail' # unlocalised GMAIL ALL
GENERIC_DRAFTS = u'\\Drafts' # unlocalised DRAFTS
GENERIC_GMAIL_CHATS = gmvault_const.GMAIL_UNLOCAL_CHATS # unlocalised Chats names
FOLDER_NAMES = ['ALLMAIL', 'CHATS', 'DRAFTS']
GMAIL_ID = 'X-GM-MSGID' #GMAIL ID attribute
GMAIL_THREAD_ID = 'X-GM-THRID'
GMAIL_LABELS = 'X-GM-LABELS'
IMAP_INTERNALDATE = 'INTERNALDATE'
IMAP_FLAGS = 'FLAGS'
IMAP_ALL = {'type':'imap', 'req':'ALL'}
EMAIL_BODY = 'BODY[]'
GMAIL_SPECIAL_DIRS = ['\\Inbox', '\\Starred', '\\Sent', '\\Draft', '\\Important']
#GMAIL_SPECIAL_DIRS_LOWER = ['\\inbox', '\\starred', '\\sent', '\\draft', '\\important']
GMAIL_SPECIAL_DIRS_LOWER = ['\\inbox', '\\starred', '\\sent', '\\draft', '\\important', '\\trash']
IMAP_BODY_PEEK = 'BODY.PEEK[]' #get body without setting msg as seen
#get the body info without setting msg as seen
IMAP_HEADER_PEEK_FIELDS = 'BODY.PEEK[HEADER.FIELDS (MESSAGE-ID SUBJECT X-GMAIL-RECEIVED)]'
#key used to find these fields in the IMAP Response
IMAP_HEADER_FIELDS_KEY = 'BODY[HEADER.FIELDS (MESSAGE-ID SUBJECT X-GMAIL-RECEIVED)]'
#GET_IM_UID_RE
APPENDUID = r'^[APPENDUID [0-9]* ([0-9]*)] \(Success\)$'
APPENDUID_RE = re.compile(APPENDUID)
GET_ALL_INFO = [ GMAIL_ID, GMAIL_THREAD_ID, GMAIL_LABELS, IMAP_INTERNALDATE, \
IMAP_BODY_PEEK, IMAP_FLAGS, IMAP_HEADER_PEEK_FIELDS]
GET_ALL_BUT_DATA = [ GMAIL_ID, GMAIL_THREAD_ID, GMAIL_LABELS, IMAP_INTERNALDATE, \
IMAP_FLAGS, IMAP_HEADER_PEEK_FIELDS]
GET_DATA_ONLY = [ GMAIL_ID, IMAP_BODY_PEEK]
GET_GMAIL_ID = [ GMAIL_ID ]
GET_GMAIL_ID_DATE = [ GMAIL_ID, IMAP_INTERNALDATE]
def __init__(self, host, port, login, credential, readonly_folder = True): #pylint:disable=R0913
'''
Constructor
'''
self.host = host
self.port = port
self.login = login
self.once_connected = False
self.credential = credential
self.ssl = True
self.use_uid = True
self.readonly_folder = readonly_folder
self.localized_folders = { 'ALLMAIL': { 'loc_dir' : None, 'friendly_name' : 'allmail'},
'CHATS' : { 'loc_dir' : None, 'friendly_name' : 'chats'},
'DRAFTS' : { 'loc_dir' : None, 'friendly_name' : 'drafts'} }
# memoize the current folder (All Mail or Chats) for reconnection management
self.current_folder = None
self.server = None
self.go_to_all_folder = True
self.total_nb_reconns = 0
# True when CHATS or other folder error msg has been already printed
self.printed_folder_error_msg = { 'ALLMAIL' : False, 'CHATS': False , 'DRAFTS':False }
#update GENERIC_GMAIL_CHATS. Should be done at the class level
self.GENERIC_GMAIL_CHATS.extend(gmvault_utils.get_conf_defaults().get_list('Localisation', 'chat_folder', []))
def spawn_connection(self):
"""
spawn a connection with the same parameters
"""
conn = GIMAPFetcher(self.host, self.port, self.login, self.credential, self.readonly_folder)
conn.connect()
return conn
def connect(self, go_to_current_folder = False):
"""
connect to the IMAP server
"""
# create imap object
self.server = mimap.MonkeyIMAPClient(self.host, port = self.port, use_uid= self.use_uid, need_ssl= self.ssl)
# connect with password or xoauth
if self.credential['type'] == 'passwd':
self.server.login(self.login, self.credential['value'])
elif self.credential['type'] == 'oauth2':
#connect with oauth2
if self.once_connected:
self.credential = credential_utils.CredentialHelper.get_oauth2_credential(self.login, renew_cred = False)
LOG.debug("credential['value'] = %s" % (self.credential['value']))
#try to login
self.server.oauth2_login(self.credential['value'])
else:
raise Exception("Unknown authentication method %s. Please use xoauth or passwd authentication " \
% (self.credential['type']))
#set connected to True to handle reconnection in case of failure
self.once_connected = True
# check gmailness
self.check_gmailness()
# find allmail chats and drafts folders
self.find_folder_names()
if go_to_current_folder and self.current_folder:
self.server.select_folder(self.current_folder, readonly = self.readonly_folder)
#enable compression
if gmvault_utils.get_conf_defaults().get_boolean('General', 'enable_imap_compression', True):
self.enable_compression()
LOG.debug("After Enabling compression.")
else:
LOG.debug("Do not enable imap compression.")
def disconnect(self):
"""
disconnect to avoid too many simultaneous connection problem
"""
if self.server:
try:
self.server.logout()
except Exception, ignored: #ignored exception but still log it in log file if activated
LOG.exception(ignored)
self.server = None
def reconnect(self):
"""
disconnect and connect again
"""
self.disconnect()
self.connect()
def enable_compression(self):
"""
Try to enable the compression
"""
#self.server.enable_compression()
pass
@retry(3,1,2) # try 3 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 4 sec
def find_folder_names(self):
"""
depending on your account the all mail folder can be named
[GMAIL]/ALL Mail or [GoogleMail]/All Mail.
Find and set the right one
"""
#use xlist because of localized dir names
folders = self.server.xlist_folders()
the_dir = None
for (flags, _, the_dir) in folders:
#non localised GMAIL_ALL
if GIMAPFetcher.GENERIC_GMAIL_ALL in flags:
#it could be a localized Dir name
self.localized_folders['ALLMAIL']['loc_dir'] = the_dir
elif the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS :
#it could be a localized Dir name
self.localized_folders['CHATS']['loc_dir'] = the_dir
elif GIMAPFetcher.GENERIC_DRAFTS in flags:
self.localized_folders['DRAFTS']['loc_dir'] = the_dir
if not self.localized_folders['ALLMAIL']['loc_dir']: # all mail error
raise Exception("Cannot find global 'All Mail' folder (maybe localized and translated into your language) ! "\
"Check whether 'Show in IMAP for 'All Mail' is enabled in Gmail (Go to Settings->Labels->All Mail)")
elif not self.localized_folders['CHATS']['loc_dir'] and \
gmvault_utils.get_conf_defaults().getboolean("General","errors_if_chat_not_visible", False):
raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
"is enabled in Gmail (Go to Settings->Labels->All Mail)")
elif not self.localized_folders['DRAFTS']['loc_dir']:
raise Exception("Cannot find global 'Drafts' folder.")
@retry(3,1,2) # try 3 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 4 sec
def find_all_mail_folder(self):
"""
depending on your account the all mail folder can be named
[GMAIL]/ALL Mail or [GoogleMail]/All Mail.
Find and set the right one
"""
#use xlist because of localized dir names
folders = self.server.xlist_folders()
the_dir = None
for (flags, _, the_dir) in folders:
#non localised GMAIL_ALL
if GIMAPFetcher.GENERIC_GMAIL_ALL in flags:
#it could be a localized Dir name
self.localized_folders['ALLMAIL']['loc_dir'] = the_dir
return the_dir
if not self.localized_folders['ALLMAIL']['loc_dir']:
#Error
raise Exception("Cannot find global 'All Mail' folder (maybe localized and translated into your language) !"\
" Check whether 'Show in IMAP for 'All Mail' is enabled in Gmail (Go to Settings->Labels->All Mail)")
@retry(3,1,2) # try 3 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 4 sec
def find_chats_folder(self):
"""
depending on your account the chats folder can be named
[GMAIL]/Chats or [GoogleMail]/Chats, [GMAIL]/tous les chats ...
Find and set the right one
Npte: Cannot use the flags as Chats is not a system label. Thanks Google
"""
#use xlist because of localized dir names
folders = self.server.xlist_folders()
LOG.debug("Folders = %s\n" % (folders))
the_dir = None
for (_, _, the_dir) in folders:
#look for GMAIL Chats
if the_dir in GIMAPFetcher.GENERIC_GMAIL_CHATS :
#it could be a localized Dir name
self.localized_folders['CHATS']['loc_dir'] = the_dir
return the_dir
#Error did not find Chats dir
if gmvault_utils.get_conf_defaults().getboolean("General", "errors_if_chat_not_visible", False):
raise Exception("Cannot find global 'Chats' folder ! Check whether 'Show in IMAP for 'Chats' "\
"is enabled in Gmail (Go to Settings->Labels->All Mail)")
return None
def is_visible(self, a_folder_name):
"""
check if a folder is visible otherwise
"""
dummy = self.localized_folders.get(a_folder_name)
if dummy and (dummy.get('loc_dir', None) is not None):
return True
if not self.printed_folder_error_msg.get(a_folder_name, None):
LOG.critical("Cannot find 'Chats' folder on Gmail Server. If you wish to backup your chats,"\
" look at the documentation to see how to configure your Gmail account.\n")
self.printed_folder_error_msg[a_folder_name] = True
return False
def get_folder_name(self, a_folder_name):
"""return real folder name from generic ones"""
if a_folder_name not in self.FOLDER_NAMES:
raise Exception("%s is not a predefined folder names. Please use one" % (a_folder_name) )
folder = self.localized_folders.get(a_folder_name, {'loc_dir' : 'GMVNONAME'})['loc_dir']
return folder
@retry(3,1,2) # try 3 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 4 sec
def select_folder(self, a_folder_name, use_predef_names = True):
"""
Select one of the existing folder
"""
if use_predef_names:
if a_folder_name not in self.FOLDER_NAMES:
raise Exception("%s is not a predefined folder names. Please use one" % (a_folder_name) )
folder = self.localized_folders.get(a_folder_name, {'loc_dir' : 'GMVNONAME'})['loc_dir']
if self.current_folder != folder:
self.server.select_folder(folder, readonly = self.readonly_folder)
self.current_folder = folder
elif self.current_folder != a_folder_name:
self.server.select_folder(a_folder_name, readonly = self.readonly_folder)
self.current_folder = a_folder_name
return self.current_folder
@retry(3,1,2) # try 3 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 4 sec
def list_all_folders(self):
"""
Return all folders mainly for debuging purposes
"""
return self.server.xlist_folders()
@retry(3,1,2) # try 3 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 4 sec
def get_capabilities(self):
"""
return the server capabilities
"""
if not self.server:
raise Exception("GIMAPFetcher not connect to the GMAIL server")
return self.server.capabilities()
@retry(3,1,2) # try 3 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 4 sec
def check_gmailness(self):
"""
Check that the server is a gmail server
"""
if not GIMAPFetcher.GMAIL_EXTENSION in self.get_capabilities():
raise Exception("GIMAPFetcher is not connected to a IMAP GMAIL server. Please check host (%s) and port (%s)" \
% (self.host, self.port))
return True
@retry(3,1,2) # try 3 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 4 sec
def search(self, a_criteria):
"""
Return all found ids corresponding to the search
"""
return self.server.search(a_criteria)
@retry(3,1,2) # try 4 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 8 sec
def fetch(self, a_ids, a_attributes):
"""
Return all attributes associated to each message
"""
return self.server.fetch(a_ids, a_attributes)
@classmethod
def _build_labels_str(cls, a_labels):
"""
Create IMAP label string from list of given labels.
Convert the labels to utf7
a_labels: List of labels
"""
# add GMAIL LABELS
labels_str = None
if a_labels and len(a_labels) > 0:
labels_str = '('
for label in a_labels:
label = gmvault_utils.remove_consecutive_spaces_and_strip(label)
#add not in self.GMAIL_SPECIAL_DIRS_LOWER
if label.lower() in cls.GMAIL_SPECIAL_DIRS_LOWER:
labels_str += '%s ' % (label)
else:
label = label.replace('"', '\\"') #replace quote with escaped quotes
labels_str += '\"%s\" ' % (label)
labels_str = '%s%s' % (labels_str[:-1],')')
return labels_str
@classmethod
def _get_dir_from_labels(cls, label):
"""
Get the dirs to create from the labels
label: label name with / in it
"""
dirs = []
i = 0
for lab in label.split('/'):
lab = gmvault_utils.remove_consecutive_spaces_and_strip(lab)
if i == 0:
dirs.append(lab)
else:
dirs.append('%s/%s' % (dirs[i-1], lab))
i += 1
return dirs
def create_gmail_labels(self, labels, existing_folders):
"""
Create folders and subfolders on Gmail in order
to recreate the label hierarchy before to upload emails
Note that adding labels with +X-GM-LABELS create only nested labels
but not nested ones. This is why this trick must be used to
recreate the label hierarchy
labels: list of labels to create
"""
#1.5-beta moved that out of the loop to minimize the number of calls
#to that method. (Could go further and memoize it)
#get existing directories (or label parts)
# get in lower case because Gmail labels are case insensitive
listed_folders = set([ directory.lower() for (_, _, directory) in self.list_all_folders() ])
existing_folders = listed_folders.union(existing_folders)
reserved_labels_map = gmvault_utils.get_conf_defaults().get_dict("Restore", "reserved_labels_map", \
{ u'migrated' : u'gmv-migrated', u'\muted' : u'gmv-muted' })
LOG.debug("Labels to create: [%s]" % (labels))
for lab in labels:
#LOG.info("Reserved labels = %s\n" % (reserved_labels))
#LOG.info("lab.lower = %s\n" % (lab.lower()))
if lab.lower() in reserved_labels_map.keys(): #exclude creation of migrated label
n_lab = reserved_labels_map.get(lab.lower(), "gmv-default-label")
LOG.info("Warning ! label '%s' (lower or uppercase) is reserved by Gmail and cannot be used."\
"Use %s instead" % (lab, n_lab))
lab = n_lab
LOG.info("translated lab = %s\n" % (lab))
#split all labels
labs = self._get_dir_from_labels(lab)
for directory in labs:
low_directory = directory.lower() #get lower case directory but store original label
if (low_directory not in existing_folders) and (low_directory not in self.GMAIL_SPECIAL_DIRS_LOWER):
try:
if self.server.create_folder(directory) != 'Success':
raise Exception("Cannot create label %s: the directory %s cannot be created." % (lab, directory))
else:
LOG.debug("============== ####### Created Labels (%s)." % (directory))
except imaplib.IMAP4.error, error:
#log error in log file if it exists
LOG.debug(gmvault_utils.get_exception_traceback())
if str(error).startswith("create failed: '[ALREADYEXISTS] Duplicate folder"):
LOG.critical("Warning: label %s already exists on Gmail and Gmvault tried to create it."\
" Ignore this issue." % (directory) )
else:
raise error
#add created folder in folders
existing_folders.add(low_directory)
#return all existing folders
return existing_folders
@retry(3,1,2)
def apply_labels_to(self, imap_ids, labels):
"""
apply one labels to x emails
"""
# go to All Mail folder
LOG.debug("Applying labels %s" % (labels))
the_timer = gmvault_utils.Timer()
the_timer.start()
#utf7 the labels as they should be
labels = [ utf7_encode(label) for label in labels ]
labels_str = self._build_labels_str(labels) # create labels str
if labels_str:
#has labels so update email
the_timer.start()
LOG.debug("Before to store labels %s" % (labels_str))
id_list = ",".join(map(str, imap_ids))
#+X-GM-LABELS.SILENT to have not returned data
try:
ret_code, data = self.server._imap.uid('STORE', id_list, '+X-GM-LABELS.SILENT', labels_str) #pylint: disable=W0212
except imaplib.IMAP4.error, original_err:
LOG.info("Error in apply_labels_to. See exception traceback")
LOG.debug(gmvault_utils.get_exception_traceback())
# try to add labels to each individual ids
faulty_ids = []
for the_id in imap_ids:
try:
ret_code, data = self.server._imap.uid('STORE', the_id, '+X-GM-LABELS.SILENT', labels_str) #pylint: disable=W0212
except imaplib.IMAP4.error, store_err:
LOG.debug("Error when trying to apply labels %s to emails with imap_id %s. Error:%s" % (labels_str, the_id, store_err))
faulty_ids.append(the_id)
#raise an error to ignore faulty emails
raise LabelError("Cannot add Labels %s to emails with uids %s. Error:%s" % (labels_str, faulty_ids, original_err), ignore = True)
#ret_code, data = self.server._imap.uid('COPY', id_list, labels[0])
LOG.debug("After storing labels %s. Operation time = %s s.\nret = %s\ndata=%s" \
% (labels_str, the_timer.elapsed_ms(),ret_code, data))
# check if it is ok otherwise exception
if ret_code != 'OK':
#update individuals emails
faulty_ids = []
for the_id in imap_ids:
try:
ret_code, data = self.server._imap.uid('STORE', the_id, '+X-GM-LABELS.SILENT', labels_str) #pylint: disable=W0212
except imaplib.IMAP4.error, store_err:
LOG.debug("Error when trying to apply labels %s to emails with imap_id %s. Error:%s" % (labels_str, the_id, store_err))
faulty_ids.append(the_id)
raise LabelError("Cannot add Labels %s to emails with uids %s. Error:%s" % (labels_str, faulty_ids, data), ignore = True)
else:
LOG.debug("Stored Labels %s for gm_ids %s" % (labels_str, imap_ids))
def delete_gmail_labels(self, labels, force_delete = False):
"""
Delete passed labels. Beware experimental and labels must be ordered
"""
for label in reversed(labels):
labs = self._get_dir_from_labels(label)
for directory in reversed(labs):
if force_delete or ( (directory.lower() not in self.GMAIL_SPECIAL_DIRS_LOWER) \
and self.server.folder_exists(directory) ): #call server exists each time
try:
self.server.delete_folder(directory)
except imaplib.IMAP4.error, _:
LOG.debug(gmvault_utils.get_exception_traceback())
def erase_mailbox(self):
"""
This is for testing purpose and cannot be used with my own mailbox
"""
if self.login == "guillaume.aubert@gmail.com":
raise Exception("Error cannot activate erase_mailbox with %s" % (self.login))
LOG.info("Erase mailbox for account %s." % (self.login))
LOG.info("Delete folders")
#delete folders
folders = self.server.xlist_folders()
LOG.debug("Folders = %s.\n" %(folders))
trash_folder_name = None
for (flags, _, the_dir) in folders:
if (u'\\Starred' in flags) or (u'\\Spam' in flags) or (u'\\Sent' in flags) \
or (u'\\Important' in flags) or (the_dir == u'[Google Mail]/Chats') \
or (the_dir == u'[Google Mail]') or (u'\\Trash' in flags) or \
(u'\\Inbox' in flags) or (GIMAPFetcher.GENERIC_GMAIL_ALL in flags) or \
(GIMAPFetcher.GENERIC_DRAFTS in flags) or (GIMAPFetcher.GENERIC_GMAIL_CHATS in flags):
LOG.info("Ignore folder %s" % (the_dir))
if (u'\\Trash' in flags): #keep trash folder name
trash_folder_name = the_dir
else:
LOG.info("Delete folder %s" % (the_dir))
self.server.delete_folder(the_dir)
self.select_folder('ALLMAIL')
#self.server.store("1:*",'+X-GM-LABELS', '\\Trash')
#self.server._imap.uid('STORE', id_list, '+X-GM-LABELS.SILENT', '\\Trash')
#self.server.add_gmail_labels(self, messages, labels)
LOG.info("Move emails to Trash.")
# get all imap ids in ALLMAIL
imap_ids = self.search(GIMAPFetcher.IMAP_ALL)
#flag all message as deleted
#print(self.server.delete_messages(imap_ids))
if len(imap_ids) > 0:
self.apply_labels_to(imap_ids, ['\\Trash'])
LOG.info("Got all imap_ids flagged to Trash : %s." % (imap_ids))
else:
LOG.info("No messages to erase.")
LOG.info("Delete emails from Trash.")
if trash_folder_name == None:
raise Exception("No trash folder ???")
self.select_folder(trash_folder_name, False)
# get all imap ids in ALLMAIL
imap_ids = self.search(GIMAPFetcher.IMAP_ALL)
if len(imap_ids) > 0:
res = self.server.delete_messages(imap_ids)
LOG.debug("Delete messages result = %s" % (res))
LOG.info("Expunge everything.")
self.server.expunge()
@retry(4,1,2) # try 4 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 8 sec
def push_data(self, a_folder, a_body, a_flags, a_internal_time):
"""
Push the data
"""
# protection against myself
if self.login == 'guillaume.aubert@gmail.com':
raise Exception("Cannot push to this account")
the_timer = gmvault_utils.Timer()
the_timer.start()
LOG.debug("Before to Append email contents")
#import sys #to print the msg in stdout
#import codecs
#sys.stdout = codecs.getwriter('utf-8')(sys.__stdout__)
#msg = "a_folder = %s, a_flags = %s" % (a_folder.encode('utf-8'), a_flags)
#msg = "a_folder = %s" % (a_folder.encode('utf-8'))
#msg = msg.encode('utf-8')
#print(msg)
res = None
try:
#a_body = self._clean_email_body(a_body)
res = self.server.append(a_folder, a_body, a_flags, a_internal_time)
except imaplib.IMAP4.abort, err:
# handle issue when there are invalid characters (This is do to the presence of null characters)
if str(err).find("APPEND => Invalid character in literal") >= 0:
LOG.critical("Invalid character detected. Try to clean the email and reconnect.")
a_body = self._clean_email_body(a_body)
self.reconnect()
res = self.server.append(a_folder, a_body, a_flags, a_internal_time)
LOG.debug("Appended data with flags %s and internal time %s. Operation time = %s.\nres = %s\n" \
% (a_flags, a_internal_time, the_timer.elapsed_ms(), res))
# check res otherwise Exception
if '(Success)' not in res:
raise PushEmailError("GIMAPFetcher cannot restore email in %s account." %(self.login))
match = GIMAPFetcher.APPENDUID_RE.match(res)
if match:
result_uid = int(match.group(1))
LOG.debug("result_uid = %s" %(result_uid))
else:
# do not quarantine it because it seems to be done by Google Mail to forbid data uploading.
raise PushEmailError("No email id returned by IMAP APPEND command. Quarantine this email.", quarantined = True)
return result_uid
def _clean_email_body(self, a_body):
"""
Clean the body of the email
"""
#for the moment just try to remove the null character brut force. In the future will have to parse the email and clean it
return a_body.replace("\0", '')
@retry(4,1,2) # try 4 times to reconnect with a sleep time of 1 sec and a backoff of 2. The fourth time will wait 8 sec
def deprecated_push_email(self, a_body, a_flags, a_internal_time, a_labels):
"""
Push a complete email body
"""
#protection against myself
if self.login == 'guillaume.aubert@gmail.com':
raise Exception("Cannot push to this account")
the_t = gmvault_utils.Timer()
the_t.start()
LOG.debug("Before to Append email contents")
try:
res = self.server.append(u'[Google Mail]/All Mail', a_body, a_flags, a_internal_time)
except imaplib.IMAP4.abort, err:
# handle issue when there are invalid characters (This is do to the presence of null characters)
if str(err).find("APPEND => Invalid character in literal") >= 0:
a_body = self._clean_email_body(a_body)
res = self.server.append(u'[Google Mail]/All Mail', a_body, a_flags, a_internal_time)
LOG.debug("Appended data with flags %s and internal time %s. Operation time = %s.\nres = %s\n" \
% (a_flags, a_internal_time, the_t.elapsed_ms(), res))
# check res otherwise Exception
if '(Success)' not in res:
raise PushEmailError("GIMAPFetcher cannot restore email in %s account." %(self.login))
match = GIMAPFetcher.APPENDUID_RE.match(res)
if match:
result_uid = int(match.group(1))
LOG.debug("result_uid = %s" %(result_uid))
else:
# do not quarantine it because it seems to be done by Google Mail to forbid data uploading.
raise PushEmailError("No email id returned by IMAP APPEND command. Quarantine this email.", quarantined = True)
labels_str = self._build_labels_str(a_labels)
if labels_str:
#has labels so update email
the_t.start()
LOG.debug("Before to store labels %s" % (labels_str))
self.server.select_folder(u'[Google Mail]/All Mail', readonly = self.readonly_folder) # go to current folder
LOG.debug("Changing folders. elapsed %s s\n" % (the_t.elapsed_ms()))
the_t.start()
ret_code, data = self.server._imap.uid('STORE', result_uid, '+X-GM-LABELS', labels_str) #pylint: disable=W0212
#ret_code = self.server._store('+X-GM-LABELS', [result_uid],labels_str)
LOG.debug("After storing labels %s. Operation time = %s s.\nret = %s\ndata=%s" \
% (labels_str, the_t.elapsed_ms(),ret_code, data))
LOG.debug("Stored Labels %s in gm_id %s" % (labels_str, result_uid))
self.server.select_folder(u'[Google Mail]/Drafts', readonly = self.readonly_folder) # go to current folder
# check if it is ok otherwise exception
if ret_code != 'OK':
raise PushEmailError("Cannot add Labels %s to email with uid %d. Error:%s" % (labels_str, result_uid, data))
return result_uid
def decode_labels(labels):
"""
Decode labels when they are received as utf7 entities or numbers
"""
new_labels = []
for label in labels:
if isinstance(label, (int, long, float, complex)):
label = str(label)
new_labels.append(utf7_decode(label))
return new_labels
# utf7 conversion functions
def utf7_encode(s): #pylint: disable=C0103
"""encode in utf7"""
if isinstance(s, str) and sum(n for n in (ord(c) for c in s) if n > 127):
raise ValueError("%r contains characters not valid in a str folder name. "
"Convert to unicode first?" % s)
r = [] #pylint: disable=C0103
_in = []
for c in s: #pylint: disable=C0103
if ord(c) in (range(0x20, 0x26) + range(0x27, 0x7f)):
if _in:
r.extend(['&', utf7_modified_base64(''.join(_in)), '-'])
del _in[:]
r.append(str(c))
elif c == '&':
if _in:
r.extend(['&', utf7_modified_base64(''.join(_in)), '-'])
del _in[:]
r.append('&-')
else:
_in.append(c)
if _in:
r.extend(['&', utf7_modified_base64(''.join(_in)), '-'])
return ''.join(r)
def utf7_decode(s): #pylint: disable=C0103
"""decode utf7"""
r = [] #pylint: disable=C0103
decode = []
for c in s: #pylint: disable=C0103
if c == '&' and not decode:
decode.append('&')
elif c == '-' and decode:
if len(decode) == 1:
r.append('&')
else:
r.append(utf7_modified_unbase64(''.join(decode[1:])))
decode = []
elif decode:
decode.append(c)
else:
r.append(c)
if decode:
r.append(utf7_modified_unbase64(''.join(decode[1:])))
out = ''.join(r)
if not isinstance(out, unicode):
out = unicode(out, 'latin-1')
return out
def utf7_modified_base64(s): #pylint: disable=C0103
"""utf7 base64"""
s_utf7 = s.encode('utf-7')
return s_utf7[1:-1].replace('/', ',')
def utf7_modified_unbase64(s): #pylint: disable=C0103
""" utf7 unbase64"""
s_utf7 = '+' + s.replace(',', '/') + '-'
return s_utf7.decode('utf-7')