Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

quarantine emails with attachments over the 20mb #360

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

quarantine emails with attachments over the 20mb #360

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions src/gmv/gmvault.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,14 @@ def handle_restore_imap_error(the_exception, gm_id, db_gmail_ids_info, gmvaulter
LOG.critical("Disconnecting and reconnecting to restart cleanly.")
gmvaulter.src.reconnect() #reconnect
else:
raise the_exception
raise
elif isinstance(the_exception, IOError) and str(the_exception).find('[Errno 2] No such file or directory:') >=0:
LOG.critical("Quarantine email with gm id %s from %s. GMAIL IMAP cannot restore it:"\
" err={%s}" % (gm_id, db_gmail_ids_info[gm_id], str(the_exception)))
gmvaulter.gstorer.quarantine_email(gm_id)
gmvaulter.error_report['emails_in_quarantine'].append(gm_id)
LOG.critical("Disconnecting and reconnecting to restart cleanly.")
gmvaulter.src.reconnect() #reconnect

gmvaulter.src.reconnect() #reconnects
elif isinstance(the_exception, imaplib.IMAP4.error):
LOG.error("Catched IMAP Error %s" % (str(the_exception)))
LOG.exception(the_exception)
Expand All @@ -64,9 +63,14 @@ def handle_restore_imap_error(the_exception, gm_id, db_gmail_ids_info, gmvaulter
LOG.critical("Quarantine email with gm id %s from %s. GMAIL IMAP cannot restore it:"\
" err={%s}" % (gm_id, db_gmail_ids_info[gm_id], str(the_exception)))
gmvaulter.gstorer.quarantine_email(gm_id)
gmvaulter.error_report['emails_in_quarantine'].append(gm_id)
elif str(the_exception) == "APPEND command error: BAD ['[TOOBIG] Message too large. https://support.google.com/mail/answer/6584#limit']":
LOG.critical("Quarantine email with gm id %s from %s. GMAIL IMAP cannot restore attachment too large:"\
" err={%s}" % (gm_id, db_gmail_ids_info[gm_id], str(the_exception)))
gmvaulter.gstorer.quarantine_email(gm_id)
gmvaulter.error_report['emails_in_quarantine'].append(gm_id)
else:
raise the_exception
raise
elif isinstance(the_exception, imap_utils.PushEmailError):
LOG.error("Catch the following exception %s" % (str(the_exception)))
LOG.exception(the_exception)
Expand All @@ -77,11 +81,11 @@ def handle_restore_imap_error(the_exception, gm_id, db_gmail_ids_info, gmvaulter
gmvaulter.gstorer.quarantine_email(gm_id)
gmvaulter.error_report['emails_in_quarantine'].append(gm_id)
else:
raise the_exception
raise
else:
LOG.error("Catch the following exception %s" % (str(the_exception)))
LOG.exception(the_exception)
raise the_exception
raise

def handle_sync_imap_error(the_exception, the_id, error_report, src):
"""
Expand Down Expand Up @@ -266,7 +270,8 @@ def __init__(self, db_root_dir, host, port, login, \
'cannot_be_fetched' : [],
'emails_in_quarantine' : [],
'reconnections' : 0,
'key_error' : []}
'key_error' : [],
'attachment_too_big' : 0 }

#instantiate gstorer
self.gstorer = gmvault_db.GmailStorer(self.db_root_dir, self.use_encryption)
Expand Down Expand Up @@ -1142,6 +1147,7 @@ def restore_emails(self, pivot_dir = None, extra_labels = [], restart = False):
labels_to_apply[ex_label] = imap_id

except Exception, err:
LOG.critical("Error on gmid %s" % (gm_id))
handle_restore_imap_error(err, gm_id, db_gmail_ids_info, self)

#create the non existing labels and update existing labels
Expand Down
9 changes: 7 additions & 2 deletions src/gmv/imap_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def reconnect(the_self, rec_nb_tries, total_nb_tries, rec_error, rec_sleep_time
LOG.exception(ignored)
else:
#cascade error
raise rec_error
raise

def inner_retry(the_func): #pylint:disable=C0111,R0912
def wrapper(*args, **kwargs): #pylint:disable=C0111,R0912
Expand Down Expand Up @@ -179,6 +179,7 @@ def wrapper(*args, **kwargs): #pylint:disable=C0111,R0912
LOG.debug("IMAP (normal) error message = %s. traceback:%s" % (err, gmvault_utils.get_exception_traceback()))

if nb_tries[0] < a_nb_tries:
LOG.critical(err)
LOG.critical("Error when reaching Gmail server. Wait %s second(s) and retry up to 2 times." \
% (m_sleep_time[0]))
else:
Expand Down Expand Up @@ -800,14 +801,18 @@ def push_data(self, a_folder, a_body, a_flags, a_internal_time):
try:
#a_body = self._clean_email_body(a_body)
res = self.server.append(a_folder, a_body, a_flags, a_internal_time)
except imaplib.IMAP4.error, erre:
if str(erre).find("APPEND command error: BAD ['[TOOBIG]") >= 0:
LOG.critical("APPEND error due to attachment too big: %s" % (len(a_body)))
raise
except imaplib.IMAP4.abort, err:
# handle issue when there are invalid characters (This is do to the presence of null characters)
if str(err).find("APPEND => Invalid character in literal") >= 0:
LOG.critical("Invalid character detected. Try to clean the email and reconnect.")
a_body = self._clean_email_body(a_body)
self.reconnect()
res = self.server.append(a_folder, a_body, a_flags, a_internal_time)

LOG.debug("Appended data with flags %s and internal time %s. Operation time = %s.\nres = %s\n" \
% (a_flags, a_internal_time, the_timer.elapsed_ms(), res))

Expand Down