From 386e656a55bcfea704b726d10f718f7b9efca9b4 Mon Sep 17 00:00:00 2001 From: Stephane de Labrusse Date: Thu, 16 Nov 2023 14:47:11 +0100 Subject: [PATCH 1/5] Generate cron and env files for getmail migration --- imageroot/actions/import-module/40migration | 136 ++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100755 imageroot/actions/import-module/40migration diff --git a/imageroot/actions/import-module/40migration b/imageroot/actions/import-module/40migration new file mode 100755 index 0000000..545cbcf --- /dev/null +++ b/imageroot/actions/import-module/40migration @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import json +import uuid +import os + +file_path = './getmail.json' + +def remove_file(): + # Remove the file + os.remove(file_path) + print(f"File {file_path} removed.") + +try: + with open(file_path, 'r') as file: + data = json.load(file) + # Extract properties for type "getmail" with unique IDs + getmail_properties = [] + + for entry in data: + if entry['type'] == 'getmail': + props = entry['props'] + unique_id = str(uuid.uuid4())[:6] + properties = { + 'ID': unique_id, + 'Account': props['Account'], + 'Password': props.get('Password', ''), + 'Server': props['Server'], + 'Username': props['Username'], + 'Retriever': props['Retriever'], + 'Delete': props.get('Delete', ''), + 'Time': props.get('Time', ''), + 'Status': props.get('status', ''), + 'FilterCheck': props.get('FilterCheck', '') + } + getmail_properties.append(properties) + + # iterate over keys and write tasks files + for properties in getmail_properties: + task_id = properties['ID'] + localuser = properties['Account'].split('@')[0] + remotehostname = properties['Server'] + # convert getmail protocol to imapsync protocol + if properties['Retriever'] == "SimplePOP3Retriever": + remoteport = "143" + security = "" + # Pop3 to IMAP not sure it will work, we disable + cron = "" + elif properties['Retriever'] == "SimplePOP3SSLRetriever": + remoteport = "993" + security = "--ssl1" + # Pop3 to IMAP not sure it will work, we disable + cron = "" + elif properties['Retriever'] == "SimpleIMAPRetriever": + remoteport = '143' + security = '' + cron = properties['Time'] + elif properties['Retriever'] == "SimpleIMAPSSLRetriever": + remoteport = '993' + security = '--ssl1' + cron = properties['Time'] + remoteusername = properties['Username'] + remotepassword = properties['Password'] + + # options not available in getmail, we disable for validation + delete_local = "" + delete_folder = "" + + # delete on remote once migrated + delete_remote = properties['Delete'] + + # in imapsync we just have an option to remove it immediately + if delete_remote != "-1": + delete_remote = "--delete1" + expunge_remote = "--noexpungeaftereach" + else: + delete_remote = "" + expunge_remote = "" + + # not available in getmail, we disable for validation + exclude = "" + + # cron + if cron != "": + if cron == '60': + cron_env = "1 */1 * * * root /usr/local/bin/syncctl start "+localuser+'_'+task_id + cron = '1h' + else: + cron_env = "*/"+cron+" * * * * root /usr/local/bin/syncctl start "+localuser+'_'+task_id + cron = cron + 'm' + f = open("./cron/"+localuser+'_'+task_id+".cron", "w", encoding="utf-8") + # MAIL_HOST is an env variable used by perl/imapsync + f.write('MAIL_HOST='+os.environ['MAIL_HOST']+"\n") + f.write(cron_env+"\n") + f.close() + + foldersynchronization = "inbox" + folder_inbox = "--folder\ 'INBOX'" + + user_env = f""" +TASK_ID={task_id} +LOCALUSER={localuser} +REMOTEUSERNAME={remoteusername} +REMOTEHOSTNAME={remotehostname} +REMOTEPORT={remoteport} +SECURITY={security} +DELETE_LOCAL={delete_local} +DELETEFOLDER={delete_folder} +EXCLUDE={exclude} +DELETE_REMOTE={delete_remote} +EXPUNGE_REMOTE={expunge_remote} +CRON={cron} +FOLDER_INBOX={folder_inbox} +FOLDERSYNCHRONIZATION={foldersynchronization} + """ + os.umask(0o77) + + f = open("./imapsync/"+localuser+'_'+task_id+".env", "w", encoding="utf-8") + f.write(user_env) + f.close() + + f = open("./imapsync/"+localuser+'_'+task_id+".pwd", "w", encoding="utf-8") + f.write(remotepassword) + f.close() + +except Exception as e: + print(f"An error occurred: {str(e)}") + +# remove +finally: + remove_file() From ea89981a7904c3dbbb2568f125f0101ccc40aa16 Mon Sep 17 00:00:00 2001 From: Stephane de Labrusse Date: Thu, 16 Nov 2023 16:06:50 +0100 Subject: [PATCH 2/5] use atexit to trap the file removal on exit --- imageroot/actions/import-module/40migration | 199 ++++++++++---------- 1 file changed, 98 insertions(+), 101 deletions(-) diff --git a/imageroot/actions/import-module/40migration b/imageroot/actions/import-module/40migration index 545cbcf..01d9ff3 100755 --- a/imageroot/actions/import-module/40migration +++ b/imageroot/actions/import-module/40migration @@ -8,6 +8,7 @@ import json import uuid import os +import atexit file_path = './getmail.json' @@ -16,93 +17,96 @@ def remove_file(): os.remove(file_path) print(f"File {file_path} removed.") -try: - with open(file_path, 'r') as file: - data = json.load(file) - # Extract properties for type "getmail" with unique IDs - getmail_properties = [] - - for entry in data: - if entry['type'] == 'getmail': - props = entry['props'] - unique_id = str(uuid.uuid4())[:6] - properties = { - 'ID': unique_id, - 'Account': props['Account'], - 'Password': props.get('Password', ''), - 'Server': props['Server'], - 'Username': props['Username'], - 'Retriever': props['Retriever'], - 'Delete': props.get('Delete', ''), - 'Time': props.get('Time', ''), - 'Status': props.get('status', ''), - 'FilterCheck': props.get('FilterCheck', '') - } - getmail_properties.append(properties) - - # iterate over keys and write tasks files - for properties in getmail_properties: - task_id = properties['ID'] - localuser = properties['Account'].split('@')[0] - remotehostname = properties['Server'] - # convert getmail protocol to imapsync protocol - if properties['Retriever'] == "SimplePOP3Retriever": - remoteport = "143" - security = "" - # Pop3 to IMAP not sure it will work, we disable - cron = "" - elif properties['Retriever'] == "SimplePOP3SSLRetriever": - remoteport = "993" - security = "--ssl1" - # Pop3 to IMAP not sure it will work, we disable - cron = "" - elif properties['Retriever'] == "SimpleIMAPRetriever": - remoteport = '143' - security = '' - cron = properties['Time'] - elif properties['Retriever'] == "SimpleIMAPSSLRetriever": - remoteport = '993' - security = '--ssl1' - cron = properties['Time'] - remoteusername = properties['Username'] - remotepassword = properties['Password'] - - # options not available in getmail, we disable for validation - delete_local = "" - delete_folder = "" - - # delete on remote once migrated - delete_remote = properties['Delete'] - - # in imapsync we just have an option to remove it immediately - if delete_remote != "-1": - delete_remote = "--delete1" - expunge_remote = "--noexpungeaftereach" +# register the function to be called on exit +atexit.register(remove_file) + +with open(file_path, 'r') as file: + data = json.load(file) + +# Extract properties for type "getmail" with unique IDs +getmail_properties = [] + +for entry in data: + if entry['type'] == 'getmail': + props = entry['props'] + unique_id = str(uuid.uuid4())[:6] + properties = { + 'ID': unique_id, + 'Account': props['Account'], + 'Password': props.get('Password', ''), + 'Server': props['Server'], + 'Username': props['Username'], + 'Retriever': props['Retriever'], + 'Delete': props.get('Delete', ''), + 'Time': props.get('Time', ''), + 'Status': props.get('status', ''), + 'FilterCheck': props.get('FilterCheck', '') + } + getmail_properties.append(properties) + +# iterate over keys and write tasks files +for properties in getmail_properties: + task_id = properties['ID'] + localuser = properties['Account'].split('@')[0] + remotehostname = properties['Server'] + # convert getmail protocol to imapsync protocol + if properties['Retriever'] == "SimplePOP3Retriever": + remoteport = "143" + security = "" + # Pop3 to IMAP not sure it will work, we disable + cron = "" + elif properties['Retriever'] == "SimplePOP3SSLRetriever": + remoteport = "993" + security = "--ssl1" + # Pop3 to IMAP not sure it will work, we disable + cron = "" + elif properties['Retriever'] == "SimpleIMAPRetriever": + remoteport = '143' + security = '' + cron = properties['Time'] + elif properties['Retriever'] == "SimpleIMAPSSLRetriever": + remoteport = '993' + security = '--ssl1' + cron = properties['Time'] + remoteusername = properties['Username'] + remotepassword = properties['Password'] + + # options not available in getmail, we disable for validation + delete_local = "" + delete_folder = "" + + # delete on remote once migrated + delete_remote = properties['Delete'] + + # in imapsync we just have an option to remove it immediately + if delete_remote != "-1": + delete_remote = "--delete1" + expunge_remote = "--noexpungeaftereach" + else: + delete_remote = "" + expunge_remote = "" + + # not available in getmail, we disable for validation + exclude = "" + + # cron + if cron != "": + if cron == '60': + cron_env = "1 */1 * * * root /usr/local/bin/syncctl start "+localuser+'_'+task_id + cron = '1h' else: - delete_remote = "" - expunge_remote = "" - - # not available in getmail, we disable for validation - exclude = "" - - # cron - if cron != "": - if cron == '60': - cron_env = "1 */1 * * * root /usr/local/bin/syncctl start "+localuser+'_'+task_id - cron = '1h' - else: - cron_env = "*/"+cron+" * * * * root /usr/local/bin/syncctl start "+localuser+'_'+task_id - cron = cron + 'm' - f = open("./cron/"+localuser+'_'+task_id+".cron", "w", encoding="utf-8") - # MAIL_HOST is an env variable used by perl/imapsync - f.write('MAIL_HOST='+os.environ['MAIL_HOST']+"\n") - f.write(cron_env+"\n") - f.close() - - foldersynchronization = "inbox" - folder_inbox = "--folder\ 'INBOX'" - - user_env = f""" + cron_env = "*/"+cron+" * * * * root /usr/local/bin/syncctl start "+localuser+'_'+task_id + cron = cron + 'm' + f = open("./cron/"+localuser+'_'+task_id+".cron", "w", encoding="utf-8") + # MAIL_HOST is an env variable used by perl/imapsync + f.write('MAIL_HOST='+os.environ['MAIL_HOST']+"\n") + f.write(cron_env+"\n") + f.close() + + foldersynchronization = "inbox" + folder_inbox = "--folder\ 'INBOX'" + + user_env = f""" TASK_ID={task_id} LOCALUSER={localuser} REMOTEUSERNAME={remoteusername} @@ -117,20 +121,13 @@ EXPUNGE_REMOTE={expunge_remote} CRON={cron} FOLDER_INBOX={folder_inbox} FOLDERSYNCHRONIZATION={foldersynchronization} - """ - os.umask(0o77) - - f = open("./imapsync/"+localuser+'_'+task_id+".env", "w", encoding="utf-8") - f.write(user_env) - f.close() - - f = open("./imapsync/"+localuser+'_'+task_id+".pwd", "w", encoding="utf-8") - f.write(remotepassword) - f.close() + """ + os.umask(0o77) -except Exception as e: - print(f"An error occurred: {str(e)}") + f = open("./imapsync/"+localuser+'_'+task_id+".env", "w", encoding="utf-8") + f.write(user_env) + f.close() -# remove -finally: - remove_file() + f = open("./imapsync/"+localuser+'_'+task_id+".pwd", "w", encoding="utf-8") + f.write(remotepassword) + f.close() From 8719c801486784d04a947942ac3bd3037993a1be Mon Sep 17 00:00:00 2001 From: Stephane de Labrusse Date: Mon, 20 Nov 2023 12:33:40 +0100 Subject: [PATCH 3/5] Fix empty account fields in getmail_properties --- imageroot/actions/import-module/40migration | 36 +++++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/imageroot/actions/import-module/40migration b/imageroot/actions/import-module/40migration index 01d9ff3..71a3afc 100755 --- a/imageroot/actions/import-module/40migration +++ b/imageroot/actions/import-module/40migration @@ -8,18 +8,10 @@ import json import uuid import os -import atexit +import sys file_path = './getmail.json' -def remove_file(): - # Remove the file - os.remove(file_path) - print(f"File {file_path} removed.") - -# register the function to be called on exit -atexit.register(remove_file) - with open(file_path, 'r') as file: data = json.load(file) @@ -32,21 +24,36 @@ for entry in data: unique_id = str(uuid.uuid4())[:6] properties = { 'ID': unique_id, - 'Account': props['Account'], + 'Account': props.get('Account',''), 'Password': props.get('Password', ''), - 'Server': props['Server'], - 'Username': props['Username'], - 'Retriever': props['Retriever'], + 'Server': props.get('Server',''), + 'Username': props.get('Username',''), + 'Retriever': props.get('Retriever',''), 'Delete': props.get('Delete', ''), 'Time': props.get('Time', ''), 'Status': props.get('status', ''), 'FilterCheck': props.get('FilterCheck', '') } + if properties['Account'] == "": + print("Account: no argument given", file=sys.stderr) + continue # skip empty accounts + elif properties['Password'] == "": + print("Password: no argument given", file=sys.stderr) + continue # skip accounts without password + elif properties['Server'] == "": + print("Server: no argument given", file=sys.stderr) + continue # skip accounts without server + elif properties['Username'] == "": + print("Username: no argument given", file=sys.stderr) + continue # skip accounts without username + elif properties['Retriever'] == "": + print("Retriever: no argument given", file=sys.stderr) + continue # skip accounts without retriever getmail_properties.append(properties) # iterate over keys and write tasks files for properties in getmail_properties: - task_id = properties['ID'] + task_id = properties['ID'] localuser = properties['Account'].split('@')[0] remotehostname = properties['Server'] # convert getmail protocol to imapsync protocol @@ -131,3 +138,4 @@ FOLDERSYNCHRONIZATION={foldersynchronization} f = open("./imapsync/"+localuser+'_'+task_id+".pwd", "w", encoding="utf-8") f.write(remotepassword) f.close() + From 5d7eb71619e00396103c36ed922979a955b4b6ea Mon Sep 17 00:00:00 2001 From: Stephane de Labrusse Date: Mon, 20 Nov 2023 12:34:01 +0100 Subject: [PATCH 4/5] Remove getmail.json only on success --- imageroot/actions/import-module/40migration | 3 +++ 1 file changed, 3 insertions(+) diff --git a/imageroot/actions/import-module/40migration b/imageroot/actions/import-module/40migration index 71a3afc..467fd88 100755 --- a/imageroot/actions/import-module/40migration +++ b/imageroot/actions/import-module/40migration @@ -139,3 +139,6 @@ FOLDERSYNCHRONIZATION={foldersynchronization} f.write(remotepassword) f.close() +# Remove the file +os.remove(file_path) +print(f"File {file_path} removed.") From 550c43ba03463dcd5b3b57834d2dc44e7db68917 Mon Sep 17 00:00:00 2001 From: Stephane de Labrusse Date: Mon, 20 Nov 2023 12:58:45 +0100 Subject: [PATCH 5/5] Print to stderr the file removal Co-authored-by: Davide Principi --- imageroot/actions/import-module/40migration | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imageroot/actions/import-module/40migration b/imageroot/actions/import-module/40migration index 467fd88..2483826 100755 --- a/imageroot/actions/import-module/40migration +++ b/imageroot/actions/import-module/40migration @@ -141,4 +141,4 @@ FOLDERSYNCHRONIZATION={foldersynchronization} # Remove the file os.remove(file_path) -print(f"File {file_path} removed.") +print(f"File {file_path} removed.", file=sys.stderr)