From 2e40b8532d3b777ffb20b2141a7c3b0ce91d808f Mon Sep 17 00:00:00 2001 From: lance Date: Sat, 21 Mar 2026 07:23:08 +0800 Subject: [PATCH] Fix Email message input ignores IMAP folder Signed-off-by: lance --- .../org/apache/hop/mail/common/MailConst.java | 43 ++ .../mail/metadata/MailServerConnection.java | 514 +++++++++--------- .../metadata/MailServerConnectionEditor.java | 21 +- .../transforms/mailinput/MailInput.java | 33 +- .../workflow/actions/getpop/ActionGetPOP.java | 11 +- .../messages/messages_en_US.properties | 4 +- 6 files changed, 350 insertions(+), 276 deletions(-) create mode 100644 plugins/misc/mail/src/main/java/org/apache/hop/mail/common/MailConst.java diff --git a/plugins/misc/mail/src/main/java/org/apache/hop/mail/common/MailConst.java b/plugins/misc/mail/src/main/java/org/apache/hop/mail/common/MailConst.java new file mode 100644 index 00000000000..48d2c94078c --- /dev/null +++ b/plugins/misc/mail/src/main/java/org/apache/hop/mail/common/MailConst.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.mail.common; + +import lombok.experimental.UtilityClass; + +/** mail const */ +@UtilityClass +public class MailConst { + public static final String MAIL_PREFIX = "mail."; + + /** mail support protocol */ + public static final String INBOX_FOLDER = "INBOX"; + + public static final String PROTOCOL_STRING_IMAP = "IMAP"; + public static final String PROTOCOL_STRING_POP3 = "POP3"; + + public static final String PROTOCOL_MBOX = "MBOX"; + + public static final String PROTOCOL_SMTP = "SMTP"; + public static final String PROTOCOL_SSL_SMTP = "smtps"; + + public static final String PROTOCOL_MSTOR = "mstor"; + + public static final String SSL_TLS = "TLS"; + public static final String SSL_TLS_12 = "TLS 1.2"; + public static final String SSL_TLS_V12 = "TLSv1.2"; +} diff --git a/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnection.java b/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnection.java index ddf7ce7a992..1befcd5a04c 100644 --- a/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnection.java +++ b/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnection.java @@ -62,6 +62,7 @@ import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.mail.common.MailConst; import org.apache.hop.metadata.api.HopMetadata; import org.apache.hop.metadata.api.HopMetadataBase; import org.apache.hop.metadata.api.HopMetadataProperty; @@ -81,63 +82,19 @@ hopMetadataPropertyType = HopMetadataPropertyType.MAIL_SERVER_CONNECTION, supportsGlobalReplace = true) public class MailServerConnection extends HopMetadataBase implements IHopMetadata { - private static final Class PKG = MailServerConnection.class; - public static final String FOLDER_SEPARATOR = "/"; - - public static final int PROTOCOL_POP3 = 0; - public static final int PROTOCOL_IMAP = 1; public static final int PROTOCOL_MBOX = 2; - public static final String INBOX_FOLDER = "INBOX"; - public static final String PROTOCOL_STRING_IMAP = "IMAP"; - public static final String PROTOCOL_STRING_POP3 = "POP3"; - public static final String[] protocolCodes = new String[] {"POP3", "IMAP", "MBOX"}; - public static final String PROTOCOL_STRING_MBOX = protocolCodes[PROTOCOL_MBOX]; - - public static final int DEFAULT_IMAP_PORT = 110; - public static final int DEFAULT_POP3_PORT = 110; - public static final int DEFAULT_SSL_POP3_PORT = 995; - public static final int DEFAULT_SSL_IMAP_PORT = 993; - - public static final String CONST_MAIL = "mail."; - private static final String CONST_POP3_UNSUPPORTED = - "MailConnection.Error.ReceivedDatePOP3Unsupported"; - - public static final String[] conditionDateCode = - new String[] {"ignore", "equal", "smaller", "greater", "between"}; - public static final int CONDITION_DATE_IGNORE = 0; public static final int CONDITION_DATE_EQUAL = 1; public static final int CONDITION_DATE_SMALLER = 2; public static final int CONDITION_DATE_GREATER = 3; public static final int CONDITION_DATE_BETWEEN = 4; - public static final String[] actionTypeDesc = - new String[] { - BaseMessages.getString(PKG, "ActionGetPOP.ActionType.GetMessages.Label"), - BaseMessages.getString(PKG, "ActionGetPOP.ActionType.MoveMessages.Label"), - BaseMessages.getString(PKG, "ActionGetPOP.ActionType.DeleteMessages.Label"), - }; - public static final String[] actionTypeCode = new String[] {"get", "move", "delete"}; public static final int ACTION_TYPE_GET = 0; public static final int ACTION_TYPE_MOVE = 1; public static final int ACTION_TYPE_DELETE = 2; - public static final String[] valueIMAPListCode = - new String[] { - "imaplistall", - "imaplistnew", - "imaplistold", - "imaplistread", - "imaplistunread", - "imaplistflagged", - "imaplistnotflagged", - "imaplistdraft", - "imaplistnotdraft", - "imaplistanswered", - "imaplistnotanswered" - }; public static final int VALUE_IMAP_LIST_ALL = 0; public static final int VALUE_IMAP_LIST_NEW = 1; public static final int VALUE_IMAP_LIST_OLD = 2; @@ -147,17 +104,7 @@ public class MailServerConnection extends HopMetadataBase implements IHopMetadat public static final int VALUE_IMAP_LIST_NOT_FLAGGED = 6; public static final int VALUE_IMAP_LIST_DRAFT = 7; public static final int VALUE_IMAP_LIST_NOT_DRAFT = 8; - public static final int VALUE_IMAP_LIST_ANWERED = 9; - public static final int VALUE_IMAP_LIST_NOT_ANSWERED = 10; - - public static final String[] afterGetIMAPDesc = - new String[] { - BaseMessages.getString(PKG, "ActionGetPOP.afterGetIMAP.Nothing.Label"), - BaseMessages.getString(PKG, "ActionGetPOP.afterGetIMAP.Delete.Label"), - BaseMessages.getString(PKG, "ActionGetPOP.afterGetIMAP.MoveTo.Label") - }; - public static final String[] afterGetIMAPCode = new String[] {"nothing", "delete", "move"}; - public static final int AFTER_GET_IMAP_NOTHING = 0; + public static final int AFTER_GET_IMAP_DELETE = 1; public static final int AFTER_GET_IMAP_MOVE = 2; @@ -235,102 +182,22 @@ public MailServerConnection(IVariables variables) { public Session getSession(IVariables variables) { this.variables = variables; - // SMTP - if (protocol.equals("SMTP")) { - // Send an e-mail... - // create some properties and get the default Session - - protocol = "smtp"; - if (useSecureAuthentication) { - if (useXOAuth2) { - props.put("mail.smtp.auth.mechanisms", "XOAUTH2"); - } - if (secureConnectionType.equals("TLS")) { - // Allow TLS authentication - props.put("mail.smtp.starttls.enable", "true"); - } else if (secureConnectionType.equals("TLS 1.2")) { - // Allow TLS 1.2 authentication - props.put("mail.smtp.starttls.enable", "true"); - props.put("mail.smtp.ssl.protocols", "TLSv1.2"); - } else { - protocol = "smtps"; - // required to get rid of a SSL exception : - // nested exception is: - // javax.net.ssl.SSLException: Unsupported record version Unknown - props.put("mail.smtps.quitwait", "false"); - } - props.setProperty( - "mail.smtp.ssl.checkServerIdentity", String.valueOf(isCheckServerIdentity())); - if (!Utils.isEmpty(trustedHosts)) { - props.put("mail.smtp.ssl.trust", variables.resolve(trustedHosts)); - } - } - - props.put(CONST_MAIL + protocol.toLowerCase() + ".host", variables.resolve(serverHost)); - if (!Utils.isEmpty(serverPort)) { - props.put(CONST_MAIL + protocol.toLowerCase() + ".port", variables.resolve(serverPort)); - } - - if (useAuthentication) { - props.put(CONST_MAIL + protocol + ".auth", "true"); - } + if (isSmtp()) { + buildSmtpProps(); } else { - String protocolString = ""; - if (isUseProxy()) { - // Need here to pass a proxy - // use SASL authentication - props.put("mail.imap.sasl.enable", "true"); - props.put("mail.imap.sasl.authorizationid", proxyUsername); - } - - if (protocol.equals("POP3")) { - props.setProperty("mail.pop3s.rsetbeforequit", "true"); - props.setProperty("mail.pop3.rsetbeforequit", "true"); - } else if (protocol.equals("MBOX")) { - props.setProperty("mstor.mbox.metadataStrategy", "none"); // none|xml|yaml - props.setProperty("mstor.cache.disabled", "true"); // prevent diskstore fail - } - - protocolString = - (protocol.equals("POP3")) ? "pop3" : protocol.equals("MBOX") ? "mstor" : "imap"; - if (useSecureAuthentication && !protocol.equals("MBOX")) { - // Supports IMAP/POP3 connection with SSL, the connection is established via SSL. - props.setProperty( - CONST_MAIL + protocolString + ".socketFactory.class", "javax.net.ssl.SSLSocketFactory"); - props.setProperty(CONST_MAIL + protocolString + ".socketFactory.fallback", "false"); - props.setProperty( - CONST_MAIL + protocolString + ".port", "" + variables.resolve(serverPort)); - props.setProperty( - CONST_MAIL + protocolString + ".socketFactory.port", - "" + variables.resolve(serverPort)); - if (useXOAuth2) { - props.setProperty(CONST_MAIL + protocolString + ".ssl.enable", "true"); - props.setProperty(CONST_MAIL + protocolString + ".auth.mechanisms", "XOAUTH2"); - } - } - } - props.setProperty("mail.imap.ssl.checkServerIdentity", String.valueOf(isCheckServerIdentity())); - props.setProperty( - "mail.imaps.ssl.checkServerIdentity", String.valueOf(isCheckServerIdentity())); - props.setProperty("mail.pop3.ssl.checkServerIdentity", String.valueOf(isCheckServerIdentity())); - props.setProperty( - "mail.pop3s.ssl.checkServerIdentity", String.valueOf(isCheckServerIdentity())); - if (!Utils.isEmpty(trustedHosts)) { - String resolvedTrusted = variables.resolve(trustedHosts); - props.setProperty("mail.imap.ssl.trust", resolvedTrusted); - props.setProperty("mail.imaps.ssl.trust", resolvedTrusted); - props.setProperty("mail.pop3.ssl.trust", resolvedTrusted); - props.setProperty("mail.pop3s.ssl.trust", resolvedTrusted); + buildStoreProps(); } - session = Session.getInstance(props); + applyCommonSecurity(); + applyTrustConfig(); + session = Session.getInstance(props); return session; } // SMTP public Transport getTransport() throws MessagingException { - Transport transport = session.getTransport(protocol); + Transport transport = session.getTransport(protocol.toLowerCase()); String authPass = getPassword(password); if (useAuthentication) { @@ -355,10 +222,10 @@ public Transport getTransport() throws MessagingException { // IMAP, POP, MBOX public Store getStore() throws MessagingException { - if (useSecureAuthentication && !protocol.equals("MBOX")) { + if (useSecureAuthentication && !protocol.equals(MailConst.PROTOCOL_MBOX)) { URLName url = new URLName( - protocol, + protocol.toLowerCase(), variables.resolve(serverHost), Integer.parseInt(variables.resolve(serverPort)), "", @@ -366,30 +233,26 @@ public Store getStore() throws MessagingException { variables.resolve(password)); switch (protocol) { - case "POP3": + case MailConst.PROTOCOL_STRING_POP3: store = new POP3SSLStore(session, url); break; - case "IMAP": + case MailConst.PROTOCOL_STRING_IMAP: store = new IMAPSSLStore(session, url); break; default: break; } } else { - if (protocol.equals("MBOX")) { - this.store = this.session.getStore(new URLName(protocol + ":" + serverHost)); + if (protocol.equalsIgnoreCase(MailConst.PROTOCOL_MBOX)) { + this.store = this.session.getStore(new URLName(MailConst.PROTOCOL_MSTOR + ":file:///")); } else { - this.store = this.session.getStore(protocol); + this.store = this.session.getStore(protocol.toLowerCase()); } } return store; } - public MailServerConnection(MailServerConnection connection) { - // no impementation - } - @Override public String toString() { return name == null ? super.toString() : name; @@ -422,7 +285,7 @@ public boolean equals(Object object) { */ public void connect() throws HopException, NoSuchProviderException { try { - if (this.useSecureAuthentication || this.protocol.equals("MBOX")) { + if (this.useSecureAuthentication || this.protocol.equals(MailConst.PROTOCOL_MBOX)) { // Supports IMAP/POP3 connection with SSL, // the connection is established via SSL. this.store.connect(); @@ -447,17 +310,34 @@ public void connect() throws HopException, NoSuchProviderException { "ActionGetMailsFromPOP.Error.Connecting", this.serverHost, this.username, - Const.NVL("" + this.serverPort, "")), + Const.NVL(this.serverPort, "")), e); } } - public boolean testConnection(Session session) { + public void testConnection(Session session) { try { this.session = session; + + // mbox[mstor], Currently not supported. + if (protocol.equalsIgnoreCase(MailConst.PROTOCOL_MBOX)) { + return; + } + + // send mail + if (protocol.equalsIgnoreCase(MailConst.PROTOCOL_SMTP)) { + getTransport(); + return; + } + + // receive mail + String host = variables.resolve(this.serverHost); + int port = Integer.parseInt(variables.resolve(Const.NVL(this.serverPort, "0"))); + String user = variables.resolve(this.username); + String pwd = variables.resolve(this.password); + Store theStore = getStore(); - theStore.connect(); - return true; + theStore.connect(host, port, user, pwd); } catch (MessagingException e) { throw new RuntimeException(e); } @@ -470,19 +350,18 @@ public String getPassword(String authPassword) { /** * Set destination folder * - * @param folderName destination foldername + * @param folderName destination folder name * @param createFolder flag create folder if needed - * @throws HopException */ public void setDestinationFolder(String folderName, boolean createFolder) throws HopException { try { - String[] folderparts = folderName.split("/"); + String[] folderParts = folderName.split("/"); if (!store.isConnected()) { store.connect(); } Folder f = store.getDefaultFolder(); // Open destination folder - for (String folderpart : folderparts) { + for (String folderpart : folderParts) { f = f.getFolder(folderpart); if (!f.exists()) { if (createFolder) { @@ -530,13 +409,13 @@ private void addSearchTerm(SearchTerm term) { } /** - * Set filter on receipient. + * Set filter on recipient. * - * @param receipient messages will be filtered on receipient + * @param recipient messages will be filtered on recipient */ - public void setReceipientTerm(String receipient) { - if (!Utils.isEmpty(receipient)) { - addSearchTerm(new RecipientStringTerm(Message.RecipientType.TO, receipient)); + public void setRecipientTerm(String recipient) { + if (!Utils.isEmpty(recipient)) { + addSearchTerm(new RecipientStringTerm(Message.RecipientType.TO, recipient)); } } @@ -557,17 +436,17 @@ public void setSubjectTerm(String subject, boolean notTerm) { } /** - * Search all messages with body containing the word bodyfilter + * Search all messages with body containing the word body filter * - * @param bodyfilter + * @param bodyFilter bodyFilter * @param notTerm negate condition */ - public void setBodyTerm(String bodyfilter, boolean notTerm) { - if (!Utils.isEmpty(bodyfilter)) { + public void setBodyTerm(String bodyFilter, boolean notTerm) { + if (!Utils.isEmpty(bodyFilter)) { if (notTerm) { - addSearchTerm(new NotTerm(new BodyTerm(bodyfilter))); + addSearchTerm(new NotTerm(new BodyTerm(bodyFilter))); } else { - addSearchTerm(new BodyTerm(bodyfilter)); + addSearchTerm(new BodyTerm(bodyFilter)); } } } @@ -575,11 +454,11 @@ public void setBodyTerm(String bodyfilter, boolean notTerm) { /** * Set filter on message received date. * - * @param receiveddate messages will be filtered on receiveddate + * @param receivedDate messages will be filtered on received date */ - public void setReceivedDateTermEQ(Date receiveddate) { - if (!this.protocol.equals("POP3")) { - addSearchTerm(new ReceivedDateTerm(ComparisonTerm.EQ, receiveddate)); + public void setReceivedDateTermEQ(Date receivedDate) { + if (!this.protocol.equals(MailConst.PROTOCOL_STRING_POP3)) { + addSearchTerm(new ReceivedDateTerm(ComparisonTerm.EQ, receivedDate)); } } @@ -589,7 +468,7 @@ public void setReceivedDateTermEQ(Date receiveddate) { * @param futureDate messages will be filtered on futureDate */ public void setReceivedDateTermLT(Date futureDate) { - if (!this.protocol.equals("POP3")) { + if (!this.protocol.equals(MailConst.PROTOCOL_STRING_POP3)) { addSearchTerm(new ReceivedDateTerm(ComparisonTerm.LT, futureDate)); } } @@ -600,13 +479,13 @@ public void setReceivedDateTermLT(Date futureDate) { * @param pastDate messages will be filtered on pastDate */ public void setReceivedDateTermGT(Date pastDate) { - if (!this.protocol.equals("POP3")) { + if (!this.protocol.equals(MailConst.PROTOCOL_STRING_POP3)) { addSearchTerm(new ReceivedDateTerm(ComparisonTerm.GT, pastDate)); } } public void setReceivedDateTermBetween(Date beginDate, Date endDate) { - if (!this.protocol.equals("POP3")) { + if (!this.protocol.equals(MailConst.PROTOCOL_STRING_POP3)) { addSearchTerm( new AndTerm( new ReceivedDateTerm(ComparisonTerm.LT, endDate), @@ -672,9 +551,8 @@ private HashSet returnSubfolders(Folder folder) throws HopException { * @return sub folders */ public String[] returnAllFolders(Folder folder) throws HopException { - HashSet list = new HashSet<>(); - list = returnSubfolders(folder); - return list.toArray(new String[list.size()]); + HashSet list = returnSubfolders(folder); + return list.toArray(new String[0]); } /** @@ -693,7 +571,6 @@ public String[] returnAllFolders() throws HopException { * @return sub folders */ public String[] returnAllFolders(String folder) throws HopException { - Folder dfolder = null; String[] retval = null; try { @@ -777,7 +654,7 @@ public void updateSavedMessagesCounter() { * Disconnect from the server and close folder, connection. * * @param expunge expunge folder - * @throws HopException + * @throws HopException ex */ public void disconnect(boolean expunge) throws HopException { try { @@ -804,7 +681,7 @@ public void disconnect(boolean expunge) throws HopException { * Close folder. * * @param expunge expunge folder - * @throws HopException + * @throws HopException ex */ public void closeFolder(boolean expunge) throws HopException { try { @@ -836,16 +713,16 @@ public void clearFilters() { /** * Disconnect from the server and close folder, connection. * - * @throws HopException + * @throws HopException ex */ public void disconnect() throws HopException { disconnect(true); } /** - * Returns the foldername. + * Returns the folder name. * - * @return foldername + * @return folder name */ public String getFolderName() { if (this.folder == null) { @@ -893,25 +770,11 @@ public void openFolder(String folderName, boolean defaultFolder, boolean write) } if (defaultFolder) { - if (protocol.equals("MBOX")) { - this.folder = this.store.getDefaultFolder(); - } else { - // get the default folder - this.folder = getRecursiveFolder(INBOX_FOLDER); - } - - if (this.folder == null) { - throw new HopException( - BaseMessages.getString(PKG, "ActionGetMailsFromPOP.InvalidDefaultFolder.Label")); - } - - if ((folder.getType() & Folder.HOLDS_MESSAGES) == 0) { - throw new HopException( - BaseMessages.getString(PKG, "MailConnection.DefaultFolderCanNotHoldMessage")); - } + this.folder = getDefaultFolder(protocol, this.store); } else { // Open specified Folder (for IMAP/MBOX) - if (this.protocol.equals("IMAP") || this.protocol.equals("MBOX")) { + if (this.protocol.equals(MailConst.PROTOCOL_STRING_IMAP) + || this.protocol.equals(MailConst.PROTOCOL_MBOX)) { this.folder = getRecursiveFolder(folderName); } if (this.folder == null || !this.folder.exists()) { @@ -935,16 +798,38 @@ public void openFolder(String folderName, boolean defaultFolder, boolean write) } } + private Folder getDefaultFolder(String protocol, Store store) + throws HopException, MessagingException { + Folder defaultFolder; + if (protocol.equals(MailConst.PROTOCOL_MBOX)) { + defaultFolder = store.getDefaultFolder(); + } else { + // get the default folder + defaultFolder = getRecursiveFolder(MailConst.INBOX_FOLDER); + } + + if (defaultFolder == null) { + throw new HopException( + BaseMessages.getString(PKG, "ActionGetMailsFromPOP.InvalidDefaultFolder.Label")); + } + + if ((defaultFolder.getType() & Folder.HOLDS_MESSAGES) == 0) { + throw new HopException( + BaseMessages.getString(PKG, "MailConnection.DefaultFolderCanNotHoldMessage")); + } + return defaultFolder; + } + private Folder getRecursiveFolder(String folderName) throws MessagingException { Folder dfolder; - String[] folderparts = folderName.split("/"); + String[] folderParts = folderName.split("/"); if (!store.isConnected()) { store.connect(); } dfolder = store.getDefaultFolder(); // Open destination folder - for (String folderpart : folderparts) { - dfolder = dfolder.getFolder(folderpart); + for (String folderPart : folderParts) { + dfolder = dfolder.getFolder(folderPart); } return dfolder; } @@ -952,7 +837,7 @@ private Folder getRecursiveFolder(String folderName) throws MessagingException { /** * Retrieve all messages from server * - * @throws HopException + * @throws HopException ex */ public void retrieveMessages() throws HopException { try { @@ -981,7 +866,7 @@ public int getMessagesCount() { /** * Get next message. * - * @throws HopException + * @throws HopException ex */ public void fetchNext() throws HopException { updateMessageNr(); @@ -996,7 +881,7 @@ public void fetchNext() throws HopException { /** * Delete current fetched message * - * @throws HopException + * @throws HopException ex */ public void deleteMessage() throws HopException { try { @@ -1013,7 +898,7 @@ public void deleteMessage() throws HopException { /** * Delete messages. * - * @throws HopException + * @throws HopException ex */ public void deleteMessages(boolean setCounter) throws HopException { try { @@ -1031,7 +916,7 @@ public void deleteMessages(boolean setCounter) throws HopException { * Move current message to a target folder. (IMAP) You must call setDestinationFolder before * calling this method * - * @throws HopException + * @throws HopException ex */ public void moveMessage() throws HopException { try { @@ -1054,7 +939,7 @@ public void moveMessage() throws HopException { /** * Move messages to a folder. You must call setDestinationFolder before calling this method * - * @throws HopException + * @throws HopException ex */ public void moveMessages() throws HopException { try { @@ -1074,7 +959,7 @@ public void moveMessages() throws HopException { * * @param filename the target filename * @param folderName the parent folder of filename - * @throws HopException + * @throws HopException ex */ public void saveMessageContentToFile(String filename, String folderName) throws HopException { OutputStream os = null; @@ -1103,9 +988,9 @@ public void saveMessageContentToFile(String filename, String folderName) throws /** * Save attached files to a folder. * - * @param folderName the target foldername + * @param folderName the target folder name * @param pattern regular expression to filter on files - * @throws HopException + * @throws HopException ex */ public void saveAttachedFiles(String folderName, Pattern pattern) throws HopException { Object content = null; @@ -1152,12 +1037,7 @@ private void handlePart(String folderName, Part part, Pattern pattern) throws Ho if (disposition.equalsIgnoreCase(Part.ATTACHMENT) || disposition.equalsIgnoreCase(Part.INLINE)) { - String mimeText = null; - try { - mimeText = MimeUtility.decodeText(part.getFileName()); - } catch (Exception e) { - // Ignore errors - } + String mimeText = decodeText(part); if (mimeText != null) { String filename = MimeUtility.decodeText(part.getFileName()); if (isWildcardMatch(filename, pattern)) { @@ -1199,11 +1079,13 @@ private static void saveFile(String folderName, String filename, InputStream inp } finally { if (bis != null) { IOUtils.closeQuietly(bis); - bis = null; // Help the GC + // Help the GC + bis = null; } if (bos != null) { IOUtils.closeQuietly(bos); - bos = null; // Help the GC + // Help the GC + bos = null; // Note - closing the BufferedOuputStream closes the underlying output stream according to // the Javadoc } @@ -1225,8 +1107,9 @@ static String findValidTarget(String folderName, final String fileName) throws H throw new IllegalArgumentException("Cannot have null arguments to findValidTarget"); } String fileNameRoot = FilenameUtils.getBaseName(fileName); + // only a "." String ext = "." + FilenameUtils.getExtension(fileName); - if ((ext.length() == 1)) { // only a "." + if ((ext.length() == 1)) { ext = ""; } String rtn = ""; @@ -1236,7 +1119,8 @@ static String findValidTarget(String folderName, final String fileName) throws H int i = -1; do { i++; - build.setLength(baseSz); // bring string back to size + // bring string back to size + build.setLength(baseSz); build.append(i > 0 ? Integer.toString(i) : "").append(ext); rtn = build.toString(); } while (HopVfs.fileExists(rtn)); @@ -1265,16 +1149,7 @@ private String getMessageBodyOrContentType(Part p, final boolean returnContentTy } if (p.isMimeType("multipart/alternative")) { - // prefer html text over plain text - Multipart mp = (Multipart) p.getContent(); - String text = null; - for (int i = 0; i < mp.getCount(); i++) { - Part bp = mp.getBodyPart(i); - if (bp.isMimeType("text/plain") && text == null) { - text = getMessageBodyOrContentType(bp, returnContentType); - } - } - return text; + return multipartAlternative(p, returnContentType); } else if (p.isMimeType("multipart/*")) { Multipart mp = (Multipart) p.getContent(); for (int i = 0; i < mp.getCount(); i++) { @@ -1288,6 +1163,20 @@ private String getMessageBodyOrContentType(Part p, final boolean returnContentTy return null; } + private String multipartAlternative(Part p, boolean returnContentType) + throws IOException, MessagingException { + // prefer html text over plain text + Multipart mp = (Multipart) p.getContent(); + String text = null; + for (int i = 0; i < mp.getCount(); i++) { + Part bp = mp.getBodyPart(i); + if (bp.isMimeType("text/plain") && text == null) { + text = getMessageBodyOrContentType(bp, returnContentType); + } + } + return text; + } + public boolean isMessageDraft(Message msg) { try { return msg.isSet(Flags.Flag.DRAFT); @@ -1368,7 +1257,7 @@ public boolean isMessageDeleted(Message msg) { * Returns attached files count for the current message * * @param pattern (optional) - * @return true if message is Draft + * @return 1 if message is Draft */ public int getAttachedFilesCount(Pattern pattern) throws HopException { return getAttachedFilesCount(getMessage(), pattern); @@ -1387,12 +1276,7 @@ public int getAttachedFilesCount(Message message, Pattern pattern) throws HopExc if ((disposition != null) && (disposition.equalsIgnoreCase(Part.ATTACHMENT) || disposition.equalsIgnoreCase(Part.INLINE))) { - String mimeText = null; - try { - mimeText = MimeUtility.decodeText(part.getFileName()); - } catch (Exception e) { - // Ignore errors - } + String mimeText = decodeText(part); if (mimeText != null) { String filename = MimeUtility.decodeText(part.getFileName()); if (isWildcardMatch(filename, pattern)) { @@ -1416,4 +1300,144 @@ public int getAttachedFilesCount(Message message, Pattern pattern) throws HopExc } return retval; } + + /** builder smtp properties */ + private void buildSmtpProps() { + String proto = protocol.toLowerCase(); + + if (useSecureAuthentication) { + applySmtpSecurity(); + } + + setHostPort(proto); + if (useAuthentication) { + props.put(mailKey(proto, "auth"), "true"); + } + } + + /** builder smtp security */ + private void applySmtpSecurity() { + if (useXOAuth2) { + props.put("mail.smtp.auth.mechanisms", "XOAUTH2"); + } + + if (MailConst.SSL_TLS.equals(secureConnectionType)) { + props.put("mail.smtp.starttls.enable", "true"); + } else if (MailConst.SSL_TLS_12.equals(secureConnectionType)) { + props.put("mail.smtp.starttls.enable", "true"); + props.put("mail.smtp.ssl.protocols", MailConst.SSL_TLS_V12); + } else { + protocol = MailConst.PROTOCOL_SSL_SMTP; + props.put("mail.smtps.quitwait", "false"); + } + + props.put("mail.smtp.ssl.checkServerIdentity", isCheckServerIdentity()); + if (!Utils.isEmpty(trustedHosts)) { + props.put("mail.smtp.ssl.trust", variables.resolve(trustedHosts)); + } + } + + /** builder store properties */ + private void buildStoreProps() { + String proto = resolveProtocol(protocol); + + if (isUseProxy()) { + props.put("mail.imap.sasl.enable", "true"); + props.put("mail.imap.sasl.authorizationid", proxyUsername); + } + + handleSpecialProtocols(); + if (useSecureAuthentication && !isMbox()) { + applyStoreSSL(proto); + } + } + + /** put store ssl */ + private void applyStoreSSL(String proto) { + props.put(mailKey(proto, "socketFactory.class"), "javax.net.ssl.SSLSocketFactory"); + props.put(mailKey(proto, "socketFactory.fallback"), "false"); + props.put(mailKey(proto, "port"), variables.resolve(serverPort)); + props.put(mailKey(proto, "socketFactory.port"), variables.resolve(serverPort)); + + if (useXOAuth2) { + props.put(mailKey(proto, "ssl.enable"), "true"); + props.put(mailKey(proto, "auth.mechanisms"), "XOAUTH2"); + } + } + + /** apply common security */ + private void applyCommonSecurity() { + String[] protocols = {"imap", "imaps", "pop3", "pop3s"}; + + for (String p : protocols) { + props.put( + MailConst.MAIL_PREFIX + p + ".ssl.checkServerIdentity", + String.valueOf(isCheckServerIdentity())); + } + } + + /** apply trust config */ + private void applyTrustConfig() { + if (Utils.isEmpty(trustedHosts)) { + return; + } + + String trusted = variables.resolve(trustedHosts); + String[] protocols = {"imap", "imaps", "pop3", "pop3s"}; + + for (String p : protocols) { + props.put(MailConst.MAIL_PREFIX + p + ".ssl.trust", trusted); + } + } + + /** handler special protocol */ + private void handleSpecialProtocols() { + if (MailConst.PROTOCOL_STRING_POP3.equals(protocol)) { + props.put("mail.pop3.rsetbeforequit", "true"); + props.put("mail.pop3s.rsetbeforequit", "true"); + } else if (MailConst.PROTOCOL_MBOX.equals(protocol)) { + props.put("mstor.mbox.metadataStrategy", "none"); + props.put("mstor.cache.disabled", "true"); + } + } + + /** resolve protocol */ + private String resolveProtocol(String protocol) { + if (MailConst.PROTOCOL_STRING_POP3.equals(protocol)) { + return MailConst.PROTOCOL_STRING_POP3.toLowerCase(); + } else if (MailConst.PROTOCOL_MBOX.equals(protocol)) { + return MailConst.PROTOCOL_MSTOR; + } else { + return MailConst.PROTOCOL_STRING_IMAP.toLowerCase(); + } + } + + private void setHostPort(String proto) { + props.put(mailKey(proto, "host"), variables.resolve(serverHost)); + + if (!Utils.isEmpty(serverPort)) { + props.put(mailKey(proto, "port"), variables.resolve(serverPort)); + } + } + + private boolean isSmtp() { + return MailConst.PROTOCOL_SMTP.equalsIgnoreCase(protocol); + } + + private boolean isMbox() { + return MailConst.PROTOCOL_MBOX.equalsIgnoreCase(protocol); + } + + private String mailKey(String protocol, String key) { + return MailConst.MAIL_PREFIX + protocol + "." + key; + } + + private String decodeText(Part part) { + try { + return MimeUtility.decodeText(part.getFileName()); + } catch (Exception e) { + // Ignore errors + } + return null; + } } diff --git a/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnectionEditor.java b/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnectionEditor.java index a777676bd4a..906c8f78d50 100644 --- a/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnectionEditor.java +++ b/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnectionEditor.java @@ -456,7 +456,7 @@ private void testConnection() { connection.setProtocol(variables.resolve(wConnectionProtocol.getText())); connection.setServerHost(variables.resolve(wServerHost.getText())); connection.setServerPort(variables.resolve(wServerPort.getText())); - connection.setUseAuthentication(wUseSecureAuthentication.getSelection()); + connection.setUseAuthentication(wUseAuthentication.getSelection()); connection.setSecureConnectionType(wSecureConnectionType.getText()); connection.setUseXOAuth2(wUseXOAuth2.getSelection()); connection.setUsername(variables.resolve(wServerUsername.getText())); @@ -466,15 +466,16 @@ private void testConnection() { connection.setProxyUsername(variables.resolve(wProxyUsername.getText())); try { - if (connection.testConnection(connection.getSession(variables))) { - MessageBox mb = new MessageBox(hopGui.getShell(), SWT.OK | SWT.ICON_INFORMATION); - mb.setMessage( - BaseMessages.getString( - PKG, "ActionGetPOP.Connected.OK", variables.resolve(wServerHost.getText())) - + Const.CR); - mb.setText(BaseMessages.getString(PKG, "ActionGetPOP.Connected.Title.Ok")); - mb.open(); - } + connection.testConnection(connection.getSession(variables)); + MessageBox mb = new MessageBox(hopGui.getShell(), SWT.OK | SWT.ICON_INFORMATION); + mb.setMessage( + BaseMessages.getString( + PKG, + "MailServerConnectionDialog.Connected.OK", + variables.resolve(wServerHost.getText())) + + Const.CR); + mb.setText(BaseMessages.getString(PKG, "MailServerConnectionDialog.Connected.Title.Ok")); + mb.open(); } catch (Exception e) { new ErrorDialog(hopGui.getShell(), "Error", "Error connecting mail server:", e); } diff --git a/plugins/misc/mail/src/main/java/org/apache/hop/mail/pipeline/transforms/mailinput/MailInput.java b/plugins/misc/mail/src/main/java/org/apache/hop/mail/pipeline/transforms/mailinput/MailInput.java index de7118a06f8..ab24de88a3d 100644 --- a/plugins/misc/mail/src/main/java/org/apache/hop/mail/pipeline/transforms/mailinput/MailInput.java +++ b/plugins/misc/mail/src/main/java/org/apache/hop/mail/pipeline/transforms/mailinput/MailInput.java @@ -19,7 +19,7 @@ import jakarta.mail.Header; import jakarta.mail.Message; -import jakarta.mail.MessagingException; +import jakarta.mail.Session; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -33,6 +33,7 @@ import org.apache.hop.core.row.RowMeta; import org.apache.hop.core.util.Utils; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.mail.common.MailConst; import org.apache.hop.mail.metadata.MailServerConnection; import org.apache.hop.mail.workflow.actions.getpop.MailConnection; import org.apache.hop.mail.workflow.actions.getpop.MailConnectionMeta; @@ -65,10 +66,9 @@ public MailInput( @Override public boolean processRow() throws HopException { - Object[] outputRowData = getOneRow(); - - if (outputRowData == null) { // no more input to be expected... + // no more input to be expected... + if (outputRowData == null) { setOutputDone(); return false; @@ -107,7 +107,7 @@ public String[] getFolders(String realIMAPFolder) throws HopException { folderslist = connection.getProtocol().equals(MailServerConnection.PROTOCOL_MBOX) ? new String[] {""} - : new String[] {Const.NVL(realIMAPFolder, MailServerConnection.INBOX_FOLDER)}; + : new String[] {Const.NVL(realIMAPFolder, MailConst.INBOX_FOLDER)}; } else { // mstor's default folder has no name folderslist = @@ -118,7 +118,7 @@ public String[] getFolders(String realIMAPFolder) throws HopException { } else { folderslist = new String[folderslist0.length + 1]; if (connection != null) { - folderslist[0] = Const.NVL(realIMAPFolder, MailServerConnection.INBOX_FOLDER); + folderslist[0] = Const.NVL(realIMAPFolder, MailConst.INBOX_FOLDER); } else { folderslist[0] = Const.NVL(realIMAPFolder, MailConnectionMeta.INBOX_FOLDER); } @@ -131,7 +131,7 @@ public String[] getFolders(String realIMAPFolder) throws HopException { folderslist = connection.getProtocol().equals(MailServerConnection.PROTOCOL_MBOX) ? new String[] {""} - : new String[] {Const.NVL(realIMAPFolder, MailServerConnection.INBOX_FOLDER)}; + : new String[] {Const.NVL(realIMAPFolder, MailConst.INBOX_FOLDER)}; } else { folderslist = data.mailConn.getProtocol() == MailConnectionMeta.PROTOCOL_MBOX @@ -158,7 +158,7 @@ private void applySearch(Date beginDate, Date endDate) { if (!Utils.isEmpty(realSearchReceipient)) { // apply TO if (connection != null) { - connection.setReceipientTerm(realSearchReceipient); + connection.setRecipientTerm(realSearchReceipient); } else { data.mailConn.setReceipientTerm(realSearchReceipient); } @@ -427,7 +427,11 @@ private boolean openNextFolder() { } } else { if (connection != null) { - connection.openFolder(false); + if (meta.getProtocol().equalsIgnoreCase(MailConst.PROTOCOL_STRING_IMAP)) { + connection.openFolder(data.folder, true); + } else { + connection.openFolder(false); + } } else { data.mailConn.openFolder(false); } @@ -489,7 +493,6 @@ private boolean openNextFolder() { @Override public boolean init() { - if (!super.init()) { return false; } @@ -525,10 +528,10 @@ public boolean init() { "Mail Server Connection " + meta.getConnectionName() + " could not be found"); } try { - connection.getSession(variables); - connection.getStore().connect(); + Session session = connection.getSession(variables); + connection.testConnection(session); connected = true; - } catch (MessagingException e) { + } catch (Exception e) { logError( "A connection to mail server connection '" + meta.getConnectionName() @@ -674,8 +677,8 @@ public boolean init() { private int getReadFirst(String protocol) { if (connection != null) { - if (protocol.equals(MailServerConnection.PROTOCOL_STRING_IMAP) - || protocol.equals(MailServerConnection.PROTOCOL_STRING_POP3)) { + if (protocol.equals(MailConst.PROTOCOL_STRING_IMAP) + || protocol.equals(MailConst.PROTOCOL_STRING_POP3)) { return Const.toInt(meta.getFirstMails(), 0); } } else { diff --git a/plugins/misc/mail/src/main/java/org/apache/hop/mail/workflow/actions/getpop/ActionGetPOP.java b/plugins/misc/mail/src/main/java/org/apache/hop/mail/workflow/actions/getpop/ActionGetPOP.java index 9084ec79b2b..000f0205473 100644 --- a/plugins/misc/mail/src/main/java/org/apache/hop/mail/workflow/actions/getpop/ActionGetPOP.java +++ b/plugins/misc/mail/src/main/java/org/apache/hop/mail/workflow/actions/getpop/ActionGetPOP.java @@ -41,6 +41,7 @@ import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.mail.common.MailConst; import org.apache.hop.mail.metadata.MailServerConnection; import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.HopMetadataPropertyType; @@ -228,7 +229,7 @@ public ActionGetPOP(String n) { firstMails = null; delete = false; if (!StringUtils.isEmpty(getConnectionName())) { - protocol = MailServerConnection.PROTOCOL_STRING_POP3; + protocol = MailConst.PROTOCOL_STRING_POP3; } else { protocol = MailConnectionMeta.PROTOCOL_STRING_POP3; } @@ -327,7 +328,7 @@ public Result execute(Result previousResult, int nr) throws HopException { boolean usePOP3; if (connectionName != null) { - usePOP3 = getProtocol().equals(MailServerConnection.PROTOCOL_STRING_POP3); + usePOP3 = getProtocol().equals(MailConst.PROTOCOL_STRING_POP3); } else { usePOP3 = getProtocol().equals(MailConnectionMeta.PROTOCOL_STRING_POP3); } @@ -343,7 +344,7 @@ public Result execute(Result previousResult, int nr) throws HopException { // Check destination folder String realMoveToIMAPFolder = resolve(getMoveToIMAPFolder()); if (StringUtils.isEmpty(connectionName)) { - if (getProtocol().equals(MailServerConnection.PROTOCOL_STRING_IMAP) + if (getProtocol().equals(MailConst.PROTOCOL_STRING_IMAP) && (getActionType() == MailServerConnection.ACTION_TYPE_MOVE) || (getActionType() == MailServerConnection.ACTION_TYPE_GET && getAfterGetIMAP() == MailServerConnection.AFTER_GET_IMAP_MOVE)) { @@ -508,7 +509,7 @@ && getAfterGetIMAP() == MailConnectionMeta.AFTER_GET_IMAP_MOVE)) { if (!Utils.isEmpty(realSearchReceipient)) { // apply TO if (connection != null) { - connection.setReceipientTerm(realSearchReceipient); + connection.setRecipientTerm(realSearchReceipient); } else { mailConn.setReceipientTerm(realSearchReceipient); } @@ -788,7 +789,7 @@ void fetchOneFolder( "ActionGetMailsFromPOP.TotalMessagesFolder.Label", "" + messagesCount, (connection != null) - ? Const.NVL(connection.getFolderName(), MailServerConnection.INBOX_FOLDER) + ? Const.NVL(connection.getFolderName(), MailConst.INBOX_FOLDER) : Const.NVL(mailConn.getFolderName(), MailConnectionMeta.INBOX_FOLDER))); } diff --git a/plugins/misc/mail/src/main/resources/org/apache/hop/mail/metadata/messages/messages_en_US.properties b/plugins/misc/mail/src/main/resources/org/apache/hop/mail/metadata/messages/messages_en_US.properties index d94f7e705bd..e52471f099d 100644 --- a/plugins/misc/mail/src/main/resources/org/apache/hop/mail/metadata/messages/messages_en_US.properties +++ b/plugins/misc/mail/src/main/resources/org/apache/hop/mail/metadata/messages/messages_en_US.properties @@ -35,4 +35,6 @@ MailServerConnectionDialog.ConnectionProtocol=Connection protocol MailServerConnectionDialog.CheckServerIdentity=Check server identity? MailServerConnectionDialog.TrustedHosts=Trusted hosts\: MailServerConnectionDialog.TrustedHosts.Tooltip=Insert the list of trusted hosts separated by a space\ - \nExample\: host1 host2 host3 \ No newline at end of file + \nExample\: host1 host2 host3 +MailServerConnectionDialog.Connected.OK=Connected to {0} +MailServerConnectionDialog.Connected.Title.Ok=Success