JAMES-1854 Sieve should be handled as a separate mailet

dockerfiles/run/guice/destination/conf/mailetcontainer.xml

@@ -85,6 +85,7 @@
             </mailet>
             <mailet match="All" class="RecipientRewriteTable" />
             <mailet match="RecipientIsLocal" class="org.apache.james.jmap.mailet.VacationMailet"/>
+            <mailet match="RecipientIsLocal" class="Sieve"/>
             <mailet match="RecipientIsLocal" class="LocalDelivery"/>
             <!--
             <mailet match="HostIsLocal" class="ToProcessor">

dockerfiles/run/spring/destination/conf/mailetcontainer.xml

@@ -81,6 +81,7 @@
                 <value>true</value>
             </mailet>
             <mailet match="All" class="RecipientRewriteTable" />
+            <mailet match="RecipientIsLocal" class="Sieve"/>
             <mailet match="RecipientIsLocal" class="LocalDelivery"/>
             <mailet match="HostIsLocal" class="ToProcessor">
                 <processor>local-address-error</processor>

server/app/src/main/resources/mailetcontainer-template.xml

@@ -430,6 +430,7 @@ Regards, Postmaster XXX.YYY
 	     </mailet>
 
        <!-- Is the recipient is for a local account, deliver it locally -->
+       <mailet match="RecipientIsLocal" class="Sieve"/>
        <mailet match="RecipientIsLocal" class="LocalDelivery"/>
 
        <!-- If the host is handled by this server and it did not get -->

server/container/guice/guice-common/src/main/java/org/apache/james/utils/ExtendedServerProbe.java

@@ -37,4 +37,6 @@ void appendMessage(String username, MailboxPath mailboxPath, InputStream message
 
     Mailbox getMailbox(String namespace, String user, String name);
 
+    void addActiveSieveScript(String user, String name, String script) throws Exception;
+
 }

server/container/guice/guice-common/src/main/java/org/apache/james/utils/GuiceServerProbe.java

@@ -62,8 +62,8 @@ public class GuiceServerProbe implements ExtendedServerProbe, GuiceProbe {
     private final MailboxMapperFactory mailboxMapperFactory;
     private final DomainList domainList;
     private final UsersRepository usersRepository;
-    private final SieveRepository sieveRepository;
     private final RecipientRewriteTable recipientRewriteTable;
+    private final SieveRepository sieveRepository;
 
     @Inject
     private GuiceServerProbe(MailboxManager mailboxManager, MailboxMapperFactory mailboxMapperFactory,
@@ -349,4 +349,9 @@ public void removeSieveQuota(String user) throws Exception {
         sieveRepository.removeQuota(user);
     }
 
+    @Override
+    public void addActiveSieveScript(String user, String name, String script) throws Exception {
+        sieveRepository.putScript(user, name, script);
+        sieveRepository.setActive(user, name);
+    }
 }

server/mailet/integration-testing/src/test/java/org/apache/james/mailets/SieveDelivery.java

@@ -0,0 +1,98 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+
+package org.apache.james.mailets;
+
+import org.apache.james.mailbox.model.MailboxConstants;
+import org.apache.james.mailets.configuration.CommonProcessors;
+import org.apache.james.mailets.configuration.MailetContainer;
+import org.apache.james.mailets.utils.IMAPMessageReader;
+import org.apache.james.mailets.utils.SMTPMessageSender;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import com.jayway.awaitility.Awaitility;
+import com.jayway.awaitility.Duration;
+import com.jayway.awaitility.core.ConditionFactory;
+
+public class SieveDelivery {
+
+    private static final String DEFAULT_DOMAIN = "james.org";
+    private static final String LOCALHOST_IP = "127.0.0.1";
+    private static final int IMAP_PORT = 1143;
+    private static final int SMTP_PORT = 1025;
+    private static final String PASSWORD = "secret";
+
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private TemporaryJamesServer jamesServer;
+    private ConditionFactory calmlyAwait;
+
+    @Before
+    public void setup() throws Exception {
+        MailetContainer mailetContainer = MailetContainer.builder()
+            .postmaster("postmaster@" + DEFAULT_DOMAIN)
+            .threads(5)
+            .addProcessor(CommonProcessors.root())
+            .addProcessor(CommonProcessors.error())
+            .addProcessor(CommonProcessors.transport())
+            .addProcessor(CommonProcessors.spam())
+            .addProcessor(CommonProcessors.localAddressError())
+            .addProcessor(CommonProcessors.relayDenied())
+            .addProcessor(CommonProcessors.bounces())
+            .addProcessor(CommonProcessors.sieveManagerCheck())
+            .build();
+
+        jamesServer = new TemporaryJamesServer(temporaryFolder, mailetContainer);
+        Duration slowPacedPollInterval = Duration.FIVE_HUNDRED_MILLISECONDS;
+        calmlyAwait = Awaitility.with().pollInterval(slowPacedPollInterval).and().with().pollDelay(slowPacedPollInterval).await();
+    }
+
+    @After
+    public void tearDown() {
+        jamesServer.shutdown();
+    }
+
+    @Test
+    public void simpleMailShouldBeSent() throws Exception {
+        String from = "user@" + DEFAULT_DOMAIN;
+        String recipient = "user2@" + DEFAULT_DOMAIN;
+        String targetedMailbox = "INBOX.any";
+
+        jamesServer.getServerProbe().addDomain(DEFAULT_DOMAIN);
+        jamesServer.getServerProbe().addUser(from, PASSWORD);
+        jamesServer.getServerProbe().addUser(recipient, PASSWORD);
+        jamesServer.getServerProbe().createMailbox(MailboxConstants.USER_NAMESPACE, recipient, "INBOX");
+        jamesServer.getServerProbe().createMailbox(MailboxConstants.USER_NAMESPACE, recipient, targetedMailbox);
+        jamesServer.getServerProbe().addActiveSieveScript(recipient, "myscript.sieve", "require \"fileinto\";\n" +
+            "\n" +
+            "fileinto \"" + targetedMailbox + "\";");
+
+        try (SMTPMessageSender messageSender = SMTPMessageSender.noAuthentication(LOCALHOST_IP, SMTP_PORT, DEFAULT_DOMAIN);
+             IMAPMessageReader imapMessageReader = new IMAPMessageReader(LOCALHOST_IP, IMAP_PORT)) {
+            messageSender.sendMessage(from, recipient);
+            calmlyAwait.atMost(Duration.ONE_MINUTE).until(messageSender::messageHasBeenSent);
+            calmlyAwait.atMost(Duration.ONE_MINUTE).until(() -> imapMessageReader.userReceivedMessageInMailbox(recipient, PASSWORD, targetedMailbox));
+        }
+    }
+}

server/mailet/integration-testing/src/test/java/org/apache/james/mailets/configuration/CommonProcessors.java

@@ -118,6 +118,10 @@ public static ProcessorConfiguration transport() {
                         .match("RecipientIsLocal")
                         .clazz("org.apache.james.jmap.mailet.VacationMailet")
                         .build())
+                .addMailet(MailetConfiguration.builder()
+                        .match("RecipientIsLocal")
+                        .clazz("Sieve")
+                        .build())
                 .addMailet(MailetConfiguration.builder()
                         .match("RecipientIsLocal")
                         .clazz("LocalDelivery")

server/mailet/integration-testing/src/test/java/org/apache/james/mailets/utils/IMAPMessageReader.java

@@ -34,8 +34,12 @@ public IMAPMessageReader(String host, int port) throws IOException {
     }
 
     public boolean userReceivedMessage(String user, String password) throws IOException {
+        return userReceivedMessageInMailbox(user, password, "INBOX");
+    }
+
+    public boolean userReceivedMessageInMailbox(String user, String password, String mailbox) throws IOException {
         imapClient.login(user, password);
-        imapClient.select("INBOX");
+        imapClient.select(mailbox);
         imapClient.fetch("1:1", "ALL");
         return imapClient.getReplyString()
             .contains("OK FETCH completed");

server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/Sieve.java

@@ -0,0 +1,135 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+
+package org.apache.james.transport.mailets;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.inject.Inject;
+import javax.mail.MessagingException;
+
+import org.apache.commons.logging.Log;
+import org.apache.james.mailbox.model.MailboxConstants;
+import org.apache.james.sieverepository.api.SieveRepository;
+import org.apache.james.transport.mailets.delivery.MailStore;
+import org.apache.james.transport.mailets.jsieve.CommonsLoggingAdapter;
+import org.apache.james.transport.mailets.jsieve.ResourceLocator;
+import org.apache.james.transport.mailets.jsieve.delivery.SieveExecutor;
+import org.apache.james.transport.mailets.jsieve.delivery.SievePoster;
+import org.apache.james.user.api.UsersRepository;
+import org.apache.james.user.api.UsersRepositoryException;
+import org.apache.mailet.Mail;
+import org.apache.mailet.MailAddress;
+import org.apache.mailet.base.GenericMailet;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Execute Sieve scripts for incoming emails, and set the result of the execution as attributes of the mail
+ */
+public class Sieve extends GenericMailet {
+
+    private final UsersRepository usersRepository;
+    private final ResourceLocator resourceLocator;
+    private SieveExecutor sieveExecutor;
+
+    @Inject
+    public Sieve(UsersRepository usersRepository, SieveRepository sieveRepository) throws MessagingException {
+        this(usersRepository, ResourceLocatorImpl.instanciate(usersRepository, sieveRepository));
+    }
+
+    public Sieve(UsersRepository usersRepository, ResourceLocator resourceLocator) throws MessagingException {
+        this.usersRepository = usersRepository;
+        this.resourceLocator = resourceLocator;
+    }
+
+    @Override
+    public String getMailetInfo() {
+        return "Sieve Mailet";
+    }
+
+    @Override
+    public void init() throws MessagingException {
+        Log log = CommonsLoggingAdapter.builder()
+            .wrappedLogger(getMailetContext().getLogger())
+            .quiet(getInitParameter("quiet", false))
+            .verbose(getInitParameter("verbose", false))
+            .build();
+        sieveExecutor = SieveExecutor.builder()
+            .resourceLocator(resourceLocator)
+            .usersRepository(usersRepository)
+            .mailetContext(getMailetContext())
+            .log(log)
+            .sievePoster(new SievePoster(usersRepository, MailboxConstants.INBOX))
+            .build();
+    }
+
+    @Override
+    public void service(Mail mail) throws MessagingException {
+        for(MailAddress recipient: mail.getRecipients()) {
+            executeSieveScript(mail, recipient);
+        }
+        mail.setRecipients(keepNonDiscardedRecipients(mail));
+    }
+
+    private void executeSieveScript(Mail mail, MailAddress recipient) {
+        try {
+            sieveExecutor.execute(recipient, mail);
+        } catch (Exception e) {
+            getMailetContext().getLogger().warn("Failed to execute Sieve script for user " + recipient.asPrettyString(), e);
+        }
+    }
+
+
+    private ImmutableList<MailAddress> keepNonDiscardedRecipients(Mail mail) {
+        final List<MailAddress> discardedRecipients = retrieveDiscardedRecipients(mail);
+        return FluentIterable.from(mail.getRecipients()).filter(new Predicate<MailAddress>() {
+                @Override
+                public boolean apply(MailAddress input) {
+                    return !discardedRecipients.contains(input);
+                }
+            }).toList();
+    }
+
+    private List<MailAddress> retrieveDiscardedRecipients(Mail mail) {
+        final List<MailAddress> discardedRecipients = new ArrayList<MailAddress>();
+        for(MailAddress recipient: mail.getRecipients()) {
+            if (isDiscarded(mail, recipient)) {
+                discardedRecipients.add(recipient);
+            }
+        }
+        return discardedRecipients;
+    }
+
+    private boolean isDiscarded(Mail mail, MailAddress recipient) {
+        return !(mail.getAttribute(MailStore.DELIVERY_PATH_PREFIX + retrieveUser(recipient)) instanceof String);
+    }
+
+    private String retrieveUser(MailAddress recipient) {
+        try {
+            return usersRepository.getUser(recipient);
+        } catch (UsersRepositoryException e) {
+            log("Can not retrieve username for mail address " + recipient.asPrettyString(), e);
+            return recipient.asString();
+        }
+    }
+}