Skip to content

Commit

Permalink
JAMES-1970 Push security of multi-mailbox search to the back-end
Browse files Browse the repository at this point in the history
  • Loading branch information
@chibenwa
Quynh Nguyen authored and chibenwa committed Mar 21, 2017
1 parent e6a71cb commit 541ee6e
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 30 deletions.
50 changes: 29 additions & 21 deletions ...lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
View file
Expand Up @@ -101,6 +101,7 @@
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
Expand Down Expand Up @@ -272,6 +273,10 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
*/
public final static String MAILBOX_ID_FIELD ="mailboxid";

/**
* {@link Field} which will contain the user of the {@link MailboxSession}
*/
public final static String USERS = "userSession";
/**
* {@link Field} which will contain the id of the {@link MessageId}
*/
Expand Down Expand Up @@ -454,11 +459,12 @@ public void setEnableSuffixMatch(boolean suffixMatch) {
public Iterator<MessageUid> search(MailboxSession session, Mailbox mailbox, SearchQuery searchQuery) throws MailboxException {
Preconditions.checkArgument(session != null, "'session' is mandatory");
MailboxId mailboxId = mailbox.getMailboxId();
return FluentIterable.from(searchMultimap(
MultimailboxesSearchQuery
.from(searchQuery)
.inMailboxes(mailboxId)
.build()))
MultimailboxesSearchQuery multimailboxesSearchQuery = MultimailboxesSearchQuery
.from(searchQuery)
.inMailboxes(mailboxId)
.build();

return FluentIterable.from(searchMultimap(multimailboxesSearchQuery, session))
.transform(new Function<SearchResult, MessageUid>() {
@Override
public MessageUid apply(SearchResult input) {
Expand All @@ -471,7 +477,7 @@ public MessageUid apply(SearchResult input) {
@Override
public List<MessageId> search(MailboxSession session, MultimailboxesSearchQuery searchQuery, long limit) throws MailboxException {
Preconditions.checkArgument(session != null, "'session' is mandatory");
return FluentIterable.from(searchMultimap(searchQuery))
return FluentIterable.from(searchMultimap(searchQuery, session))
.transform(new Function<SearchResult, MessageId>() {
@Override
public MessageId apply(SearchResult input) {
Expand All @@ -483,7 +489,7 @@ public MessageId apply(SearchResult input) {
.toList();
}

private List<SearchResult> searchMultimap(MultimailboxesSearchQuery searchQuery) throws MailboxException {
private List<SearchResult> searchMultimap(MultimailboxesSearchQuery searchQuery, MailboxSession session) throws MailboxException {
ImmutableList.Builder<SearchResult> results = ImmutableList.builder();
IndexSearcher searcher = null;

Expand All @@ -495,6 +501,7 @@ private List<SearchResult> searchMultimap(MultimailboxesSearchQuery searchQuery)
query.add(inMailboxes, BooleanClause.Occur.MUST);
// Not return flags documents
query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST_NOT);
query.add(new TermQuery(new Term(USERS, session.getUser().getUserName().toUpperCase(Locale.US))), Occur.MUST);
List<Criterion> crits = searchQuery.getSearchQuery().getCriterias();
for (Criterion crit : crits) {
query.add(createQuery(crit, inMailboxes, searchQuery.getSearchQuery().getRecentMessageUids()), BooleanClause.Occur.MUST);
Expand Down Expand Up @@ -555,13 +562,14 @@ private Query buildQueryFromMailboxes(ImmutableSet<MailboxId> mailboxIds) {
private Document createMessageDocument(final MailboxSession session, final MailboxMessage membership) throws MailboxException{
final Document doc = new Document();
// TODO: Better handling
doc.add(new Field(MAILBOX_ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.ENGLISH), Store.YES, Index.NOT_ANALYZED));
doc.add(new Field(USERS, session.getUser().getUserName().toUpperCase(Locale.US), Store.YES, Index.NOT_ANALYZED));
doc.add(new Field(MAILBOX_ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.US), Store.YES, Index.NOT_ANALYZED));
doc.add(new NumericField(UID_FIELD,Store.YES, true).setLongValue(membership.getUid().asLong()));
doc.add(new Field(HAS_ATTACHMENT_FIELD, Boolean.toString(hasAttachment(membership)), Store.YES, Index.NOT_ANALYZED));
doc.add(new Field(MESSAGE_ID_FIELD, SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(membership), Store.YES, Index.NOT_ANALYZED));

// create an unqiue key for the document which can be used later on updates to find the document
doc.add(new Field(ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.ENGLISH) +"-" + Long.toString(membership.getUid().asLong()), Store.YES, Index.NOT_ANALYZED));
doc.add(new Field(ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.US) +"-" + Long.toString(membership.getUid().asLong()), Store.YES, Index.NOT_ANALYZED));

doc.add(new Field(INTERNAL_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.YEAR), Store.NO, Index.NOT_ANALYZED));
doc.add(new Field(INTERNAL_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MONTH), Store.NO, Index.NOT_ANALYZED));
Expand Down Expand Up @@ -590,9 +598,9 @@ public void headers(Header header) {
Iterator<org.apache.james.mime4j.stream.Field> fields = header.iterator();
while(fields.hasNext()) {
org.apache.james.mime4j.stream.Field f = fields.next();
String headerName = f.getName().toUpperCase(Locale.ENGLISH);
String headerValue = f.getBody().toUpperCase(Locale.ENGLISH);
String fullValue = f.toString().toUpperCase(Locale.ENGLISH);
String headerName = f.getName().toUpperCase(Locale.US);
String headerValue = f.getBody().toUpperCase(Locale.US);
String fullValue = f.toString().toUpperCase(Locale.US);
doc.add(new Field(HEADERS_FIELD, fullValue, Store.NO, Index.ANALYZED));
doc.add(new Field(PREFIX_HEADER_FIELD + headerName, headerValue, Store.NO, Index.ANALYZED));

Expand Down Expand Up @@ -633,7 +641,7 @@ public void headers(Header header) {
Address address = aList.get(i);
if (address instanceof org.apache.james.mime4j.dom.address.Mailbox) {
org.apache.james.mime4j.dom.address.Mailbox mailbox = (org.apache.james.mime4j.dom.address.Mailbox) address;
String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.ENGLISH);
String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.US);
doc.add(new Field(field, value, Store.NO, Index.ANALYZED));
if (i == 0) {
String mailboxAddress = SearchUtil.getMailboxAddress(mailbox);
Expand All @@ -655,7 +663,7 @@ public void headers(Header header) {
MailboxList mList = ((Group) address).getMailboxes();
for (int a = 0; a < mList.size(); a++) {
org.apache.james.mime4j.dom.address.Mailbox mailbox = mList.get(a);
String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.ENGLISH);
String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.US);
doc.add(new Field(field, value, Store.NO, Index.ANALYZED));

if (i == 0 && a == 0) {
Expand Down Expand Up @@ -727,7 +735,7 @@ public void body(BodyDescriptor desc, InputStream in) throws MimeException, IOEx
BufferedReader bodyReader = new BufferedReader(new InputStreamReader(in, charset));
String line = null;
while((line = bodyReader.readLine()) != null) {
doc.add(new Field(BODY_FIELD, line.toUpperCase(Locale.ENGLISH),Store.NO, Index.ANALYZED));
doc.add(new Field(BODY_FIELD, line.toUpperCase(Locale.US),Store.NO, Index.ANALYZED));
}

}
Expand Down Expand Up @@ -794,7 +802,7 @@ private String toSentDateField(DateResolution res) {


private static Calendar getGMT() {
return Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ENGLISH);
return Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US);
}


Expand Down Expand Up @@ -884,20 +892,20 @@ private Query createTermQuery(String fieldName, String value) {
*/
private Query createHeaderQuery(SearchQuery.HeaderCriterion crit) throws UnsupportedSearchException {
HeaderOperator op = crit.getOperator();
String name = crit.getHeaderName().toUpperCase(Locale.ENGLISH);
String name = crit.getHeaderName().toUpperCase(Locale.US);
String fieldName = PREFIX_HEADER_FIELD + name;
if (op instanceof SearchQuery.ContainsOperator) {
ContainsOperator cop = (ContainsOperator) op;
return createTermQuery(fieldName, cop.getValue().toUpperCase(Locale.ENGLISH));
return createTermQuery(fieldName, cop.getValue().toUpperCase(Locale.US));
} else if (op instanceof SearchQuery.ExistsOperator){
return new PrefixQuery(new Term(fieldName, ""));
} else if (op instanceof SearchQuery.DateOperator) {
DateOperator dop = (DateOperator) op;
String field = toSentDateField(dop.getDateResultion());
return createQuery(field, dop);
} else if (op instanceof SearchQuery.AddressOperator) {
String field = name.toLowerCase(Locale.ENGLISH);
return createTermQuery(field, ((SearchQuery.AddressOperator) op).getAddress().toUpperCase(Locale.ENGLISH));
String field = name.toLowerCase(Locale.US);
return createTermQuery(field, ((SearchQuery.AddressOperator) op).getAddress().toUpperCase(Locale.US));
} else {
// Operator not supported
throw new UnsupportedSearchException();
Expand Down Expand Up @@ -1184,7 +1192,7 @@ private String toString(Flag flag) {
* @throws UnsupportedSearchException
*/
private Query createTextQuery(SearchQuery.TextCriterion crit) throws UnsupportedSearchException {
String value = crit.getOperator().getValue().toUpperCase(Locale.ENGLISH);
String value = crit.getOperator().getValue().toUpperCase(Locale.US);
switch(crit.getType()) {
case BODY:
return createTermQuery(BODY_FIELD, value);
Expand Down
12 changes: 6 additions & 6 deletions ...test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java
View file
Expand Up @@ -94,6 +94,7 @@ protected boolean useLenient() {

@Before
public void setUp() throws Exception {
session = new MockMailboxSession("username");
TestMessageId.Factory factory = new TestMessageId.Factory();
id1 = factory.generate();
id2 = factory.generate();
Expand Down Expand Up @@ -122,23 +123,23 @@ public void setUp() throws Exception {

uid1 = MessageUid.of(1);
SimpleMailboxMembership m = new SimpleMailboxMembership(id1, mailbox.getMailboxId(), uid1, 0, new Date(), 200, new Flags(Flag.ANSWERED), "My Body".getBytes(), headersSubject);
index.add(null, mailbox, m);
index.add(session, mailbox, m);

uid2 = MessageUid.of(1);
SimpleMailboxMembership m2 = new SimpleMailboxMembership(id2, mailbox2.getMailboxId(), uid2, 0, new Date(), 20, new Flags(Flag.ANSWERED), "My Body".getBytes(), headersSubject);
index.add(null, mailbox2, m2);
index.add(session, mailbox2, m2);

uid3 = MessageUid.of(2);
Calendar cal = Calendar.getInstance();
cal.set(1980, 2, 10);
SimpleMailboxMembership m3 = new SimpleMailboxMembership(id3, mailbox.getMailboxId(), uid3, 0, cal.getTime(), 20, new Flags(Flag.DELETED), "My Otherbody".getBytes(), headersTest);
index.add(null, mailbox, m3);
index.add(session, mailbox, m3);

uid4 = MessageUid.of(3);
Calendar cal2 = Calendar.getInstance();
cal2.set(8000, 2, 10);
SimpleMailboxMembership m4 = new SimpleMailboxMembership(id4, mailbox.getMailboxId(), uid4, 0, cal2.getTime(), 20, new Flags(Flag.DELETED), "My Otherbody2".getBytes(), headersTestSubject);
index.add(null, mailbox, m4);
index.add(session, mailbox, m4);

uid5 = MessageUid.of(10);
MessageBuilder builder = new MessageBuilder();
Expand All @@ -150,9 +151,8 @@ public void setUp() throws Exception {
builder.uid = uid5;
builder.mailboxId = mailbox3.getMailboxId();

index.add(null, mailbox3, builder.build(id5));
index.add(session, mailbox3, builder.build(id5));

session = new MockMailboxSession("username");
}


Expand Down
65 changes: 62 additions & 3 deletions ...e/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java
View file
Expand Up @@ -31,6 +31,7 @@
import org.apache.james.mailbox.MessageIdManager;
import org.apache.james.mailbox.exception.MailboxException;
import org.apache.james.mailbox.model.ComposedMessageId;
import org.apache.james.mailbox.model.MailboxConstants;
import org.apache.james.mailbox.model.MailboxPath;
import org.apache.james.mailbox.model.MessageId;
import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
Expand All @@ -50,6 +51,10 @@

public abstract class AbstractMessageSearchIndexTest {

private static final String INBOX = "INBOX";
private static final String OTHERUSER = "otheruser";
private static final String USERNAME = "benwa";

private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMessageSearchIndexTest.class);
public static final long LIMIT = 100L;

Expand All @@ -58,7 +63,9 @@ public abstract class AbstractMessageSearchIndexTest {
protected MessageIdManager messageIdManager;
private Mailbox mailbox;
private Mailbox mailbox2;
private Mailbox otherMailbox;
private MailboxSession session;
private MailboxSession otherSession;

private ComposedMessageId m1;
private ComposedMessageId m2;
Expand All @@ -72,23 +79,32 @@ public abstract class AbstractMessageSearchIndexTest {
private ComposedMessageId mOther;
private ComposedMessageId mailWithAttachment;
private ComposedMessageId mailWithInlinedAttachment;
private ComposedMessageId m10;
private StoreMessageManager myFolderMessageManager;


@Before
public void setUp() throws Exception {
initializeMailboxManager();

session = storeMailboxManager.createSystemSession("benwa", LOGGER);
session = storeMailboxManager.createSystemSession(USERNAME, LOGGER);
otherSession = storeMailboxManager.createSystemSession(OTHERUSER, LOGGER);

MailboxPath inboxPath = new MailboxPath(MailboxConstants.USER_NAMESPACE, USERNAME, INBOX);
MailboxPath otherInboxPath = new MailboxPath(MailboxConstants.USER_NAMESPACE, OTHERUSER, INBOX);

MailboxPath inboxPath = new MailboxPath("#private", "benwa", "INBOX");
storeMailboxManager.createMailbox(inboxPath, session);
storeMailboxManager.createMailbox(otherInboxPath, otherSession);

StoreMessageManager inboxMessageManager = (StoreMessageManager) storeMailboxManager.getMailbox(inboxPath, session);
MailboxPath myFolderPath = new MailboxPath("#private", "benwa", "MyFolder");
StoreMessageManager otherInboxMessageManager = (StoreMessageManager) storeMailboxManager.getMailbox(otherInboxPath, otherSession);

MailboxPath myFolderPath = new MailboxPath(MailboxConstants.USER_NAMESPACE, USERNAME, "MyFolder");
storeMailboxManager.createMailbox(myFolderPath, session);
myFolderMessageManager = (StoreMessageManager) storeMailboxManager.getMailbox(myFolderPath, session);
mailbox = inboxMessageManager.getMailboxEntity();
mailbox2 = myFolderMessageManager.getMailboxEntity();
otherMailbox = otherInboxMessageManager.getMailboxEntity();

m1 = inboxMessageManager.appendMessage(
ClassLoader.getSystemResourceAsStream("eml/spamMail.eml"),
Expand Down Expand Up @@ -181,6 +197,12 @@ public void setUp() throws Exception {
true,
new Flags("Hello you"));

m10 = otherInboxMessageManager.appendMessage(
ClassLoader.getSystemResourceAsStream("eml/mail1.eml"),
new Date(1391295600000L),
otherSession,
true,
new Flags());
await();
}

Expand Down Expand Up @@ -1052,4 +1074,41 @@ public void sortShouldNotDiscardResultWhenSearchingFieldIsIdentical() throws Exc
assertThat(actual).containsOnly(m1.getMessageId(), m2.getMessageId(), m3.getMessageId(), m4.getMessageId(), m5.getMessageId(),
m6.getMessageId(), m7.getMessageId(), m8.getMessageId(), m9.getMessageId(), mOther.getMessageId(), mailWithAttachment.getMessageId(), mailWithInlinedAttachment.getMessageId());
}

@Test
public void searchInMultiMailboxShouldReturnMessagesBelongingToUserSession() throws Exception {
SearchQuery query = new SearchQuery();
query.andCriteria(SearchQuery.all());

MultimailboxesSearchQuery multiMailboxesQuery = MultimailboxesSearchQuery.from(query).build();

assertThat(messageSearchIndex.search(otherSession, multiMailboxesQuery, LIMIT))
.hasSize(1)
.containsOnly(m10.getMessageId());
}

@Test
public void searchInMultiMailboxShouldNotReturnMessagesBelongingToAnotherUserSession() throws Exception {
SearchQuery query = new SearchQuery();
query.andCriteria(SearchQuery.all());

MultimailboxesSearchQuery multiMailboxesQuery = MultimailboxesSearchQuery.from(query).build();

assertThat(messageSearchIndex.search(session, multiMailboxesQuery, LIMIT))
.doesNotContain(m10.getMessageId());
}

@Test
public void searchShouldFilterMailboxBelongingToMailboxSession() throws Exception {
SearchQuery searchQuery = new SearchQuery();

assertThat(messageSearchIndex.search(otherSession, otherMailbox, searchQuery)).containsOnly(m10.getUid());
}

@Test
public void searchShouldReturnEmptyWhenMailboxBelongingToAnotherMailboxSession() throws Exception {
SearchQuery searchQuery = new SearchQuery();

assertThat(messageSearchIndex.search(otherSession, mailbox, searchQuery)).isEmpty();
}
}

0 comments on commit 541ee6e

Please sign in to comment.