Skip to content

Commit

Permalink
[PERFORMANCE] Pre-convert Cassandra rows to lower case (#501)
Browse files Browse the repository at this point in the history
  • Loading branch information
chibenwa committed Jun 23, 2021
1 parent 7b829b7 commit 6c7fc9e
Show file tree
Hide file tree
Showing 13 changed files with 79 additions and 43 deletions.
Expand Up @@ -30,9 +30,12 @@
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.BODY_CONTENT;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.BODY_OCTECTS;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.BODY_START_OCTET;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.BODY_START_OCTET_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.FULL_CONTENT_OCTETS;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.FULL_CONTENT_OCTETS_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.HEADER_CONTENT;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.INTERNAL_DATE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.INTERNAL_DATE_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.Properties.CONTENT_DESCRIPTION;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.Properties.CONTENT_DISPOSITION_PARAMETERS;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.Properties.CONTENT_DISPOSITION_TYPE;
Expand All @@ -46,6 +49,7 @@
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.Properties.SUB_TYPE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.TABLE_NAME;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.TEXTUAL_LINE_COUNT;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.TEXTUAL_LINE_COUNT_LOWERCASE;

import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -310,9 +314,9 @@ private Mono<MessageRepresentation> message(Row row, CassandraMessageId cassandr
return buildContentRetriever(fetchType, headerId, bodyId, bodyStartOctet).map(content ->
new MessageRepresentation(
cassandraMessageId,
row.getTimestamp(INTERNAL_DATE),
row.getLong(FULL_CONTENT_OCTETS),
row.getInt(BODY_START_OCTET),
row.getTimestamp(INTERNAL_DATE_LOWERCASE),
row.getLong(FULL_CONTENT_OCTETS_LOWERCASE),
row.getInt(BODY_START_OCTET_LOWERCASE),
new ByteContent(content),
getProperties(row),
getAttachments(row).collect(Guavate.toImmutableList()),
Expand All @@ -333,7 +337,7 @@ private Properties getProperties(Row row) {
property.setContentLanguage(row.get(CONTENT_LANGUAGE, LIST_OF_STRINGS_CODEC));
property.setContentDispositionParameters(row.get(CONTENT_DISPOSITION_PARAMETERS, MAP_OF_STRINGS_CODEC));
property.setContentTypeParameters(row.get(CONTENT_TYPE_PARAMETERS, MAP_OF_STRINGS_CODEC));
property.setTextualLineCount(row.getLong(TEXTUAL_LINE_COUNT));
property.setTextualLineCount(row.getLong(TEXTUAL_LINE_COUNT_LOWERCASE));
return property.build();
}

Expand Down
Expand Up @@ -31,7 +31,9 @@
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIdTable.TABLE_NAME;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.IMAP_UID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.Flag.ANSWERED;
import static org.apache.james.mailbox.cassandra.table.Flag.DELETED;
import static org.apache.james.mailbox.cassandra.table.Flag.DRAFT;
Expand All @@ -41,6 +43,7 @@
import static org.apache.james.mailbox.cassandra.table.Flag.USER;
import static org.apache.james.mailbox.cassandra.table.Flag.USER_FLAGS;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.MOD_SEQ;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.MOD_SEQ_LOWERCASE;
import static org.apache.james.util.ReactorUtils.publishIfPresent;

import java.util.Optional;
Expand Down Expand Up @@ -344,11 +347,11 @@ private Optional<ComposedMessageIdWithMetaData> fromRowToComposedMessageIdWithFl
}
return Optional.of(ComposedMessageIdWithMetaData.builder()
.composedMessageId(new ComposedMessageId(
CassandraId.of(row.getUUID(MAILBOX_ID)),
messageIdFactory.of(row.getUUID(MESSAGE_ID)),
CassandraId.of(row.getUUID(MAILBOX_ID_LOWERCASE)),
messageIdFactory.of(row.getUUID(MESSAGE_ID_LOWERCASE)),
MessageUid.of(row.getLong(IMAP_UID))))
.flags(FlagsExtractor.getFlags(row))
.modSeq(ModSeq.of(row.getLong(MOD_SEQ)))
.modSeq(ModSeq.of(row.getLong(MOD_SEQ_LOWERCASE)))
.build());
}
}
Expand Up @@ -27,7 +27,9 @@
import static com.datastax.driver.core.querybuilder.QueryBuilder.update;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.IMAP_UID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.Flag.ANSWERED;
import static org.apache.james.mailbox.cassandra.table.Flag.DELETED;
import static org.apache.james.mailbox.cassandra.table.Flag.DRAFT;
Expand All @@ -38,6 +40,7 @@
import static org.apache.james.mailbox.cassandra.table.Flag.USER_FLAGS;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.FIELDS;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.MOD_SEQ;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.MOD_SEQ_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.TABLE_NAME;

import java.util.Optional;
Expand Down Expand Up @@ -155,7 +158,7 @@ private PreparedStatement prepareUpdate(Session session) {
private PreparedStatement prepareSelectAll(Session session) {
return session.prepare(select(FIELDS)
.from(TABLE_NAME)
.where(eq(MESSAGE_ID, bindMarker(MESSAGE_ID))));
.where(eq(MESSAGE_ID_LOWERCASE, bindMarker(MESSAGE_ID_LOWERCASE))));
}

private PreparedStatement prepareList(Session session) {
Expand All @@ -165,8 +168,8 @@ private PreparedStatement prepareList(Session session) {
private PreparedStatement prepareSelect(Session session) {
return session.prepare(select(FIELDS)
.from(TABLE_NAME)
.where(eq(MESSAGE_ID, bindMarker(MESSAGE_ID)))
.and(eq(MAILBOX_ID, bindMarker(MAILBOX_ID))));
.where(eq(MESSAGE_ID_LOWERCASE, bindMarker(MESSAGE_ID_LOWERCASE)))
.and(eq(MAILBOX_ID_LOWERCASE, bindMarker(MAILBOX_ID_LOWERCASE))));
}

public Mono<Void> delete(CassandraMessageId messageId, CassandraId mailboxId) {
Expand Down Expand Up @@ -240,19 +243,19 @@ public Flux<ComposedMessageIdWithMetaData> retrieveAllMessages() {
private ComposedMessageIdWithMetaData toComposedMessageIdWithMetadata(Row row) {
return ComposedMessageIdWithMetaData.builder()
.composedMessageId(new ComposedMessageId(
CassandraId.of(row.getUUID(MAILBOX_ID)),
messageIdFactory.of(row.getUUID(MESSAGE_ID)),
CassandraId.of(row.getUUID(MAILBOX_ID_LOWERCASE)),
messageIdFactory.of(row.getUUID(MESSAGE_ID_LOWERCASE)),
MessageUid.of(row.getLong(IMAP_UID))))
.flags(FlagsExtractor.getFlags(row))
.modSeq(ModSeq.of(row.getLong(MOD_SEQ)))
.modSeq(ModSeq.of(row.getLong(MOD_SEQ_LOWERCASE)))
.build();
}

private Statement selectStatement(CassandraMessageId messageId, Optional<CassandraId> mailboxId) {
return mailboxId
.map(cassandraId -> select.bind()
.setUUID(MESSAGE_ID, messageId.get())
.setUUID(MAILBOX_ID, cassandraId.asUuid()))
.orElseGet(() -> selectAll.bind().setUUID(MESSAGE_ID, messageId.get()));
.setUUID(MESSAGE_ID_LOWERCASE, messageId.get())
.setUUID(MAILBOX_ID_LOWERCASE, cassandraId.asUuid()))
.orElseGet(() -> selectAll.bind().setUUID(MESSAGE_ID_LOWERCASE, messageId.get()));
}
}
Expand Up @@ -37,12 +37,12 @@ public class FlagsExtractor {

public static Flags getFlags(Row row) {
Flags flags = new Flags();
for (String flag : Flag.ALL) {
for (String flag : Flag.ALL_LOWERCASE) {
if (row.getBool(flag)) {
flags.add(Flag.JAVAX_MAIL_FLAG.get(flag));
}
}
row.get(Flag.USER_FLAGS, SET_OF_STRINGS_CODEC)
row.get(Flag.USER_FLAGS_LOWERCASE, SET_OF_STRINGS_CODEC)
.forEach(flags::add);
return flags;
}
Expand Down
Expand Up @@ -19,9 +19,13 @@

package org.apache.james.mailbox.cassandra.table;

import java.util.Locale;

public interface CassandraMessageIds {

String MESSAGE_ID = "messageId";
String MESSAGE_ID_LOWERCASE = MESSAGE_ID.toLowerCase(Locale.US);
String MAILBOX_ID = "mailboxId";
String MAILBOX_ID_LOWERCASE = MAILBOX_ID.toLowerCase(Locale.US);
String IMAP_UID = "uid";
}
Expand Up @@ -19,16 +19,24 @@

package org.apache.james.mailbox.cassandra.table;

import java.util.Locale;

public interface CassandraMessageV3Table {

String TABLE_NAME = "messageV3";
String INTERNAL_DATE = "internalDate";
String INTERNAL_DATE_LOWERCASE = INTERNAL_DATE.toLowerCase(Locale.US);
String BODY_START_OCTET = "bodyStartOctet";
String BODY_START_OCTET_LOWERCASE = BODY_START_OCTET.toLowerCase(Locale.US);
String FULL_CONTENT_OCTETS = "fullContentOctets";
String FULL_CONTENT_OCTETS_LOWERCASE = FULL_CONTENT_OCTETS.toLowerCase(Locale.US);
String BODY_OCTECTS = "bodyOctets";
String TEXTUAL_LINE_COUNT = "textualLineCount";
String TEXTUAL_LINE_COUNT_LOWERCASE = TEXTUAL_LINE_COUNT.toLowerCase(Locale.US);
String BODY_CONTENT = "bodyContent";
String BODY_CONTENT_LOWERCASE = BODY_START_OCTET.toLowerCase(Locale.US);
String HEADER_CONTENT = "headerContent";
String HEADER_CONTENT_LOWERCASE = HEADER_CONTENT.toLowerCase(Locale.US);
String ATTACHMENTS = "attachments";

interface Properties {
Expand Down
Expand Up @@ -18,6 +18,8 @@
****************************************************************/
package org.apache.james.mailbox.cassandra.table;

import java.util.Locale;

import javax.mail.Flags;

import com.google.common.collect.ImmutableMap;
Expand All @@ -32,26 +34,25 @@ public interface Flag {
String FLAGGED = "flagFlagged";
String USER = "flagUser";
String USER_FLAGS = "userFlags";
String[] ALL = { ANSWERED, DELETED, DRAFT, RECENT, SEEN, FLAGGED, USER };
String[] ALL_APPLICABLE_FLAG = { ANSWERED, DELETED, DRAFT, SEEN, FLAGGED };
String USER_FLAGS_LOWERCASE = USER_FLAGS.toLowerCase(Locale.US);

ImmutableMap<String, Flags.Flag> JAVAX_MAIL_FLAG = ImmutableMap.<String, Flags.Flag>builder()
.put(ANSWERED, Flags.Flag.ANSWERED)
.put(DELETED, Flags.Flag.DELETED)
.put(DRAFT, Flags.Flag.DRAFT)
.put(RECENT, Flags.Flag.RECENT)
.put(SEEN, Flags.Flag.SEEN)
.put(FLAGGED, Flags.Flag.FLAGGED)
.put(USER, Flags.Flag.USER)
.build();
String[] ALL_LOWERCASE = {
ANSWERED.toLowerCase(Locale.US),
DELETED.toLowerCase(Locale.US),
DRAFT.toLowerCase(Locale.US),
RECENT.toLowerCase(Locale.US),
SEEN.toLowerCase(Locale.US),
FLAGGED.toLowerCase(Locale.US),
USER.toLowerCase(Locale.US)
};

ImmutableMap<Flags.Flag, String> FLAG_TO_STRING_MAP = ImmutableMap.<Flags.Flag, String>builder()
.put(Flags.Flag.ANSWERED, ANSWERED)
.put(Flags.Flag.DELETED, DELETED)
.put(Flags.Flag.DRAFT, DRAFT)
.put(Flags.Flag.RECENT, RECENT)
.put(Flags.Flag.SEEN, SEEN)
.put(Flags.Flag.FLAGGED, FLAGGED)
.put(Flags.Flag.USER, USER)
ImmutableMap<String, Flags.Flag> JAVAX_MAIL_FLAG = ImmutableMap.<String, Flags.Flag>builder()
.put(ANSWERED.toLowerCase(Locale.US), Flags.Flag.ANSWERED)
.put(DELETED.toLowerCase(Locale.US), Flags.Flag.DELETED)
.put(DRAFT.toLowerCase(Locale.US), Flags.Flag.DRAFT)
.put(RECENT.toLowerCase(Locale.US), Flags.Flag.RECENT)
.put(SEEN.toLowerCase(Locale.US), Flags.Flag.SEEN)
.put(FLAGGED.toLowerCase(Locale.US), Flags.Flag.FLAGGED)
.put(USER.toLowerCase(Locale.US), Flags.Flag.USER)
.build();
}
Expand Up @@ -23,11 +23,14 @@
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID;

import java.util.Locale;

public interface MessageIdToImapUid {

String TABLE_NAME = "imapUidTable";

String MOD_SEQ = "modSeq";
String MOD_SEQ_LOWERCASE = MOD_SEQ.toLowerCase(Locale.US);

String[] FIELDS = { MESSAGE_ID, MAILBOX_ID, IMAP_UID, MOD_SEQ,
Flag.ANSWERED, Flag.DELETED, Flag.DRAFT, Flag.FLAGGED, Flag.RECENT, Flag.SEEN, Flag.USER, Flag.USER_FLAGS };
Expand Down
Expand Up @@ -341,7 +341,7 @@ void fixMessageInconsistenciesShouldUpdateContextWhenFailedToRetrieveImapUidReco
cassandra.getConf()
.registerScenario(fail()
.times(1)
.whenQueryStartsWith("SELECT messageId,mailboxId,uid,modSeq,flagAnswered,flagDeleted,flagDraft,flagFlagged,flagRecent,flagSeen,flagUser,userFlags FROM imapUidTable WHERE messageId=:messageId AND mailboxId=:mailboxId"));
.whenQueryStartsWith("SELECT messageId,mailboxId,uid,modSeq,flagAnswered,flagDeleted,flagDraft,flagFlagged,flagRecent,flagSeen,flagUser,userFlags FROM messageIdTable WHERE mailboxId=:mailboxId AND uid=:uid;"));

testee.fixMessageInconsistencies(context, RunningOptions.DEFAULT).block();

Expand Down Expand Up @@ -536,7 +536,7 @@ void fixMailboxInconsistenciesShouldUpdateContextWhenFailedToRetrieveImapUidReco
cassandra.getConf()
.registerScenario(fail()
.times(1)
.whenQueryStartsWith("SELECT messageId,mailboxId,uid,modSeq,flagAnswered,flagDeleted,flagDraft,flagFlagged,flagRecent,flagSeen,flagUser,userFlags FROM imapUidTable WHERE messageId=:messageId AND mailboxId=:mailboxId"));
.whenQueryStartsWith("SELECT messageId,mailboxId,uid,modSeq,flagAnswered,flagDeleted,flagDraft,flagFlagged,flagRecent,flagSeen,flagUser,userFlags FROM imapUidTable WHERE messageid=:messageid AND mailboxid=:mailboxid"));

testee.fixMessageInconsistencies(context, RunningOptions.DEFAULT).block();

Expand Down
Expand Up @@ -28,6 +28,7 @@
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.DATE_LOOKUP_TABLE;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.MAILBOX_ID;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.MESSAGE_ID;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.MESSAGE_ID_LOWERCASE;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.RECEIVED_AT;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.SENT_AT;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.TABLE_NAME_RECEIVED_AT;
Expand Down Expand Up @@ -198,7 +199,7 @@ public Flux<MessageId> listMailboxContentSinceSentAt(MailboxId mailboxId, ZonedD
.setUUID(MAILBOX_ID, cassandraId.asUuid())
.setInt(LIMIT_MARKER, limit.getLimit().get())
.setTimestamp(SENT_AT, sinceDate))
.map(row -> messageIdFactory.of(row.getUUID(MESSAGE_ID)));
.map(row -> messageIdFactory.of(row.getUUID(MESSAGE_ID_LOWERCASE)));
}

@Override
Expand Down
Expand Up @@ -25,7 +25,9 @@
import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
import static com.datastax.driver.core.querybuilder.QueryBuilder.truncate;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.HAS_ATTACHMENT;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.HAS_ATTACHMENT_LOWERCASE;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.MESSAGE_ID;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.MESSAGE_ID_LOWERCASE;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.PREVIEW;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.TABLE_NAME;

Expand Down Expand Up @@ -79,7 +81,7 @@ public class CassandraMessageFastViewProjection implements MessageFastViewProjec

this.retrieveStatement = session.prepare(select()
.from(TABLE_NAME)
.where(eq(MESSAGE_ID, bindMarker(MESSAGE_ID))));
.where(eq(MESSAGE_ID_LOWERCASE, bindMarker(MESSAGE_ID_LOWERCASE))));

this.truncateStatement = session.prepare(truncate(TABLE_NAME));

Expand All @@ -103,7 +105,7 @@ public Mono<MessageFastViewPrecomputedProperties> retrieve(MessageId messageId)
checkMessage(messageId);

return cassandraAsyncExecutor.executeSingleRow(retrieveStatement.bind()
.setUUID(MESSAGE_ID, ((CassandraMessageId) messageId).get())
.setUUID(MESSAGE_ID_LOWERCASE, ((CassandraMessageId) messageId).get())
.setConsistencyLevel(ConsistencyLevel.ONE))
.map(this::fromRow)
.doOnNext(preview -> metricRetrieveHitCount.increment())
Expand Down Expand Up @@ -136,7 +138,7 @@ private void checkMessage(MessageId messageId) {
private MessageFastViewPrecomputedProperties fromRow(Row row) {
return MessageFastViewPrecomputedProperties.builder()
.preview(Preview.from(row.getString(PREVIEW)))
.hasAttachment(row.getBool(HAS_ATTACHMENT))
.hasAttachment(row.getBool(HAS_ATTACHMENT_LOWERCASE))
.build();
}
}
Expand Up @@ -19,13 +19,16 @@

package org.apache.james.jmap.cassandra.projections.table;

import java.util.Locale;

public interface CassandraEmailQueryViewTable {
String TABLE_NAME_SENT_AT = "email_query_view_sent_at";
String TABLE_NAME_RECEIVED_AT = "email_query_view_received_at";
String DATE_LOOKUP_TABLE = "email_query_view_date_lookup";

String MAILBOX_ID = "mailboxId";
String MESSAGE_ID = "messageId";
String MESSAGE_ID_LOWERCASE = MESSAGE_ID.toLowerCase(Locale.US);
String RECEIVED_AT = "receivedAt";
String SENT_AT = "sentAt";
}
Expand Up @@ -19,10 +19,14 @@

package org.apache.james.jmap.cassandra.projections.table;

import java.util.Locale;

public interface CassandraMessageFastViewProjectionTable {
String TABLE_NAME = "message_fast_view_projection";

String MESSAGE_ID = "messageId";
String MESSAGE_ID_LOWERCASE = MESSAGE_ID.toLowerCase(Locale.US);
String PREVIEW = "preview";
String HAS_ATTACHMENT = "hasAttachment";
String HAS_ATTACHMENT_LOWERCASE = HAS_ATTACHMENT.toLowerCase(Locale.US);
}

0 comments on commit 6c7fc9e

Please sign in to comment.