Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PERFORMANCE] Pre-convert Cassandra rows to lower case #501

Merged
merged 2 commits into from Jun 23, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -30,9 +30,12 @@
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.BODY_CONTENT;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.BODY_OCTECTS;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.BODY_START_OCTET;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.BODY_START_OCTET_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.FULL_CONTENT_OCTETS;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.FULL_CONTENT_OCTETS_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.HEADER_CONTENT;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.INTERNAL_DATE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.INTERNAL_DATE_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.Properties.CONTENT_DESCRIPTION;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.Properties.CONTENT_DISPOSITION_PARAMETERS;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.Properties.CONTENT_DISPOSITION_TYPE;
Expand All @@ -46,6 +49,7 @@
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.Properties.SUB_TYPE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.TABLE_NAME;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.TEXTUAL_LINE_COUNT;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageV3Table.TEXTUAL_LINE_COUNT_LOWERCASE;

import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -310,9 +314,9 @@ private Mono<MessageRepresentation> message(Row row, CassandraMessageId cassandr
return buildContentRetriever(fetchType, headerId, bodyId, bodyStartOctet).map(content ->
new MessageRepresentation(
cassandraMessageId,
row.getTimestamp(INTERNAL_DATE),
row.getLong(FULL_CONTENT_OCTETS),
row.getInt(BODY_START_OCTET),
row.getTimestamp(INTERNAL_DATE_LOWERCASE),
row.getLong(FULL_CONTENT_OCTETS_LOWERCASE),
row.getInt(BODY_START_OCTET_LOWERCASE),
new ByteContent(content),
getProperties(row),
getAttachments(row).collect(Guavate.toImmutableList()),
Expand All @@ -333,7 +337,7 @@ private Properties getProperties(Row row) {
property.setContentLanguage(row.get(CONTENT_LANGUAGE, LIST_OF_STRINGS_CODEC));
property.setContentDispositionParameters(row.get(CONTENT_DISPOSITION_PARAMETERS, MAP_OF_STRINGS_CODEC));
property.setContentTypeParameters(row.get(CONTENT_TYPE_PARAMETERS, MAP_OF_STRINGS_CODEC));
property.setTextualLineCount(row.getLong(TEXTUAL_LINE_COUNT));
property.setTextualLineCount(row.getLong(TEXTUAL_LINE_COUNT_LOWERCASE));
return property.build();
}

Expand Down
Expand Up @@ -31,7 +31,9 @@
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIdTable.TABLE_NAME;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.IMAP_UID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.Flag.ANSWERED;
import static org.apache.james.mailbox.cassandra.table.Flag.DELETED;
import static org.apache.james.mailbox.cassandra.table.Flag.DRAFT;
Expand All @@ -41,6 +43,7 @@
import static org.apache.james.mailbox.cassandra.table.Flag.USER;
import static org.apache.james.mailbox.cassandra.table.Flag.USER_FLAGS;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.MOD_SEQ;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.MOD_SEQ_LOWERCASE;
import static org.apache.james.util.ReactorUtils.publishIfPresent;

import java.util.Optional;
Expand Down Expand Up @@ -344,11 +347,11 @@ private Optional<ComposedMessageIdWithMetaData> fromRowToComposedMessageIdWithFl
}
return Optional.of(ComposedMessageIdWithMetaData.builder()
.composedMessageId(new ComposedMessageId(
CassandraId.of(row.getUUID(MAILBOX_ID)),
messageIdFactory.of(row.getUUID(MESSAGE_ID)),
CassandraId.of(row.getUUID(MAILBOX_ID_LOWERCASE)),
messageIdFactory.of(row.getUUID(MESSAGE_ID_LOWERCASE)),
MessageUid.of(row.getLong(IMAP_UID))))
.flags(FlagsExtractor.getFlags(row))
.modSeq(ModSeq.of(row.getLong(MOD_SEQ)))
.modSeq(ModSeq.of(row.getLong(MOD_SEQ_LOWERCASE)))
.build());
}
}
Expand Up @@ -27,7 +27,9 @@
import static com.datastax.driver.core.querybuilder.QueryBuilder.update;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.IMAP_UID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.Flag.ANSWERED;
import static org.apache.james.mailbox.cassandra.table.Flag.DELETED;
import static org.apache.james.mailbox.cassandra.table.Flag.DRAFT;
Expand All @@ -38,6 +40,7 @@
import static org.apache.james.mailbox.cassandra.table.Flag.USER_FLAGS;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.FIELDS;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.MOD_SEQ;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.MOD_SEQ_LOWERCASE;
import static org.apache.james.mailbox.cassandra.table.MessageIdToImapUid.TABLE_NAME;

import java.util.Optional;
Expand Down Expand Up @@ -155,7 +158,7 @@ private PreparedStatement prepareUpdate(Session session) {
private PreparedStatement prepareSelectAll(Session session) {
return session.prepare(select(FIELDS)
.from(TABLE_NAME)
.where(eq(MESSAGE_ID, bindMarker(MESSAGE_ID))));
.where(eq(MESSAGE_ID_LOWERCASE, bindMarker(MESSAGE_ID_LOWERCASE))));
}

private PreparedStatement prepareList(Session session) {
Expand All @@ -165,8 +168,8 @@ private PreparedStatement prepareList(Session session) {
private PreparedStatement prepareSelect(Session session) {
return session.prepare(select(FIELDS)
.from(TABLE_NAME)
.where(eq(MESSAGE_ID, bindMarker(MESSAGE_ID)))
.and(eq(MAILBOX_ID, bindMarker(MAILBOX_ID))));
.where(eq(MESSAGE_ID_LOWERCASE, bindMarker(MESSAGE_ID_LOWERCASE)))
.and(eq(MAILBOX_ID_LOWERCASE, bindMarker(MAILBOX_ID_LOWERCASE))));
}

public Mono<Void> delete(CassandraMessageId messageId, CassandraId mailboxId) {
Expand Down Expand Up @@ -240,19 +243,19 @@ public Flux<ComposedMessageIdWithMetaData> retrieveAllMessages() {
private ComposedMessageIdWithMetaData toComposedMessageIdWithMetadata(Row row) {
return ComposedMessageIdWithMetaData.builder()
.composedMessageId(new ComposedMessageId(
CassandraId.of(row.getUUID(MAILBOX_ID)),
messageIdFactory.of(row.getUUID(MESSAGE_ID)),
CassandraId.of(row.getUUID(MAILBOX_ID_LOWERCASE)),
messageIdFactory.of(row.getUUID(MESSAGE_ID_LOWERCASE)),
MessageUid.of(row.getLong(IMAP_UID))))
.flags(FlagsExtractor.getFlags(row))
.modSeq(ModSeq.of(row.getLong(MOD_SEQ)))
.modSeq(ModSeq.of(row.getLong(MOD_SEQ_LOWERCASE)))
.build();
}

private Statement selectStatement(CassandraMessageId messageId, Optional<CassandraId> mailboxId) {
return mailboxId
.map(cassandraId -> select.bind()
.setUUID(MESSAGE_ID, messageId.get())
.setUUID(MAILBOX_ID, cassandraId.asUuid()))
.orElseGet(() -> selectAll.bind().setUUID(MESSAGE_ID, messageId.get()));
.setUUID(MESSAGE_ID_LOWERCASE, messageId.get())
.setUUID(MAILBOX_ID_LOWERCASE, cassandraId.asUuid()))
.orElseGet(() -> selectAll.bind().setUUID(MESSAGE_ID_LOWERCASE, messageId.get()));
}
}
Expand Up @@ -37,12 +37,12 @@ public class FlagsExtractor {

public static Flags getFlags(Row row) {
Flags flags = new Flags();
for (String flag : Flag.ALL) {
for (String flag : Flag.ALL_LOWERCASE) {
if (row.getBool(flag)) {
flags.add(Flag.JAVAX_MAIL_FLAG.get(flag));
}
}
row.get(Flag.USER_FLAGS, SET_OF_STRINGS_CODEC)
row.get(Flag.USER_FLAGS_LOWERCASE, SET_OF_STRINGS_CODEC)
.forEach(flags::add);
return flags;
}
Expand Down
Expand Up @@ -19,9 +19,13 @@

package org.apache.james.mailbox.cassandra.table;

import java.util.Locale;

public interface CassandraMessageIds {

String MESSAGE_ID = "messageId";
String MESSAGE_ID_LOWERCASE = MESSAGE_ID.toLowerCase(Locale.US);
String MAILBOX_ID = "mailboxId";
String MAILBOX_ID_LOWERCASE = MAILBOX_ID.toLowerCase(Locale.US);
String IMAP_UID = "uid";
}
Expand Up @@ -19,16 +19,24 @@

package org.apache.james.mailbox.cassandra.table;

import java.util.Locale;

public interface CassandraMessageV3Table {

String TABLE_NAME = "messageV3";
String INTERNAL_DATE = "internalDate";
String INTERNAL_DATE_LOWERCASE = INTERNAL_DATE.toLowerCase(Locale.US);
String BODY_START_OCTET = "bodyStartOctet";
String BODY_START_OCTET_LOWERCASE = BODY_START_OCTET.toLowerCase(Locale.US);
String FULL_CONTENT_OCTETS = "fullContentOctets";
String FULL_CONTENT_OCTETS_LOWERCASE = FULL_CONTENT_OCTETS.toLowerCase(Locale.US);
String BODY_OCTECTS = "bodyOctets";
String TEXTUAL_LINE_COUNT = "textualLineCount";
String TEXTUAL_LINE_COUNT_LOWERCASE = TEXTUAL_LINE_COUNT.toLowerCase(Locale.US);
String BODY_CONTENT = "bodyContent";
String BODY_CONTENT_LOWERCASE = BODY_START_OCTET.toLowerCase(Locale.US);
String HEADER_CONTENT = "headerContent";
String HEADER_CONTENT_LOWERCASE = HEADER_CONTENT.toLowerCase(Locale.US);
String ATTACHMENTS = "attachments";

interface Properties {
Expand Down
Expand Up @@ -18,6 +18,8 @@
****************************************************************/
package org.apache.james.mailbox.cassandra.table;

import java.util.Locale;

import javax.mail.Flags;

import com.google.common.collect.ImmutableMap;
Expand All @@ -32,26 +34,25 @@ public interface Flag {
String FLAGGED = "flagFlagged";
String USER = "flagUser";
String USER_FLAGS = "userFlags";
String[] ALL = { ANSWERED, DELETED, DRAFT, RECENT, SEEN, FLAGGED, USER };
String[] ALL_APPLICABLE_FLAG = { ANSWERED, DELETED, DRAFT, SEEN, FLAGGED };
String USER_FLAGS_LOWERCASE = USER_FLAGS.toLowerCase(Locale.US);

ImmutableMap<String, Flags.Flag> JAVAX_MAIL_FLAG = ImmutableMap.<String, Flags.Flag>builder()
.put(ANSWERED, Flags.Flag.ANSWERED)
.put(DELETED, Flags.Flag.DELETED)
.put(DRAFT, Flags.Flag.DRAFT)
.put(RECENT, Flags.Flag.RECENT)
.put(SEEN, Flags.Flag.SEEN)
.put(FLAGGED, Flags.Flag.FLAGGED)
.put(USER, Flags.Flag.USER)
.build();
String[] ALL_LOWERCASE = {
ANSWERED.toLowerCase(Locale.US),
DELETED.toLowerCase(Locale.US),
DRAFT.toLowerCase(Locale.US),
RECENT.toLowerCase(Locale.US),
SEEN.toLowerCase(Locale.US),
FLAGGED.toLowerCase(Locale.US),
USER.toLowerCase(Locale.US)
};

ImmutableMap<Flags.Flag, String> FLAG_TO_STRING_MAP = ImmutableMap.<Flags.Flag, String>builder()
.put(Flags.Flag.ANSWERED, ANSWERED)
.put(Flags.Flag.DELETED, DELETED)
.put(Flags.Flag.DRAFT, DRAFT)
.put(Flags.Flag.RECENT, RECENT)
.put(Flags.Flag.SEEN, SEEN)
.put(Flags.Flag.FLAGGED, FLAGGED)
.put(Flags.Flag.USER, USER)
ImmutableMap<String, Flags.Flag> JAVAX_MAIL_FLAG = ImmutableMap.<String, Flags.Flag>builder()
.put(ANSWERED.toLowerCase(Locale.US), Flags.Flag.ANSWERED)
.put(DELETED.toLowerCase(Locale.US), Flags.Flag.DELETED)
.put(DRAFT.toLowerCase(Locale.US), Flags.Flag.DRAFT)
.put(RECENT.toLowerCase(Locale.US), Flags.Flag.RECENT)
.put(SEEN.toLowerCase(Locale.US), Flags.Flag.SEEN)
.put(FLAGGED.toLowerCase(Locale.US), Flags.Flag.FLAGGED)
.put(USER.toLowerCase(Locale.US), Flags.Flag.USER)
.build();
}
Expand Up @@ -23,11 +23,14 @@
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MAILBOX_ID;
import static org.apache.james.mailbox.cassandra.table.CassandraMessageIds.MESSAGE_ID;

import java.util.Locale;

public interface MessageIdToImapUid {

String TABLE_NAME = "imapUidTable";

String MOD_SEQ = "modSeq";
String MOD_SEQ_LOWERCASE = MOD_SEQ.toLowerCase(Locale.US);

String[] FIELDS = { MESSAGE_ID, MAILBOX_ID, IMAP_UID, MOD_SEQ,
Flag.ANSWERED, Flag.DELETED, Flag.DRAFT, Flag.FLAGGED, Flag.RECENT, Flag.SEEN, Flag.USER, Flag.USER_FLAGS };
Expand Down
Expand Up @@ -28,6 +28,7 @@
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.DATE_LOOKUP_TABLE;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.MAILBOX_ID;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.MESSAGE_ID;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.MESSAGE_ID_LOWERCASE;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.RECEIVED_AT;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.SENT_AT;
import static org.apache.james.jmap.cassandra.projections.table.CassandraEmailQueryViewTable.TABLE_NAME_RECEIVED_AT;
Expand Down Expand Up @@ -198,7 +199,7 @@ public Flux<MessageId> listMailboxContentSinceSentAt(MailboxId mailboxId, ZonedD
.setUUID(MAILBOX_ID, cassandraId.asUuid())
.setInt(LIMIT_MARKER, limit.getLimit().get())
.setTimestamp(SENT_AT, sinceDate))
.map(row -> messageIdFactory.of(row.getUUID(MESSAGE_ID)));
.map(row -> messageIdFactory.of(row.getUUID(MESSAGE_ID_LOWERCASE)));
}

@Override
Expand Down
Expand Up @@ -25,7 +25,9 @@
import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
import static com.datastax.driver.core.querybuilder.QueryBuilder.truncate;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.HAS_ATTACHMENT;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.HAS_ATTACHMENT_LOWERCASE;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.MESSAGE_ID;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.MESSAGE_ID_LOWERCASE;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.PREVIEW;
import static org.apache.james.jmap.cassandra.projections.table.CassandraMessageFastViewProjectionTable.TABLE_NAME;

Expand Down Expand Up @@ -79,7 +81,7 @@ public class CassandraMessageFastViewProjection implements MessageFastViewProjec

this.retrieveStatement = session.prepare(select()
.from(TABLE_NAME)
.where(eq(MESSAGE_ID, bindMarker(MESSAGE_ID))));
.where(eq(MESSAGE_ID_LOWERCASE, bindMarker(MESSAGE_ID_LOWERCASE))));

this.truncateStatement = session.prepare(truncate(TABLE_NAME));

Expand All @@ -103,7 +105,7 @@ public Mono<MessageFastViewPrecomputedProperties> retrieve(MessageId messageId)
checkMessage(messageId);

return cassandraAsyncExecutor.executeSingleRow(retrieveStatement.bind()
.setUUID(MESSAGE_ID, ((CassandraMessageId) messageId).get())
.setUUID(MESSAGE_ID_LOWERCASE, ((CassandraMessageId) messageId).get())
.setConsistencyLevel(ConsistencyLevel.ONE))
.map(this::fromRow)
.doOnNext(preview -> metricRetrieveHitCount.increment())
Expand Down Expand Up @@ -136,7 +138,7 @@ private void checkMessage(MessageId messageId) {
private MessageFastViewPrecomputedProperties fromRow(Row row) {
return MessageFastViewPrecomputedProperties.builder()
.preview(Preview.from(row.getString(PREVIEW)))
.hasAttachment(row.getBool(HAS_ATTACHMENT))
.hasAttachment(row.getBool(HAS_ATTACHMENT_LOWERCASE))
.build();
}
}
Expand Up @@ -19,13 +19,16 @@

package org.apache.james.jmap.cassandra.projections.table;

import java.util.Locale;

public interface CassandraEmailQueryViewTable {
String TABLE_NAME_SENT_AT = "email_query_view_sent_at";
String TABLE_NAME_RECEIVED_AT = "email_query_view_received_at";
String DATE_LOOKUP_TABLE = "email_query_view_date_lookup";

String MAILBOX_ID = "mailboxId";
String MESSAGE_ID = "messageId";
String MESSAGE_ID_LOWERCASE = MESSAGE_ID.toLowerCase(Locale.US);
String RECEIVED_AT = "receivedAt";
String SENT_AT = "sentAt";
}
Expand Up @@ -19,10 +19,14 @@

package org.apache.james.jmap.cassandra.projections.table;

import java.util.Locale;

public interface CassandraMessageFastViewProjectionTable {
String TABLE_NAME = "message_fast_view_projection";

String MESSAGE_ID = "messageId";
String MESSAGE_ID_LOWERCASE = MESSAGE_ID.toLowerCase(Locale.US);
String PREVIEW = "preview";
String HAS_ATTACHMENT = "hasAttachment";
String HAS_ATTACHMENT_LOWERCASE = HAS_ATTACHMENT.toLowerCase(Locale.US);
}