Skip to content

Commit

Permalink
Merge pull request #4947 from nextcloud/fix/chunking-in-statements-fo…
Browse files Browse the repository at this point in the history 
…r-oracle

Chunking `in` statements for Oracle Compliance
  • Loading branch information
@ChristophWurst
ChristophWurst authored May 27, 2021
2 parents 0be3e59 + 93a9a4b commit 6810fd0
Show file tree
Hide file tree
Showing 6 changed files with 139 additions and 95 deletions.
10 changes: 6 additions & 4 deletions lib/Db/AliasMapper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,9 +102,11 @@ public function deleteOrphans(): void {
$result->closeCursor();

$qb2 = $this->db->getQueryBuilder();
$query = $qb2
->delete($this->getTableName())
->where($qb2->expr()->in('id', $qb2->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$query->execute();
foreach (array_chunk($ids, 1000) as $chunk) {
$query = $qb2
->delete($this->getTableName())
->where($qb2->expr()->in('id', $qb2->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$query->execute();
}
}
}
7 changes: 5 additions & 2 deletions lib/Db/CollectedAddressMapper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,10 @@ public function deleteOrphans(): void {
$qb2 = $this->db->getQueryBuilder();
$query = $qb2
->delete($this->getTableName())
->where($qb2->expr()->in('id', $qb2->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$query->execute();
->where($qb2->expr()->in('id', $qb2->createParameter('ids')));
foreach (array_chunk($ids, 1000) as $chunk) {
$query->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
$query->execute();
}
}
}
10 changes: 6 additions & 4 deletions lib/Db/MailboxMapper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,10 +256,12 @@ public function deleteOrphans(): void {
$result->closeCursor();

$qb2 = $this->db->getQueryBuilder();
$query = $qb2
->delete($this->getTableName())
->where($qb2->expr()->in('id', $qb2->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$query->execute();
foreach (array_chunk($ids, 1000) as $chunk) {
$query = $qb2
->delete($this->getTableName())
->where($qb2->expr()->in('id', $qb2->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$query->execute();
}
}

/**
Expand Down
135 changes: 86 additions & 49 deletions lib/Db/MessageMapper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
use OCA\Mail\IMAP\Threading\DatabaseMessage;
use OCA\Mail\Support\PerformanceLogger;
use OCP\AppFramework\Db\DoesNotExistException;
use function OCA\Mail\array_flat_map;
use function \OCA\Mail\array_flat_map;

/**
* @template-extends QBMapper<Message>
Expand Down Expand Up @@ -494,11 +494,15 @@ public function deleteAll(Mailbox $mailbox): void {
return (int)$row['id'];
}, $messageIds);

// delete all related recipient entries
$deleteRecipientsQuery = $this->db->getQueryBuilder();
$deleteRecipientsQuery->delete('mail_recipients')
->where($deleteRecipientsQuery->expr()->in('message_id', $deleteRecipientsQuery->createNamedParameter($messageIds, IQueryBuilder::PARAM_INT_ARRAY)));
$deleteRecipientsQuery->execute();
->where($deleteRecipientsQuery->expr()->in('message_id', $deleteRecipientsQuery->createParameter('ids')));

foreach (array_chunk($messageIds, 1000) as $chunk) {
// delete all related recipient entries
$deleteRecipientsQuery->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
$deleteRecipientsQuery->execute();
}

$query = $this->db->getQueryBuilder();

Expand All @@ -510,36 +514,40 @@ public function deleteAll(Mailbox $mailbox): void {

public function deleteByUid(Mailbox $mailbox, int ...$uids): void {
$messageIdQuery = $this->db->getQueryBuilder();
$deleteRecipientsQuery = $this->db->getQueryBuilder();
$deleteMessagesQuery = $this->db->getQueryBuilder();

// Get all message ids query
$messageIdQuery->select('id')
->from($this->getTableName())
->where(
$messageIdQuery->expr()->eq('mailbox_id', $messageIdQuery->createNamedParameter($mailbox->getId())),
$messageIdQuery->expr()->in('uid', $messageIdQuery->createNamedParameter($uids, IQueryBuilder::PARAM_INT_ARRAY))
$messageIdQuery->expr()->in('uid', $messageIdQuery->createParameter('uids'))
);

$cursor = $messageIdQuery->execute();
$messageIds = $cursor->fetchAll();
$cursor->closeCursor();
$deleteRecipientsQuery->delete('mail_recipients')
->where($deleteRecipientsQuery->expr()->in('message_id', $deleteRecipientsQuery->createParameter('messageIds')));

$messageIds = array_map(function (array $row) {
return (int)$row['id'];
}, $messageIds);
$deleteMessagesQuery->delete($this->getTableName())
->where($deleteMessagesQuery->expr()->in('id', $deleteMessagesQuery->createParameter('messageIds')));

// delete all related recipient entries
$deleteRecipientsQuery = $this->db->getQueryBuilder();
$deleteRecipientsQuery->delete('mail_recipients')
->where($deleteRecipientsQuery->expr()->in('message_id', $deleteRecipientsQuery->createNamedParameter($messageIds, IQueryBuilder::PARAM_INT_ARRAY)));
$deleteRecipientsQuery->execute();
foreach (array_chunk($uids, 1000) as $chunk) {
$messageIdQuery->setParameter('uids', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
$cursor = $messageIdQuery->execute();

$query = $this->db->getQueryBuilder();
$messageIds = array_map(function (array $message) {
return $message['id'];
}, $cursor->fetchAll());
$cursor->closeCursor();

$query->delete($this->getTableName())
->where(
$query->expr()->eq('mailbox_id', $query->createNamedParameter($mailbox->getId())),
$query->expr()->in('uid', $query->createNamedParameter($uids, IQueryBuilder::PARAM_INT_ARRAY))
);
// delete all related recipient entries
$deleteRecipientsQuery->setParameter('messageIds', $messageIds, IQueryBuilder::PARAM_INT_ARRAY);
$deleteRecipientsQuery->execute();

$query->execute();
// delete all messages
$deleteMessagesQuery->setParameter('messageIds', $messageIds, IQueryBuilder::PARAM_INT_ARRAY);
$deleteMessagesQuery->execute();
}
}

/**
Expand Down Expand Up @@ -647,9 +655,10 @@ public function findIdsByQuery(Mailbox $mailbox, SearchQuery $query, ?int $limit
$qb->expr()->lt('sent_at', $qb->createNamedParameter($query->getCursor(), IQueryBuilder::PARAM_INT))
);
}
// createParameter
if ($uids !== null) {
$select->andWhere(
$qb->expr()->in('uid', $qb->createNamedParameter($uids, IQueryBuilder::PARAM_INT_ARRAY))
$qb->expr()->in('uid', $qb->createParameter('uids'))
);
}
foreach ($query->getFlags() as $flag) {
Expand All @@ -670,6 +679,15 @@ public function findIdsByQuery(Mailbox $mailbox, SearchQuery $query, ?int $limit
$select = $select->setMaxResults($limit);
}

if ($uids !== null) {
return array_flat_map(function (array $chunk) use ($qb, $select) {
$qb->setParameter('uids', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
return array_map(function (Message $message) {
return $message->getId();
}, $this->findEntities($select));
}, array_chunk($uids, 1000));
}

return array_map(function (Message $message) {
return $message->getId();
}, $this->findEntities($select));
Expand Down Expand Up @@ -747,7 +765,7 @@ public function findIdsGloballyByQuery(IUser $user, SearchQuery $query, ?int $li
}
if ($uids !== null) {
$select->andWhere(
$qb->expr()->in('uid', $qb->createNamedParameter($uids, IQueryBuilder::PARAM_INT_ARRAY))
$qb->expr()->in('uid', $qb->createParameter('uids'))
);
}
foreach ($query->getFlags() as $flag) {
Expand All @@ -768,6 +786,15 @@ public function findIdsGloballyByQuery(IUser $user, SearchQuery $query, ?int $li
$select = $select->setMaxResults($limit);
}

if ($uids !== null) {
return array_flat_map(function (array $chunk) use ($select) {
$select->setParameter('uids', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
return array_map(function (Message $message) {
return $message->getId();
}, $this->findEntities($select));
}, array_chunk($uids, 1000));
}

return array_map(function (Message $message) {
return $message->getId();
}, $this->findEntities($select));
Expand Down Expand Up @@ -839,17 +866,21 @@ public function findByIds(string $userId, array $ids): array {
if (empty($ids)) {
return [];
}
$qb = $this->db->getQueryBuilder();

$select = $qb
->select('*')
$qb = $this->db->getQueryBuilder();
$qb->select('*')
->from($this->getTableName())
->where(
$qb->expr()->in('id', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY)
$qb->expr()->in('id', $qb->createParameter('ids'))
)
->orderBy('sent_at', 'desc');

return $this->findRelatedData($this->findEntities($select), $userId);
$results = [];
foreach (array_chunk($ids, 1000) as $chunk) {
$qb->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
$results[] = $this->findRelatedData($this->findEntities($qb), $userId);
}
return array_merge(...$results);
}

/**
Expand Down Expand Up @@ -919,30 +950,34 @@ public function findRelatedData(array $messages, string $userId): array {

/**
* @param Mailbox $mailbox
* @param int $highest
*
* @param array $ids
* @return int[]
*/
public function findNewIds(Mailbox $mailbox, array $ids): array {
$qb = $this->db->getQueryBuilder();
$sub = $this->db->getQueryBuilder();
$qb = $this->db->getQueryBuilder();

$subSelect = $sub
->select($sub->func()->max('uid'))
->from($this->getTableName())
->where(
$sub->expr()->eq('mailbox_id', $qb->createNamedParameter($mailbox->getId(), IQueryBuilder::PARAM_INT)),
$sub->expr()->in('id', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY)
$sub->expr()->in('id', $qb->createParameter('ids'))
);
$select = $qb
->select('id')

$qb->select('id')
->from($this->getTableName())
->where(
$qb->expr()->eq('mailbox_id', $qb->createNamedParameter($mailbox->getId(), IQueryBuilder::PARAM_INT)),
$qb->expr()->gt('uid', $qb->createFunction('(' . $subSelect->getSQL() . ')'), IQueryBuilder::PARAM_INT)
$qb->expr()->eq('mailbox_id', $qb->createNamedParameter($mailbox->getId(), IQueryBuilder::PARAM_INT))
);

return $this->findIds($select);
return array_flat_map(function (array $chunk) use ($qb, $subSelect) {
$qb->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
$select = $qb->andWhere(
$qb->expr()->gt('uid', $qb->createFunction('(' . $subSelect->getSQL() . ')'), IQueryBuilder::PARAM_INT)
);
return $this->findIds($select);
}, array_chunk($ids, 1000));
}

/**
Expand All @@ -958,7 +993,6 @@ public function findChanged(Account $account, Mailbox $mailbox, int $since): arr
$qb->expr()->eq('mailbox_id', $qb->createNamedParameter($mailbox->getId(), IQueryBuilder::PARAM_INT)),
$qb->expr()->gt('updated_at', $qb->createNamedParameter($since, IQueryBuilder::PARAM_INT))
);
// TODO: change this to findRelatedData
return $this->findRelatedData($this->findEntities($select), $account->getUserId());
}

Expand Down Expand Up @@ -998,11 +1032,12 @@ public function deleteOrphans(): void {
$result->closeCursor();

$qb2 = $this->db->getQueryBuilder();
$query = $qb2
->delete($this->getTableName())
->where($qb2->expr()->in('id', $qb2->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$query->execute();

foreach (array_chunk($ids, 1000) as $chunk) {
$query = $qb2
->delete($this->getTableName())
->where($qb2->expr()->in('id', $qb2->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$query->execute();
}
$qb3 = $this->db->getQueryBuilder();
$recipientIdsQuery = $qb3->selectDistinct('r.id')
->from('mail_recipients', 'r')
Expand All @@ -1015,10 +1050,12 @@ public function deleteOrphans(): void {
$result->closeCursor();

$qb4 = $this->db->getQueryBuilder();
$recipientsQuery = $qb4
->delete('mail_recipients')
->where($qb4->expr()->in('id', $qb4->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$recipientsQuery->execute();
foreach (array_chunk($ids, 1000) as $chunk) {
$recipientsQuery = $qb4
->delete('mail_recipients')
->where($qb4->expr()->in('id', $qb4->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY), IQueryBuilder::PARAM_INT_ARRAY));
$recipientsQuery->execute();
}
}

public function getIdForUid(Mailbox $mailbox, $uid): ?int {
Expand Down
38 changes: 21 additions & 17 deletions lib/Db/TagMapper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,33 +134,37 @@ public function getAllTagsForMessages(array $messages, string $userId): array {
return $message->getMessageId();
}, $messages);

$tags = [];
$qb = $this->db->getQueryBuilder();
$tagsQuery = $qb->selectDistinct(['t.*', 'mt.imap_message_id'])
->from($this->getTableName(), 't')
->join('t', 'mail_message_tags', 'mt', $qb->expr()->eq('t.id', 'mt.tag_id', IQueryBuilder::PARAM_INT))
->where(
$qb->expr()->in('mt.imap_message_id', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_STR_ARRAY)),
$qb->expr()->in('mt.imap_message_id', $qb->createParameter('ids')),
$qb->expr()->eq('t.user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR))
);
$queryResult = $tagsQuery->execute();

$tags = [];
while (($row = $queryResult->fetch()) !== false) {
$messageId = $row['imap_message_id'];
if (!isset($tags[$messageId])) {
$tags[$messageId] = [];
}
foreach (array_chunk($ids, 1000) as $chunk) {
$tagsQuery->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
$queryResult = $tagsQuery->execute();

while (($row = $queryResult->fetch()) !== false) {
$messageId = $row['imap_message_id'];
if (!isset($tags[$messageId])) {
$tags[$messageId] = [];
}

// Construct a Tag instance but omit any other joined columns
$tags[$messageId][] = Tag::fromRow(array_filter(
$row,
function (string $key) {
return $key !== 'imap_message_id';
},
ARRAY_FILTER_USE_KEY
));
// Construct a Tag instance but omit any other joined columns
$tags[$messageId][] = Tag::fromRow(array_filter(
$row,
function (string $key) {
return $key !== 'imap_message_id';
},
ARRAY_FILTER_USE_KEY
));
}
$queryResult->closeCursor();
}
$queryResult->closeCursor();
return $tags;
}

Expand Down
Loading

0 comments on commit 6810fd0

Please sign in to comment.