From 8e9463e42526b5617b465a12fbc4ad565e016d08 Mon Sep 17 00:00:00 2001 From: Thomas Jarosch Date: Wed, 20 Mar 2013 14:38:06 +0100 Subject: [PATCH] Improve IMAP folder UIDVALIDITY change detection. Fixes #11807. We stored the whole list of IMAP UIDs of a folder next to every object. This turned out to be a major memory and CPU time hog (see #11807). Instead we detect a change of UIDVALIDITY at the IMAP folder cache level (where it belongs) and pass the information about a reset situation to the history 'query' as additional parameter. --- .../lib/Horde/Kolab/Storage/Data/Cached.php | 13 ++++++--- .../Kolab/Storage/Data/Query/History/Base.php | 27 ++++++++++--------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Cached.php b/framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Cached.php index 95bb02001d8..9d07bb7a511 100644 --- a/framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Cached.php +++ b/framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Cached.php @@ -7,6 +7,7 @@ * @category Kolab * @package Kolab_Storage * @author Gunnar Wrobel + * @author Thomas Jarosch * @license http://www.horde.org/licenses/lgpl21 LGPL 2.1 * @link http://pear.horde.org/index.php?package=Kolab_Storage */ @@ -22,6 +23,7 @@ * @category Kolab * @package Kolab_Storage * @author Gunnar Wrobel + * @author Thomas Jarosch * @license http://www.horde.org/licenses/lgpl21 LGPL 2.1 * @link http://pear.horde.org/index.php?package=Kolab_Storage */ @@ -248,9 +250,14 @@ public function synchronize($params = array()) } $previous = unserialize($this->_data_cache->getStamp()); - if ($previous === false || $previous->isReset($current)) { - $this->_logger->notice(sprintf("Complete folder sync: user: %s, folder: %s, is_reset: %d", $user, $folder_path, $is_reset)); - $this->_completeSynchronization($current); + + // check if UIDVALIDITY changed + $is_reset = false; + if ($previous !== false) + $is_reset = $previous->isReset($current); + + if ($previous === false || $is_reset) { + $this->_completeSynchronization($current, array('is_reset' => $is_reset)); return; } diff --git a/framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Query/History/Base.php b/framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Query/History/Base.php index ecee0b394d6..09a36a5163f 100644 --- a/framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Query/History/Base.php +++ b/framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Query/History/Base.php @@ -8,6 +8,7 @@ * @category Kolab * @package Kolab_Storage * @author Gunnar Wrobel + * @author Thomas Jarosch * @license http://www.horde.org/licenses/lgpl21 LGPL 2.1 * @link http://pear.horde.org/index.php?package=Kolab_Storage */ @@ -24,6 +25,7 @@ * @category Kolab * @package Kolab_Storage * @author Gunnar Wrobel + * @author Thomas Jarosch * @license http://www.horde.org/licenses/lgpl21 LGPL 2.1 * @link http://pear.horde.org/index.php?package=Kolab_Storage */ @@ -90,10 +92,12 @@ public function synchronize($params = array()) if (empty($prefix)) return; - $stamp = $this->data->getStamp(); - if (isset($params['changes'])) { + // check if IMAP uidvalidity changed + $is_reset = !empty($params['is_reset']); + + if (isset($params['changes']) && !$is_reset) { foreach ($params['changes'][Horde_Kolab_Storage_Folder_Stamp::ADDED] as $bid => $object) { - $this->_updateLog($prefix.$object['uid'], $bid, $stamp); + $this->_updateLog($prefix.$object['uid'], $bid); } foreach ($params['changes'][Horde_Kolab_Storage_Folder_Stamp::DELETED] as $bid => $object_uid) { // Check if the object is really gone from the folder. @@ -103,12 +107,12 @@ public function synchronize($params = array()) continue; $this->history->log( - $prefix.$object_uid, array('action' => 'delete', 'bid' => $bid, 'stamp' => $stamp), true + $prefix.$object_uid, array('action' => 'delete', 'bid' => $bid), true ); } } else { foreach ($this->data->getObjectToBackend() as $object => $bid) { - $this->_updateLog($prefix.$object, $bid, $stamp); + $this->_updateLog($prefix.$object, $bid); } } } @@ -179,16 +183,16 @@ private function _type2app($type) * @param string $object The object ID. * @param string $bid The backend ID of * the object. - * @param Horde_Kolab_Storage_Folder_Stamp $stamp The folder stamp. + * @param bool $force Force update * * @return NULL */ - private function _updateLog($object, $bid, $stamp) + private function _updateLog($object, $bid, $force=false) { $log = $this->history->getHistory($object); if (count($log) == 0) { $this->history->log( - $object, array('action' => 'add', 'bid' => $bid, 'stamp' => $stamp), true + $object, array('action' => 'add', 'bid' => $bid), true ); } else { $last = array('ts' => 0); @@ -208,14 +212,13 @@ private function _updateLog($object, $bid, $stamp) // (a foreign client is sending an update over a slow link) if ($last['action'] == 'delete') { $this->history->log( - $object, array('action' => 'add', 'bid' => $bid, 'stamp' => $stamp), true + $object, array('action' => 'add', 'bid' => $bid), true ); } - if (!isset($last['bid']) || $last['bid'] != $bid - || (isset($last['stamp']) && $last['stamp']->isReset($stamp))) { + if (!isset($last['bid']) || $last['bid'] != $bid || $force) { $this->history->log( - $object, array('action' => 'modify', 'bid' => $bid, 'stamp' => $stamp), true + $object, array('action' => 'modify', 'bid' => $bid), true ); } }