Improve IMAP folder UIDVALIDITY change detection. Fixes #11807.

We stored the whole list of IMAP UIDs of a folder next
to every object. This turned out to be a major memory
and CPU time hog (see #11807).

Instead we detect a change of UIDVALIDITY at the
IMAP folder cache level (where it belongs) and pass
the information about a reset situation to the
history 'query' as additional parameter.

framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Cached.php

@@ -7,6 +7,7 @@
  * @category Kolab
  * @package  Kolab_Storage
  * @author   Gunnar Wrobel <wrobel@pardus.de>
+ * @author   Thomas Jarosch <thomas.jarosch@intra2net.com>
  * @license  http://www.horde.org/licenses/lgpl21 LGPL 2.1
  * @link     http://pear.horde.org/index.php?package=Kolab_Storage
  */
@@ -22,6 +23,7 @@
  * @category Kolab
  * @package  Kolab_Storage
  * @author   Gunnar Wrobel <wrobel@pardus.de>
+ * @author   Thomas Jarosch <thomas.jarosch@intra2net.com>
  * @license  http://www.horde.org/licenses/lgpl21 LGPL 2.1
  * @link     http://pear.horde.org/index.php?package=Kolab_Storage
  */
@@ -248,9 +250,14 @@ public function synchronize($params = array())
         }
 
         $previous = unserialize($this->_data_cache->getStamp());
-        if ($previous === false || $previous->isReset($current)) {
-            $this->_logger->notice(sprintf("Complete folder sync: user: %s, folder: %s, is_reset: %d", $user, $folder_path, $is_reset));
-            $this->_completeSynchronization($current);
+
+        // check if UIDVALIDITY changed
+        $is_reset = false;
+        if ($previous !== false)
+            $is_reset = $previous->isReset($current);
+
+        if ($previous === false || $is_reset) {
+            $this->_completeSynchronization($current, array('is_reset' => $is_reset));
             return;
         }
 

framework/Kolab_Storage/lib/Horde/Kolab/Storage/Data/Query/History/Base.php

@@ -8,6 +8,7 @@
  * @category Kolab
  * @package  Kolab_Storage
  * @author   Gunnar Wrobel <wrobel@pardus.de>
+ * @author   Thomas Jarosch <thomas.jarosch@intra2net.com>
  * @license  http://www.horde.org/licenses/lgpl21 LGPL 2.1
  * @link     http://pear.horde.org/index.php?package=Kolab_Storage
  */
@@ -24,6 +25,7 @@
  * @category Kolab
  * @package  Kolab_Storage
  * @author   Gunnar Wrobel <wrobel@pardus.de>
+ * @author   Thomas Jarosch <thomas.jarosch@intra2net.com>
  * @license  http://www.horde.org/licenses/lgpl21 LGPL 2.1
  * @link     http://pear.horde.org/index.php?package=Kolab_Storage
  */
@@ -90,10 +92,12 @@ public function synchronize($params = array())
         if (empty($prefix))
             return;
 
-        $stamp = $this->data->getStamp();
-        if (isset($params['changes'])) {
+        // check if IMAP uidvalidity changed
+        $is_reset = !empty($params['is_reset']);
+
+        if (isset($params['changes']) && !$is_reset) {
             foreach ($params['changes'][Horde_Kolab_Storage_Folder_Stamp::ADDED] as $bid => $object) {
-                $this->_updateLog($prefix.$object['uid'], $bid, $stamp);
+                $this->_updateLog($prefix.$object['uid'], $bid);
             }
             foreach ($params['changes'][Horde_Kolab_Storage_Folder_Stamp::DELETED] as $bid => $object_uid) {
                 // Check if the object is really gone from the folder.
@@ -103,12 +107,12 @@ public function synchronize($params = array())
                     continue;
 
                 $this->history->log(
-                    $prefix.$object_uid, array('action' => 'delete', 'bid' => $bid, 'stamp' => $stamp), true
+                    $prefix.$object_uid, array('action' => 'delete', 'bid' => $bid), true
                 );
             }
         } else {
             foreach ($this->data->getObjectToBackend() as $object => $bid) {
-                $this->_updateLog($prefix.$object, $bid, $stamp);
+                $this->_updateLog($prefix.$object, $bid);
             }
         }
     }
@@ -179,16 +183,16 @@ private function _type2app($type)
      * @param string                           $object The object ID.
      * @param string                           $bid    The backend ID of
      *                                                 the object.
-     * @param Horde_Kolab_Storage_Folder_Stamp $stamp  The folder stamp.
+     * @param bool                             $force  Force update
      *
      * @return NULL
      */
-    private function _updateLog($object, $bid, $stamp)
+    private function _updateLog($object, $bid, $force=false)
     {
         $log = $this->history->getHistory($object);
         if (count($log) == 0) {
             $this->history->log(
-                $object, array('action' => 'add', 'bid' => $bid, 'stamp' => $stamp), true
+                $object, array('action' => 'add', 'bid' => $bid), true
             );
         } else {
             $last = array('ts' => 0);
@@ -208,14 +212,13 @@ private function _updateLog($object, $bid, $stamp)
             // (a foreign client is sending an update over a slow link)
             if ($last['action'] == 'delete') {
                 $this->history->log(
-                    $object, array('action' => 'add', 'bid' => $bid, 'stamp' => $stamp), true
+                    $object, array('action' => 'add', 'bid' => $bid), true
                 );
             }
 
-            if (!isset($last['bid']) || $last['bid'] != $bid
-                || (isset($last['stamp']) && $last['stamp']->isReset($stamp))) {
+            if (!isset($last['bid']) || $last['bid'] != $bid || $force) {
                 $this->history->log(
-                    $object, array('action' => 'modify', 'bid' => $bid, 'stamp' => $stamp), true
+                    $object, array('action' => 'modify', 'bid' => $bid), true
                 );
             }
         }