ENH: fixed bug #179 Added lucene search

Author: Charles Ma

core/Notification.php

@@ -14,11 +14,13 @@
 class Notification extends MIDAS_Notification
   {
   public $_components = array('Utility');
+  public $_models = array('User', 'Item');
 
   /** init notification process*/
   public function init()
     {
     $this->addCallBack('CALLBACK_CORE_GET_DASHBOARD', 'getDasboard');
+    $this->addTask('TASK_CORE_RESET_ITEM_INDEXES', 'resetItemIndexes', 'Recompute lucene indexes');
     }//end init  
 
   /** generate Dasboard information */
@@ -33,5 +35,26 @@ public function getDasboard()
 
     return $return;
     }//end _getDasboard
+    
+    
+  /** reset item indexes */
+  public function resetItemIndexes()
+    {
+    $users = $this->User->getAll();
+    foreach($users as $user)
+      {
+      $items = $this->Item->getOwnedByUser($user, 999999);
+      foreach($items as $item)
+        {
+        $this->Item->save($item);
+        }
+      }
+      
+    require_once BASE_PATH.'/core/controllers/components/SearchComponent.php';
+    $component = new SearchComponent();    
+    $index = $component->getLuceneItemIndex();
+    
+    $index->optimize();
+    }
   } //end class
 ?>

core/controllers/AdminController.php

@@ -40,6 +40,34 @@ function resetdemoAction()
     $this->disableView();
     }
 
+  /** run a task **/
+  function taskAction()
+    {
+    set_time_limit(0);
+    if(!$this->logged)
+      {
+      $this->haveToBeLogged();
+      return;
+      }
+    if(!$this->userSession->Dao->getAdmin() == 1)
+      {
+      throw new Zend_Exception("You should be an administrator");
+      }
+      
+    $task = $this->_getParam("task");
+    $params = $this->_getParam("params");
+    if(isset($params))
+      {
+      $params = JsonComponent::decode($params);
+      }
+      
+    $modules = Zend_Registry::get('notifier')->modules;
+    $tasks = Zend_Registry::get('notifier')->tasks;
+    call_user_func(array($modules[$tasks[$task]['module']],$tasks[$task]['method']), $params);
+    $this->disableLayout();
+    $this->disableView();
+    }
+    
   /** index*/
   function indexAction()
     {

core/controllers/ItemController.php

@@ -113,7 +113,7 @@ function viewAction()
         $element = $this->_getParam('element');
         $qualifier = $this->_getParam('qualifier');
         $value = $this->_getParam('value');
-        if(isset($metadataId))
+        if(isset($metadataId) && !empty($metadataIds))
           {
           $this->ItemRevision->deleteMetadata($itemRevision, $metadataId);
           }

core/controllers/components/SearchComponent.php

@@ -13,6 +13,26 @@
 /** Search */
 class SearchComponent extends AppComponent
 {  
+  
+  /** get Zend Lucene index */
+  public function getLuceneItemIndex()
+    {
+    $path = BASE_PATH.'/tmp/cache/searchIndex';
+    if(!file_exists($path))
+      {
+      mkdir($path);
+      }
+    
+    $path .= '/item';
+    if(!file_exists($path))
+      {
+      mkdir($path);
+      Zend_Search_Lucene::create($path);
+      }
+      
+    return Zend_Search_Lucene::open($path);
+    }
+    
   /** search all the results */
   public function searchAll($userDao, $search, $order)
     {

core/models/base/ItemModelBase.php

@@ -63,6 +63,40 @@ public function save($dao)
       }
     $dao->setDateUpdate(date('c'));
     parent::save($dao);
+    
+    require_once BASE_PATH.'/core/controllers/components/SearchComponent.php';
+    $component = new SearchComponent();    
+    $index = $component->getLuceneItemIndex();
+    
+    $hits = $index->find("item_id:".$dao->getKey());
+    foreach ($hits as $hit) 
+      {
+      $index->delete($hit->id);
+      }
+    $doc = new Zend_Search_Lucene_Document();
+    $doc->addField(Zend_Search_Lucene_Field::Text('title', $dao->getName()));
+    $doc->addField(Zend_Search_Lucene_Field::Keyword('item_id', $dao->getKey()));    
+    $doc->addField(Zend_Search_Lucene_Field::UnStored('description', $dao->getDescription()));    
+    
+    $modelLoad = new MIDAS_ModelLoader();
+    $revisionModel = $modelLoad->loadModel('ItemRevision');    
+    $revision = $this->getLastRevision($dao);
+    
+    $metadata = $revisionModel->getMetadata($revision);
+    $metadataString = '';
+    
+    foreach($metadata as $m)
+      {
+      $doc->addField(Zend_Search_Lucene_Field::Keyword($m->getElement().'-'.$m->getQualifier(), $m->getValue())); 
+      if(!is_numeric($m->getValue()))
+        {
+        $metadataString.=' '. $m->getValue();
+        }
+      }
+      
+    $doc->addField(Zend_Search_Lucene_Field::Text('metadata', $metadataString));
+    $index->addDocument($doc);
+    $index->commit();
     }
 
   /** copy parent folder policies*/

core/models/base/MetadataModelBase.php

@@ -85,6 +85,18 @@ function addMetadataValue($itemRevisionDao, $type, $element, $qualifier, $value)
       {
       throw new Zend_Exception("This metadata value already exists for that revision.");   
       }
+     
+    $item = $itemRevisionDao->getItem();
+    $modelLoader = new MIDAS_ModelLoader();
+    $itemModel = $modelLoader->loadModel('Item');
+    $lastrevision = $itemModel->getLastRevision($item);
+    
+    //refresh zend search index
+    if($lastrevision->getKey() == $itemRevisionDao->getKey())
+      {
+      $itemModel->save($item);
+      }
+      
     $this->saveMetadataValue($metadataDao);
     } // end addMetadataValue()  
 

core/models/pdo/ItemKeywordModel.php

@@ -45,30 +45,21 @@ function getItemsFromSearch($searchterm, $userDao, $limit = 14, $group = true, $
         }
       }
 
-    $searchterms = explode(' ', $searchterm);
-    // Apparently it's slow to do a like in a subquery so we run it first  
-    $sql = $this->database->select()->from(array('i' => 'itemkeyword'), array())
-                ->setIntegrityCheck(false);
-    $sql->join(array('i2k' => 'item2keyword'), 'i.keyword_id = i2k.keyword_id', array('item_id'));
-    
-    if(empty($searchterms))
+    require_once BASE_PATH.'/core/controllers/components/SearchComponent.php';
+    $component = new SearchComponent();    
+    $index = $component->getLuceneItemIndex();
+    Zend_Search_Lucene_Search_QueryParser::setDefaultOperator(Zend_Search_Lucene_Search_QueryParser::B_AND);
+    Zend_Search_Lucene::setResultSetLimit($limit * 3);
+    if($group && strpos($searchterm, ':') === false)
       {
-      return array();
+      $rowset = $index->find('title:'.$searchterm);
       }
-    
-    foreach($searchterms as $key => $term)
+    else
       {
-      if($key == 0)
-        {
-        $sql->where('value LIKE ?', '%'.$term.'%'); 
-        }
-      else
-        {
-        $sql->orWhere('value LIKE ?', '%'.$term.'%'); 
-        }
+      $rowset = $index->find($searchterm);
       }
-                                   
-    $rowset = $this->database->fetchAll($sql);
+
+    
     $return = array();
     $itemIdsCount = array();
     $itemIds = array();
@@ -85,14 +76,7 @@ function getItemsFromSearch($searchterm, $userDao, $limit = 14, $group = true, $
       }
     foreach($itemIdsCount as $key => $n)
       {
-      if($n < count($searchterms))
-        {
-        unset($itemIdsCount[$key]);
-        }
-      else
-        {
-        $itemIds[] = $key;
-        }
+      $itemIds[] = $key;
       }
 
     if(empty($itemIds))

core/models/pdo/ItemModel.php

@@ -204,6 +204,17 @@ function delete($itemdao)
       $policy_user_model->delete($policy);
       }
 
+      
+    require_once BASE_PATH.'/core/controllers/components/SearchComponent.php';
+    $component = new SearchComponent();    
+    $index = $component->getLuceneItemIndex();
+    
+    $hits = $index->find("item_id:".$itemdao->getKey());
+    foreach ($hits as $hit) 
+      {
+      $index->delete($hit->id);
+      }
+      
     parent::delete($itemdao);
     unset($itemdao->item_id);
     $itemdao->saved = false;

core/models/pdo/ItemRevisionModel.php

@@ -60,6 +60,17 @@ function deleteMetadata($revisiondao, $metadataId)
       }
 
     Zend_Registry::get('dbAdapter')->delete('metadatavalue', 'itemrevision_id = '.$revisiondao->getKey().' AND metadata_id = '.$metadataId);
+    
+    $item = $revisiondao->getItem();
+    $modelLoader = new MIDAS_ModelLoader();
+    $itemModel = $modelLoader->loadModel('Item');
+    $lastrevision = $itemModel->getLastRevision($item);
+    
+    //refresh zend search index
+    if($lastrevision->getKey() == $revisiondao->getKey())
+      {
+      $itemModel->save($item);
+      }
     return;
     }  // end getMetadata
 

notification/MIDASNotifier.php

@@ -25,7 +25,7 @@ public function getNotifications()
     }
 
   /** get Tasks */
-  public function getTaks()
+  public function getTasks()
     {
     return $this->tasks;
     }