library/Zend/Search/Lucene/Index/TermsPriorityQueue.php library/Zend/Search/Lucene/Index/TermsStream/Interface.php tests/Zend/Search/Lucene/Index/TermsPriorityQueueTest.php @@ -63,8 +63,11 @@ require_once 'Zend/Search/Lucene/Search/QueryHit.php'; /** Zend_Search_Lucene_Search_Similarity */ require_once 'Zend/Search/Lucene/Search/Similarity.php'; -/** Zend_Search_Lucene_Index_SegmentInfoPriorityQueue */ -require_once 'Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php'; +/** Zend_Search_Lucene_Index_TermsPriorityQueue */ +require_once 'Zend/Search/Lucene/Index/TermsPriorityQueue.php'; + +/** Zend_Search_Lucene_TermStreamsPriorityQueue */ +require_once 'Zend/Search/Lucene/TermStreamsPriorityQueue.php'; /** Zend_Search_Lucene_Index_DocsFilter */ require_once 'Zend/Search/Lucene/Index/DocsFilter.php'; @@ -1413,10 +1416,10 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface { $result = array(); - $segmentInfoQueue = new Zend_Search_Lucene_Index_SegmentInfoPriorityQueue(); + $segmentInfoQueue = new Zend_Search_Lucene_Index_TermsPriorityQueue(); foreach ($this->_segmentInfos as $segmentInfo) { - $segmentInfo->reset(); + $segmentInfo->resetTermsStream(); // Skip "empty" segments if ($segmentInfo->currentTerm() !== null) { @@ -1443,36 +1446,22 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface /** - * Terms stream queue - * - * @var Zend_Search_Lucene_Index_SegmentInfoPriorityQueue - */ - private $_termsStreamQueue = null; - - /** - * Last Term in a terms stream + * Terms stream priority queue object * - * @var Zend_Search_Lucene_Index_Term + * @var Zend_Search_Lucene_termStreamsPriorityQueue */ - private $_lastTerm = null; + private $_termsStream = null; /** * Reset terms stream. */ public function resetTermsStream() { - $this->_termsStreamQueue = new Zend_Search_Lucene_Index_SegmentInfoPriorityQueue(); - - foreach ($this->_segmentInfos as $segmentInfo) { - $segmentInfo->reset(); - - // Skip "empty" segments - if ($segmentInfo->currentTerm() !== null) { - $this->_termsStreamQueue->put($segmentInfo); - } - } - - $this->nextTerm(); + if ($this->_termsStream === null) { + $this->_termsStream = new Zend_Search_Lucene_termStreamsPriorityQueue($this->_segmentInfos); + } else { + $this->_termsStream->resetTermsStream(); + } } /** @@ -1484,21 +1473,7 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface */ public function skipTo(Zend_Search_Lucene_Index_Term $prefix) { - $segments = array(); - - while (($segmentInfo = $this->_termsStreamQueue->pop()) !== null) { - $segments[] = $segmentInfo; - } - - foreach ($segments as $segmentInfo) { - $segmentInfo->skipTo($prefix); - - if ($segmentInfo->currentTerm() !== null) { - $this->_termsStreamQueue->put($segmentInfo); - } - } - - $this->nextTerm(); + $this->_termsStream->skipTo($prefix); } /** @@ -1508,31 +1483,7 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface */ public function nextTerm() { - while (($segmentInfo = $this->_termsStreamQueue->pop()) !== null) { - if ($this->_termsStreamQueue->top() === null || - $this->_termsStreamQueue->top()->currentTerm()->key() != - $segmentInfo->currentTerm()->key()) { - // We got new term - $this->_lastTerm = $segmentInfo->currentTerm(); - - if ($segmentInfo->nextTerm() !== null) { - // Put segment back into the priority queue - $this->_termsStreamQueue->put($segmentInfo); - } - - return $this->_lastTerm; - } - - if ($segmentInfo->nextTerm() !== null) { - // Put segment back into the priority queue - $this->_termsStreamQueue->put($segmentInfo); - } - } - - // End of stream - $this->_lastTerm = null; - - return null; + return $this->_termsStream->nextTerm(); } /** @@ -1542,7 +1493,7 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface */ public function currentTerm() { - return $this->_lastTerm; + return $this->_termsStream->currentTerm(); } /** @@ -1552,12 +1503,8 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface */ public function closeTermsStream() { - while (($segmentInfo = $this->_termsStreamQueue->pop()) !== null) { - $segmentInfo->closeTermsStream(); - } - - $this->_termsStreamQueue = null; - $this->_lastTerm = null; + $this->_termsStream->closeTermsStream(); + $this->_termsStream = null; } library/Zend/Search/Lucene.php @@ -25,6 +25,9 @@ require_once 'Zend/Search/Lucene/Index/DictionaryLoader.php'; /** Zend_Search_Lucene_Index_DocsFilter */ require_once 'Zend/Search/Lucene/Index/DocsFilter.php'; +/** Zend_Search_Lucene_Index_TermsStream_Interface */ +require_once 'Zend/Search/Lucene/Index/TermsStream/Interface.php'; + /** * @category Zend * @package Zend_Search_Lucene @@ -32,7 +35,7 @@ require_once 'Zend/Search/Lucene/Index/DocsFilter.php'; * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ -class Zend_Search_Lucene_Index_SegmentInfo +class Zend_Search_Lucene_Index_SegmentInfo implements Zend_Search_Lucene_Index_TermsStream_Interface { /** * "Full scan vs fetch" boundary. @@ -1750,8 +1753,28 @@ class Zend_Search_Lucene_Index_SegmentInfo * @throws Zend_Search_Lucene_Exception * @return integer */ - public function reset($startId = 0, $mode = self::SM_TERMS_ONLY) + public function resetTermsStream(/** $startId = 0, $mode = self::SM_TERMS_ONLY */) { + /** + * SegmentInfo->resetTermsStream() method actually takes two optional parameters: + * $startId (default value is 0) + * $mode (default value is self::SM_TERMS_ONLY) + */ + $argList = func_get_args(); + if (count($argList) > 2) { + require_once 'Zend/Search/Lucene/Exception.php'; + throw new Zend_Search_Lucene_Exception('Wrong number of arguments'); + } else if (count($argList) == 2) { + $startId = $argList[0]; + $mode = $argList[1]; + } else if (count($argList) == 1) { + $startId = $argList[0]; + $mode = self::SM_TERMS_ONLY; + } else { + $startId = 0; + $mode = self::SM_TERMS_ONLY; + } + if ($this->_tisFile !== null) { $this->_tisFile = null; } library/Zend/Search/Lucene/Index/SegmentInfo.php @@ -25,8 +25,8 @@ require_once 'Zend/Search/Lucene/Index/SegmentInfo.php'; /** Zend_Search_Lucene_Index_SegmentWriter_StreamWriter */ require_once 'Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php'; -/** Zend_Search_Lucene_Index_SegmentInfoPriorityQueue */ -require_once 'Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php'; +/** Zend_Search_Lucene_Index_TermsPriorityQueue */ +require_once 'Zend/Search/Lucene/Index/TermsPriorityQueue.php'; /** * @category Zend @@ -225,11 +225,11 @@ class Zend_Search_Lucene_Index_SegmentMerger */ private function _mergeTerms() { - $segmentInfoQueue = new Zend_Search_Lucene_Index_SegmentInfoPriorityQueue(); + $segmentInfoQueue = new Zend_Search_Lucene_Index_TermsPriorityQueue(); $segmentStartId = 0; foreach ($this->_segmentInfos as $segName => $segmentInfo) { - $segmentStartId = $segmentInfo->reset($segmentStartId, Zend_Search_Lucene_Index_SegmentInfo::SM_MERGE_INFO); + $segmentStartId = $segmentInfo->resetTermsStream($segmentStartId, Zend_Search_Lucene_Index_SegmentInfo::SM_MERGE_INFO); // Skip "empty" segments if ($segmentInfo->currentTerm() !== null) { library/Zend/Search/Lucene/Index/SegmentMerger.php @@ -18,6 +18,9 @@ * @license http://framework.zend.com/license/new-bsd New BSD License */ +/** Zend_Search_Lucene_Index_TermsStream_Interface */ +require_once 'Zend/Search/Lucene/Index/TermsStream/Interface.php'; + /** * @category Zend @@ -25,7 +28,7 @@ * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ -interface Zend_Search_Lucene_Interface +interface Zend_Search_Lucene_Interface extends Zend_Search_Lucene_Index_TermsStream_Interface { /** * Get current generation number @@ -376,43 +379,6 @@ interface Zend_Search_Lucene_Interface */ public function terms(); - - /** - * Reset terms stream. - */ - public function resetTermsStream(); - - /** - * Skip terms stream up to specified term preffix. - * - * Prefix contains fully specified field info and portion of searched term - * - * @param Zend_Search_Lucene_Index_Term $prefix - */ - public function skipTo(Zend_Search_Lucene_Index_Term $prefix); - - /** - * Scans terms dictionary and returns next term - * - * @return Zend_Search_Lucene_Index_Term|null - */ - public function nextTerm(); - - /** - * Returns term in current position - * - * @return Zend_Search_Lucene_Index_Term|null - */ - public function currentTerm(); - - /** - * Close terms stream - * - * Should be used for resources clean up if stream is not read up to the end - */ - public function closeTermsStream(); - - /** * Undeletes all documents currently marked as deleted in this index. */ library/Zend/Search/Lucene/Interface.php @@ -18,6 +18,11 @@ * @license http://framework.zend.com/license/new-bsd New BSD License */ +/** Zend_Search_Lucene_TermStreamsPriorityQueue */ +require_once 'Zend/Search/Lucene/TermStreamsPriorityQueue.php'; + +/** Zend_Search_Lucene_Interface */ +require_once 'Zend/Search/Lucene/Interface.php'; /** * Multisearcher allows to search through several independent indexes. @@ -29,10 +34,6 @@ */ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_Interface { - const MULTISEARCHER_FUNCTIONALITY_LIMITATION_MESSAGE = - "Multisearcher is intended for searching only and doesn't update index or provide update related information."; - - /** * List of indices for searching. * Array of Zend_Search_Lucene_Interface objects @@ -84,7 +85,7 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I public static function getActualGeneration(Zend_Search_Lucene_Storage_Directory $directory) { require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception(self::MULTISEARCHER_FUNCTIONALITY_LIMITATION_MESSAGE); + throw new Zend_Search_Lucene_Exception("Generation number can't be retrieved for multi-searcher"); } /** @@ -92,12 +93,10 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I * * @param integer $generation * @return string - * @throws Zend_Search_Lucene_Exception */ public static function getSegmentFileName($generation) { - require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception(self::MULTISEARCHER_FUNCTIONALITY_LIMITATION_MESSAGE); + return Zend_Search_Lucene::getSegmentFileName($generation); } /** @@ -109,7 +108,7 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I public function getFormatVersion() { require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception(self::MULTISEARCHER_FUNCTIONALITY_LIMITATION_MESSAGE); + throw new Zend_Search_Lucene_Exception("Format version can't be retrieved for multi-searcher"); } /** @@ -117,12 +116,12 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I * Index is converted to this format at the nearest upfdate time * * @param int $formatVersion - * @throws Zend_Search_Lucene_Exception */ public function setFormatVersion($formatVersion) { - require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception(self::MULTISEARCHER_FUNCTIONALITY_LIMITATION_MESSAGE); + foreach ($this->_indices as $index) { + $index->setFormatVersion($formatVersion); + } } /** @@ -133,7 +132,7 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I public function getDirectory() { require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception(self::MULTISEARCHER_FUNCTIONALITY_LIMITATION_MESSAGE); + throw new Zend_Search_Lucene_Exception("Index directory can't be retrieved for multi-searcher"); } /** @@ -775,6 +774,40 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I throw new Zend_Search_Lucene_Exception('Document id is out of the range.'); } + + /** + * Callback used to choose target index for new documents + * + * Function/method signature: + * Zend_Search_Lucene_Interface callbackFunction(Zend_Search_Lucene_Document $document, array $indices); + * + * null means "default documents distributing algorithm" + * + * @var callback + */ + protected $_documentDistributorCallBack = null; + + /** + * Set callback for choosing target index. + * + * @param callback $callback + */ + public function setDocumentDistributorCallback($callback) + { + if ($callback !== null && !is_callable($callback)) + $this->_documentDistributorCallBack = $callback; + } + + /** + * Get callback for choosing target index. + * + * @return callback + */ + public function getDocumentDistributorCallback() + { + return $this->_documentDistributorCallBack; + } + /** * Adds a document to this index. * @@ -783,8 +816,13 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I */ public function addDocument(Zend_Search_Lucene_Document $document) { - require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception(self::MULTISEARCHER_FUNCTIONALITY_LIMITATION_MESSAGE); + if ($this->_documentDistributorCallBack !== null) { + $index = call_user_func($this->_documentDistributorCallBack, $document, $this->_indices); + } else { + $index = $this->_indices[ array_rand($this->_indices) ]; + } + + $index->addDocument($document); } /** @@ -827,12 +865,21 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I /** + * Terms stream priority queue object + * + * @var Zend_Search_Lucene_termStreamsPriorityQueue + */ + private $_termsStream = null; + + /** * Reset terms stream. */ public function resetTermsStream() { - foreach ($this->_indices as $index) { - $index->resetTermsStream(); + if ($this->_termsStream === null) { + $this->_termsStream = new Zend_Search_Lucene_termStreamsPriorityQueue($this->_indices); + } else { + $this->_termsStream->resetTermsStream(); } } @@ -845,12 +892,9 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I */ public function skipTo(Zend_Search_Lucene_Index_Term $prefix) { - foreach ($this->_indices as $index) { - $index->skipTo($prefix); - } + $this->_termsStream->skipTo($prefix); } - /** * Scans terms dictionary and returns next term * @@ -858,10 +902,7 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I */ public function nextTerm() { - /** @todo Unify terms streaming queue */ - - require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception('Stream terms processing is not supported for the multisearcher.'); + return $this->_termsStream->nextTerm(); } /** @@ -871,8 +912,7 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I */ public function currentTerm() { - require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception('Stream terms processing is not supported for the multisearcher.'); + return $this->_termsStream->currentTerm(); } /** @@ -882,8 +922,8 @@ class Zend_Search_Lucene_Interface_MultiSearcher implements Zend_Search_Lucene_I */ public function closeTermsStream() { - require_once 'Zend/Search/Lucene/Exception.php'; - throw new Zend_Search_Lucene_Exception('Stream terms processing is not supported for the multisearcher.'); + $this->_termsStream->closeTermsStream(); + $this->_termsStream = null; } library/Zend/Search/Lucene/MultiSearcher.php tests/Zend/Paginator/_files/test.sqlite @@ -22,7 +22,7 @@ require_once 'Zend/Search/Lucene/AnalysisTest.php'; require_once 'Zend/Search/Lucene/Index/DictionaryLoaderTest.php'; require_once 'Zend/Search/Lucene/Index/FieldInfoTest.php'; -require_once 'Zend/Search/Lucene/Index/SegmentInfoPriorityQueueTest.php'; +require_once 'Zend/Search/Lucene/Index/TermsPriorityQueueTest.php'; require_once 'Zend/Search/Lucene/Index/SegmentInfoTest.php'; require_once 'Zend/Search/Lucene/Index/SegmentMergerTest.php'; require_once 'Zend/Search/Lucene/Index/TermInfoTest.php'; @@ -57,7 +57,7 @@ class Zend_Search_Lucene_AllTests $suite->addTestSuite('Zend_Search_Lucene_Index_DictionaryLoaderTest'); $suite->addTestSuite('Zend_Search_Lucene_Index_FieldInfoTest'); - $suite->addTestSuite('Zend_Search_Lucene_Index_SegmentInfoPriorityQueueTest'); + $suite->addTestSuite('Zend_Search_Lucene_Index_TermsPriorityQueueTest'); $suite->addTestSuite('Zend_Search_Lucene_Index_SegmentInfoTest'); $suite->addTestSuite('Zend_Search_Lucene_Index_SegmentMergerTest'); $suite->addTestSuite('Zend_Search_Lucene_Index_TermInfoTest'); tests/Zend/Search/Lucene/AllTests.php @@ -265,7 +265,7 @@ class Zend_Search_Lucene_Index_SegmentInfoTest extends PHPUnit_Framework_TestCas $segmentInfo = new Zend_Search_Lucene_Index_SegmentInfo($directory, '_3', 2); - $this->assertEquals($segmentInfo->reset(6, Zend_Search_Lucene_Index_SegmentInfo::SM_FULL_INFO), 8); + $this->assertEquals($segmentInfo->resetTermsStream(6, Zend_Search_Lucene_Index_SegmentInfo::SM_FULL_INFO), 8); $terms = array(); @@ -453,7 +453,7 @@ class Zend_Search_Lucene_Index_SegmentInfoTest extends PHPUnit_Framework_TestCas $segmentInfo1 = new Zend_Search_Lucene_Index_SegmentInfo($directory, '_3', 2); - $this->assertEquals($segmentInfo1->reset(6, Zend_Search_Lucene_Index_SegmentInfo::SM_MERGE_INFO), 7); + $this->assertEquals($segmentInfo1->resetTermsStream(6, Zend_Search_Lucene_Index_SegmentInfo::SM_MERGE_INFO), 7); } public function testTermStreamStyleReadingSkipTo() @@ -462,7 +462,7 @@ class Zend_Search_Lucene_Index_SegmentInfoTest extends PHPUnit_Framework_TestCas $segmentInfo = new Zend_Search_Lucene_Index_SegmentInfo($directory, '_3', 2); - $this->assertEquals($segmentInfo->reset(6, Zend_Search_Lucene_Index_SegmentInfo::SM_FULL_INFO), 8); + $this->assertEquals($segmentInfo->resetTermsStream(6, Zend_Search_Lucene_Index_SegmentInfo::SM_FULL_INFO), 8); $segmentInfo->skipTo(new Zend_Search_Lucene_Index_Term('prefetch', 'contents')); @@ -545,7 +545,7 @@ class Zend_Search_Lucene_Index_SegmentInfoTest extends PHPUnit_Framework_TestCas $segmentInfo1 = new Zend_Search_Lucene_Index_SegmentInfo($directory, '_3', 2); - $this->assertEquals($segmentInfo1->reset(6, Zend_Search_Lucene_Index_SegmentInfo::SM_MERGE_INFO), 7); + $this->assertEquals($segmentInfo1->resetTermsStream(6, Zend_Search_Lucene_Index_SegmentInfo::SM_MERGE_INFO), 7); } } tests/Zend/Search/Lucene/Index/SegmentInfoTest.php library/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php tests/Zend/Search/Lucene/Index/SegmentInfoPriorityQueueTest.php 0ed579c2a5f9970f7b3d22d305692c6dbaf5931b alexander alexander@44c647ce-9c0f-0410-b52a-842ac1e357ba http://github.com/tjohns/zf/commit/6ca798be84e6f8c1aa3e0ebff6edbd84d44bb09d 6ca798be84e6f8c1aa3e0ebff6edbd84d44bb09d 2009-04-07T11:42:21-07:00 2009-04-07T11:42:21-07:00 Zend_Search_Lucene: Multi-searcher index update functionality. [ZF-525] related. git-svn-id: http://framework.zend.com/svn/framework/standard/trunk@14744 44c647ce-9c0f-0410-b52a-842ac1e357ba 3ce0b32e8d1582a46c884b9c176bfcb347aa42c5 alexander alexander@44c647ce-9c0f-0410-b52a-842ac1e357ba