diff --git a/DefaultSettings.php b/DefaultSettings.php index a059a99d5a..ee39412d0f 100644 --- a/DefaultSettings.php +++ b/DefaultSettings.php @@ -1004,6 +1004,7 @@ # DB back-end to use special fulltext index operations. # # - Tested with MySQL/MariaDB +# - Tested with SQLite # # @since 2.5 ## @@ -1041,13 +1042,19 @@ # from MariaDB 10.0.5 with InnoDB tables and from MariaDB 10.0.15 # with Mroonga tables (according to sources) # +# - SQLite FTS3 has been available since version 3.5, FTS4 were added with +# version 3.7.4, and FTS5 is available with version 3.9.0 (according to +# sources); The setting allows to specify extra arguments after the module +# engine such as array( 'FTS4', 'tokenize=porter' ). +# # It is possible to extend the option decription (MySQL 5.7+) with # 'mysql' => array( 'ENGINE=MyISAM, DEFAULT CHARSET=utf8', 'WITH PARSER ngram' ) # # @since 2.5 ## $GLOBALS['smwgFulltextSearchTableOptions'] = array( - 'mysql' => array( 'ENGINE=MyISAM, DEFAULT CHARSET=utf8' ) + 'mysql' => array( 'ENGINE=MyISAM, DEFAULT CHARSET=utf8' ), + 'sqlite' => array( 'FTS4' ) ); ## diff --git a/src/DeferredRequestDispatchManager.php b/src/DeferredRequestDispatchManager.php index 5625e21072..dd93542a4c 100644 --- a/src/DeferredRequestDispatchManager.php +++ b/src/DeferredRequestDispatchManager.php @@ -44,7 +44,12 @@ class DeferredRequestDispatchManager { * * @var boolean */ - private $enabledHttpDeferredJobRequestState = true; + private $enabledHttpDeferredRequest = true; + + /** + * @var boolean + */ + private $preferredWithJobQueue = false; /** * @since 2.3 @@ -60,16 +65,29 @@ public function __construct( HttpRequest $httpRequest ) { */ public function reset() { self::$canConnectToUrl = null; - $this->enabledHttpDeferredJobRequestState = true; + $this->enabledHttpDeferredRequest = true; } /** * @since 2.3 * - * @param boolean $enabledHttpDeferredJobRequestState + * @param boolean $enabledHttpDeferredRequest + */ + public function setEnabledHttpDeferredRequest( $enabledHttpDeferredRequest ) { + $this->enabledHttpDeferredRequest = (bool)$enabledHttpDeferredRequest; + } + + /** + * Certain types of jobs or tasks may prefer to be executed using the job + * queue therefore indicate whether the dispatcher should try opening a + * http request or not. + * + * @since 2.5 + * + * @param boolean $preferredWithJobQueue */ - public function setEnabledHttpDeferredJobRequestState( $enabledHttpDeferredJobRequestState ) { - $this->enabledHttpDeferredJobRequestState = (bool)$enabledHttpDeferredJobRequestState; + public function setPreferredWithJobQueue( $preferredWithJobQueue ) { + $this->preferredWithJobQueue = (bool)$preferredWithJobQueue; } /** @@ -116,7 +134,7 @@ public function dispatchJobRequestFor( $type, Title $title, $parameters = array( $parameters['timestamp'] = time(); $parameters['requestToken'] = SpecialDeferredRequestDispatcher::getRequestToken( $parameters['timestamp'] ); - if ( $this->enabledHttpDeferredJobRequestState && $this->canConnectToUrl() ) { + if ( !$this->preferredWithJobQueue && $this->enabledHttpDeferredRequest && $this->canConnectToUrl() ) { return $this->doPostJobWith( $type, $title, $parameters, $dispatchableCallbackJob ); } diff --git a/src/MediaWiki/Hooks/HookRegistry.php b/src/MediaWiki/Hooks/HookRegistry.php index f483f2235c..e06f670c47 100644 --- a/src/MediaWiki/Hooks/HookRegistry.php +++ b/src/MediaWiki/Hooks/HookRegistry.php @@ -106,10 +106,17 @@ private function addCallbackHandlers( $basePath, $globalVars ) { $httpRequestFactory->newSocketRequest() ); - $deferredRequestDispatchManager->setEnabledHttpDeferredJobRequestState( + $deferredRequestDispatchManager->setEnabledHttpDeferredRequest( $applicationFactory->getSettings()->get( 'smwgEnabledHttpDeferredJobRequest' ) ); + // SQLite has no lock manager making table lock contention very common + // hence use the JobQueue to enqueue any change request and avoid + // a rollback due to canceled DB transactions + $deferredRequestDispatchManager->setPreferredWithJobQueue( + $GLOBALS['wgDBtype'] === 'sqlite' + ); + $permissionPthValidator = new PermissionPthValidator(); /** diff --git a/src/SQLStore/QueryEngine/Fulltext/SQLiteValueMatchConditionBuilder.php b/src/SQLStore/QueryEngine/Fulltext/SQLiteValueMatchConditionBuilder.php new file mode 100644 index 0000000000..27d9899674 --- /dev/null +++ b/src/SQLStore/QueryEngine/Fulltext/SQLiteValueMatchConditionBuilder.php @@ -0,0 +1,148 @@ +searchTable = $searchTable; + } + + /** + * @since 2.5 + * + * @return boolean + */ + public function isEnabled() { + return $this->searchTable->isEnabled(); + } + + /** + * @since 2.5 + * + * @return string + */ + public function getTableName() { + return $this->searchTable->getTableName(); + } + + /** + * @since 2.5 + * + * @param string $value + * + * @return boolean + */ + public function hasMinTokenLength( $value ) { + return mb_strlen( $value ) >= $this->searchTable->getMinTokenSize(); + } + + /** + * @since 2.5 + * + * @param string $temporaryTable + * + * @return string + */ + public function getSortIndexField( $temporaryTable = '' ) { + return ( $temporaryTable !== '' ? $temporaryTable . '.' : '' ) . $this->searchTable->getSortField(); + } + + /** + * @since 2.5 + * + * @param ValueDescription $description + * + * @return boolean + */ + public function canApplyFulltextSearchMatchCondition( ValueDescription $description ) { + + if ( !$this->isEnabled() || $description->getProperty() === null ) { + return false; + } + + if ( $this->searchTable->isExemptedProperty( $description->getProperty() ) ) { + return false; + } + + $matchableText = $this->getMatchableTextFromDescription( + $description + ); + + $comparator = $description->getComparator(); + + if ( $matchableText && ( $comparator === SMW_CMP_LIKE || $comparator === SMW_CMP_NLKE ) ) { + return $this->hasMinTokenLength( str_replace( '*', '', $matchableText ) ); + } + + return false; + } + + /** + * @since 2.5 + * + * @param ValueDescription $description + * @param string $temporaryTable + * + * @return string + */ + public function getWhereCondition( ValueDescription $description, $temporaryTable = '' ) { + + $matchableText = $this->getMatchableTextFromDescription( + $description + ); + + $value = $this->searchTable->getTextSanitizer()->sanitize( + $matchableText, + true + ); + + // A leading or trailing minus sign indicates that this word must not + // be present in any of the rows that are returned. + // InnoDB only supports leading minus signs. + if ( $description->getComparator() === SMW_CMP_NLKE ) { + $value = '-' . $value; + } + + // Something like [[Has text::!~database]] will cause a + // "malformed MATCH expression" due to "An FTS query may not consist + // entirely of terms or term-prefix queries with unary "-" operators + // attached to them." and doing "NOT database" will result in an empty + // result set + + $temporaryTable = $temporaryTable !== '' ? $temporaryTable . '.' : ''; + $column = $temporaryTable . $this->searchTable->getIndexField(); + + $property = $description->getProperty(); + $propertyCondition = ''; + + // Full text is collected in a single table therefore limit the match + // process by adding the PID as an additional condition + if ( $property !== null ) { + $propertyCondition = ' AND ' . $temporaryTable . 'p_id=' . $this->searchTable->addQuotes( + $this->searchTable->getPropertyID( $property ) + ); + } + + return $column . " MATCH " . $this->searchTable->addQuotes( $value ) . "$propertyCondition"; + } + +} diff --git a/src/SQLStore/QueryEngine/Fulltext/TextByChangeUpdater.php b/src/SQLStore/QueryEngine/Fulltext/TextByChangeUpdater.php index ffb1d4a8af..71e4af40b0 100644 --- a/src/SQLStore/QueryEngine/Fulltext/TextByChangeUpdater.php +++ b/src/SQLStore/QueryEngine/Fulltext/TextByChangeUpdater.php @@ -123,9 +123,13 @@ public function pushUpdatesFromJobParameters( array $parameters ) { return; } + $start = microtime( true ); + foreach ( $parameters['diff'] as $tableName => $changeOp ) { $this->doUpdateFromTableChangeOp( new TableChangeOp( $tableName, $changeOp ) ); } + + wfDebugLog( 'smw', __METHOD__ . ' procTime (sec): '. round( ( microtime( true ) - $start ), 5 ) ); } /** @@ -145,7 +149,6 @@ public function pushUpdatesFromPropertyTableDiff( CompositePropertyTableDiffIter $this->doUpdateFromTableChangeOp( $tableChangeOp ); } - wfDebugLog( 'smw', __METHOD__ . ' procTime (sec): '. round( ( microtime( true ) - $start ), 5 ) ); } @@ -221,8 +224,13 @@ private function doAggregateFromFieldChangeOp( $type, $fieldChangeOp, &$aggregat } private function doUpdateOnAggregatedValues( $inserts, $deletes ) { - // Remove any "deletes" first + $this->doUpdateOnDeletes( $deletes ); + $this->doUpdateOnInserts( $inserts ); + } + + private function doUpdateOnDeletes( $deletes ) { + foreach ( $deletes as $key => $values ) { list( $sid, $pid ) = explode( ':', $key, 2 ); @@ -243,6 +251,9 @@ private function doUpdateOnAggregatedValues( $inserts, $deletes ) { $this->searchTableUpdater->update( $sid, $pid, $text ); } + } + + private function doUpdateOnInserts( $inserts ) { foreach ( $inserts as $key => $value ) { list( $sid, $pid ) = explode( ':', $key, 2 ); diff --git a/src/SQLStore/QueryEngine/FulltextSearchTableFactory.php b/src/SQLStore/QueryEngine/FulltextSearchTableFactory.php index 049174d914..d0f3e5f874 100644 --- a/src/SQLStore/QueryEngine/FulltextSearchTableFactory.php +++ b/src/SQLStore/QueryEngine/FulltextSearchTableFactory.php @@ -6,6 +6,7 @@ use SMW\ApplicationFactory; use SMW\SQLStore\QueryEngine\Fulltext\ValueMatchConditionBuilder; use SMW\SQLStore\QueryEngine\Fulltext\MySQLValueMatchConditionBuilder; +use SMW\SQLStore\QueryEngine\Fulltext\SQLiteValueMatchConditionBuilder; use SMW\SQLStore\QueryEngine\Fulltext\TextByChangeUpdater; use SMW\SQLStore\QueryEngine\Fulltext\TextSanitizer; use SMW\SQLStore\QueryEngine\Fulltext\SearchTable; @@ -50,6 +51,11 @@ public function newValueMatchConditionBuilderByType( SQLStore $store ) { $this->newSearchTable( $store ) ); break; + case 'sqlite': + return new SQLiteValueMatchConditionBuilder( + $this->newSearchTable( $store ) + ); + break; } return new ValueMatchConditionBuilder(); diff --git a/tests/phpunit/Integration/ByJsonScript/ByJsonScriptFixtureTestCaseRunnerTest.php b/tests/phpunit/Integration/ByJsonScript/ByJsonScriptFixtureTestCaseRunnerTest.php index 76b0bde9a1..c8d8918469 100644 --- a/tests/phpunit/Integration/ByJsonScript/ByJsonScriptFixtureTestCaseRunnerTest.php +++ b/tests/phpunit/Integration/ByJsonScript/ByJsonScriptFixtureTestCaseRunnerTest.php @@ -266,8 +266,13 @@ private function tryToProcessQueryTestCase( $jsonTestCaseFileHandler ) { $jsonTestCaseFileHandler->getDebugMode() ); - foreach ( $jsonTestCaseFileHandler->findTestCasesFor( 'query-testcases' ) as $queryCase ) { - $this->queryTestCaseProcessor->processQueryCase( new QueryTestCaseInterpreter( $queryCase ) ); + foreach ( $jsonTestCaseFileHandler->findTestCasesFor( 'query-testcases' ) as $case ) { + + if ( $jsonTestCaseFileHandler->requiredToSkipFor( $case, $this->connectorId ) ) { + continue; + } + + $this->queryTestCaseProcessor->processQueryCase( new QueryTestCaseInterpreter( $case ) ); } foreach ( $jsonTestCaseFileHandler->findTestCasesFor( 'concept-testcases' ) as $conceptCase ) { diff --git a/tests/phpunit/Integration/ByJsonScript/Fixtures/q-0104.json b/tests/phpunit/Integration/ByJsonScript/Fixtures/q-0104.json index 65b9609a91..7f19aac010 100644 --- a/tests/phpunit/Integration/ByJsonScript/Fixtures/q-0104.json +++ b/tests/phpunit/Integration/ByJsonScript/Fixtures/q-0104.json @@ -1,5 +1,5 @@ { - "description": "Test `_txt`/`~` with enabled Fulltext search support (only enabled for MySQL)", + "description": "Test `_txt`/`~` with enabled full-text search support (only enabled for MySQL, SQLite)", "properties": [ { "name": "Has text", @@ -151,6 +151,9 @@ }, { "about": "#8 free search (wide proximity)", + "skip-on": { + "sqlite": "works different in comparison to MySQL, see #9" + }, "condition": "[[~~with a category]]", "printouts" : [], "parameters" : { @@ -166,6 +169,21 @@ }, { "about": "#9 free search (wide proximity)", + "condition": "[[~~with* a category]] [[~Example/Q0104/*]]", + "printouts" : [], + "parameters" : { + "limit" : "10" + }, + "queryresult": { + "count": 2, + "results": [ + "Example/Q0104/4#0##_5a524a435267f6e6d2d45d64a419c1da", + "Example/Q0104/4#0##_d4fe48d7241e6530c628f32168815beb" + ] + } + }, + { + "about": "#10 free search (wide proximity)", "condition": "[[~~with a category]] [[Category:Q0104]]", "printouts" : [], "parameters" : { @@ -179,7 +197,7 @@ } }, { - "about": "#10 retain spaces on +/- operators", + "about": "#11 retain spaces on +/- operators", "condition": "[[Has text::~+*maria* -postgres*]]", "printouts" : [], "parameters" : { @@ -202,7 +220,6 @@ "meta": { "skip-on": { "postgres": "Not supported by PostgreSQL.", - "sqlite": "Not supported by SQLite.", "sesame": "Not supported by SPARQLStore (Sesame).", "virtuoso": "Not supported by SPARQLStore (Virtuoso).", "fuseki": "Not supported by SPARQLStore (Fuskei).", diff --git a/tests/phpunit/Unit/DeferredRequestDispatchManagerTest.php b/tests/phpunit/Unit/DeferredRequestDispatchManagerTest.php index f451650b28..f9610e367d 100644 --- a/tests/phpunit/Unit/DeferredRequestDispatchManagerTest.php +++ b/tests/phpunit/Unit/DeferredRequestDispatchManagerTest.php @@ -43,7 +43,7 @@ public function testDispatchJobFor( $type, $deferredJobRequestState, $parameters $instance = new DeferredRequestDispatchManager( $httpRequest ); $instance->reset(); - $instance->setEnabledHttpDeferredJobRequestState( $deferredJobRequestState ); + $instance->setEnabledHttpDeferredRequest( $deferredJobRequestState ); $this->assertTrue( $instance->dispatchJobRequestFor( $type, DIWikiPage::newFromText( __METHOD__ )->getTitle(), $parameters ) @@ -62,7 +62,7 @@ public function testDispatchParserCachePurgeJob() { $instance = new DeferredRequestDispatchManager( $httpRequest ); $instance->reset(); - $instance->setEnabledHttpDeferredJobRequestState( true ); + $instance->setEnabledHttpDeferredRequest( true ); $parameters = array( 'idlist' => '1|2' ); $title = DIWikiPage::newFromText( __METHOD__ )->getTitle(); diff --git a/tests/phpunit/Unit/SQLStore/QueryEngine/Fulltext/SQLiteValueMatchConditionBuilderTest.php b/tests/phpunit/Unit/SQLStore/QueryEngine/Fulltext/SQLiteValueMatchConditionBuilderTest.php new file mode 100644 index 0000000000..5ebfb50d2e --- /dev/null +++ b/tests/phpunit/Unit/SQLStore/QueryEngine/Fulltext/SQLiteValueMatchConditionBuilderTest.php @@ -0,0 +1,206 @@ +dataItemFactory = new DataItemFactory(); + + $this->searchTable = $this->getMockBuilder( '\SMW\SQLStore\QueryEngine\Fulltext\SearchTable' ) + ->disableOriginalConstructor() + ->getMock(); + } + + public function testCanConstruct() { + + $this->assertInstanceOf( + '\SMW\SQLStore\QueryEngine\Fulltext\SQLiteValueMatchConditionBuilder', + new SQLiteValueMatchConditionBuilder( $this->searchTable ) + ); + } + + public function testIsEnabled() { + + $this->searchTable->expects( $this->once() ) + ->method( 'isEnabled' ) + ->will( $this->returnValue( true ) ); + + $instance = new SQLiteValueMatchConditionBuilder( + $this->searchTable + ); + + $this->assertTrue( + $instance->isEnabled() + ); + } + + public function testGetTableName() { + + $this->searchTable->expects( $this->once() ) + ->method( 'getTableName' ) + ->will( $this->returnValue( 'Foo' ) ); + + $instance = new SQLiteValueMatchConditionBuilder( + $this->searchTable + ); + + $this->assertEquals( + 'Foo', + $instance->getTableName() + ); + } + + public function testHasMinTokenLength() { + + $this->searchTable->expects( $this->any() ) + ->method( 'getMinTokenSize' ) + ->will( $this->returnValue( 4 ) ); + + $instance = new SQLiteValueMatchConditionBuilder( + $this->searchTable + ); + + $this->assertFalse( + $instance->hasMinTokenLength( 'bar' ) + ); + + $this->assertFalse( + $instance->hasMinTokenLength( 'ใƒ†ใ‚นใƒˆ' ) + ); + + $this->assertTrue( + $instance->hasMinTokenLength( 'test' ) + ); + } + + public function testGetSortIndexField() { + + $this->searchTable->expects( $this->any() ) + ->method( 'getSortField' ) + ->will( $this->returnValue( 's_id' ) ); + + $instance = new SQLiteValueMatchConditionBuilder( + $this->searchTable + ); + + $this->assertEquals( + 'Foo.s_id', + $instance->getSortIndexField( 'Foo' ) + ); + } + + public function testCanApplyFulltextSearchMatchCondition() { + + $this->searchTable->expects( $this->once() ) + ->method( 'isEnabled' ) + ->will( $this->returnValue( true ) ); + + $this->searchTable->expects( $this->once() ) + ->method( 'isExemptedProperty' ) + ->will( $this->returnValue( false ) ); + + $instance = new SQLiteValueMatchConditionBuilder( + $this->searchTable + ); + + $description = $this->getMockBuilder( '\SMW\Query\Language\ValueDescription' ) + ->disableOriginalConstructor() + ->getMock(); + + $description->expects( $this->atLeastOnce() ) + ->method( 'getProperty' ) + ->will( $this->returnValue( $this->dataItemFactory->newDIProperty( 'Foo' ) ) ); + + $description->expects( $this->atLeastOnce() ) + ->method( 'getDataItem' ) + ->will( $this->returnValue( $this->dataItemFactory->newDIBlob( 'Bar' ) ) ); + + $description->expects( $this->once() ) + ->method( 'getComparator' ) + ->will( $this->returnValue( SMW_CMP_LIKE ) ); + + $this->assertTrue( + $instance->canApplyFulltextSearchMatchCondition( $description ) + ); + } + + /** + * @dataProvider searchTermProvider + */ + public function testGetWhereConditionWithoutProperty( $text, $indexField, $expected ) { + + $textSanitizer = $this->getMockBuilder( '\SMW\SQLStore\QueryEngine\Fulltext\TextSanitizer' ) + ->disableOriginalConstructor() + ->getMock(); + + $textSanitizer->expects( $this->once() ) + ->method( 'sanitize' ) + ->will( $this->returnValue( $text ) ); + + $this->searchTable->expects( $this->any() ) + ->method( 'isEnabled' ) + ->will( $this->returnValue( true ) ); + + $this->searchTable->expects( $this->once() ) + ->method( 'getTextSanitizer' ) + ->will( $this->returnValue( $textSanitizer ) ); + + $this->searchTable->expects( $this->once() ) + ->method( 'getIndexField' ) + ->will( $this->returnValue( $indexField ) ); + + $this->searchTable->expects( $this->once() ) + ->method( 'addQuotes' ) + ->will( $this->returnArgument( 0 ) ); + + $instance = new SQLiteValueMatchConditionBuilder( + $this->searchTable + ); + + $description = $this->getMockBuilder( '\SMW\Query\Language\ValueDescription' ) + ->disableOriginalConstructor() + ->getMock(); + + $description->expects( $this->atLeastOnce() ) + ->method( 'getDataItem' ) + ->will( $this->returnValue( $this->dataItemFactory->newDIBlob( 'Bar' ) ) ); + + $description->expects( $this->once() ) + ->method( 'getComparator' ) + ->will( $this->returnValue( SMW_CMP_LIKE ) ); + + $this->assertEquals( + $expected, + $instance->getWhereCondition( $description ) + ); + } + + public function searchTermProvider() { + + $provider[] = array( + 'foooo', + 'barColumn', + "barColumn MATCH foooo" + ); + + return $provider; + } + +} diff --git a/tests/phpunit/Utils/MwDatabaseTableBuilder.php b/tests/phpunit/Utils/MwDatabaseTableBuilder.php index fa376580f1..71a7a7acd0 100644 --- a/tests/phpunit/Utils/MwDatabaseTableBuilder.php +++ b/tests/phpunit/Utils/MwDatabaseTableBuilder.php @@ -279,7 +279,7 @@ private function isNotUnittest( $table ) { } private function isNotSearchindex( $table ) { - return strpos( $table, 'searchindex' ) === false; + return strpos( $table, 'searchindex' ) === false && strpos( $table, 'smw_ft_search' ) === false; } private function isAvailableDatabaseType() {