Support for QueryResult caching

DefaultSettings.php

@@ -1030,18 +1030,20 @@
 $GLOBALS['smwgEnabledFulltextSearch'] = false;
 
 ##
-# Throttle the amount of expected index updates.
+# Throttle index updates
 #
-# It can be of advantage to postpone the update using a deferred job execution to
-# decouple changes to the storage back-end and the fulltext index table.
+# The objective is to postpone an update by relying on a deferred process that
+# runs the index update decoupled from the storage back-end update.
 #
 # In case `smwgFulltextDeferredUpdate` and `$GLOBALS['smwgEnabledDeferredUpdate']` are
-# enabled then the updater will try to open a new process for posting instructions
-# to execute the `SearchTableUpdateJob` immediately otherwise `SearchTableUpdateJob`
-# will be enqueued and `runJobs.php` is required to be schedule for execution.
+# both enabled then the updater will try to open a new request and posting instructions
+# to execute the `SearchTableUpdateJob` immediately in background. If the request
+# cannot be executed then the `SearchTableUpdateJob` will be enqueued and requires
+# `runJobs.php` to schedule the index table update.
 #
-# If a user wants to avoid the JobQueue for executing updates via `SearchTableUpdateJob`
-# then this setting should be disabled.
+# If a user wants to push updates to the updater immediately then this setting needs
+# to be disabled but by disabling this setting update lag may increase due to having
+# the process being executed synchronously to the wikipage, store-backend storage.
 #
 # @since 2.5
 # @default true
@@ -1051,10 +1053,10 @@
 ##
 # Fulltext search table options
 #
-# This setting directly influences how a ft table is created therefore please
-# change the content with caution.
+# This setting directly influences how a ft table is created therefore change
+# the content with caution.
 #
-# - MySQL version 5.6 or later with only MyISAM and InnoDB storage engines
+# - MySQL version 5.5 or later with only MyISAM and InnoDB storage engines
 # to support full-text search (according to sources)
 #
 # - MariaDB full-text indexes can be used only with MyISAM and Aria tables,
@@ -1138,3 +1140,60 @@
 # @default false
 ##
 $GLOBALS['smwgQTemporaryTablesAutoCommitMode'] = false;
+
+###
+# Support to store a computed subject list that were fetched from the QueryEngine
+# (not the string result generated from a result printer) and improve general
+# page-loading time for articles that contain embedded queries and decrease
+# server load on query requests.
+#
+# It is recommended that `smwgEnabledQueryDependencyLinksStore` is enabled to make
+# use of automatic query results cache eviction.
+#
+# @since 2.5 (experimental)
+#
+# @default: CACHE_NONE (== that this feature is disabled)
+##
+$GLOBALS['smwgQueryResultCacheType'] = CACHE_NONE;
+##
+
+###
+# Specifies the lifetime of embedded query and their results fetched from a
+# QueryEngine for when `smwgQueryResultCacheType` is enabled.
+#
+# @since 2.5
+##
+$GLOBALS['smwgQueryResultCacheLifetime'] = 60 * 60 * 24; // a day
+##
+
+###
+# Specifies the lifetime of non-embedded queries (Special:Ask, API etc.) and their
+# results that are fetched from a QueryEngine for when `smwgQueryResultCacheType` is
+# enabled.
+#
+# This setting can also be used to minimize a possible DoS vector by preventing
+# an advisory to make unlimited query requests from either Special:Ask or the
+# API that may lock the DB due to complex query answering and instead being
+# rerouted to the cache once a result has been computed.
+#
+# @note Non-embedded queries cannot not be tracked using the `QueryDependencyLinksStore`
+# (subject is being missing that would identify the entity) therefore
+# an auto-purge mechanism as in case of an embedded entity is not possible hence
+# the lifetime should be carefully selected to provide the necessary means for a
+# user and the application.
+#
+# 0/false as setting to disable caching of non-embedded queries.
+#
+# @since 2.5
+##
+$GLOBALS['smwgQueryResultNonEmbeddedCacheLifetime'] = 60 * 10; // 10 min
+##
+
+###
+# Enables the manual refresh for embedded queries when the action=purge event is
+# triggered.
+#
+# @since 2.5
+##
+$GLOBALS['smwgQueryResultCacheRefreshOnPurge'] = true;
+##

includes/ParserData.php

@@ -296,7 +296,7 @@ public function updateStore( $enabledDeferredUpdate = false ) {
 			$storeUpdater->doUpdate();
 		} );
 
-		$deferredCallableUpdate->setOrigin( __METHOD__ );
+		$deferredCallableUpdate->setOrigin( __METHOD__ . ' :: ' . $this->semanticData->getSubject()->getHash() );
 
 		$deferredCallableUpdate->enabledDeferredUpdate(
 			$enabledDeferredUpdate

includes/Settings.php

@@ -148,6 +148,10 @@ public static function newFromGlobals() {
 			'smwgFulltextSearchMinTokenSize' => $GLOBALS['smwgFulltextSearchMinTokenSize'],
 			'smwgFulltextLanguageDetection' => $GLOBALS['smwgFulltextLanguageDetection'],
 			'smwgQTemporaryTablesAutoCommitMode' => $GLOBALS['smwgQTemporaryTablesAutoCommitMode'],
+			'smwgQueryResultCacheType' => $GLOBALS['smwgQueryResultCacheType'],
+			'smwgQueryResultCacheLifetime' => $GLOBALS['smwgQueryResultCacheLifetime'],
+			'smwgQueryResultNonEmbeddedCacheLifetime' => $GLOBALS['smwgQueryResultNonEmbeddedCacheLifetime'],
+			'smwgQueryResultCacheRefreshOnPurge' => $GLOBALS['smwgQueryResultCacheRefreshOnPurge'],
 		);
 
 		$settings = $settings + array(

includes/query/SMW_Query.php

@@ -35,6 +35,8 @@
  */
 class SMWQuery implements QueryContext {
 
+	const ID_PREFIX = '_QUERY';
+
 	/**
 	 * The time the QueryEngine required to answer a query condition
 	 */
@@ -396,7 +398,7 @@ public function toArray() {
 		// @2.4 Keep the queryID stable with previous versions unless
 		// a query source is selected. The "same" query executed on different
 		// remote systems requires a different queryID
-		if ( $this->querySource !== '' ) {
+		if ( $this->querySource !== null && $this->querySource !== '' ) {
 			$serialized['parameters']['source'] = $this->querySource;
 		}
 
@@ -411,12 +413,44 @@ public function toArray() {
 	}
 
 	/**
+	 * @note Before 2.5, toArray was used to generate the content, as of 2.5
+	 * only parameters that influence the result of an query is included.
+	 *
 	 * @since 2.1
 	 *
 	 * @return string
 	 */
 	public function getHash() {
-		return HashBuilder::createHashIdForContent( $this->toArray() );
+
+		// FIXME 3.0 Leave the hash unchanged to avoid unnecessary BC issues in
+		// case the cache is not used.
+		if ( $GLOBALS['smwgQueryResultCacheType'] === false ||
+			$GLOBALS['smwgQueryResultCacheType'] === CACHE_NONE ||
+			$GLOBALS['smwgQueryResultCacheType'] === 'hash' ) {
+			return HashBuilder::createFromArray( $this->toArray() );
+		}
+
+		// For an optimal (less fragmentation) use of the cache, only use
+		// elements that directly influence the result list
+		$serialized = array();
+
+		$serialized['conditions'] = $this->getQueryString();
+		$serialized['parameters'] = array(
+			'limit'     => $this->limit,
+			'offset'    => $this->offset,
+			'sortkeys'  => $this->sortkeys,
+			'querymode' => $this->querymode
+		);
+
+		if ( $this->querySource !== null && $this->querySource !== '' ) {
+			$serialized['parameters']['source'] = $this->querySource;
+		}
+
+		// printouts are avoided as part of the hash as they not influence the
+		// match process and are only resolved after the query result has been
+		// retrieved
+
+		return HashBuilder::createFromArray( $serialized );
 	}
 
 	/**
@@ -434,7 +468,7 @@ public function getAsString() {
 	 * @return string
 	 */
 	public function getQueryId() {
-		return '_QUERY' . $this->getHash();
+		return self::ID_PREFIX . $this->getHash();
 	}
 
 }

includes/storage/SQLStore/SMW_SQLStore3.php

@@ -313,7 +313,7 @@ public function getQueryResult( SMWQuery $query ) {
 		$result = null;
 		$start = microtime( true );
 
-		if ( \Hooks::run( 'SMW::Store::BeforeQueryResultLookupComplete', array( $this, $query, &$result ) ) ) {
+		if ( \Hooks::run( 'SMW::Store::BeforeQueryResultLookupComplete', array( $this, $query, &$result, $this->factory->newSlaveQueryEngine() ) ) ) {
 			$result = $this->fetchQueryResult( $query );
 		}
 

phpunit.xml.dist

@@ -67,6 +67,7 @@
        <var name="smwgEnabledFulltextSearch" value="false"/>
        <var name="smwgEnabledHttpDeferredJobRequest" value="false"/>
        <var name="smwgEnabledQueryDependencyLinksStore" value="true"/>
+       <var name="smwgQueryResultCacheType" value="hash"/>
        <var name="benchmarkQueryRepetitionExecutionThreshold" value="5"/>
        <var name="benchmarkQueryLimit" value="500"/>
        <var name="benchmarkQueryOffset" value="0"/>

src/ApplicationFactory.php

@@ -165,6 +165,15 @@ public function newCacheFactory() {
 		return $this->callbackLoader->load( 'CacheFactory', $this->getSettings()->get( 'smwgCacheType' ) );
 	}
 
+	/**
+	 * @since 2.2
+	 *
+	 * @return CacheFactory
+	 */
+	public function getCacheFactory() {
+		return $this->callbackLoader->singleton( 'CacheFactory', $this->getSettings()->get( 'smwgCacheType' ) );
+	}
+
 	/**
 	 * @since 2.5
 	 *
@@ -344,6 +353,15 @@ public function getCachedPropertyValuesPrefetcher() {
 		return $this->callbackLoader->singleton( 'CachedPropertyValuesPrefetcher' );
 	}
 
+	/**
+	 * @since 2.5
+	 *
+	 * @return CachedQueryResultPrefetcher
+	 */
+	public function getCachedQueryResultPrefetcher() {
+		return $this->callbackLoader->singleton( 'CachedQueryResultPrefetcher' );
+	}
+
 	/**
 	 * @since 2.4
 	 *
@@ -431,6 +449,15 @@ public function getQueryFactory() {
 		return $this->callbackLoader->singleton( 'QueryFactory' );
 	}
 
+	/**
+	 * @since 2.5
+	 *
+	 * @return LoggerInterface
+	 */
+	public function getMediaWikiLogger() {
+		return $this->callbackLoader->singleton( 'MediaWikiLogger' );
+	}
+
 	private static function registerBuilder( CallbackLoader $callbackLoader = null ) {
 
 		if ( $callbackLoader === null ) {

src/CacheFactory.php

@@ -146,15 +146,7 @@ public function newMediaWikiCompositeCache( $mediaWikiCacheType = null ) {
 	 * @return BlobStore
 	 */
 	public function newBlobStore( $namespace, $cacheType = null, $cacheLifetime = 0 ) {
-
-		$blobStore = $this->callbackInstantiator->load( 'BlobStore', $namespace, $cacheType, $cacheLifetime );
-
-		// If CACHE_NONE is selected, disable the usage
-		$blobStore->setUsageState(
-			$cacheType !== CACHE_NONE
-		);
-
-		return $blobStore;
+		return $this->callbackInstantiator->load( 'BlobStore', $namespace, $cacheType, $cacheLifetime );
 	}
 
 }

src/CachedPropertyValuesPrefetcher.php

@@ -30,6 +30,11 @@ class CachedPropertyValuesPrefetcher {
 	 */
 	const VERSION = '0.4.1';
 
+	/**
+	 * Namespace occupied by the BlobStore
+	 */
+	const CACHE_NAMESPACE = 'smw:pvp:store';
+
 	/**
 	 * @var Store
 	 */