diff --git a/SemanticMediaWiki.settings.php b/SemanticMediaWiki.settings.php index c99b5110c6..6655c47b99 100644 --- a/SemanticMediaWiki.settings.php +++ b/SemanticMediaWiki.settings.php @@ -881,7 +881,7 @@ # @since 2.3 (experimental) # @default false ## -$GLOBALS['smwgEnabledQueryDependencyLinksStore'] = false; +$GLOBALS['smwgEnabledQueryDependencyLinksStore'] = true; ## ### @@ -900,9 +900,51 @@ $GLOBALS['smwgPropertyDependencyDetectionBlacklist'] = array( '_MDAT', '_SOBJ' ); ## +### +# This stores subjects that were fetched from the QueryEngine (not the string result +# generated from a result printer) and allows to improve general page-loading time +# for articles that contain embedded queries and decrease server load on query requests +# due to subject list (as to the answer of the query) being served from cache +# instead of running a live DB/SPARQL query. +# +# It is suggested that `smwgEnabledQueryDependencyLinksStore` is enabled to make +# use of the automatic query results update (and hereof the invalidation of the +# cache). +# +# CACHE_NONE as default means that this feature is disabled +# +# @since 2.4 (experimental) +# +# @default: CACHE_NONE, users need to actively enable it in order +# to make use of it +## +$GLOBALS['smwgEmbeddedQueryResultCacheType'] = CACHE_NONE; +## + +### +# Declares the lifetime of a cached item for when `smwgEmbeddedQueryResultCacheType` +# is enabled. +# +# @since 2.4 +## +$GLOBALS['smwgEmbeddedQueryResultCacheLifetime'] = 60 * 60 * 24; // a day +## + +### +# If `smwgEnabledQueryDependencyLinksStore` is enabled this setting is not required +# as results are expected to be automatically purged. In case it is disabled and +# a user still wants to take advantage of the cache then it is suggested to enable +# this setting so that results are refreshed (or better the cache is going to be +# invalidated) by force on an article purge event. +# +# @since 2.4 +## +$GLOBALS['smwgEmbeddedQueryResultCacheRefreshOnPurge'] = false; +## + ### # The setting is introduced the keep backwards compatibility with existing Rdf/Turtle -# exports. The `aux` marker is epxected only used to be used for selected properties +# exports. The `aux` marker is expected to be used only for selected properties # to generate a helper value and not for any other predefined property. # # Any property that does not explicitly require an auxiliary value (such `_dat`/ diff --git a/includes/Settings.php b/includes/Settings.php index c272b6bb57..faf1121744 100644 --- a/includes/Settings.php +++ b/includes/Settings.php @@ -122,6 +122,9 @@ public static function newFromGlobals() { 'smwgValueLookupCacheType' => $GLOBALS['smwgValueLookupCacheType'], 'smwgValueLookupCacheLifetime' => $GLOBALS['smwgValueLookupCacheLifetime'], 'smwgValueLookupFeatures' => $GLOBALS['smwgValueLookupFeatures'], + 'smwgEmbeddedQueryResultCacheType' => $GLOBALS['smwgEmbeddedQueryResultCacheType'], + 'smwgEmbeddedQueryResultCacheLifetime' => $GLOBALS['smwgEmbeddedQueryResultCacheLifetime'], + 'smwgEmbeddedQueryResultCacheRefreshOnPurge' => $GLOBALS['smwgEmbeddedQueryResultCacheRefreshOnPurge'], 'smwgFixedProperties' => $GLOBALS['smwgFixedProperties'], 'smwgPropertyLowUsageThreshold' => $GLOBALS['smwgPropertyLowUsageThreshold'], 'smwgPropertyZeroCountDisplay' => $GLOBALS['smwgPropertyZeroCountDisplay'], diff --git a/includes/storage/SMW_QueryResult.php b/includes/storage/SMW_QueryResult.php index 1677b83a9f..8c4e069a82 100644 --- a/includes/storage/SMW_QueryResult.php +++ b/includes/storage/SMW_QueryResult.php @@ -61,6 +61,11 @@ class SMWQueryResult { */ private $countValue; + /** + * @var boolean + */ + private $isFromCache = false; + /** * Initialise the object with an array of SMWPrintRequest objects, which * define the structure of the result "table" (one for each column). @@ -208,6 +213,24 @@ public function getCountValue() { return $this->countValue; } + /** + * @since 2.4 + * + * @param boolean $isFromCache + */ + public function setFromCache( $isFromCache ) { + $this->isFromCache = (bool)$isFromCache; + } + + /** + * @since 2.4 + * + * @return boolean + */ + public function isFromCache() { + return $this->isFromCache; + } + /** * Return error array, possibly empty. * diff --git a/includes/storage/SMW_Store.php b/includes/storage/SMW_Store.php index 6ce6eec42c..c22b8dad13 100644 --- a/includes/storage/SMW_Store.php +++ b/includes/storage/SMW_Store.php @@ -215,6 +215,7 @@ public function updateData( SemanticData $semanticData ) { /** * @since 1.6 */ + wfRunHooks( 'SMW::Store::BeforeDataUpdateComplete', array( $this, $semanticData ) ); wfRunHooks( 'SMWStore::updateDataBefore', array( $this, $semanticData ) ); // Invalidate the page, so data stored on it gets displayed immediately in queries. @@ -232,6 +233,7 @@ public function updateData( SemanticData $semanticData ) { * @since 1.6 */ wfRunHooks( 'SMWStore::updateDataAfter', array( $this, $semanticData ) ); + wfRunHooks( 'SMW::Store::AfterDataUpdateComplete', array( $this, $semanticData ) ); } /** diff --git a/includes/storage/SQLStore/SMW_SQLStore3.php b/includes/storage/SQLStore/SMW_SQLStore3.php index 2270210933..9cb8e3b38e 100644 --- a/includes/storage/SQLStore/SMW_SQLStore3.php +++ b/includes/storage/SQLStore/SMW_SQLStore3.php @@ -367,7 +367,7 @@ public function getQueryResult( SMWQuery $query ) { $result = null; - if ( wfRunHooks( 'SMW::Store::BeforeQueryResultLookupComplete', array( $this, $query, &$result ) ) ) { + if ( wfRunHooks( 'SMW::Store::BeforeQueryResultLookupComplete', array( $this, $query, &$result, $this->factory->newSlaveQueryEngine() ) ) ) { $result = $this->fetchQueryResult( $query ); } diff --git a/src/CacheFactory.php b/src/CacheFactory.php index da4990fafe..d6d5912815 100644 --- a/src/CacheFactory.php +++ b/src/CacheFactory.php @@ -3,6 +3,7 @@ namespace SMW; use Onoi\Cache\CacheFactory as OnoiCacheFactory; +use Onoi\BlobStore\BlobStore; use SMW\ApplicationFactory; use ObjectCache; use RuntimeException; @@ -127,4 +128,85 @@ public function newMediaWikiCompositeCache( $mediaWikiCacheType = null ) { return $compositeCache; } + /** + * @since 2.3 + * + * @param integer|string $mbeddedQueryResultCacheType + * @param integer $embeddedQueryResultCacheLifetime + * + * @return BlobStore + */ + public function newEmbeddedQueryResultBlobstore( $embeddedQueryResultCacheType = null, $embeddedQueryResultCacheLifetime = 3600 ) { + + $blobStore = new BlobStore( + 'smw:qrc:store', + $this->newMediaWikiCompositeCache( $embeddedQueryResultCacheType ) + ); + + // If CACHE_NONE is selected, disable the usage + $blobStore->setUsageState( + $embeddedQueryResultCacheType !== CACHE_NONE + ); + + $blobStore->setExpiryInSeconds( + $embeddedQueryResultCacheLifetime + ); + + $blobStore->setNamespacePrefix( + $this->getCachePrefix() + ); + + return $blobStore; + } + + /** + * @since 2.4 + * + * @return EmbeddedQueryResultCache + */ + public function newEmbeddedQueryResultCache( $embeddedQueryResultCacheType = null, $embeddedQueryResultCacheLifetime = 3600 ) { + + $embeddedQueryResultBlobstore = $this->newEmbeddedQueryResultBlobstore( + $embeddedQueryResultCacheType, + $embeddedQueryResultCacheLifetime + ); + + $embeddedQueryResultCache = new EmbeddedQueryResultCache( + $embeddedQueryResultBlobstore + ); + + return $embeddedQueryResultCache; + } + + /** + * @since 2.3 + * + * @param integer|string $mediaWikiCacheType + * @param integer $valueLookupCacheLifetime + * + * @return BlobStore + */ + public function newValueLookupBlobstore( $valueLookupCacheType = null, $valueLookupCacheLifetime = 3600 ) { + + $blobStore = new BlobStore( + 'smw:vl:store', + $this->newMediaWikiCompositeCache( $valueLookupCacheType ) + ); + + // If CACHE_NONE is selected, disable the usage + $blobStore->setUsageState( + $valueLookupCacheType !== CACHE_NONE + ); + + $blobStore->setExpiryInSeconds( + $valueLookupCacheLifetime + ); + + $blobStore->setNamespacePrefix( + $this->getCachePrefix() + ); + + return $blobStore; + } + } diff --git a/src/EmbeddedQueryResultCache.php b/src/EmbeddedQueryResultCache.php new file mode 100644 index 0000000000..fe22e97cd1 --- /dev/null +++ b/src/EmbeddedQueryResultCache.php @@ -0,0 +1,172 @@ +blobStore = $blobStore; + } + + /** + * @since 2.4 + * + * @param boolean $enabledState + */ + public function setEnabledState( $enabledState ) { + $this->enabledState = (bool)$enabledState; + } + + /** + * @since 2.4 + * + * @return boolean + */ + public function isEnabled() { + return $this->enabledState && $this->blobStore->canUse(); + } + + /** + * @note Called from 'SMW::Store::BeforeQueryResultLookupComplete' + * + * @since 2.4 + * + * @param Store $store + * @param Query $query + * @param QueryEngine $queryEngine + * + * @return QueryResult|string + */ + public function fetchQueryResult( Store $store, Query $query, QueryEngine $queryEngine ) { + + if ( !$this->isEnabled() || $query->getLimit() < 1 || $query->getSubject() === null ) { + return $queryEngine->getQueryResult( $query ); + } + + // The queryID is used without a subject to access query content with the same + // query signature + $queryID = md5( $query->getQueryId() . self::VERSION ); + $container = $this->blobStore->read( $queryID ); + + if ( $container->has( 'results' ) ) { + wfDebugLog( 'smw', 'Using EmbeddedQueryResultCache for ' . $queryID ); + + $queryResult = new QueryResult( + $container->get( 'printrequests' ), + $query, + $container->get( 'results' ), + $store, + $container->get( 'furtherresults' ), + true + ); + + $queryResult->setCountValue( $container->get( 'countvalue' ) ); + $queryResult->setFromCache( true ); + + return $queryResult; + } + + $queryResult = $queryEngine->getQueryResult( $query ); + + // E.g. format=debug returns a string therefore it is not going to be cached + if ( !$queryResult instanceof QueryResult ) { + return $queryResult; + } + + $container->set( 'printrequests', $queryResult->getPrintRequests() ); + $container->set( 'results', $queryResult->getResults() ); + $container->set( 'furtherresults', $queryResult->hasFurtherResults() ); + $container->set( 'countvalue', $queryResult->getCountValue() ); + + $queryResult->reset(); + + $this->blobStore->save( + $container + ); + + // We can not ensure that EmbeddedQueryDependencyLinksStore is + // enabled and yet we still allow to use the cache and store subjects and + // queryID's separately to make them easily discoverable and removable + // per subject + $container = $this->blobStore->read( + md5( $query->getSubject()->getHash() . self::VERSION ) + ); + + $container->append( + 'list', + array( $queryID => true ) + ); + + $this->blobStore->save( + $container + ); + + return $queryResult; + } + + /** + * @since 2.4 + * + * @param array $queryList + */ + public function purgeCacheByQueryList( array $queryList ) { + foreach ( $queryList as $queryID ) { + $this->blobStore->delete( md5( $queryID . self::VERSION ) ); + } + } + + /** + * @since 2.4 + * + * @param DIWikiPage $subject + */ + public function purgeCacheBySubject( DIWikiPage $subject ) { + + $id = md5( $subject->getHash() . self::VERSION ); + $container = $this->blobStore->read( $id ); + + if ( !$container->has( 'list' ) ) { + return; + } + + $list = array_keys( $container->get( 'list' ) ); + + foreach ( $list as $queryID ) { + $this->blobStore->delete( $queryID ); + } + + $this->blobStore->delete( $id ); + } + +} diff --git a/src/EventListenerRegistry.php b/src/EventListenerRegistry.php index 43a9951da3..a1b9e61284 100644 --- a/src/EventListenerRegistry.php +++ b/src/EventListenerRegistry.php @@ -6,6 +6,7 @@ use Onoi\EventDispatcher\EventDispatcherFactory; use SMWExporter as Exporter; use SMW\Query\QueryComparator; +use SMW\DIWikiPage; /** * @license GNU GPL v2+ @@ -40,18 +41,48 @@ public function getCollection() { private function addListenersToCollection() { + $applicationFactory = ApplicationFactory::getInstance(); + $this->eventListenerCollection->registerCallback( - 'factbox.cache.delete', function( $dispatchContext ) { + 'factbox.cache.delete', function( $dispatchContext ) use ( $applicationFactory ) { $title = $dispatchContext->get( 'title' ); - $cache = ApplicationFactory::getInstance()->getCache(); + $cache = $applicationFactory->getCache(); $cache->delete( - ApplicationFactory::getInstance()->newCacheFactory()->getFactboxCacheKey( $title->getArticleID() ) + $applicationFactory->newCacheFactory()->getFactboxCacheKey( $title->getArticleID() ) ); } ); + /** + * Listen to when an ArticlePurge event is emitted + */ + $this->eventListenerCollection->registerCallback( + 'cache.delete.on.article.purge', function( $dispatchContext ) use ( $applicationFactory ) { + + $title = $dispatchContext->get( 'title' ); + $cache = $applicationFactory->getCache(); + $cacheFactory = $applicationFactory->newCacheFactory(); + + if ( $applicationFactory->getSettings()->get( 'smwgFactboxCacheRefreshOnPurge' ) ) { + $cache->delete( + $cacheFactory->getFactboxCacheKey( $title->getArticleID() ) + ); + } + + if ( $applicationFactory->getSettings()->get( 'smwgEmbeddedQueryResultCacheRefreshOnPurge' ) ) { + + $embeddedQueryResultCache = $cacheFactory->newEmbeddedQueryResultCache( + $applicationFactory->getSettings()->get( 'smwgEmbeddedQueryResultCacheType' ), + $applicationFactory->getSettings()->get( 'smwgEmbeddedQueryResultCacheLifetime' ) + ); + + $embeddedQueryResultCache->purgeCacheBySubject( DIWikiPage::newFromTitle( $title ) ); + } + } + ); + $this->eventListenerCollection->registerCallback( 'exporter.reset', function() { Exporter::getInstance()->clear(); diff --git a/src/MediaWiki/Hooks/ArticlePurge.php b/src/MediaWiki/Hooks/ArticlePurge.php index 07be9b0c5e..c9c742f305 100644 --- a/src/MediaWiki/Hooks/ArticlePurge.php +++ b/src/MediaWiki/Hooks/ArticlePurge.php @@ -5,6 +5,7 @@ use SMW\ApplicationFactory; use SMW\Cache\CacheFactory; use WikiPage; +use SMW\EventHandler; /** * A function hook being executed before running "&action=purge" @@ -43,11 +44,13 @@ public function process( WikiPage &$wikiPage ) { ); } - if ( $settings->get( 'smwgFactboxCacheRefreshOnPurge' ) ) { - $cache->delete( - $cacheFactory->getFactboxCacheKey( $pageId ) - ); - } + $dispatchContext = EventHandler::getInstance()->newDispatchContext(); + $dispatchContext->set( 'title', $wikiPage->getTitle() ); + + EventHandler::getInstance()->getEventDispatcher()->dispatch( + 'cache.delete.on.article.purge', + $dispatchContext + ); return true; } diff --git a/src/MediaWiki/Hooks/HookRegistry.php b/src/MediaWiki/Hooks/HookRegistry.php index 78c3c41697..dcc5327ac7 100644 --- a/src/MediaWiki/Hooks/HookRegistry.php +++ b/src/MediaWiki/Hooks/HookRegistry.php @@ -10,6 +10,7 @@ use SMW\NamespaceManager; use SMW\SQLStore\EmbeddedQueryDependencyLinksStore; use SMW\SQLStore\EmbeddedQueryDependencyListResolver; +use SMW\EmbeddedQueryResultCache; use SMW\DeferredRequestDispatchManager; use SMW\PropertyHierarchyLookup; use Onoi\HttpRequest\HttpRequestFactory; @@ -521,6 +522,23 @@ private function addCallbackHandlers( $basePath, $globalVars ) { return true; }; + $this->handlers['SMW::Store::BeforeQueryResultLookupComplete'] = function ( $store, $query, &$result = null, $queryEngine ) use ( $applicationFactory ) { + + $embeddedQueryResultCache = $applicationFactory->newCacheFactory()->newEmbeddedQueryResultCache( + $applicationFactory->getSettings()->get( 'smwgEmbeddedQueryResultCacheType' ), + $applicationFactory->getSettings()->get( 'smwgEmbeddedQueryResultCacheLifetime' ) + ); + + $result = $embeddedQueryResultCache->fetchQueryResult( + $store, + $query, + $queryEngine + ); + + // Suspend further processing by returning false + return false; + }; + $this->handlers['SMW::Store::AfterQueryResultLookupComplete'] = function ( $store, &$result ) use ( $applicationFactory, $propertyHierarchyLookup ) { $embeddedQueryDependencyListResolver = new EmbeddedQueryDependencyListResolver( diff --git a/src/MediaWiki/Jobs/ParserCachePurgeJob.php b/src/MediaWiki/Jobs/ParserCachePurgeJob.php index e767badf3b..c76b171ba9 100644 --- a/src/MediaWiki/Jobs/ParserCachePurgeJob.php +++ b/src/MediaWiki/Jobs/ParserCachePurgeJob.php @@ -4,6 +4,7 @@ use SMW\ApplicationFactory; use SMW\SQLStore\EmbeddedQueryDependencyLinksStore; +use SMW\EmbeddedQueryResultCache; use SMW\DIWikiPage; use SMW\HashBuilder; use Title; @@ -23,6 +24,11 @@ class ParserCachePurgeJob extends JobBase { */ const CHUNK_SIZE = 300; + /** + * @var ApplicationFactory + */ + private $applicationFactory; + /** * @var integer */ @@ -46,6 +52,7 @@ class ParserCachePurgeJob extends JobBase { */ public function __construct( Title $title, $params = array() ) { parent::__construct( 'SMW\ParserCachePurgeJob', $title, $params ); + $this->applicationFactory = ApplicationFactory::getInstance(); } /** @@ -55,8 +62,8 @@ public function __construct( Title $title, $params = array() ) { */ public function run() { - $this->pageUpdater = ApplicationFactory::getInstance()->newMwCollaboratorFactory()->newPageUpdater(); - $this->store = ApplicationFactory::getInstance()->getStore(); + $this->pageUpdater = $this->applicationFactory->newMwCollaboratorFactory()->newPageUpdater(); + $this->store = $this->applicationFactory->getStore(); if ( $this->hasParameter( 'limit' ) ) { $this->limit = $this->getParameter( 'limit' ); @@ -130,20 +137,31 @@ private function findEmbeddedQueryTargetLinksBatches( $idList ) { wfDebugLog( 'smw', __METHOD__ . " counted: {$countedHashListEntries} | offset: {$this->offset} for " . $this->getTitle()->getPrefixedDBKey() . "\n" ); - $hashList = $this->doBuildUniqueTargetLinksHashList( + list( $hashList, $queryList ) = $this->doBuildUniqueTargetLinksHashList( $hashList ); + $embeddedQueryResultCache = $this->applicationFactory->newCacheFactory()->newEmbeddedQueryResultCache( + $this->applicationFactory->getSettings()->get( 'smwgEmbeddedQueryResultCacheType' ), + $this->applicationFactory->getSettings()->get( 'smwgEmbeddedQueryResultCacheLifetime' ) + ); + + $embeddedQueryResultCache->purgeCacheByQueryList( $queryList ); + $this->addPagesToUpdater( $hashList ); } private function doBuildUniqueTargetLinksHashList( array $targetLinksHashList ) { $uniqueTargetLinksHashList = array(); + $uniqueQueryList = array(); foreach ( $targetLinksHashList as $targetLinkHash ) { - list( $title, $namespace, $iw ) = explode( '#', $targetLinkHash, 4 ); + list( $title, $namespace, $iw, $subobjectname ) = explode( '#', $targetLinkHash, 4 ); + + // EmbeddedQueryResultCache stores queries with they queryID = $subobjectname + $uniqueQueryList[$subobjectname] = true; // We make an assumption (as we avoid to query the DB) about that a // query is bind to its subject by simply removing the subobject @@ -152,7 +170,7 @@ private function doBuildUniqueTargetLinksHashList( array $targetLinksHashList ) $uniqueTargetLinksHashList[HashBuilder::createHashIdFromSegments( $title, $namespace, $iw )] = true; } - return array_keys( $uniqueTargetLinksHashList ); + return array( array_keys( $uniqueTargetLinksHashList ), array_keys( $uniqueQueryList ) ); } private function addPagesToUpdater( array $hashList ) { diff --git a/src/QueryEngine.php b/src/QueryEngine.php new file mode 100644 index 0000000000..286df0fbef --- /dev/null +++ b/src/QueryEngine.php @@ -0,0 +1,29 @@ +factory->newMasterQueryEngine() ) ) ) { $result = $this->fetchQueryResult( $query ); } diff --git a/src/SQLStore/QueryEngine/QueryEngine.php b/src/SQLStore/QueryEngine/QueryEngine.php index c48390080e..246ee4aac2 100644 --- a/src/SQLStore/QueryEngine/QueryEngine.php +++ b/src/SQLStore/QueryEngine/QueryEngine.php @@ -15,6 +15,7 @@ use SMWDataItem as DataItem; use SMWPropertyValue as PropertyValue; use SMW\InvalidPredefinedPropertyException; +use SMW\QueryEngine as IQueryEngine; use RuntimeException; /** @@ -27,7 +28,7 @@ * @author Jeroen De Dauw * @author mwjames */ -class QueryEngine { +class QueryEngine implements IQueryEngine { /** * @var SQLStore diff --git a/src/SQLStore/SQLStoreFactory.php b/src/SQLStore/SQLStoreFactory.php index 17eb197016..ddc02c6abc 100644 --- a/src/SQLStore/SQLStoreFactory.php +++ b/src/SQLStore/SQLStoreFactory.php @@ -265,29 +265,14 @@ public function newCachedValueLookupStore() { $circularReferenceGuard = new CircularReferenceGuard( 'vl:store' ); $circularReferenceGuard->setMaxRecursionDepth( 2 ); - $cacheFactory = ApplicationFactory::getInstance()->newCacheFactory(); - - $blobStore = new BlobStore( - 'smw:vl:store', - $cacheFactory->newMediaWikiCompositeCache( $GLOBALS['smwgValueLookupCacheType'] ) - ); - - // If CACHE_NONE is selected, disable the usage - $blobStore->setUsageState( - $GLOBALS['smwgValueLookupCacheType'] !== CACHE_NONE - ); - - $blobStore->setExpiryInSeconds( + $valueLookupBlobstore = ApplicationFactory::getInstance()->newCacheFactory()->newValueLookupBlobstore( + $GLOBALS['smwgValueLookupCacheType'], $GLOBALS['smwgValueLookupCacheLifetime'] ); - $blobStore->setNamespacePrefix( - $cacheFactory->getCachePrefix() - ); - $cachedValueLookupStore = new CachedValueLookupStore( $this->store, - $blobStore + $valueLookupBlobstore ); $cachedValueLookupStore->setValueLookupFeatures(