Skip to content

Commit

Permalink
[WIP] Add EmbeddedQueryResultCache
Browse files Browse the repository at this point in the history
Most of the job is already done by the `EmbeddedQueryDependencyLinksStore`
to track and update query dependencies.

To eliminate necessary SQL/SPARQL connections `EmbeddedQueryResultCache` ought to
cache a subject list returned from the `QueryEngine`.

We don't cache the string result of a printer == it means we don't
interfere with how the printer manipulates the data, `EmbeddedQueryResultCache` as
the name suggests caches the return object (aka `QueryResult`) from
the QueryEngine before it is forwarded to an individual printer.

If for some reason `EmbeddedQueryDependencyLinksStore` is not enabled then
auto-invalidation of the `EmbeddedQueryResultCache` can not occur but instead
(same as of now == manual intervention using the purge button) setting
`$GLOBALS['smwgEmbeddedQueryResultCacheRefreshOnPurge'] = true;` can
be used to emit a purge action event to invalidate all query cache items stored
with the corresponding article (aka subject).

`$GLOBALS['smwgEmbeddedQueryResultCacheLifetime'] = 60 * 60 * 24; // a day` declares the lifetime
of a cached item
  • Loading branch information
mwjames committed Nov 6, 2015
1 parent cba1b3c commit 3384eb0
Show file tree
Hide file tree
Showing 16 changed files with 422 additions and 38 deletions.
46 changes: 44 additions & 2 deletions SemanticMediaWiki.settings.php
Expand Up @@ -881,7 +881,7 @@
# @since 2.3 (experimental)
# @default false
##
$GLOBALS['smwgEnabledQueryDependencyLinksStore'] = false;
$GLOBALS['smwgEnabledQueryDependencyLinksStore'] = true;
##

###
Expand All @@ -900,9 +900,51 @@
$GLOBALS['smwgPropertyDependencyDetectionBlacklist'] = array( '_MDAT', '_SOBJ' );
##

###
# This stores subjects that were fetched from the QueryEngine (not the string result
# generated from a result printer) and allows to improve general page-loading time
# for articles that contain embedded queries and decrease server load on query requests
# due to subject list (as to the answer of the query) being served from cache
# instead of running a live DB/SPARQL query.
#
# It is suggested that `smwgEnabledQueryDependencyLinksStore` is enabled to make
# use of the automatic query results update (and hereof the invalidation of the
# cache).
#
# CACHE_NONE as default means that this feature is disabled
#
# @since 2.4 (experimental)
#
# @default: CACHE_NONE, users need to actively enable it in order
# to make use of it
##
$GLOBALS['smwgEmbeddedQueryResultCacheType'] = CACHE_NONE;
##

###
# Declares the lifetime of a cached item for when `smwgEmbeddedQueryResultCacheType`
# is enabled.
#
# @since 2.4
##
$GLOBALS['smwgEmbeddedQueryResultCacheLifetime'] = 60 * 60 * 24; // a day
##

###
# If `smwgEnabledQueryDependencyLinksStore` is enabled this setting is not required
# as results are expected to be automatically purged. In case it is disabled and
# a user still wants to take advantage of the cache then it is suggested to enable
# this setting so that results are refreshed (or better the cache is going to be
# invalidated) by force on an article purge event.
#
# @since 2.4
##
$GLOBALS['smwgEmbeddedQueryResultCacheRefreshOnPurge'] = false;
##

###
# The setting is introduced the keep backwards compatibility with existing Rdf/Turtle
# exports. The `aux` marker is epxected only used to be used for selected properties
# exports. The `aux` marker is expected to be used only for selected properties
# to generate a helper value and not for any other predefined property.
#
# Any property that does not explicitly require an auxiliary value (such `_dat`/
Expand Down
3 changes: 3 additions & 0 deletions includes/Settings.php
Expand Up @@ -122,6 +122,9 @@ public static function newFromGlobals() {
'smwgValueLookupCacheType' => $GLOBALS['smwgValueLookupCacheType'],
'smwgValueLookupCacheLifetime' => $GLOBALS['smwgValueLookupCacheLifetime'],
'smwgValueLookupFeatures' => $GLOBALS['smwgValueLookupFeatures'],
'smwgEmbeddedQueryResultCacheType' => $GLOBALS['smwgEmbeddedQueryResultCacheType'],
'smwgEmbeddedQueryResultCacheLifetime' => $GLOBALS['smwgEmbeddedQueryResultCacheLifetime'],
'smwgEmbeddedQueryResultCacheRefreshOnPurge' => $GLOBALS['smwgEmbeddedQueryResultCacheRefreshOnPurge'],
'smwgFixedProperties' => $GLOBALS['smwgFixedProperties'],
'smwgPropertyLowUsageThreshold' => $GLOBALS['smwgPropertyLowUsageThreshold'],
'smwgPropertyZeroCountDisplay' => $GLOBALS['smwgPropertyZeroCountDisplay'],
Expand Down
2 changes: 2 additions & 0 deletions includes/storage/SMW_Store.php
Expand Up @@ -215,6 +215,7 @@ public function updateData( SemanticData $semanticData ) {
/**
* @since 1.6
*/
wfRunHooks( 'SMW::Store::BeforeDataUpdateComplete', array( $this, $semanticData ) );
wfRunHooks( 'SMWStore::updateDataBefore', array( $this, $semanticData ) );

// Invalidate the page, so data stored on it gets displayed immediately in queries.
Expand All @@ -232,6 +233,7 @@ public function updateData( SemanticData $semanticData ) {
* @since 1.6
*/
wfRunHooks( 'SMWStore::updateDataAfter', array( $this, $semanticData ) );
wfRunHooks( 'SMW::Store::AfterDataUpdateComplete', array( $this, $semanticData ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion includes/storage/SQLStore/SMW_SQLStore3.php
Expand Up @@ -367,7 +367,7 @@ public function getQueryResult( SMWQuery $query ) {

$result = null;

if ( wfRunHooks( 'SMW::Store::BeforeQueryResultLookupComplete', array( $this, $query, &$result ) ) ) {
if ( wfRunHooks( 'SMW::Store::BeforeQueryResultLookupComplete', array( $this, $query, &$result, $this->factory->newSlaveQueryEngine() ) ) ) {
$result = $this->fetchQueryResult( $query );
}

Expand Down
2 changes: 1 addition & 1 deletion src/ApplicationFactory.php
Expand Up @@ -268,7 +268,7 @@ public function newMwCollaboratorFactory() {
* @return NamespaceExaminer
*/
public function getNamespaceExaminer() {
return NamespaceExaminer::newFromArray( $this->getSettings()->get( 'smwgNamespacesWithSemanticLinks' ) );
return $this->callbackLoader->load( 'NamespaceExaminer' );//NamespaceExaminer::newFromArray( $this->getSettings()->get( 'smwgNamespacesWithSemanticLinks' ) );
}

/**
Expand Down
82 changes: 82 additions & 0 deletions src/CacheFactory.php
Expand Up @@ -3,6 +3,7 @@
namespace SMW;

use Onoi\Cache\CacheFactory as OnoiCacheFactory;
use Onoi\BlobStore\BlobStore;
use SMW\ApplicationFactory;
use ObjectCache;
use RuntimeException;
Expand Down Expand Up @@ -127,4 +128,85 @@ public function newMediaWikiCompositeCache( $mediaWikiCacheType = null ) {
return $compositeCache;
}

/**
* @since 2.3
*
* @param integer|string $mbeddedQueryResultCacheType
* @param integer $embeddedQueryResultCacheLifetime
*
* @return BlobStore
*/
public function newEmbeddedQueryResultBlobstore( $embeddedQueryResultCacheType = null, $embeddedQueryResultCacheLifetime = 3600 ) {

$blobStore = new BlobStore(
'smw:qrc:store',
$this->newMediaWikiCompositeCache( $embeddedQueryResultCacheType )
);

// If CACHE_NONE is selected, disable the usage
$blobStore->setUsageState(
$embeddedQueryResultCacheType !== CACHE_NONE
);

$blobStore->setExpiryInSeconds(
$embeddedQueryResultCacheLifetime
);

$blobStore->setNamespacePrefix(
$this->getCachePrefix()
);

return $blobStore;
}

/**
* @since 2.4
*
* @return EmbeddedQueryResultCache
*/
public function newEmbeddedQueryResultCache( $embeddedQueryResultCacheType = null, $embeddedQueryResultCacheLifetime = 3600 ) {

$embeddedQueryResultBlobstore = $this->newEmbeddedQueryResultBlobstore(
$embeddedQueryResultCacheType,
$embeddedQueryResultCacheLifetime
);

$embeddedQueryResultCache = new EmbeddedQueryResultCache(
$embeddedQueryResultBlobstore
);

return $embeddedQueryResultCache;
}

/**
* @since 2.3
*
* @param integer|string $mediaWikiCacheType
* @param integer $valueLookupCacheLifetime
*
* @return BlobStore
*/
public function newValueLookupBlobstore( $valueLookupCacheType = null, $valueLookupCacheLifetime = 3600 ) {

$blobStore = new BlobStore(
'smw:vl:store',
$this->newMediaWikiCompositeCache( $valueLookupCacheType )
);

// If CACHE_NONE is selected, disable the usage
$blobStore->setUsageState(
$valueLookupCacheType !== CACHE_NONE
);

$blobStore->setExpiryInSeconds(
$valueLookupCacheLifetime
);

$blobStore->setNamespacePrefix(
$this->getCachePrefix()
);

return $blobStore;
}

}
169 changes: 169 additions & 0 deletions src/EmbeddedQueryResultCache.php
@@ -0,0 +1,169 @@
<?php

namespace SMW;

use Onoi\BlobStore\BlobStore;
use SMWQuery as Query;
use SMWQueryResult as QueryResult;

/**
* @license GNU GPL v2+
* @since 2.4
*
* @author mwjames
*/
class EmbeddedQueryResultCache {

/**
* Update this version number when the serialization format
* changes.
*/
const VERSION = '0.5';

/**
* @var BlobStore
*/
private $blobStore;

/**
* @var boolean
*/
private $enabledState = true;

/**
* @since 2.4
*
* @param BlobStore $blobStore
*/
public function __construct( BlobStore $blobStore ) {
$this->blobStore = $blobStore;
}

/**
* @since 2.4
*
* @param boolean $enabledState
*/
public function setEnabledState( $enabledState ) {
$this->enabledState = (bool)$enabledState;
}

/**
* @since 2.4
*
* @return boolean
*/
public function isEnabled() {
return $this->enabledState && $this->blobStore->canUse();
}

/**
* @note Called from 'SMW::Store::BeforeQueryResultLookupComplete'
*
* @since 2.4
*
* @param Store $store
* @param Query $query
* @param QueryEngine $queryEngine
*
* @return QueryResult|string
*/
public function fetchQueryResult( Store $store, Query $query, QueryEngine $queryEngine ) {

if ( !$this->isEnabled() || $query->getLimit() < 1 || $query->getSubject() === null ) {
return $queryEngine->getQueryResult( $query );
}

// The queryID is used without a subject to access query content with the same
// query signature
$queryID = md5( $query->getQueryId() . self::VERSION );
$container = $this->blobStore->read( $queryID );

if ( $container->has( 'results' ) ) {
wfDebugLog( 'smw', 'Using EmbeddedQueryResultCache for ' . $queryID );

$queryResult = new QueryResult(
$container->get( 'printrequests' ),
$query,
$container->get( 'results' ),
$store,
$container->get( 'furtherresults' )
);

$queryResult->setCountValue( $container->get( 'countvalue' ) );

return $queryResult;
}

$queryResult = $queryEngine->getQueryResult( $query );

if ( !$queryResult instanceof QueryResult ) {
return $queryResult;
}

$container->set( 'printrequests', $queryResult->getPrintRequests() );
$container->set( 'results', $queryResult->getResults() );
$container->set( 'furtherresults', $queryResult->hasFurtherResults() );
$container->set( 'countvalue', $queryResult->getCountValue() );

$queryResult->reset();

$this->blobStore->save(
$container
);

// We can not ensure that EmbeddedQueryDependencyLinksStore is
// enabled and yet we still allow to use the cache and store subjects and
// queryID's separately to make them easily discoverable and removable
// per subject
$container = $this->blobStore->read(
md5( $query->getSubject()->getHash() . self::VERSION )
);

$container->append(
'list',
array( $queryID => true )
);

$this->blobStore->save(
$container
);

return $queryResult;
}

/**
* @since 2.4
*
* @param array $queryList
*/
public function purgeCacheByQueryList( array $queryList ) {
foreach ( $queryList as $queryID ) {
$this->blobStore->delete( md5( $queryID . self::VERSION ) );
}
}

/**
* @since 2.4
*
* @param DIWikiPage $subject
*/
public function purgeCacheBySubject( DIWikiPage $subject ) {

$id = md5( $subject->getHash() . self::VERSION );
$container = $this->blobStore->read( $id );

if ( !$container->has( 'list' ) ) {
return;
}

$list = array_keys( $container->get( 'list' ) );

foreach ( $list as $queryID ) {
$this->blobStore->delete( $queryID );
}

$this->blobStore->delete( $id );
}

}

0 comments on commit 3384eb0

Please sign in to comment.