Skip to content

Commit

Permalink
Added object cache for Swift ext storage
Browse files Browse the repository at this point in the history 
This avoid calling the remote API for repeated calls to is_dir,
filetype, etc
  • Loading branch information
Vincent Petry committed May 24, 2016
1 parent 4ffc936 commit f77a11a
Showing 1 changed file with 71 additions and 17 deletions.
88 changes: 71 additions & 17 deletions apps/files_external/lib/Lib/Storage/Swift.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,14 @@ class Swift extends \OC\Files\Storage\Common {
*/
private static $tmpFiles = array();

/**
* Key value cache mapping path to data object. Maps path to
* \OpenCloud\OpenStack\ObjectStorage\Resource\DataObject for existing
* paths and path to false for not existing paths.
* @var \OCP\ICache
*/
private $objectCache;

/**
* @param string $path
*/
Expand All @@ -96,18 +104,31 @@ private function normalizePath($path) {
* @param string $path
* @return string
*/
private function getContainerName($path) {
$path = trim(trim($this->root, '/') . "/" . $path, '/.');
return str_replace('/', '\\', $path);
}

/**
* Fetches an object from the API.
* If the object is cached already or a
* failed "doesn't exist" response was cached,
* that one will be returned.
*
* @param string $path
* @return \OpenCloud\OpenStack\ObjectStorage\Resource\DataObject|bool object
* or false if the object did not exist
*/
private function doesObjectExist($path) {
private function fetchObject($path) {
if ($this->objectCache->hasKey($path)) {
// might be "false" if object did not exist from last check
return $this->objectCache->get($path);
}
try {
$this->getContainer()->getPartialObject($path);
return true;
$object = $this->getContainer()->getPartialObject($path);
$this->objectCache->set($path, $object);
return $object;
} catch (ClientErrorResponseException $e) {
// this exception happens when the object does not exist, which
// is expected in most cases
$this->objectCache->set($path, false);
return false;
} catch (ClientErrorResponseException $e) {
// Expected response is "404 Not Found", so only log if it isn't
if ($e->getResponse()->getStatusCode() !== 404) {
Expand All @@ -117,6 +138,17 @@ private function doesObjectExist($path) {
}
}

/**
* Returns whether the given path exists.
*
* @param string $path
*
* @return bool true if the object exist, false otherwise
*/
private function doesObjectExist($path) {
return $this->fetchObject($path) !== false;
}

public function __construct($params) {
if ((empty($params['key']) and empty($params['password']))
or empty($params['user']) or empty($params['bucket'])
Expand Down Expand Up @@ -144,6 +176,8 @@ public function __construct($params) {
}

$this->params = $params;
// FIXME: private class...
$this->objectCache = new \OC\Cache\CappedMemoryCache();
}

public function mkdir($path) {
Expand All @@ -162,6 +196,9 @@ public function mkdir($path) {
$metadataHeaders = DataObject::stockHeaders(array());
$allHeaders = $customHeaders + $metadataHeaders;
$this->getContainer()->uploadObject($path, '', $allHeaders);
// invalidate so that the next access gets the real object
// with all properties
$this->objectCache->remove($path);
} catch (Exceptions\CreateUpdateError $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
Expand Down Expand Up @@ -202,6 +239,7 @@ public function rmdir($path) {

try {
$this->getContainer()->dataObject()->setName($path . '/')->delete();
$this->objectCache->remove($path . '/');
} catch (Exceptions\DeleteError $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
Expand Down Expand Up @@ -256,7 +294,10 @@ public function stat($path) {

try {
/** @var DataObject $object */
$object = $this->getContainer()->getPartialObject($path);
$object = $this->fetchObject($path);
if (!$object) {
return false;
}
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
Expand Down Expand Up @@ -310,8 +351,12 @@ public function unlink($path) {

try {
$this->getContainer()->dataObject()->setName($path)->delete();
$this->objectCache->remove($path);
$this->objectCache->remove($path . '/');
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
if ($e->getResponse()->getStatusCode() !== 404) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
}
return false;
}

Expand Down Expand Up @@ -391,15 +436,20 @@ public function touch($path, $mtime = null) {
$path .= '/';
}

$object = $this->getContainer()->getPartialObject($path);
$object->saveMetadata($metadata);
$object = $this->fetchObject($path);
if ($object->saveMetadata($metadata)) {
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
}
return true;
} else {
$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
$customHeaders = array('content-type' => $mimeType);
$metadataHeaders = DataObject::stockHeaders($metadata);
$allHeaders = $customHeaders + $metadataHeaders;
$this->getContainer()->uploadObject($path, '', $allHeaders);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
return true;
}
}
Expand All @@ -415,8 +465,11 @@ public function copy($path1, $path2) {
$this->unlink($path2);

try {
$source = $this->getContainer()->getPartialObject($path1);
$source = $this->fetchObject($path1);
$source->copy($this->bucket . '/' . $path2);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path2);
$this->objectCache->remove($path2 . '/');
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
Expand All @@ -428,8 +481,11 @@ public function copy($path1, $path2) {
$this->unlink($path2);

try {
$source = $this->getContainer()->getPartialObject($path1 . '/');
$source = $this->fetchObject($path1 . '/');
$source->copy($this->bucket . '/' . $path2 . '/');
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path2);
$this->objectCache->remove($path2 . '/');
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
Expand Down Expand Up @@ -461,10 +517,6 @@ public function rename($path1, $path2) {
$fileType = $this->filetype($path1);

if ($fileType === 'dir' || $fileType === 'file') {

// make way
$this->unlink($path2);

// copy
if ($this->copy($path1, $path2) === false) {
return false;
Expand Down Expand Up @@ -564,6 +616,8 @@ public function writeBack($tmpFile) {
}
$fileData = fopen($tmpFile, 'r');
$this->getContainer()->uploadObject(self::$tmpFiles[$tmpFile], $fileData);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove(self::$tmpFiles[$tmpFile]);
unlink($tmpFile);
}

Expand Down

0 comments on commit f77a11a

Please sign in to comment.