From 08aaa637fb1873ee7f75212f6e41936bd7853838 Mon Sep 17 00:00:00 2001 From: Sam Hemelryk Date: Fri, 2 Nov 2012 12:06:44 +1300 Subject: [PATCH] MDL-36120 cachestore_file: improved file storage method + new setting --- cache/stores/file/addinstanceform.php | 4 + cache/stores/file/lang/en/cachestore_file.php | 15 ++++ cache/stores/file/lib.php | 82 ++++++++++++++++--- 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/cache/stores/file/addinstanceform.php b/cache/stores/file/addinstanceform.php index a5e3125e801cb..18eb3e49b7c91 100644 --- a/cache/stores/file/addinstanceform.php +++ b/cache/stores/file/addinstanceform.php @@ -52,6 +52,10 @@ protected function configuration_definition() { $form->addHelpButton('autocreate', 'autocreate', 'cachestore_file'); $form->disabledIf('autocreate', 'path', 'eq', ''); + $form->addElement('checkbox', 'singledirectory', get_string('singledirectory', 'cachestore_file')); + $form->setType('singledirectory', PARAM_BOOL); + $form->addHelpButton('singledirectory', 'singledirectory', 'cachestore_file'); + $form->addElement('checkbox', 'prescan', get_string('prescan', 'cachestore_file')); $form->setType('prescan', PARAM_BOOL); $form->addHelpButton('prescan', 'prescan', 'cachestore_file'); diff --git a/cache/stores/file/lang/en/cachestore_file.php b/cache/stores/file/lang/en/cachestore_file.php index 8a17b71f95a36..f0097c250c6d8 100644 --- a/cache/stores/file/lang/en/cachestore_file.php +++ b/cache/stores/file/lang/en/cachestore_file.php @@ -35,3 +35,18 @@ $string['pluginname'] = 'File cache'; $string['prescan'] = 'Prescan directory'; $string['prescan_help'] = 'If enabled the directory is scanned when the cache is first used and requests for files are first checked against the scan data. This can help if you have a slow file system and are finding that file operations are causing you a bottle neck.'; +$string['singledirectory'] = 'Single directory store'; +$string['singledirectory_help'] = 'If enabled files (cached items) will be stored in a single directory rather than being broken up into multiple directories.
+Enabling this will speed up file interactions but comes at the cost of increased risk of hitting file system limitations.
+It is advisable to only turn this on if the following is true:
+ - If you know the number of items in the cache is going to be small enough that it won\'t cause issues on the file system you are running with.
+ - The data being cached is not expensive to generate. If it is then sticking with the default may still be the better option as it reduces the chance of issues.'; + +/** + * This is is like the file store, but designed for siutations where: + * - many more things are likely to be stored in the cache, so CRC hashing is + * too likely to give collisions, and storing everything in a completely flat + * directory structure is inadvisable. + * - the things we are caching are more expensive to calculate, so the extra + * time to computer a better hash is a worthwhile trade-off. + */ \ No newline at end of file diff --git a/cache/stores/file/lib.php b/cache/stores/file/lib.php index 03d2b32ffca5f..b50d0b3efc226 100644 --- a/cache/stores/file/lib.php +++ b/cache/stores/file/lib.php @@ -57,6 +57,14 @@ class cachestore_file implements cache_store, cache_is_key_aware { */ protected $prescan = false; + /** + * Set to true if we should store files within a single directory. + * By default we use a nested structure in order to reduce the chance of conflicts and avoid any file system + * limitations such as maximum files per directory. + * @var bool + */ + protected $singledirectory = false; + /** * Set to true when the path should be automatically created if it does not yet exist. * @var bool @@ -122,7 +130,20 @@ public function __construct($name, array $configuration = array()) { } $this->isready = $path !== false; $this->path = $path; - $this->prescan = array_key_exists('prescan', $configuration) ? (bool)$configuration['prescan'] : false; + // Check if we should prescan the directory. + if (array_key_exists('prescan', $configuration)) { + $this->prescan = (bool)$configuration['prescan']; + } else { + // Default is no, we should not prescan. + $this->prescan = false; + } + // Check if we should be storing in a single directory. + if (array_key_exists('singledirectory', $configuration)) { + $this->singledirectory = (bool)$configuration['singledirectory']; + } else { + // Default: No, we will use multiple directories. + $this->singledirectory = false; + } } /** @@ -226,10 +247,52 @@ public function initialise(cache_definition $definition) { $this->prescan = false; } if ($this->prescan) { - $pattern = $this->path.'/*.cache'; - foreach (glob($pattern, GLOB_MARK | GLOB_NOSORT) as $filename) { - $this->keys[basename($filename)] = filemtime($filename); + $this->prescan_keys(); + } + } + + /** + * Pre-scan the cache to see which keys are present. + */ + protected function prescan_keys() { + foreach (glob($this->glob_keys_pattern(), GLOB_MARK | GLOB_NOSORT) as $filename) { + $this->keys[basename($filename)] = filemtime($filename); + } + } + + /** + * Gets a pattern suitable for use with glob to find all keys in the cache. + * @return string The pattern. + */ + protected function glob_keys_pattern() { + if ($this->singledirectory) { + return $this->path . '/*.cache'; + } else { + return $this->path . '/*/*/*.cache'; + } + } + + /** + * Returns the file path to use for the given key. + * + * @param string $key The key to generate a file path for. + * @param bool $create If set to the true the directory structure the key requires will be created. + * @return string The full path to the file that stores a particular cache key. + */ + protected function file_path_for_key($key, $create = false) { + if ($this->singledirectory) { + // Its a single directory, easy, just the store instances path + the file name. + return $this->path . '/' . $key . '.cache'; + } else { + // We are using multiple subdirectories. We want two levels. + $subdir1 = substr($key, 0, 2); + $subdir2 = substr($key, 2, 2); + $dir = $this->path . '/' . $subdir1 .'/'. $subdir2; + if ($create) { + // Create the directory. This function does it recursivily! + make_writable_directory($dir); } + return $dir . '/' . $key . '.cache'; } } @@ -241,7 +304,7 @@ public function initialise(cache_definition $definition) { */ public function get($key) { $filename = $key.'.cache'; - $file = $this->path.'/'.$filename; + $file = $this->file_path_for_key($key); $ttl = $this->definition->get_ttl(); if ($ttl) { $maxtime = cache::now() - $ttl; @@ -307,7 +370,7 @@ public function get_many($keys) { */ public function delete($key) { $filename = $key.'.cache'; - $file = $this->path.'/'.$filename; + $file = $this->file_path_for_key($key); $result = @unlink($file); unset($this->keys[$filename]); return $result; @@ -339,7 +402,7 @@ public function delete_many(array $keys) { public function set($key, $data) { $this->ensure_path_exists(); $filename = $key.'.cache'; - $file = $this->path.'/'.$filename; + $file = $this->file_path_for_key($key, true); $result = $this->write_file($file, $this->prep_data_before_save($data)); if (!$result) { // Couldn't write the file. @@ -404,11 +467,11 @@ public function set_many(array $keyvaluearray) { */ public function has($key) { $filename = $key.'.cache'; - $file = $this->path.'/'.$key.'.cache'; $maxtime = cache::now() - $this->definition->get_ttl(); if ($this->prescan) { return array_key_exists($filename, $this->keys) && $this->keys[$filename] >= $maxtime; } + $file = $this->file_path_for_key($key); return (file_exists($file) && ($this->definition->get_ttl() == 0 || filemtime($file) >= $maxtime)); } @@ -448,8 +511,7 @@ public function has_any(array $keys) { * @return boolean True on success. False otherwise. */ public function purge() { - $pattern = $this->path.'/*.cache'; - foreach (glob($pattern, GLOB_MARK | GLOB_NOSORT) as $filename) { + foreach (glob($this->glob_keys_pattern(), GLOB_MARK | GLOB_NOSORT) as $filename) { @unlink($filename); } $this->keys = array();