Merge pull request #2902 from vanilla/hotfix/cache-auto-shard

Fix underlying sharding issue (sort order) and add auto sharding + randomization
vanilla · Jul 22, 2015 · 0694374 · 0694374
2 parents 148ace6 + 67631cd
commit 0694374
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 24 deletions.
diff --git a/library/core/class.cache.php b/library/core/class.cache.php
@@ -101,8 +101,14 @@ abstract class Gdn_Cache {
     /** Seconds. */
     const APC_CACHE_DURATION = 300;
 
-    /** Max number of shards. */
-    const CACHE_MAX_SHARDS = 6;
+    /** Max number of shards. 0 = no limit */
+    const CACHE_SHARD_MAX_SHARDS = 0;
+
+    /** Min size for a shard, in bytes. 0 = no limit */
+    const CACHE_SHARD_MIN_SIZE = 10000;
+
+    /** Auto shard keys that are larger than this, in bytes. */
+    const CACHE_SHARD_AUTO_SIZE = 100000;
 
     /**  @var array Local in-memory cache of fetched data. This prevents duplicate gets to memcache. */
     protected static $localCache = array();

diff --git a/library/core/class.memcached.php b/library/core/class.memcached.php
@@ -277,45 +277,63 @@ public function canAutoShard() {
      *
      * @param string $key data key
      * @param mixed $value data value to shard
+     * @param integer $keySize value size
      * @param int|boolean $shards number of shards, or simply bool true
      * @return array
      */
-    public function shard($key, $value, $shards = true) {
+    public function shard($key, $value, $keySize, $shards = true) {
 
         $shardMap = $this->shardMap();
-
-        // Limit the number of shards to a sane value to reduce overhead (but only if $shards wasn't explicitly specified)
-        if (!is_numeric($shards) && count($shardMap) > Gdn_Cache::CACHE_MAX_SHARDS) {
-            if (mt_rand(1,2) % 2) {
-                $shardMap = array_slice($shardMap, 0, Gdn_Cache::CACHE_MAX_SHARDS);
-            } else {
-                $shardMap = array_slice($shardMap, -Gdn_Cache::CACHE_MAX_SHARDS);
-            }
-        }
-
         $mapSize = count($shardMap);
-        $shardMap = array_values($shardMap);
 
-        // Calculate automatic shard count
+        // Apply automated shard limits if shard allocation is automatic
         if (!is_numeric($shards)) {
+
+            // By default, shard to all servers
             $shards = $mapSize;
 
-            // If we're not precisely targeting keys, add a shard for safety
-            if (!$this->canAutoShard()) {
-                $shards = $mapSize + 1;
+            // Don't over-shard (make sure shards are large enough to warrant their own key)
+            $shardSize = ceil($keySize / $shards);
+            if (Gdn_Cache::CACHE_SHARD_MIN_SIZE > 0 && $shardSize < Gdn_Cache::CACHE_SHARD_MIN_SIZE) {
+                $shards = ceil($keySize / Gdn_Cache::CACHE_SHARD_MIN_SIZE);
+            }
+
+            // Limit the number of shards to a sane value to reduce overhead (but only if $shards wasn't explicitly specified)
+            if (Gdn_Cache::CACHE_SHARD_MAX_SHARDS > 0 && count($shardMap) > Gdn_Cache::CACHE_SHARD_MAX_SHARDS) {
+                $shards = Gdn_Cache::CACHE_SHARD_MAX_SHARDS;
             }
+
         }
 
+        // Don't shard to more servers than we know about
+        if ($shards > $mapSize) {
+            $shards = $mapSize;
+        }
+
+        // If we're sharding to less servers than we know about, pick some random ones
+        if ($shards < $mapSize) {
+            $shardSlices = array_rand($shardMap, $shards);
+            $shardMap = array_intersect_key($shardMap, array_fill_keys($shardSlices, true));
+        }
+        $mapSize = count($shardMap);
+
+        // If we're not precisely targeting keys, add a shard for safety
+        if (!$this->canAutoShard()) {
+            $shards = $mapSize + 1;
+        }
+
+        $shardMap = array_values($shardMap);
+
         // Prepare manifest
         $data = serialize($value);
         $hash = md5($data);
-        $size = strlen($data);
+        $keySize = strlen($data);
         $manifest = new MemcachedShard();
         $manifest->hash = $hash;
-        $manifest->size = $size;
+        $manifest->size = $keySize;
 
         // Determine chunk size
-        $chunk = ceil($size / $shards);
+        $chunk = ceil($keySize / $shards);
 
         // Write keys
         $chunks = str_split($data, $chunk);
@@ -424,9 +442,15 @@ public function store($key, $value, $options = array()) {
 
         $realKey = $this->makeKey($key, $finalOptions);
 
+        // Should auto sharding be enabled?
+        $keySize = strlen(serialize($value));
+        if ($this->hasFeature(Gdn_Cache::FEATURE_SHARD) && $keySize > Gdn_Cache::CACHE_SHARD_AUTO_SIZE) {
+            $finalOptions[Gdn_Cache::FEATURE_SHARD] = true;
+        }
+
         // Sharding, write real keys and manifest
         if (array_key_exists(Gdn_Cache::FEATURE_SHARD, $finalOptions) && $shards = $finalOptions[Gdn_Cache::FEATURE_SHARD]) {
-            $manifest = $this->shard($realKey, $value, $shards);
+            $manifest = $this->shard($realKey, $value, $keySize, $shards);
             $shards = $manifest->shards;
             unset($manifest->shards);
 
@@ -521,7 +545,8 @@ public function get($key, $options = array()) {
 
         $data = array();
         $hitCache = false;
-        if ($numKeys = sizeof($realKeys)) {
+        $numKeys = sizeof($realKeys);
+        if ($numKeys) {
             $hitCache = true;
             if ($numKeys > 1) {
                 $data = $this->memcache->getMulti($realKeys);
@@ -549,7 +574,7 @@ public function get($key, $options = array()) {
                         $serverKeys = $this->memcache->getMultiByKey($serverKey, $keys);
                         $shardKeys = array_merge($shardKeys, $serverKeys);
                     }
-                    ksort($shardKeys);
+                    ksort($shardKeys, SORT_NATURAL);
 
                     // Check subkeys for validity
                     $shardData = implode('', array_values($shardKeys));