Skip to content

Loading…

Added vbuckets config per shard #7

Merged
merged 4 commits into from

2 participants

@geisbruch

Hi, We've added the functionality of create a shard with an specific amount of vnodes, so you could modify the amount of vnodes in a new shard to add it progresively, then you'll get a better distribution of misses when a new shard is added.

This change is 100% backward compatible

Gabriel Eisb... added some commits
Gabriel Eisbruch Fix variable to run test ecfbcdc
Gabriel Eisbruch Added the capability to select virtual nodes per shard.
This feature enable to do a progressive miss on shard added, so you could
modify the amount of new vnodes progressively getting a better distribution
of misses on shard added
0447337
Gabriel Eisbruch Add vnodes config to readme ec369a7
@3rd-Eden
Owner

@geisbruch Thanks for the pull request, I will review it shortly but as there are not backwards compatibility issues, it should be fine :)

Gabriel Eisbruch rollback makefile 44ec4c5
@3rd-Eden 3rd-Eden merged commit 56b92ad into 3rd-Eden:master

1 check passed

Details default The Travis build passed
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Sep 18, 2012
  1. Fix variable to run test

    Gabriel Eisbruch committed
  2. Added the capability to select virtual nodes per shard.

    Gabriel Eisbruch committed
    This feature enable to do a progressive miss on shard added, so you could
    modify the amount of new vnodes progressively getting a better distribution
    of misses on shard added
  3. Add vnodes config to readme

    Gabriel Eisbruch committed
  4. rollback makefile

    Gabriel Eisbruch committed
Showing with 58 additions and 7 deletions.
  1. +12 −0 README.md
  2. +28 −7 lib/hashring.js
  3. +18 −0 tests/hashring.test.js
View
12 README.md
@@ -41,6 +41,18 @@ var ring = new hashring({
});
```
+Creating a hash ring with multiple servers an vnodes selected per nodes
+
+``` javascript
+var hashring = require('hashring');
+var ring = new hashring({
+ '192.168.0.102:11212': {"vnodes": 5}
+, '192.168.0.103:11212': {"vnodes": 10}
+, '192.168.0.104:11212': {"vnodes": 7}
+});
+```
+Optionaly you could add the weigth property to the object.
+
By default the hash ring uses a JavaScript crc32 implementation hashing algorithm. But this can be overwritten by adding a second argument to the constructor. This can be anything that is supported as hashing algorithm by the crypto module.
``` javascript
View
35 lib/hashring.js
@@ -23,15 +23,27 @@ var createHash = require('crypto').createHash
function HashRing (args, algorithm, options) {
var nodes = []
- , weights = {};
+ , weights = {}
+ , vnodes = {};
switch (Object.prototype.toString.call(args)){
case '[object String]':
nodes.push(args);
break;
case '[object Object]':
- weights = args;
nodes = Object.keys(args);
+ for(var i in args){
+ if(typeof args[i] === "number"){
+ weights[i] = args[i];
+ }else{
+ if(args[i].weigth != undefined){
+ weights[i] = args[i].weigth;
+ }
+ if(args[i].vnodes != undefined){
+ vnodes[i] = args[i].vnodes;
+ }
+ }
+ }
break;
case '[object Undefined]':
break;
@@ -49,6 +61,7 @@ function HashRing (args, algorithm, options) {
this.nodes = nodes || [];
this.weights = weights;
+ this.vnodes = vnodes;
this.algorithm = algorithm || 'crc32';
// overwrite the hashKey method if crc32 is chosen
@@ -105,11 +118,10 @@ HashRing.prototype.generateRing = function generateRing () {
// The factor is based on the weight, the more weight the more space a item
// will get in our hash ring
- factor = Math.floor((this.options.vnode_count * len * weight) / totalweight);
-
+ var vnodes_count = this.vnodes[node] || this.options.vnode_count
+ factor = Math.floor((vnodes_count * len * weight) / totalweight);
for (j = 0; j < factor; j++) {
tmp = this.hashKey(node + '-' + j);
-
for (k = 0; k < 3; k++) {
key = this.hashValue(tmp, function hash (x) {
return x + k * 4;
@@ -120,7 +132,6 @@ HashRing.prototype.generateRing = function generateRing () {
}
}
}
-
// Sort the keys, nummeric !important. I forgot it at first and took me
// 2 hours to debug \o/
this.sortedKeys.sort(function sort (a, b) {
@@ -207,7 +218,7 @@ HashRing.prototype.replaceServer = function replaceServer (oldServer, newServer)
* @api public
*/
-HashRing.prototype.addServer = function addServer (server, weights) {
+HashRing.prototype.addServer = function addServer (server, weights, vnodes) {
if (this.nodes.indexOf(server) !== -1) return; // prevents duplicates
// add weights
@@ -217,6 +228,12 @@ HashRing.prototype.addServer = function addServer (server, weights) {
}
}
+ // add vnodes
+ if (vnodes) {
+ for(var key in vnodes) {
+ this.vnodes[key] = vnodes[key];
+ }
+ }
if (!Array.isArray(server)) {
server = [server];
}
@@ -247,6 +264,10 @@ HashRing.prototype.removeServer = function removeServer (server) {
delete this.weights[server];
}
+ if (this.vnodes[server]) {
+ delete this.weights[server];
+ }
+
// clear all old caches and regenerate
this.ring = {};
this.cache = {};
View
18 tests/hashring.test.js
@@ -38,7 +38,25 @@ module.exports = {
ring.sortedKeys.length.should.be.above(1);
Object.keys(ring.weights).should.have.length(3);
}
+, 'Constructing with a object with pernode vnodes': function(){
+ var ring = new Hashring({
+ '192.168.0.102:11212': {"vnodes":40}
+ , '192.168.0.103:11212': {"vnodes":50}
+ , '192.168.0.104:11212': {"vnodes":5}
+ });
+ ring.nodes.should.have.length(3);
+ ring.sortedKeys.length.should.be.equal((40+50+5)*3)
+
+ ring = new Hashring({
+ '192.168.0.102:11212': {"vnodes":4}
+ , '192.168.0.103:11212': {"vnodes":3}
+ , '192.168.0.104:11212': {"vnodes":5}
+ });
+ ring.nodes.should.have.length(3);
+ ring.sortedKeys.length.should.be.equal((4+3+5)*3)
+
+ }
, 'Constructing with a different algorithm': function () {
var ring = new Hashring('192.168.0.102:11212', 'md5');
Something went wrong with that request. Please try again.