Skip to content

Commit

Permalink
Move storage group handling into db.js & some tweaks
Browse files Browse the repository at this point in the history 
Storage group handling depends on the configuration and is stateful, so seems
to be better placed in the DB class.

Also:

- Add keyspaceNameCache: hashing is quite expensive, and did show up at around
  1% of total CPU usage in profiling.

- Move `InternalRequest` to the end of the file, so that it doesn't break the
  flow of DB. The previous JSHint config patch makes that possible.

- Make `_buildStorageGroups` side-effect free.

- Optimize deleted row removal. Deletions are fairly rare, so filtering is not
  the fastest option in the common (no deletions) case. The filter function
  accounted for 50% of the time spent in db.js, and around 0.6% of total time.
  • Loading branch information
@gwicke
gwicke committed Mar 3, 2015
1 parent 9b6fee2 commit 07447dd
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 124 deletions.
169 changes: 137 additions & 32 deletions lib/db.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,51 +22,36 @@ function DB (client, options) {

// cache keyspace -> schema
this.schemaCache = {};
this.keyspaceNameCache = {};

// read the storage groups configuration
dbu.readStorageGroups(options.conf.storage_groups);
/* Process the array of storage groups declared in the config */
this.storageGroups = this._buildStorageGroups(options.conf.storage_groups);
/* The cache holding the already-resolved domain-to-group mappings */
this.storageGroupsCache = {};
}

/**
* Wrap common internal request state
*/
function InternalRequest (opts) {
this.domain = opts.domain;
this.table = opts.table;
this.keyspace = opts.keyspace || dbu.keyspaceName(opts.domain, opts.table);
this.query = opts.query || null;
this.consistency = opts.consistency;
this.schema = opts.schema || null;
this.columnfamily = opts.columnfamily || 'data';
}

/**
* Construct a new InternalRequest based on an existing one, optionally
* overriding existing properties.
* Set up internal request-related information and wrap it into an
* InternalRequest instance.
*/
InternalRequest.prototype.extend = function(opts) {
var req = new InternalRequest(this);
Object.keys(opts).forEach(function(key) {
req[key] = opts[key];
});
return req;
};

DB.prototype._makeInternalRequest = function (domain, table, query, consistency) {
var self = this;
consistency = consistency || this.defaultConsistency;
if (query.consistency && query.consistency in {all:1, localQuorum:1}) {
consistency = cass.types.consistencies[query.consistency];
}
var cacheKey = JSON.stringify([domain,table]);
var req = new InternalRequest({
domain: domain,
table: table,
keyspace: this.keyspaceNameCache[cacheKey]
|| this._keyspaceName(domain, table),
query: query,
consistency: consistency,
columnfamily: 'data'
columnfamily: 'data',
schema: this.schemaCache[cacheKey]
});
var schemaCacheKey = JSON.stringify([req.keyspace, domain]);
req.schema = this.schemaCache[schemaCacheKey];
if (req.schema) {
return P.resolve(req);
} else {
Expand All @@ -87,7 +72,8 @@ DB.prototype._makeInternalRequest = function (domain, table, query, consistency)
// Need to parse the JSON manually here as we are using the
// internal _get(), which doesn't apply transforms.
var schema = JSON.parse(res.items[0].value);
self.schemaCache[schemaCacheKey] = req.schema = dbu.makeSchemaInfo(schema);
self.keyspaceNameCache[cacheKey] = req.keyspace;
self.schemaCache[cacheKey] = req.schema = dbu.makeSchemaInfo(schema);
}
return req;
}, function(err) {
Expand All @@ -108,6 +94,94 @@ DB.prototype._makeInternalRequest = function (domain, table, query, consistency)
}
};

/**
* Process the storage group configuration.
*
* @param {Array} the array of group objects to read, each must contain
* at least the name and domains keys
* @return {Array} Array of storage group objects
*/
DB.prototype._buildStorageGroups = function (groups) {
var storageGroups = [];
if(!Array.isArray(groups)) {
return storageGroups;
}
groups.forEach(function(group) {
var grp = extend(true, {}, group);
if(!Array.isArray(grp.domains)) {
grp.domains = [grp.domains];
}
grp.domains = grp.domains.map(function(domain) {
if(/^\/.*\/$/.test(domain)) {
return new RegExp(domain.slice(1, -1));
}
return domain;
});
storageGroups.push(grp);
});
return storageGroups;
};

/**
* Derive a valid keyspace name from a random bucket name. Try to use valid
* chars from the requested name as far as possible, but fall back to a sha1
* if not possible. Also respect Cassandra's limit of 48 or fewer alphanum
* chars & first char being an alpha char.
*
* @param {string} domain in dot notation
* @param {string} table, the logical table name
* @return {string} Valid Cassandra keyspace key
*/
DB.prototype._keyspaceName = function (domain, table) {
var name = this._resolveStorageGroup(domain).name;
var reversedName = name.toLowerCase().split('.').reverse().join('.');
var prefix = dbu.makeValidKey(reversedName, Math.max(26, 48 - table.length - 3));
return prefix
// 6 chars _hash_ to prevent conflicts between domains & table names
+ '_T_' + dbu.makeValidKey(table, 48 - prefix.length - 3);
};

/**
* Finds the storage group for a given domain.
*
* @param {String} domain the domain's name
* @return {Object} the group object matching the domain
*/
DB.prototype._resolveStorageGroup = function (domain) {
var group = this.storageGroupsCache[domain];
var idx;
if(group) {
return group;
}
// not found in cache, find it
for(idx = 0; idx < this.storageGroups.length; idx++) {
var curr = this.storageGroups[idx];
var domIdx;
for(domIdx = 0; domIdx < curr.domains.length; domIdx++) {
var dom = curr.domains[domIdx];
if(((dom instanceof RegExp) && dom.test(domain)) ||
(typeof dom === 'string' && dom === domain)) {
group = curr;
break;
}
}
if(group) {
break;
}
}
if(!group) {
// no group found, assume the domain is to
// be grouped by itself
group = {
name: domain,
domain: [domain]
};
}
// save it in the cache
this.storageGroupsCache[domain] = group;
return group;
};


// Info table schema
DB.prototype.infoSchema = dbu.validateAndNormalizeSchema({
Expand Down Expand Up @@ -155,10 +229,17 @@ DB.prototype._get = function (req) {
return self.client.execute_p(buildResult.cql, buildResult.params,
{consistency: req.consistency, prepare: true})
.then(function(result){
var rows = result.rows;
var length = rows.length;
for (var i = 0; i < length; i++) {
if (rows[i]._del) {
rows.splice(i,1);
i--;
length--;
}
}
return {
items: result.rows.filter(function(row) {
return !row._del;
})
items: rows
};
});

Expand Down Expand Up @@ -804,10 +885,34 @@ DB.prototype._createKeyspace = function (req, options) {


DB.prototype.dropTable = function (domain, table) {
var keyspace = dbu.keyspaceName(domain, table);
var keyspace = this._keyspaceName(domain, table);
return this.client.execute_p('drop keyspace ' + cassID(keyspace), [],
{consistency: this.defaultConsistency});
};

/**
* Wrap common internal request state
*/
function InternalRequest (opts) {
this.domain = opts.domain;
this.table = opts.table;
this.keyspace = opts.keyspace;
this.query = opts.query || null;
this.consistency = opts.consistency;
this.schema = opts.schema || null;
this.columnfamily = opts.columnfamily || 'data';
}

/**
* Construct a new InternalRequest based on an existing one, optionally
* overriding existing properties.
*/
InternalRequest.prototype.extend = function(opts) {
var req = new InternalRequest(this);
Object.keys(opts).forEach(function(key) {
req[key] = opts[key];
});
return req;
};

module.exports = DB;
92 changes: 0 additions & 92 deletions lib/dbutils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,98 +110,6 @@ dbu.makeValidKey = function makeValidKey (key, length) {
};


/* The array of storage groups declared in the config */
dbu.storageGroups = [];
/* The cache holding the already-resolved domain-to-group mappings */
dbu.storageGroupsCache = {};


/**
* Reads the storage groups configuration.
*
* @param {Array} the array of group objects to read, each must contain
* at least the name and domains keys
*/
dbu.readStorageGroups = function readStorageGroups (groups) {
if(!Array.isArray(groups)) {
return;
}
groups.forEach(function(group) {
var grp = extend(true, {}, group);
if(!Array.isArray(grp.domains)) {
grp.domains = [grp.domains];
}
grp.domains = grp.domains.map(function(domain) {
if(/^\/.*\/$/.test(domain)) {
return new RegExp(domain.slice(1, -1));
}
return domain;
});
dbu.storageGroups.push(grp);
});
};


/**
* Finds the storage group for a given domain.
*
* @param {String} domain the domain's name
* @return {Object} the group object matching the domain
*/
dbu.resolveStorageGroup = function resolveStorageGroup (domain) {
var group = dbu.storageGroupsCache[domain];
var idx;
if(group) {
return group;
}
// not found in cache, find it
for(idx = 0; idx < dbu.storageGroups.length; idx++) {
var curr = dbu.storageGroups[idx];
var domIdx;
for(domIdx = 0; domIdx < curr.domains.length; domIdx++) {
var dom = curr.domains[domIdx];
if(((dom instanceof RegExp) && dom.test(domain)) ||
(typeof dom === 'string' && dom === domain)) {
group = curr;
break;
}
}
if(group) {
break;
}
}
if(!group) {
// no group found, assume the domain is to
// be grouped by itself
group = {
name: domain,
domain: [domain]
};
}
// save it in the cache
dbu.storageGroupsCache[domain] = group;
return group;
};


/**
* Derive a valid keyspace name from a random bucket name. Try to use valid
* chars from the requested name as far as possible, but fall back to a sha1
* if not possible. Also respect Cassandra's limit of 48 or fewer alphanum
* chars & first char being an alpha char.
*
* @param {string} domain in dot notation
* @param {string} table, the logical table name
* @return {string} Valid Cassandra keyspace key
*/
dbu.keyspaceName = function keyspaceName (domain, table) {
var prefix = dbu.makeValidKey(dbu.resolveStorageGroup(domain).name, Math.max(26, 48 - table.length - 3));
return prefix
// 6 chars _hash_ to prevent conflicts between domains & table names
+ '_T_' + dbu.makeValidKey(table, 48 - prefix.length - 3);
};


/*
* # Section 2: Schema validation, normalization and -handling
*/
Expand Down

0 comments on commit 07447dd

Please sign in to comment.