From f09df7c02a2e917c64e358319fda7404d2a1e525 Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Fri, 4 Dec 2015 15:17:54 +0300 Subject: [PATCH 1/2] Update dependencies and integrate jscs --- index.js | 16 ++-- lib/db.js | 205 +++++++++++++--------------------------- lib/dbutils.js | 86 ++++++++--------- lib/index.js | 4 +- lib/revisionPolicy.js | 2 +- lib/schemaMigration.js | 3 +- lib/secondaryIndexes.js | 6 +- package.json | 25 ++--- test/index.js | 2 + 9 files changed, 138 insertions(+), 211 deletions(-) diff --git a/index.js b/index.js index 7a1f90b..0e5983d 100644 --- a/index.js +++ b/index.js @@ -6,7 +6,7 @@ // global includes var spec = require('restbase-mod-table-spec').spec; -function RBCassandra (options) { +function RBCassandra(options) { this.options = options; this.conf = options.conf; this.log = options.log; @@ -24,7 +24,7 @@ function RBCassandra (options) { }; } -RBCassandra.prototype.createTable = function (rb, req) { +RBCassandra.prototype.createTable = function(rb, req) { var store = this.store; // XXX: decide on the interface req.body.table = req.params.table; @@ -64,7 +64,7 @@ RBCassandra.prototype.createTable = function (rb, req) { }; // Query a table -RBCassandra.prototype.get = function (rb, req) { +RBCassandra.prototype.get = function(rb, req) { var rp = req.params; if (!rp.rest && !req.body) { // Return the entire table @@ -97,7 +97,7 @@ RBCassandra.prototype.get = function (rb, req) { }; // Update a table -RBCassandra.prototype.put = function (rb, req) { +RBCassandra.prototype.put = function(rb, req) { var domain = req.params.domain; // XXX: Use the path to determine the primary key? return this.store.put(domain, req.body) @@ -120,7 +120,7 @@ RBCassandra.prototype.put = function (rb, req) { }); }; -RBCassandra.prototype.dropTable = function (rb, req) { +RBCassandra.prototype.dropTable = function(rb, req) { var domain = req.params.domain; return this.store.dropTable(domain, req.params.table) .then(function(res) { @@ -142,7 +142,7 @@ RBCassandra.prototype.dropTable = function (rb, req) { }); }; -RBCassandra.prototype.getTableSchema = function (rb, req) { +RBCassandra.prototype.getTableSchema = function(rb, req) { var domain = req.params.domain; return this.store.getTableSchema(domain, req.params.table) .then(function(res) { @@ -171,7 +171,7 @@ RBCassandra.prototype.getTableSchema = function (rb, req) { * * @return {Promise} */ -RBCassandra.prototype.setup = function setup () { +RBCassandra.prototype.setup = function setup() { var self = this; // Set up storage backend var backend = require('./lib/index'); @@ -190,7 +190,7 @@ RBCassandra.prototype.setup = function setup () { * @return {Promise} with registration being the registration * object */ -function makeRBCassandra (options) { +function makeRBCassandra(options) { var rb = new RBCassandra(options); return rb.setup(); } diff --git a/lib/db.js b/lib/db.js index 8847975..b9cda3d 100644 --- a/lib/db.js +++ b/lib/db.js @@ -11,9 +11,9 @@ var SchemaMigrator = require('./schemaMigration'); var secIndexes = require('./secondaryIndexes'); /** @const */ -var validTextConsistencies = {all:1, localOne: 1, localQuorum:1}; +var validTextConsistencies = { all: 1, localOne: 1, localQuorum: 1 }; -function DB (client, options) { +function DB(client, options) { this.conf = options.conf; this.log = options.log; @@ -48,7 +48,7 @@ DB.prototype._initCaches = function() { * Set up internal request-related information and wrap it into an * InternalRequest instance. */ -DB.prototype._makeInternalRequest = function (domain, table, query, consistency) { +DB.prototype._makeInternalRequest = function(domain, table, query, consistency) { consistency = consistency || this.defaultConsistency; if (query.consistency && query.consistency in validTextConsistencies) { consistency = cass.types.consistencies[query.consistency]; @@ -112,7 +112,7 @@ DB.prototype._fetchSchema = function(req) { return self.client.execute_p('SELECT columnfamily_name FROM ' + 'system.schema_columnfamilies WHERE keyspace_name=? ' + 'and columnfamily_name=?', [req.keyspace, 'meta']) - .then(function (res) { + .then(function(res) { if (res && res.rows.length === 0) { // meta column family doesn't exist yet return req; @@ -128,22 +128,22 @@ DB.prototype._fetchSchema = function(req) { /** * Process the storage group configuration. * - * @param {Array} the array of group objects to read, each must contain + * @param {Array} groups the array of group objects to read, each must contain * at least the name and domains keys * @return {Array} Array of storage group objects */ -DB.prototype._buildStorageGroups = function (groups) { +DB.prototype._buildStorageGroups = function(groups) { var storageGroups = []; - if(!Array.isArray(groups)) { + if (!Array.isArray(groups)) { return storageGroups; } groups.forEach(function(group) { var grp = extend(true, {}, group); - if(!Array.isArray(grp.domains)) { + if (!Array.isArray(grp.domains)) { grp.domains = [grp.domains]; } grp.domains = grp.domains.map(function(domain) { - if(/^\/.*\/$/.test(domain)) { + if (/^\/.*\/$/.test(domain)) { return new RegExp(domain.slice(1, -1)); } return domain; @@ -163,7 +163,7 @@ DB.prototype._buildStorageGroups = function (groups) { * @param {string} table, the logical table name * @return {string} Valid Cassandra keyspace key */ -DB.prototype._keyspaceName = function (domain, table) { +DB.prototype._keyspaceName = function(domain, table) { var cacheKey = JSON.stringify([domain,table]); var cachedName = this.keyspaceNameCache[cacheKey]; if (cachedName) { @@ -186,29 +186,29 @@ DB.prototype._keyspaceName = function (domain, table) { * @param {String} domain the domain's name * @return {Object} the group object matching the domain */ -DB.prototype._resolveStorageGroup = function (domain) { +DB.prototype._resolveStorageGroup = function(domain) { var group = this.storageGroupsCache[domain]; var idx; - if(group) { + if (group) { return group; } // not found in cache, find it - for(idx = 0; idx < this.storageGroups.length; idx++) { + for (idx = 0; idx < this.storageGroups.length; idx++) { var curr = this.storageGroups[idx]; var domIdx; - for(domIdx = 0; domIdx < curr.domains.length; domIdx++) { + for (domIdx = 0; domIdx < curr.domains.length; domIdx++) { var dom = curr.domains[domIdx]; - if(((dom instanceof RegExp) && dom.test(domain)) || + if (((dom instanceof RegExp) && dom.test(domain)) || (typeof dom === 'string' && dom === domain)) { group = curr; break; } } - if(group) { + if (group) { break; } } - if(!group) { + if (!group) { // no group found, assume the domain is to // be grouped by itself group = { @@ -239,7 +239,7 @@ DB.prototype.infoSchema = dbu.validateAndNormalizeSchema({ DB.prototype.infoSchemaInfo = dbu.makeSchemaInfo(DB.prototype.infoSchema, true); -DB.prototype.get = function (domain, query) { +DB.prototype.get = function(domain, query) { var self = this; return this._makeInternalRequest(domain, query.table, query) .then(function(req) { @@ -252,7 +252,7 @@ DB.prototype.get = function (domain, query) { }); }; -DB.prototype._getRaw = function (req, options) { +DB.prototype._getRaw = function(req, options) { var self = this; options = options || {}; @@ -267,24 +267,24 @@ DB.prototype._getRaw = function (req, options) { // Index queries are currently handled in buildGetQuery. See // https://phabricator.wikimedia.org/T78722 for secondary index TODOs. - //if (req.index) { + // if (req.index) { // return this._getSecondaryIndex(keyspace, req, consistency, table, buildResult); - //} + // } // Paging request: - var cassOpts = {consistency: req.consistency, prepare: true}; + var cassOpts = { consistency: req.consistency, prepare: true }; if (req.query.limit) { cassOpts.fetchSize = req.query.limit; if (req.query.next) { - cassOpts.pageState = new Buffer( req.query.next, 'base64'); + cassOpts.pageState = new Buffer(req.query.next, 'base64'); } } var buildResult = dbu.buildGetQuery(req, options); return self.client.execute_p(buildResult.cql, buildResult.params, cassOpts) - .then(function(result){ + .then(function(result) { var rows = result.rows; // Decorate the row result with the _ttl attribute. if (options.withTTLs) { @@ -312,77 +312,6 @@ DB.prototype._getRaw = function (req, options) { }); }; -/* - Handler for request GET requests on secondary indexes. - This is currently not used. TODO: fix. -*/ -//DB.prototype._getSecondaryIndex = function(keyspace, domain, req, -// consistency, table, buildResult){ -// -// // TODO: handle '_tid' cases -// var self = this; -// return self.client.execute_p(buildResult.cql, buildResult.params, -// {consistency: consistency, prepare: true}) -// .then(function(results) { -// var queries = []; -// var cachedSchema = self.schemaCache[keyspace]; -// -// // convert the result values -// results.rows.forEach(function (row) { -// dbu.convertRow(row, cachedSchema); -// }); -// -// var newReq = { -// table: table, -// attributes: {}, -// limit: req.limit + Math.ceil(req.limit/4) -// }; -// -// // build main data queries -// for ( var rowno in results.rows ) { -// for ( var attr in cachedSchema.iKeyMap ) { -// newReq.attributes[attr] = results.rows[rowno][attr]; -// } -// queries.push(dbu.buildGetQuery(keyspace, domain, newReq, consistency, table, cachedSchema)); -// newReq.attributes = {}; -// } -// -// // prepare promises for batch execution -// var batchPromises = []; -// queries.forEach(function(item) { -// batchPromises.push(self.client.execute_p(item.cql, item.params, -// item.options || {consistency: consistency, prepare: true})); -// }); -// -// // execute batch and check if limit is fulfilled -// return P.all(batchPromises).then(function(batchResults){ -// var finalRows = []; -// batchResults.forEach(function(item){ -// if (finalRows.length < req.limit) { -// finalRows.push(dbu.convertRow(item.rows[0], cachedSchema)); -// } -// }); -// return [finalRows, results.rows[rowno]]; -// }); -// }) -// .then(function(rows){ -// //TODO: handle case when limit > no of entries in table -// if (rows[0].length max) { + } else if (row[key] > max) { max = row[key]; } } @@ -221,7 +220,7 @@ dbu.validateAndNormalizeSchema = function validateAndNormalizeSchema(schema, con }; // Extract the index keys from a table schema -dbu.indexKeys = function indexKeys (index) { +dbu.indexKeys = function indexKeys(index) { var res = []; index.forEach(function(elem) { if (elem.type === 'hash' || elem.type === 'range') { @@ -231,7 +230,7 @@ dbu.indexKeys = function indexKeys (index) { return res; }; -dbu.makeIndexSchema = function makeIndexSchema (dataSchema, indexName) { +dbu.makeIndexSchema = function makeIndexSchema(dataSchema, indexName) { var index = dataSchema.secondaryIndexes[indexName]; var s = { @@ -286,7 +285,7 @@ dbu.makeIndexSchema = function makeIndexSchema (dataSchema, indexName) { return s; }; -function encodeBlob (blob) { +function encodeBlob(blob) { if (blob instanceof Buffer) { return blob; } else { @@ -296,31 +295,31 @@ function encodeBlob (blob) { var schemaTypeToCQLTypeMap = { - 'blob': 'blob', + blob: 'blob', 'set': 'set', - 'decimal': 'decimal', + decimal: 'decimal', 'set': 'set', - 'double': 'double', + double: 'double', 'set': 'set', - 'float': 'float', + float: 'float', 'set': 'set', - 'boolean': 'boolean', + boolean: 'boolean', 'set': 'set', - 'int': 'int', + int: 'int', 'set': 'set', - 'varint': 'varint', + varint: 'varint', 'set': 'set', - 'string': 'text', + string: 'text', 'set': 'set', - 'timeuuid': 'timeuuid', + timeuuid: 'timeuuid', 'set': 'set', - 'uuid': 'uuid', + uuid: 'uuid', 'set': 'set', - 'timestamp': 'timestamp', + timestamp: 'timestamp', 'set': 'set', - 'json': 'text', + json: 'text', 'set': 'set', - 'long': 'bigint', + long: 'bigint', 'set': 'set' }; @@ -340,7 +339,7 @@ dbu.schemaTypeToCQLType = function(schemaType) { * @param {Object} convObj the conversion object to use for individual values (from dbu.conversions) * @returns {Object} an object with 'read' and 'write' attributes */ -function generateSetConvertor (convObj) { +function generateSetConvertor(convObj) { if (!convObj) { return { write: function(arr) { @@ -360,7 +359,7 @@ function generateSetConvertor (convObj) { read: null }; if (convObj.write) { - res.write = function (valArray) { + res.write = function(valArray) { if (!Array.isArray(valArray) || valArray.length === 0) { // Empty set is equivalent to null in Cassandra return null; @@ -370,7 +369,7 @@ function generateSetConvertor (convObj) { }; } if (convObj.read) { - res.read = function (valArray) { + res.read = function(valArray) { return valArray.map(convObj.read); }; } @@ -394,7 +393,7 @@ dbu.conversions = { json: { write: JSON.stringify, read: JSON.parse }, decimal: { read: toString() }, timestamp: { - read: function (date) { + read: function(date) { return date.toISOString(); } }, @@ -427,10 +426,10 @@ dbu.makeSchemaInfo = function makeSchemaInfo(schema, isMetaCF) { psi.conversions = {}; for (var att in psi.attributes) { var type = psi.attributes[att]; - var set_type = /^set<(\w+)>$/.exec(type); - if (set_type) { + var setType = /^set<(\w+)>$/.exec(type); + if (setType) { // this is a set-typed attribute - type = set_type[1]; + type = setType[1]; // generate the convertors only if the underlying type has them defined psi.conversions[att] = generateSetConvertor(dbu.conversions[type]); } else if (dbu.conversions[type]) { @@ -511,7 +510,7 @@ dbu.makeSchemaInfo = function makeSchemaInfo(schema, isMetaCF) { // define a 'hash' string representation for the schema for quick schema // comparisons. - psi.hash = stable_stringify(psi); + psi.hash = stableStringify(psi); return psi; }; @@ -524,7 +523,7 @@ dbu.makeSchemaInfo = function makeSchemaInfo(schema, isMetaCF) { * @param {object} schema the schema info to use for conversion * @returns {array} a converted array of result rows */ -dbu.convertRows = function convertRows (rows, schema) { +dbu.convertRows = function convertRows(rows, schema) { var conversions = schema.conversions; var newRows = new Array(rows.length); for (var i = 0; i < rows.length; i++) { @@ -560,12 +559,12 @@ dbu.makeRawRequest = function(internalReq, extendFields) { extendFields.query = extend(true, {}, internalReq.query); var clonedReq = internalReq.extend(extendFields); var attrs = clonedReq.query.attributes; - if(!conversions || !attrs) { + if (!conversions || !attrs) { return clonedReq; } Object.keys(attrs).forEach(function(key) { var conv = conversions[key]; - if(conv && conv.write) { + if (conv && conv.write) { attrs[key] = conv.write(attrs[key]); } }); @@ -583,7 +582,7 @@ dbu.makeRawRequest = function(internalReq, extendFields) { * @param {Boolean} noConvert if true, no attribute value conversion will take place * @return {object} queryInfo object with cql and params attributes */ -dbu.buildCondition = function buildCondition (predicates, schema, noConvert) { +dbu.buildCondition = function buildCondition(predicates, schema, noConvert) { function convert(key, val) { var convObj = schema.conversions[key]; if (!noConvert && convObj && convObj.write) { @@ -594,7 +593,7 @@ dbu.buildCondition = function buildCondition (predicates, schema, noConvert) { } // make sure we have got a predicate object - if(!predicates || predicates.constructor !== Object) { + if (!predicates || predicates.constructor !== Object) { throw new Error('The condition predicate has not been supplied or is not an Object.'); } @@ -660,9 +659,6 @@ dbu.buildCondition = function buildCondition (predicates, schema, noConvert) { * @return {object} queryInfo object with cql and params attributes */ dbu.buildPutQuery = function(req, noConvert) { - - //table = schema.table; - if (!req.schema) { throw new Error('Table not found!'); } @@ -707,7 +703,8 @@ dbu.buildPutQuery = function(req, noConvert) { params.push(val); } placeholders.push('?'); - } else if (!/^_ttl.*/.test(key) && !schema.attributes[key]) { // Allow TTL fields not in the schema + } else if (!/^_ttl.*/.test(key) && !schema.attributes[key]) { + // Allow TTL fields not in the schema throw new Error('Unknown attribute ' + key); } }); @@ -736,7 +733,8 @@ dbu.buildPutQuery = function(req, noConvert) { // - Need to verify that all primary key members are supplied as well, // else error. - var cql = '', condResult; + var cql = ''; + var condResult; if (query.if && query.if.constructor === String) { query.if = query.if.trim().split(/\s+/).join(' ').toLowerCase(); @@ -920,7 +918,7 @@ dbu.buildGetQuery = function(req, options) { cql += limit ? ' limit ' + limit : ''; } - return {cql: cql, params: params}; + return { cql: cql, params: params }; }; dbu.getOptionCQL = function(options, db) { @@ -938,7 +936,7 @@ var updateOptionMap = { 'write-once': "{ 'class' : 'SizeTieredCompactionStrategy', " + "'tombstone_threshold': '0.02' }", 'random-update': "{ 'class' : 'LeveledCompactionStrategy' }", - 'timeseries': "{ 'class': 'DateTieredCompactionStrategy', " + timeseries: "{ 'class': 'DateTieredCompactionStrategy', " // Change base_time_seconds from 60s default to 45, for better // alignment with two-day expiry (24 hour TTL + 24 hour // gc_grace_seconds). See also: diff --git a/lib/index.js b/lib/index.js index 8a9675f..a3329fb 100644 --- a/lib/index.js +++ b/lib/index.js @@ -9,7 +9,7 @@ var DB = require('./db'); P.promisifyAll(cass, { suffix: '_p' }); -function validateAndNormalizeDcConf (conf) { +function validateAndNormalizeDcConf(conf) { // Default to 'datacenter1' if (!conf.localDc) { conf.localDc = 'datacenter1'; } if (!conf.datacenters) { conf.datacenters = ['datacenter1']; } @@ -42,7 +42,7 @@ function sslOptions(sslConf) { return sslOpts; } -function makeClient (options) { +function makeClient(options) { var clientOpts = {}; var conf = options.conf; validateAndNormalizeDcConf(conf); diff --git a/lib/revisionPolicy.js b/lib/revisionPolicy.js index 2b416e7..b3a1dfc 100644 --- a/lib/revisionPolicy.js +++ b/lib/revisionPolicy.js @@ -32,7 +32,7 @@ RevisionPolicyManager.prototype._setTtl = function(item) { self.request.query.timestamp = null; var query = dbu.buildPutQuery(self.request, true); - var queryOptions = {consistency: self.request.consistency, prepare: true}; + var queryOptions = { consistency: self.request.consistency, prepare: true }; return self.db.client.execute_p(query.cql, query.params, queryOptions) .catch(function(e) { diff --git a/lib/schemaMigration.js b/lib/schemaMigration.js index ea4fcde..31acf2e 100644 --- a/lib/schemaMigration.js +++ b/lib/schemaMigration.js @@ -173,7 +173,8 @@ Index.prototype.validate = function(req, current, proposed) { } }); - // If index deleted the column is not deleted, need to remove it and add back to change index. + // If index deleted the column is not deleted, + // need to remove it and add back to change index. // Not supported. delIndexes.forEach(function(index) { if (proposed.attributes[index.attribute]) { diff --git a/lib/secondaryIndexes.js b/lib/secondaryIndexes.js index 30703cc..0185770 100644 --- a/lib/secondaryIndexes.js +++ b/lib/secondaryIndexes.js @@ -5,7 +5,7 @@ var cass = require('cassandra-driver'); var TimeUuid = cass.types.TimeUuid; var dbu = require('./dbutils'); -function IndexRebuilder (db, req, secondaryKeys, timestamp) { +function IndexRebuilder(db, req, secondaryKeys, timestamp) { this.db = db; this.req = dbu.makeRawRequest(req); this.primaryKeys = this.req.schema.iKeys; @@ -21,7 +21,7 @@ function IndexRebuilder (db, req, secondaryKeys, timestamp) { * for each diff: find / update indexes * _del is included to handle deleted primary table items */ -IndexRebuilder.prototype.diffRow = function (row) { +IndexRebuilder.prototype.diffRow = function(row) { var diff = {}; var i, att; for (i = 0; i < this.primaryKeys.length; i++) { @@ -62,7 +62,7 @@ IndexRebuilder.prototype.diffRow = function (row) { * @param {object} row; a row object * @return a promise that resolves when the update is complete */ -IndexRebuilder.prototype.handleRow = function (row) { +IndexRebuilder.prototype.handleRow = function(row) { if (!this.prevRow) { // In normal operation there is no need to update the index for the // first row, as we are only interested in diffs, and the new data was diff --git a/package.json b/package.json index 361eb91..1823d41 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,16 @@ { "name": "restbase-mod-table-cassandra", "description": "RESTBase table storage on Cassandra", - "version": "0.8.12", + "version": "0.8.13", "dependencies": { "bluebird": "~2.8.2", - "cassandra-driver": "~2.2.1", - "core-js": "^1.1.1", - "extend": "^2.0.0", - "js-yaml": "^3.2.5", + "cassandra-driver": "^2.2.2", + "core-js": "^1.2.6", + "extend": "^3.0.0", + "js-yaml": "^3.4.6", "json-stable-stringify": "git+https://github.com/wikimedia/json-stable-stringify#master", "restbase-mod-table-spec": "^0.1.3", - "semver": "^5.0.3" + "semver": "^5.1.0" }, "repository": { "type": "git", @@ -22,11 +22,12 @@ "coveralls": "cat ./coverage/lcov.info | coveralls" }, "devDependencies": { - "coveralls": "2.11.2", - "istanbul": "0.3.5", - "mocha": "x.x.x", - "mocha-jshint": "0.0.9", - "mocha-lcov-reporter": "0.0.1", - "preq": "^0.4.4" + "coveralls": "^2.11.4", + "istanbul": "^0.4.1", + "mocha": "^2.3.4", + "mocha-jshint": "^2.2.5", + "mocha-jscs": "^4.0.0", + "mocha-lcov-reporter": "^1.0.0", + "preq": "^0.4.7" } } diff --git a/test/index.js b/test/index.js index 4fcbe83..fe26f98 100644 --- a/test/index.js +++ b/test/index.js @@ -8,6 +8,8 @@ var fs = require("fs"); // Run jshint as part of normal testing require('mocha-jshint')(); +require('mocha-jscs')(); + describe('Functional', function() { var conf = yaml.safeLoad(fs.readFileSync(__dirname + '/utils/test_client.conf.yaml')); From d23c0c9a4f4f3d9c57de2ae45f3d26ff2581e0a3 Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Mon, 7 Dec 2015 12:45:12 +0300 Subject: [PATCH 2/2] Added a .jscs config file --- .jscs.json | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .jscs.json diff --git a/.jscs.json b/.jscs.json new file mode 100644 index 0000000..9adad94 --- /dev/null +++ b/.jscs.json @@ -0,0 +1,32 @@ +{ + "preset": "node-style-guide", + "requireCurlyBraces": [ + "if", + "else", + "for", + "while", + "do", + "try", + "catch" + ], + "validateIndentation": 4, + "requireCamelCaseOrUpperCaseIdentifiers": "ignoreProperties", + "requireCapitalizedComments": null, + "maximumLineLength": { + "value": 100, + "allExcept": ["urlComments"] + }, + "validateQuoteMarks": null, + "requireTrailingComma": null, + "disallowTrailingComma": null, + "requireSpacesInsideObjectBrackets": "all", + + "excludeFiles": [ + "node_modules/**", + "test/**", + "coverage/**", + "scripts/**", + "maintenance/**", + "doc/**" + ] +} \ No newline at end of file