Permalink
Browse files

First public

  • Loading branch information...
0 parents commit 1eb65ca2a1a97acb4c36b90c14e990fbdb4c4d36 @mariocasciaro committed Sep 21, 2013
Showing with 459 additions and 0 deletions.
  1. +7 −0 .gitignore
  2. +21 −0 LICENSE
  3. +62 −0 README.md
  4. +58 −0 lib/IndexToObjectStream.js
  5. +44 −0 lib/index.js
  6. +67 −0 lib/indexManager.js
  7. +51 −0 package.json
  8. +149 −0 test/test.js
7 .gitignore
@@ -0,0 +1,7 @@
+node_modules
+.c9revisions
+.settings
+.idea
+npm-debug.log
+generated
+tmpdb
21 LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Mario Casciaro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
62 README.md
@@ -0,0 +1,62 @@
+level-indico
+============
+
+Create and manage indices for you leveldb database.
+
+* No generic query system, provides just some flexible low level methods to access indexed data. You can use those to build your custom queries.
+* Uses [bytewise](https://github.com/deanlandolt/bytewise) encoding for indices.
+* Supports automatic indexing (using hooks).
+* Works with changing fields.
+
+[![NPM](https://nodei.co/npm/level-indico.png?downloads=true)](https://nodei.co/npm/level-indico/)
+
+[![Build Status](https://travis-ci.org/mariocasciaro/level-indico.png)](https://travis-ci.org/mariocasciaro/level-indico)
+
+
+## Usage
+
+```javascript
+//Indico require a sublevel-enabled db
+var db = indico(sublevel(level('db', { valueEncoding: 'json' }));
+
+//set indices on a sublevel
+var posts = db.sublevel('posts');
+
+//set a single index
+db.ensureIndex('title');
+db.ensureIndex('commentCount');
+//set a compound index
+db.ensureIndex('title', 'commentCount');
+
+//[...] Put some data
+
+//Now query...
+
+//Find all all posts having title = "Hello"
+db.findBy(['title'], {start: ['Hello'], end: ['Hello']}, function (err, data) {
+ //...
+});
+
+//Find all all posts having title = "Hello" AND commentCount >= "1"
+db.findBy(['title', 'commentCount'], {start: ['Hello', 1], end: ['Hello', undefined]}, function (err, data) {
+ //...
+});
+
+//Get all posts sorted by commentCount desc
+db.findBy(['commentCount'], {start: [null], end: [undefined]}, function (err, data) {
+ //...
+});
+
+//Streaming version
+db.streamBy(['commentCount'], {start: [null], end: [undefined]})
+.on('data', function(data) {
+//...
+})
+.on('close, function() {
+//...
+})
+
+```
+
+
+### ...More to come
58 lib/IndexToObjectStream.js
@@ -0,0 +1,58 @@
+var Transform = require('stream').Transform || require('readable-stream').Transform,
+ _ = require('lodash'),
+ util = require('util');
+
+util.inherits(IndexToObjectStream, Transform);
+
+function IndexToObjectStream(mainDb, indexDb, options) {
+ if (!(this instanceof IndexToObjectStream))
+ return new IndexToObjectStream(mainDb, indexDb, options);
+
+ options = options ? _.clone(options) : {};
+ options.objectMode = true;
+
+ Transform.call(this, options);
+ this._mainDb = mainDb;
+ this._indexDb = indexDb;
+ this._options = options;
+}
+
+IndexToObjectStream.prototype._transform = function(chunk, encoding, callback) {
+ var self = this;
+ var key = chunk.value;
+
+ if (self._options.values === false) {
+ self.push(key);
+ return callback();
+ }
+
+ self._mainDb.get(key, function(err, value) {
+ //if the object does not exists or contain not up to date info...then do some cleanup
+ if((err && err.type === 'NotFoundError') ||
+ self._indexDb._indico.indexManager.encodeObject(key, value).toString() !== chunk.key)
+ {
+ self._indexDb.del(chunk.key, function(err) {
+ if(err) {
+ return callback(err);
+ } else {
+ //nothing to return
+ return callback();
+ }
+ });
+ } else if (err) {
+ return callback();
+ } else {
+ if(self._options.keys === false) {
+ self.push(value);
+ } else {
+ self.push({
+ key: key,
+ value: value
+ });
+ }
+ return callback();
+ }
+ });
+};
+
+module.exports = IndexToObjectStream;
44 lib/index.js
@@ -0,0 +1,44 @@
+var _ = require('lodash'),
+ IndexToObjectStream = require('./IndexToObjectStream'),
+ endpoint = require('endpoint'),
+ indexManager = require('./indexManager');
+
+
+module.exports = function(db) {
+ db.ensureIndex = function() {
+ var properties = Array.prototype.slice.call(arguments);
+ indexManager(db, properties);
+ };
+
+ db.findBy = function(props, options, callback) {
+ db.streamBy(props, options).pipe(endpoint({objectMode: true}, callback));
+ };
+
+
+ db.streamBy = function(props, options) {
+ props = _.isArray(props) ? props : [props];
+ var idxManager = indexManager(db, props);
+
+ var transformStreamOptions = _.pick(_.extend({values: true, keys: false}, options), ['values', 'keys']);
+ var indexToValue = new IndexToObjectStream(db, idxManager.indexDb, transformStreamOptions);
+
+ var indexStreamOptions = _.extend({}, options);
+ indexStreamOptions.keys = true;
+ indexStreamOptions.values = true;
+
+ indexStreamOptions.start = _.isArray(indexStreamOptions.start) ?
+ indexStreamOptions.start : [indexStreamOptions.start];
+ indexStreamOptions.end = _.isArray(indexStreamOptions.end) ?
+ indexStreamOptions.end : [indexStreamOptions.end];
+
+ indexStreamOptions.start.push(null);
+ indexStreamOptions.end.push(undefined);
+
+ indexStreamOptions.start = idxManager.encode(indexStreamOptions.start);
+ indexStreamOptions.end = idxManager.encode(indexStreamOptions.end);
+ idxManager.indexDb.createReadStream(indexStreamOptions).pipe(indexToValue);
+ return indexToValue;
+ };
+
+ return db;
+};
67 lib/indexManager.js
@@ -0,0 +1,67 @@
+var _ = require('lodash'),
+ objectPath = require('object-path'),
+ bytewise = require('bytewise');
+
+
+module.exports = function(mainDb, properties) {
+ var indexName = propertiesName(properties);
+ //check the index managers already registered into the mainDb
+ if(!mainDb._indico || !mainDb._indico.indexManagers || !mainDb._indico.indexManagers[indexName]) {
+ objectPath.set(mainDb, "_indico.indexManagers."+indexName, new IndexManager(mainDb, properties));
+ }
+ return mainDb._indico.indexManagers[indexName];
+};
+
+function propertiesName(properties) {
+ return properties && ("IDX_" + properties.join('|'));
+}
+
+
+function IndexManager(mainDb, properties) {
+ if (!(this instanceof IndexManager))
+ return new IndexManager(mainDb, properties);
+
+ this.mainDb = mainDb;
+ this.properties = properties;
+ this.indexName = properties && ("IDX_" + properties.join('|'));
+ this.indexDb = mainDb.sublevel(this.indexName);
+ objectPath.set(this.indexDb, "_indico.indexManager", this);
+ this.registerHooks();
+}
+
+
+IndexManager.prototype.encode = function(values) {
+ return bytewise.encode(values);
+};
+
+
+
+IndexManager.prototype.encodeObject = function(key, entity) {
+ var compositeKey = [];
+ _.each(this.properties, function(prop) {
+ compositeKey.push(objectPath.get(entity, prop));
+ });
+ //we append the key so we can create a stream of entities with same indexes (but different keys)
+ compositeKey.push(key);
+ return bytewise.encode(compositeKey);
+};
+
+
+
+IndexManager.prototype.registerHooks = function() {
+ var self = this;
+
+ //register hook
+ self.removeHook = self.mainDb.pre(function(change, add) {
+ if(change.type === 'put') {
+ add({
+ type: 'put',
+ key: self.encodeObject(change.key, change.value),
+ value: change.key,
+ prefix: self.indexDb
+ });
+ } else if(change.type === 'del') {
+ //TODO cleanup
+ }
+ });
+};
51 package.json
@@ -0,0 +1,51 @@
+{
+ "name": "level-indico",
+ "author": {
+ "name": "Mario Casciaro"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git://github.com/mariocasciaro/level-indico"
+ },
+ "bugs": {
+ "url": "https://github.com/mariocasciaro/level-indico/issues"
+ },
+ "licenses": [
+ {
+ "type": "MIT",
+ "url": "https://github.com/mariocasciaro/level-indico/blob/master/LICENSE"
+ }
+ ],
+ "engines": {
+ "node": ">= 0.8.0"
+ },
+ "description": "Simple indexing and querying for leveldb",
+ "version": "0.1.0",
+ "main": "lib",
+ "dependencies": {
+ "lodash": "~2.0.0",
+ "level-sublevel": "~5.1.1",
+ "bytewise": "~0.6.0",
+ "object-path": "~0.0.1",
+ "readable-stream": "~1.1.9",
+ "endpoint": "~0.4.2"
+ },
+ "devDependencies": {
+ "levelup": "~0.16.0",
+ "leveldown": "~0.8.3",
+ "mocha": "~1.13.0",
+ "chai": "~1.8.0",
+ "async": "~0.2.9"
+ },
+ "scripts": {
+ "test": "node_modules/.bin/mocha test/*.js --reporter spec"
+ },
+ "keywords": [
+ "leveldb",
+ "levelup",
+ "index",
+ "query",
+ "indices",
+ "indexer"
+ ]
+}
149 test/test.js
@@ -0,0 +1,149 @@
+
+var expect = require('chai').expect,
+ indico = require('../lib'),
+ level = require('levelup'),
+ endpoint = require('endpoint'),
+ async = require('async'),
+ sublevel = require('level-sublevel');
+
+
+describe('findBy', function() {
+ var db;
+
+ beforeEach(function(done) {
+ async.series([
+ function(callback) {
+ require('leveldown').destroy('tmpdb', callback);
+ },
+ function(callback) {
+ level('tmpdb', { valueEncoding: 'json' }, function(err, db2) {
+ db = indico(sublevel(db2));
+ callback(err);
+ });
+ }
+ ], done);
+ });
+
+ afterEach(function(done) {
+ db.close(done);
+ });
+
+
+ it('should return empty result if no match found', function(done) {
+ var results = [];
+ db.findBy('title', 'H', function(err, data) {
+ expect(results).to.have.length(0);
+ done();
+ });
+ });
+
+
+ it('should return all objects for the given index', function(done) {
+ db.ensureIndex('title');
+
+ async.waterfall([
+ function(callback) {
+ db.put('123', {title: "Hello", "content": "World"}, callback);
+ },
+ function(callback) {
+ db.put('124', {title: "Hello", "content": "World2"}, callback);
+ },
+ function(callback) {
+ db.put('125', {title: "Helloo", "content": "World3"}, callback);
+ },
+ function(callback) {
+ db.findBy('title', {start: 'Hello', end: 'Hello'}, function (err, data) {
+ expect(err).to.not.exist;
+ expect(data).to.have.length(2);
+ expect(data).to.have.deep.property("0.content", "World");
+ expect(data).to.have.deep.property("1.content", "World2");
+ callback(err);
+ });
+ }
+ ], done);
+ });
+
+
+ it('should work with changing data', function(done) {
+ db.ensureIndex('title');
+
+ async.waterfall([
+ function(callback) {
+ db.put('123', {title: "Hello", "content": "World"}, callback);
+ },
+ function(callback) {
+ db.put('124', {title: "Hello", "content": "World2"}, callback);
+ },
+ function(callback) {
+ db.put('124', {title: "Hello2", "content": "World2"}, callback);
+ },
+ function(callback) {
+ db.put('125', {title: "Helloo", "content": "World3"}, callback);
+ },
+ function(callback) {
+ db.findBy('title', {start: 'Hello', end: 'Hello'}, function (err, data) {
+ expect(err).to.not.exist;
+ expect(data).to.have.length(1);
+ expect(data).to.have.deep.property("0.content", "World");
+ callback(err);
+ });
+ }
+ ], done);
+ });
+
+
+
+ it('should work with multiple indicies', function(done) {
+ db.ensureIndex('title', 'tag');
+
+ async.series([
+ function(callback){
+ db.put('123', {title: "Hello", tag: "M",content: "World"}, callback);
+ },
+ function(callback){
+ db.put('124', {title: "Hello\x00", tag: "M", "content": "World2"}, callback);
+ },
+ function(callback){
+ db.put('125', {title: "Hello", tag: "\00M", "content": "World3"}, callback);
+ },
+ function(callback) {
+ db.findBy(['title', 'tag'], {start: ['Hello', 'M'], end: ['Hello', 'M']}, function (err, data) {
+ expect(err).to.not.exist;
+ expect(data).to.have.length(1);
+ expect(data).to.have.deep.property("0.content", "World");
+ callback(err);
+ });
+ }
+ ], done);
+ });
+
+
+ it('can be used to sort by', function(done) {
+ db.ensureIndex('title', 'len');
+
+ async.series([
+ function(callback){
+ db.put('123', {title: "Hello", len: 31,content: "World1"}, callback);
+ },
+ function(callback){
+ db.put('124', {title: "Helloo", len: 2, "content": "World2"}, callback);
+ },
+ function(callback){
+ db.put('125', {title: "Hello", len: 123, "content": "World3"}, callback);
+ },
+ function(callback){
+ db.put('126', {title: "Hello", len: 1, "content": "World4"}, callback);
+ },
+ function(callback) {
+ db.findBy(['title', 'len'], {start: ['Hello', null], end: ['Hello', undefined]}, function (err, data) {
+ expect(err).to.not.exist;
+ expect(data).to.have.length(3);
+ expect(data).to.have.deep.property("0.content", "World4");
+ expect(data).to.have.deep.property("1.content", "World1");
+ expect(data).to.have.deep.property("2.content", "World3");
+ callback(err);
+ });
+ }
+ ], done);
+ });
+});

0 comments on commit 1eb65ca

Please sign in to comment.