async map-reduce functions for nodejs
JavaScript
Latest commit 3060b58 Jul 31, 2016 @dominictarr committed on GitHub Merge pull request #25 from jamesbrown0/patch-1
Fix a typo
Permalink
Failed to load latest commit information.
test more debugable Jul 19, 2014
.gitignore dotfiles Nov 17, 2012
.travis.yml travis May 28, 2013
LICENSE open source Nov 17, 2012
README.md Fix a typo Jul 30, 2016
example.js and == 0 test Jun 24, 2013
index.js tidy Sep 11, 2014
package.json 6.0.2 Sep 11, 2014
range.js tidy Jul 19, 2014
testem.json testem support Nov 18, 2012

README.md

Map Reduce for leveldb (via levelup)

Incremental map-reduces and real-time results.

Build Status

Waat?

An "incremental map reduce" means when you update one key, only a relevant portion of the data needs to be recalculated.

"real-time results" means that you can listen to the database, and recieve change notifications on the fly! a la level-live-stream

If you just want something very simple, like mapping the date a blog post is created to the blog, then level-index may be enough.

Example

create a simple map-reduce

var LevelUp   = require('levelup')
var SubLevel  = require('level-sublevel')
var MapReduce = require('map-reduce')

var db = SubLevel(LevelUp(file))

var mapDb = 
  MapReduce(
    db, //the parent db
    'example',  //name.
    function (key, value, emit) {
      //perform some mapping.
      var obj = JSON.parse(value)
      //emit(key, value)
      //key may be an array of strings. 
      //value must be a string or buffer.
      emit(['all', obj.group], ''+obj.lines.length)
    },
    function (acc, value, key) {
      //reduce little into big
      //must return a string or buffer.
      return ''+(Number(acc) + Number(value))
    },
    //pass in the initial value for the reduce.
    //*must* be a string or buffer.
    '0'
  })
})

map-reduce uses level-trigger to make map reduces durable.

querying results.

  //get all the results in a specific group
  //start:[...] implies end:.. to be the end of that group.
  mapDb.createReadStream({range: ['all', group]}) 

  //get all the results in under a group.
  mapDb.createReadStream({range: ['all', true]}) 

  //get all the top level 
  mapDb.createReadStream({range: [true]})

complex aggregations

map-reduce with multiple levels of aggregation.

suppose we are building a database of all the street-food in the world. the data looks like this:

{
  country: USA | Germany | Cambodia, etc...
  state:   CA | NY | '', etc...
  city: Oakland | New York | Berlin | Phnom Penh, etc...
  type: taco | chili-dog | doner | noodles, etc...
}

We will aggregate to counts per-region, that look like this:

//say: under the key USA
{
  'taco': 23497,
  'chili-dog': 5643,
  etc...
}

first we'll map the raw data to ([country, state, city],type) tuples. then we'll count up all the instances of a particular type in that region!

var LevelUp   = require('levelup')
var SubLevel  = require('level-sublevel')
var MapReduce = require('map-reduce')

var db = SubLevel(LevelUp(file))
var mapDb = 
  MapReduce(
    db,
    'streetfood',
    function (key, value, emit) {
      //perform some mapping.
      var obj = JSON.parse(value)
      //emit(key, value)
      //key may be an array of strings. 
      //value must be a string or buffer.
      emit(
        [obj.country, obj.state || '', obj.city],
        //notice that we are just returning a string.
        JSON.stringify(obj.type)
      )
    },
    function (acc, value) {
      acc = JSON.parse(acc)
      value = JSON.parse(value)
      //check if this is top level data, like 'taco' or 'noodle'
      if('string' === typeof value) {
        //increment by one (remember to set as a number if it was undefined)
        acc[value] = (acc[value] || 0) ++
        return JSON.stringify(acc)
      }
      //if we get to here, we are combining two aggregates.
      //say, all the cities in a state, or all the countries in the world.
      //value and acc will both be objects {taco: number, doner: number2, etc...}

      for(var type in value) {
        //add the counts for each type together...
        //remembering to check that it is set as a value...
        acc[type] = (acc[type] || 0) + value[type]
      }
      //stringify the object, so that it can be written to disk!
      return JSON.stringify(acc)
    },
    '{}')

then query it like this:

mapDb.createReadStream({range: ['USA', 'CA', true]})
  .pipe(...)

retrive a specific result

pass db.get an array, and you can retrive a specific value, by group.

var userMapping = require("map-reduce")(
    db,
    "userPoints",
    function(key, value, emit){
        value = JSON.parse(value);
        var date = new Date(value.created);
        emit([value.user, date.getYear(), date.getMonth()], value.amount);
    },
    function(acc, value){
        return (Number(acc) + Number(value)).toString();
    },
    0
);

function getTotalPointsForUser(user, year, month, cb){
    userMapping.get([user, year, month], cb);
}

License

MIT