Skip to content
Browse files

initial

  • Loading branch information...
0 parents commit 5afb8d8e3de8195263595b2981b1baa9293d3ed8 @Raynos Raynos committed Feb 14, 2013
Showing with 487 additions and 0 deletions.
  1. +15 −0 .gitignore
  2. +14 −0 .npmignore
  3. +11 −0 .testem.json
  4. +4 −0 .travis.yml
  5. +19 −0 LICENCE
  6. +108 −0 README.md
  7. +30 −0 index.js
  8. +70 −0 package.json
  9. +156 −0 test/index.js
  10. +11 −0 test/static/index.html
  11. +49 −0 test/static/test-adapter.js
15 .gitignore
@@ -0,0 +1,15 @@
+.DS_Store
+.monitor
+.*.swp
+.nodemonignore
+releases
+*.log
+*.err
+fleet.json
+public/browserify
+bin/*.json
+.bin
+build
+compile
+.lock-wscript
+node_modules
14 .npmignore
@@ -0,0 +1,14 @@
+.DS_Store
+.monitor
+.*.swp
+.nodemonignore
+releases
+*.log
+*.err
+fleet.json
+public/browserify
+bin/*.json
+.bin
+build
+compile
+.lock-wscript
11 .testem.json
@@ -0,0 +1,11 @@
+{
+ "launchers": {
+ "node": {
+ "command": "npm test"
+ }
+ },
+ "src_files": [
+ "./**/*.js"
+ ],
+ "launch_in_dev": ["node"]
+}
4 .travis.yml
@@ -0,0 +1,4 @@
+language: node_js
+node_js:
+ - 0.8
+ - 0.9
19 LICENCE
@@ -0,0 +1,19 @@
+Copyright (c) 2013 Colingo.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
108 README.md
@@ -0,0 +1,108 @@
+# incremental-map-reduce
+
+[![build status][1]][2] [![dependency status][3]][4]
+
+[![browser support][5]][6]
+
+incremental map reduce function for mongodb
+
+## Example
+
+To run an incremental map reduce you have to have a raw collection
+that you want to map reduce and a target collection where you
+reduce the data into.
+
+The raw collection must have some kind of timestamp so that we
+can query it to only reduce values since the last timestamp.
+
+The output into the reduced collection must also have some
+kind of timestamp which we can use to query the reduced collection
+for the latest document that was reduced.
+
+As long as those two are upheld we can run an incremental map reduce
+which means only re-reducing new data which allows for efficient
+map reduces on large append only collections.
+
+```js
+var incrementalMapReduce = require("incremental-map-reduce")
+var passback = require("callback-reduce/passback")
+var mongo = require("mongodb")
+
+// Get a mongo db in whatever way you prefer
+var db = someMongoDb
+/* A collection containing
+
+{
+ id: someId
+ , count: numberOfItemsOrSomething
+ , timestamp: lastUpdatedTime
+}
+
+*/
+var rawCollection = db.collection("raw-data")
+var reducedCollection = db.collection("reduced.some-data")
+
+var result = incrementalMapReduce(rawCollection, {
+ reducedCollection: reducedCollection
+ , map: map
+ , reduce: reduce
+ , options: {
+ out: {
+ reduce: "reduced.some-data"
+ }
+ , finalize: finalize
+ }
+})
+
+passback(result, function (err, collection) {
+ /* result of rawCollection.mapReduce(...) */
+})
+
+function map() {
+ /*global emit*/
+ emit(this.id, this)
+}
+
+function reduce(key, values) {
+ var result = {
+ id: values[0].id
+ , count: 0
+ , timestamp: 0
+ }
+
+ values.forEach(function (value) {
+ result.count += value.count
+
+ if (value.timestamp > result.timestamp) {
+ result.timestamp = value.timestamp
+ }
+ })
+
+ return result
+}
+
+function finalize(key, value) {
+ return {
+ id: value.id
+ , count: value.count
+ , timestamp: value.timestamp
+ }
+}
+```
+
+## Installation
+
+`npm install incremental-map-reduce`
+
+## Contributors
+
+ - Raynos
+
+## MIT Licenced
+
+ [1]: https://secure.travis-ci.org/Colingo/incremental-map-reduce.png
+ [2]: http://travis-ci.org/Colingo/incremental-map-reduce
+ [3]: http://david-dm.org/Colingo/incremental-map-reduce/status.png
+ [4]: http://david-dm.org/Colingo/incremental-map-reduce
+ [5]: http://ci.testling.com/Colingo/incremental-map-reduce.png
+ [6]: http://ci.testling.com/Colingo/incremental-map-reduce
30 index.js
@@ -0,0 +1,30 @@
+var findOne = require("mongo-client/findOne")
+var mapReduce = require("mongo-client/mapReduce")
+var expand = require("reducers/expand")
+
+module.exports = incrementalMapReduce
+
+function incrementalMapReduce(col, opts) {
+ var timestampPath = opts.timestampPath || "timestamp"
+ var lastTimestampPath = opts.lastTimestampPath || "timestamp"
+
+ var lastValue = findOne(opts.reducedCollection, {}, {
+ sort: [["value." + lastTimestampPath, -1]]
+ })
+
+ var result = expand(lastValue, function (doc) {
+ var lastTimestamp = doc && doc.value && doc.value[lastTimestampPath]
+ var options = opts.options || {}
+
+ if (lastTimestamp) {
+ options.query = options.query || {}
+ options.query[timestampPath] = {
+ $gt: lastTimestamp
+ }
+ }
+
+ return mapReduce(col, opts.map, opts.reduce, options)
+ })
+
+ return result
+}
70 package.json
@@ -0,0 +1,70 @@
+{
+ "name": "incremental-map-reduce",
+ "version": "0.1.0",
+ "description": "incremental map reduce function for mongodb",
+ "keywords": [],
+ "author": "Raynos <raynos2@gmail.com>",
+ "repository": "git://github.com/Colingo/incremental-map-reduce.git",
+ "main": "index",
+ "homepage": "https://github.com/Colingo/incremental-map-reduce",
+ "contributors": [
+ {
+ "name": "Raynos"
+ }
+ ],
+ "bugs": {
+ "url": "https://github.com/Colingo/incremental-map-reduce/issues",
+ "email": "raynos2@gmail.com"
+ },
+ "dependencies": {
+ "reducible": "~1.0.5",
+ "mongo-client": "~0.1.5",
+ "reducers": "git://github.com/Raynos/reducers"
+ },
+ "devDependencies": {
+ "tape": "~0.2.2",
+ "browserify": "https://github.com/raynos/node-browserify/tarball/master",
+ "testem": "~0.2.56",
+ "node-uuid": "~1.4.0",
+ "callback-reduce": "~1.1.0"
+ },
+ "licenses": [
+ {
+ "type": "MIT",
+ "url": "http://github.com/Colingo/incremental-map-reduce/raw/master/LICENSE"
+ }
+ ],
+ "scripts": {
+ "test": "node ./test",
+ "build": "browserify test/index.js -o test/static/bundle.js",
+ "testem": "testem"
+ },
+ "testling": {
+ "files": "test/index.js",
+ "browsers": {
+ "ie": [
+ "8",
+ "9",
+ "10"
+ ],
+ "firefox": [
+ "16",
+ "17",
+ "nightly"
+ ],
+ "chrome": [
+ "22",
+ "23",
+ "canary"
+ ],
+ "opera": [
+ "12",
+ "next"
+ ],
+ "safari": [
+ "5.1"
+ ]
+ }
+ },
+ "private": true
+}
156 test/index.js
@@ -0,0 +1,156 @@
+var test = require("tape")
+var uuid = require("node-uuid")
+
+var expand = require("reducers/expand")
+var take = require("reducers/take")
+var mongo = require("mongo-client")
+var insert = require("mongo-client/insert")
+var find = require("mongo-client/find")
+var passback = require("callback-reduce/passback")
+
+var incrementalMapReduce = require("../index")
+
+var client = mongo("mongodb://localhost/incremental-map-reduce:test")
+var reducedCollectionName = uuid()
+var rawCollection = client(uuid())
+var reducedCollection = client(reducedCollectionName)
+var ts = Date.now()
+
+test("can run incremental map reduce", function (assert) {
+ var insertion = insert(rawCollection, [{
+ id: "1"
+ , count: 22
+ , timestamp: ts + 1
+ }, {
+ id: "2"
+ , count: 23
+ , timestamp: ts + 2
+ }, {
+ id: "1"
+ , count: 40
+ , timestamp: ts + 3
+ }])
+
+ var mapReduce = expand(take(insertion, 1), function () {
+ return incrementalMapReduce(rawCollection, {
+ reducedCollection: reducedCollection
+ , map: map
+ , reduce: reduce
+ , options: {
+ out: {
+ reduce: reducedCollectionName
+ }
+ , finalize: finalize
+ }
+ })
+ })
+
+ var results = expand(mapReduce, function () {
+ return find(reducedCollection, {})
+ })
+
+ passback(results, Array, function (err, results) {
+ assert.ifError(err)
+
+ assert.deepEqual(results, [{
+ _id: "1"
+ , value: {
+ id: "1"
+ , count: 62
+ , timestamp: ts + 3
+ }
+ }, {
+ _id: "2"
+ , value: {
+ id: "2"
+ , count: 23
+ , timestamp: ts + 2
+ }
+ }])
+
+ assert.end()
+ })
+})
+
+test("doesn't re-run old data", function (assert) {
+ var insertion = insert(rawCollection, [{
+ id: "1"
+ , count: 100
+ , timestamp: ts + 2
+ }, {
+ id: "2"
+ , count: 50
+ , timestamp: ts + 4
+ }])
+
+ var mapReduce = expand(take(insertion, 1), function () {
+ return incrementalMapReduce(rawCollection, {
+ reducedCollection: reducedCollection
+ , map: map
+ , reduce: reduce
+ , options: {
+ out: {
+ reduce: reducedCollectionName
+ }
+ , finalize: finalize
+ }
+ })
+ })
+
+ var results = expand(mapReduce, function () {
+ return find(reducedCollection, {})
+ })
+
+ passback(results, Array, function (err, results) {
+ assert.ifError(err)
+
+ assert.deepEqual(results, [{
+ _id: "1"
+ , value: {
+ id: "1"
+ , count: 62
+ , timestamp: ts + 3
+ }
+ }, {
+ _id: "2"
+ , value: {
+ id: "2"
+ , count: 73
+ , timestamp: ts + 4
+ }
+ }])
+
+ assert.end()
+ })
+})
+
+function map() {
+ /*global emit*/
+ emit(this.id, this)
+}
+
+function reduce(key, values) {
+ var result = {
+ id: values[0].id
+ , count: 0
+ , timestamp: 0
+ }
+
+ values.forEach(function (value) {
+ result.count += value.count
+
+ if (value.timestamp > result.timestamp) {
+ result.timestamp = value.timestamp
+ }
+ })
+
+ return result
+}
+
+function finalize(key, value) {
+ return {
+ id: value.id
+ , count: value.count
+ , timestamp: value.timestamp
+ }
+}
11 test/static/index.html
@@ -0,0 +1,11 @@
+<!doctype html>
+<html>
+<head>
+ <title>TAPE Example</title>
+ <script src="/testem.js"></script>
+ <script src="test-adapter.js"></script>
+ <script src="bundle.js"></script>
+</head>
+<body>
+</body>
+</html>
49 test/static/test-adapter.js
@@ -0,0 +1,49 @@
+(function () {
+ var Testem = window.Testem
+ var regex = /^((?:not )?ok) (\d+) (.+)$/
+
+ Testem.useCustomAdapter(tapAdapter)
+
+ function tapAdapter(socket){
+ var results = {
+ failed: 0
+ , passed: 0
+ , total: 0
+ , tests: []
+ }
+
+ socket.emit('tests-start')
+
+ Testem.handleConsoleMessage = function(msg){
+ var m = msg.match(regex)
+ if (m) {
+ var passed = m[1] === 'ok'
+ var test = {
+ passed: passed ? 1 : 0,
+ failed: passed ? 0 : 1,
+ total: 1,
+ id: m[2],
+ name: m[3],
+ items: []
+ }
+
+ if (passed) {
+ results.passed++
+ } else {
+ results.failed++
+ }
+
+ results.total++
+
+ socket.emit('test-result', test)
+ results.tests.push(test)
+ } else if (msg === '# ok' || msg.match(/^# tests \d+/)){
+ socket.emit('all-test-results', results)
+ }
+
+ // return false if you want to prevent the console message from
+ // going to the console
+ // return false
+ }
+ }
+}())

0 comments on commit 5afb8d8

Please sign in to comment.
Something went wrong with that request. Please try again.