New directions in LMDB

.gitignore

@@ -1,2 +1,16 @@
 node_modules/
-build/
+build/
+build-pre-gyp/
+Release/
+libleveldb.so
+libleveldb.a
+leakydb
+*.sln
+*.vcxproj
+*.vcxproj.filters
+*.tlog
+*.obj
+*.1sdk.pdb
+*.lastbuildstate
+npm-debug.log
+prebuilds/

bench/db-bench-plot.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+gnuplot <<EOF
+  reset
+  set terminal pngcairo truecolor enhanced font "Ubuntu Mono,13" size 1920, 1080
+  set output "/tmp/5mbench.png"
+  set datafile separator ','
+
+  set logscale y
+  set nologscale y2
+  unset log y2
+  set autoscale y
+  set autoscale y2
+  set ytics nomirror
+  set y2tics
+  set tics out
+
+  set xlabel "Minutes" tc rgb "#777777"
+  set ylabel "Milliseconds per write" tc rgb "#777777"
+  set y2label "Throughput MB/s" tc rgb "#777777"
+
+  set title "Node.js LevelDB (LevelDOWN): 100,000,000 random writes, 64M write buffer, HDD RAID1" tc rgb "#777777"
+  set key left tc rgb "#777777"
+  set border lc rgb "#777777"
+
+  set style line 1 lt 7 ps 1.2 lc rgb "#55019FD7"
+  set style line 2 lt 7 ps 0.1 lc rgb "#55019FD7"
+  set style line 3 lt 1 lw 2   lc rgb "#55019FD7"
+
+  set style line 4 lt 7 ps 1.2 lc rgb "#559ECC3C"
+  set style line 5 lt 7 ps 0.1 lc rgb "#559ECC3C"
+  set style line 6 lt 1 lw 2   lc rgb "#559ECC3C"
+
+  set style line 7 lt 7 ps 1.2 lc rgb "#55CC3C3C"
+  set style line 8 lt 7 ps 0.1 lc rgb "#55CC3C3C"
+  set style line 9 lt 1 lw 2   lc rgb "#55CC3C3C"
+
+  set style line 10 lt 7 ps 1.2 lc rgb "#553C3C3C"
+  set style line 11 lt 7 ps 0.1 lc rgb "#553C3C3C"
+  set style line 12 lt 1 lw 2   lc rgb "#553C3C3C"
+
+  plot \
+      1/0 with points title "Google LevelDB" ls 1 \
+    , 1/0 with points title "Hyper LevelDB"  ls 4 \
+    , 1/0 with points title "Basho LevelDB"  ls 7 \
+    , 1/0 with points title "LMDB"  ls 10 \
+    , "5m_google.csv" using (\$1/1000/60):(\$4/1000000) notitle         ls 2 axes x1y1 \
+    , "5m_hyper.csv"  using (\$1/1000/60):(\$4/1000000) notitle         ls 5 axes x1y1 \
+    , "5m_basho.csv"  using (\$1/1000/60):(\$4/1000000) notitle         ls 8 axes x1y1 \
+    , "5m_lmdb.csv"   using (\$1/1000/60):(\$4/1000000) notitle         ls 11 axes x1y1 \
+    , "5m_google.csv" using (\$1/1000/60):(\$5)         w lines notitle ls 3 axes x1y2 \
+    , "5m_hyper.csv"  using (\$1/1000/60):(\$5)         w lines notitle ls 6 axes x1y2 \
+    , "5m_basho.csv"  using (\$1/1000/60):(\$5)         w lines notitle ls 9 axes x1y2 \
+    , "5m_lmdb.csv"   using (\$1/1000/60):(\$5)         w lines notitle ls 12 axes x1y2 \
+
+EOF

bench/db-bench.js

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+
+const leveldown    = require('../')
+    , fs           = require('fs')
+    , du           = require('du')
+    , rimraf       = require('rimraf')
+
+    , argv         = require('optimist').argv
+
+    , options      = {
+          benchmark        : argv.benchmark
+        , useExisting      : argv.use_existing
+        , db               : argv.db              || __dirname + '/db'
+        , num              : argv.num             || 1000000
+        , concurrency      : argv.concurrency     || 4
+        , cacheSize        : argv.cacheSize       || 8
+        , writeBufferSize  : argv.writeBufferSize || 4
+        , valueSize        : argv.valueSize       || 100
+        , timingOutput     : argv.timingOutput    || __dirname + '/timingOutput'
+        , throughputOutput : argv.throughputOutput
+      }
+
+    , randomString = require('slump').string
+    , keyTmpl      = '0000000000000000'
+
+if (!options.useExisting) {
+  leveldown.destroy(options.db, function () {})
+}
+
+var db          = leveldown(options.db)
+  , timesStream = fs.createWriteStream(options.timingOutput, 'utf8')
+
+function make16CharPaddedKey () {
+  var r = Math.floor(Math.random() * options.num)
+    , k = keyTmpl + r
+  return k.substr(k.length - 16)
+}
+
+timesStream.write('Elapsed (ms), Entries, Bytes, Last 1000 Avg Time, MB/s\n')
+
+function start () {
+  var inProgress  = 0
+    , totalWrites = 0
+    , totalBytes  = 0
+    , startTime   = Date.now()
+    , timesAccum  = 0
+    , writeBuf    = ''
+    , elapsed
+
+  function report () {
+    console.log(
+        'Wrote'
+      , options.num
+      , 'entries in'
+      , Math.floor((Date.now() - startTime) / 1000) + 's,'
+      , (Math.floor((totalBytes / 1048576) * 100) / 100) + 'MB'
+    )
+    timesStream.end()
+
+    du(options.db, function (err, size) {
+      if (err)
+        throw err
+      console.log('Database size:', Math.floor(size / 1024 / 1024) + 'M')
+    })
+  }
+
+
+  function write () {
+    if (totalWrites++ == options.num) {
+      db.close(function () {
+        report(Date.now() - startTime)
+      })
+    }
+    if (inProgress >= options.concurrency || totalWrites > options.num)
+      return
+
+    inProgress++
+
+    if (totalWrites % 100000 === 0)
+      console.log('' + inProgress, totalWrites, Math.round(totalWrites / options.num * 100) + '%')
+
+    if (totalWrites % 1000 === 0) {
+      elapsed = Date.now() - startTime
+      timesStream.write(
+                elapsed
+        + ',' + totalWrites
+        + ',' + totalBytes
+        + ',' + Math.floor(timesAccum / 1000)
+        + ',' + (Math.floor(((totalBytes / 1048576) / (elapsed / 1000)) * 100) / 100)
+        + '\n')
+      timesAccum = 0
+    }
+
+    var time = process.hrtime()
+
+    db.put(make16CharPaddedKey(), randomString({ length: options.valueSize }), function (err) {
+      if (err)
+        throw err
+
+      totalBytes += keyTmpl.length + options.valueSize
+      timesAccum += process.hrtime(time)[1]
+      inProgress--
+      process.nextTick(write)
+    })
+  }
+
+  for (var i = 0; i < options.concurrency; i++)
+    write()
+}
+
+setTimeout(function () {
+  db.open({
+        errorIfExists   : false
+      , createIfMissing : true
+      , cacheSize       : options.cacheSize << 20
+      , writeBufferSize : options.writeBufferSize << 20
+  }, function (err) {
+    if (err)
+      throw err
+
+    start()
+
+  })
+}, 500)

bench/memory.js

@@ -0,0 +1,100 @@
+var leveldown = require('../')
+
+var addr = '1111111111111111111114oLvT2'
+
+var db = leveldown(process.env.HOME + '/iterleak.db')
+var records = {
+  'w/a/14r6JPSJNzBXXJEM2jnmoybQCw3arseKuY/primary': '00',
+  'w/a/17nJuKqjTyAeujSJnPCebpSTEz1v9kjNKg/primary': '00',
+  'w/a/18cxWLCiJMESL34Ev1LJ2meGTgL14bAxfj/primary': '00',
+  'w/a/18pghEAnqCRTrjd7iyUj6XNKmSNx4fAywB/primary': '00',
+  'w/a/19EWPPzY6XkQeM7DxqJ4THbY3DGekRQKbt/primary': '00',
+  'w/a/1DKDeLQyBCkV5aMG15XbAdpwwHnxqw9rvY/primary': '00',
+  'w/a/1HSJAoU5TaGKhAJxwpTC1imiMM1ab8SFGW/primary': '00',
+  'w/a/1HkeafxVvStf2Np6wxUjkTpCt1gTDJSLpi/primary': '00',
+  'w/a/1Hr5JduPFdZ4n4dHBUqRoxLQEG4P93C658/primary': '00',
+  'w/a/1KATodK9Ko8MchJZzDxLhjWz4d8oAuCqEh/primary': '00',
+  'w/a/1NhRKhiAAJrmwXoLhL9dGG1z6oMiFGrxZ7/primary': '00',
+  'w/a/1yTq3DpyUNqUCxDttczGjufbEBKAXMTSq/primary': '00',
+  'w/w/primary': '00'
+}
+
+db.open({
+  createIfMissing: true,
+  errorIfExists: false,
+  compression: true,
+  cacheSize: 8 << 20,
+  writeBufferSize: 4 << 20,
+  maxOpenFiles: 8192
+}, function(err) {
+  if (err)
+    throw err
+
+  memory()
+
+  var batch = db.batch();
+  Object.keys(records).forEach(function(key) {
+    var value = new Buffer(records[key], 'hex')
+    batch.put(key, value)
+  })
+
+  batch.write(function(err) {
+    if (err)
+      throw err
+
+    // This will leak roughly 1mb per call.
+    setTimeout(function self() {
+      var i = 0
+      ;(function next(err) {
+        if (err)
+          throw err
+        if (i++ >= 10000) {
+          memory()
+          return setTimeout(self, 1000)
+        }
+        iterate(addr, next)
+      })()
+    }, 1000)
+  })
+})
+
+function memory() {
+  var mem = process.memoryUsage()
+  console.log('Memory: rss=%dmb, js-heap=%d/%dmb native-heap=%dmb',
+              mb(mem.rss),
+              mb(mem.heapUsed),
+              mb(mem.heapTotal),
+              mb(mem.rss - mem.heapTotal))
+}
+
+function mb(size) {
+  return size / 1024 / 1024 | 0
+}
+
+function iterate(address, callback) {
+  var iter = db.iterator({
+    gte: 'w/a/' + address,
+    lte: 'w/a/' + address + '~',
+    keys: true,
+    values: false,
+    fillCache: false,
+    keyAsBuffer: false
+  })
+
+  ;(function next() {
+    iter.next(function(err, key, value) {
+      if (err) {
+        return iter.end(function(e) {
+          if (e)
+            throw e
+          callback(err)
+        })
+      }
+
+      if (key === undefined)
+        return iter.end(callback)
+
+      next()
+    })
+  })()
+}

bench/write-random-plot.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+gnuplot <<EOF
+  reset
+  set terminal png size 1920, 1080
+  set output "write_random_times.png"
+  set datafile separator ','
+
+  #set yrange [0:0.6]
+  set logscale y
+
+  set xlabel "Seconds"
+  set ylabel "Milliseconds per write"
+
+  set title "1.3G / 10,000,000 writes"
+  set key below
+  set grid
+
+  plot "write_random_times_g32.csv" using (\$1/1000):(\$2/1000000) title "Google LevelDB" lc rgb "red" lt 7 ps 0.3, \
+       "write_random_times_h32.csv" using (\$1/1000):(\$2/1000000) title "HyperDex LevelDB" lc rgb "green" lt 7 ps 0.3, \
+       "write_random_times_b32.csv" using (\$1/1000):(\$2/1000000) title "Basho LevelDB" lc rgb "blue" lt 7 ps 0.3, \
+       "write_random_times.csv" using (\$1/1000):(\$2/1000000) title "LMDB" lc rgb "black" lt 7 ps 0.3
+
+EOF

bench/write-random.js

@@ -0,0 +1,65 @@
+const leveldown   = require('../')
+    , crypto      = require('crypto')
+    , fs          = require('fs')
+    , du          = require('du')
+    , uuid        = require('node-uuid')
+
+    , entryCount  = 10000000
+    , concurrency = 10
+    , timesFile   = './write_random_times.csv'
+    , dbDir       = './write_random.db'
+    , data        = crypto.randomBytes(256) // buffer
+
+var db          = leveldown(dbDir)
+  , timesStream = fs.createWriteStream(timesFile, 'utf8')
+
+function report (ms) {
+  console.log('Wrote', entryCount, 'in', Math.floor(ms / 1000) + 's')
+  timesStream.end()
+  du(dbDir, function (err, size) {
+    if (err)
+      throw err
+    console.log('Database size:', Math.floor(size / 1024 / 1024) + 'M')
+  })
+  console.log('Wrote times to ', timesFile)
+}
+
+db.open(function (err) {
+  if (err)
+    throw err
+
+  var inProgress  = 0
+    , totalWrites = 0
+    , startTime   = Date.now()
+    , writeBuf    = ''
+
+  function write() {
+    if (totalWrites % 100000 == 0) console.log(inProgress, totalWrites)
+
+    if (totalWrites % 1000 == 0) {
+      timesStream.write(writeBuf)
+      writeBuf = ''
+    }
+
+    if (totalWrites++ == entryCount)
+      return report(Date.now() - startTime)
+
+    if (inProgress >= concurrency || totalWrites > entryCount)
+      return
+
+    var time = process.hrtime()
+    inProgress++
+
+    db.put(uuid.v4(), data, function (err) {
+      if (err)
+        throw err
+      writeBuf += (Date.now() - startTime) + ',' + process.hrtime(time)[1] + '\n'
+      inProgress--
+      process.nextTick(write)
+    })
+
+    process.nextTick(write)
+  }
+
+  write()
+})