id -> key

docs/api.md

@@ -48,11 +48,11 @@ Gets a key, calls callback with `(error, value)`. `value` is a JS object
db.put([key], value, [opts], cb)
```

Puts value into the database by key. Specify the key you want by setting it as either `key` or `value.id`, e.g. `db.put({id: 'bob'} ... )`. `key` is optional to be compatible with the levelup API
Puts value into the database by key. Specify the key you want by setting it as either `key` or `value.key`, e.g. `db.put({key: 'bob'} ... )`. `key` is optional to be compatible with the levelup API

`cb` will be called with `(error, newVersion)` where `newVersion` will be be a JS object with `id` and `version` properties.
`cb` will be called with `(error, newVersion)` where `newVersion` will be be a JS object with `key` and `version` properties.

If something already exists in the database with the key you specified you may receive a conflict error. To ensure you do not overwrite data accidentally you must pass in the current version of the key you wish to update, e.g. if `bob` is in the database at version 1 and you want to update it to add a `foo` key: `db.put({id: 'bob', version: 1, 'foo': 'bar'})`, which will update the row to version 2.
If something already exists in the database with the key you specified you may receive a conflict error. To ensure you do not overwrite data accidentally you must pass in the current version of the key you wish to update, e.g. if `bob` is in the database at version 1 and you want to update it to add a `foo` key: `db.put({key: 'bob', version: 1, 'foo': 'bar'})`, which will update the row to version 2.

All versions of all rows are persisted and replicated.

@@ -117,7 +117,7 @@ var keyStream = db.createKeyStream([opts])

Returns a [key stream](https://github.com/rvagg/node-levelup#createKeyStream) over the most recent version of all keys in the dat store.

By default the returned stream is a readable object stream that will emit 1 JS object per row in the form `{id: key, version: number, deleted: boolean}`. This differs slightly from levelup where the value stream is not an object stream by default. Dat stores the id, version and deleted status in the key on disk which is why all 3 properties are returned by this stream.
By default the returned stream is a readable object stream that will emit 1 JS object per row in the form `{key: key, version: number, deleted: boolean}`. This differs slightly from levelup where the value stream is not an object stream by default. Dat stores the key, version and deleted status in the key on disk which is why all 3 properties are returned by this stream.

### Options
* `start` (defaults to the beginning of the possible keyspace) - key to start iterating from
@@ -133,7 +133,7 @@ var changes = db.createChangesStream([opts])

Returns a read stream that iterates over the dat store change log (a log of all CRUD in the history of the database).

Changes are emitted as JS objects that look like `{change: 352, id: 'foo', version: 2}`
Changes are emitted as JS objects that look like `{change: 352, key: 'foo', version: 2}`

### Options

@@ -163,8 +163,8 @@ You can write:
* `csv` - setting to true is equivalent to `{format: 'csv'}`
* `json` - setting to true is equivalent to `{format: 'json'}`
* `objects` - setting to true is equivalent to `{format: 'objects'}`
* `primary` (default `id`) - the column or array of columns to use as the primary key
* `hash` (default `false`) - if true `id` will be set to the md5 hex hash of the string of the primary key(s)
* `primary` (default `key`) - the column or array of columns to use as the primary key
* `hash` (default `false`) - if true `key` will be set to the md5 hex hash of the string of the primary key(s)
* `primaryFormat` - a function that formats the key before it gets inserted. accepts `(val)` and must return a string to set as the key.
* `columns` - specify the column names to use when parsing multibuffer/csv. Mandatory for multibuffer, optional for csv (csv headers are automatically parsed but this can be used to override them)
* `headerRow` (default `true`) - set to false if your csv doesn't have a header row. you'll also have to manually specify `columns`
@@ -174,7 +174,7 @@ You can write:
## createVersionStream

```js
var versions = db.createVersionStream(id, [opts])
var versions = db.createVersionStream(key, [opts])
```

Returns a read stream that emits all versions of a given key

lib/commands.js

@@ -375,7 +375,7 @@ dat.headers = function(options, cb) {
    return col.name
  });

  var headers = ['id', 'version'].concat(cols)
  var headers = ['key', 'version'].concat(cols)
  if (!cb) return headers
  cb(null, headers)
}
@@ -401,7 +401,7 @@ dat.createReadStream = dat.readStream = function(opts) {
  
  function keyValueFormatter() {
    return through.obj(function(obj, enc, cb) {
      this.push({key: obj.id, value: obj})
      this.push({key: obj.key, value: obj})
      cb()
    })
  }
@@ -452,7 +452,7 @@ dat.createBlobWriteStream = function(options, doc, cb) {
  }
  
  if (typeof doc === 'string') {
    doc = { id: doc }
    doc = { key: doc }
  }
  
  if (typeof options === 'string') {
@@ -509,8 +509,8 @@ dat.createWriteStream = dat.writeStream = function(options) {
  return writeStream(this, options)
}

dat.createVersionStream = function(id, options) {
  return this.storage.createVersionStream(id, options)
dat.createVersionStream = function(key, options) {
  return this.storage.createVersionStream(key, options)
}

dat.close = function(cb) {

lib/document.js

@@ -15,27 +15,27 @@ module.exports.decodeChange = decodeChange
module.exports.decodeKey = decodeKey

function updateVersion(doc, rowBuffer, columns) {
  var id = doc.id
  if (typeof id !== 'string') id = cuid()
  var key = doc.key
  if (typeof key !== 'string') key = cuid()
  var prev = 0
  if (doc.version) {
    prev = +doc.version
  }
  var ver = prev + 1
  return extend({}, doc, {version: ver, id: id})
  return extend({}, doc, {version: ver, key: key})
}

function rowKeys(keys, sep, id, version, change, deleted) {
function rowKeys(keys, sep, keyString, version, change, deleted) {
  var versionNum = +version
  var varChange = pack(change)
  var changeKey = key(sep, keys.change, varChange)
  var verString = pack(versionNum)
  var row = sep + keys.data + sep + id + sep + verString
  var row = sep + keys.data + sep + keyString + sep + verString
  if (deleted) row += sep + '1' // deleted is boolean
  return {
    row: row,
    change: changeKey,
    cur: key(sep, keys.cur, id)
    cur: key(sep, keys.cur, keyString)
  }
}

@@ -79,7 +79,7 @@ function decodeKey(key) {
  var parts = key.split('ÿ')
  var ver = parts[3]
  var obj = {
    'id': parts[2],
    'key': parts[2],
    'version': unpack(ver)
  }
  if (parts[5]) obj.deleted = true
@@ -90,7 +90,7 @@ function decodeKey(key) {
function decodeChange(key) {
  var parts = JSON.parse(key)
  return {
    'id': parts[1],
    'key': parts[1],
    'version': parts[2],
    'change': +parts[0],
    'deleted': parts[3]

lib/meta.js

@@ -12,7 +12,7 @@ function Meta(dat, ready) {
  if (!(this instanceof Meta)) return new Meta(dat, ready)
  if (!ready) ready = function noop() {}
  this.dat = dat
  this.reserved = ['id', 'change', 'version', 'deleted']
  this.reserved = ['key', 'change', 'version', 'deleted']
  this.rowCount = 0
  this.file = path.join(this.dat.dir, '.dat', 'schema.json')
  

lib/replicator.js

@@ -31,7 +31,7 @@ Replicator.prototype.createPullStream = function(remote, options) {
    for (var i = 0; i < attachments.length; i++) {
      var filename = attachments[i]
      var attachment = obj.attachments[filename]
      var attachmentReq = request(remote + '/api/' + obj.id + '/' + filename)
      var attachmentReq = request(remote + '/api/' + obj.key + '/' + filename)
      attachmentReq.on('error', function(err) {
        blobFetcher.emit('error', err)
      })

lib/rest-handler.js

@@ -34,17 +34,17 @@ RestHandler.prototype.createRoutes = function() {
  router.addRoute("/api/json", this.exportJson.bind(this))
  router.addRoute("/api/archive", this.archive.bind(this))
  router.addRoute("/api", this.document.bind(this))
  router.addRoute("/api/:id", this.document.bind(this))
  router.addRoute("/api/:id/:filename", this.blob.bind(this))
  router.addRoute("/api/:key", this.document.bind(this))
  router.addRoute("/api/:key/:filename", this.blob.bind(this))
  router.addRoute("*", this.notFound.bind(this))
  return router
}

RestHandler.prototype.blob = function(req, res, opts) {
  var self = this
  if (req.method === 'GET') {
    var id = opts.id
    var blob = self.dat.createBlobReadStream(opts.id, opts.filename, opts)
    var key = opts.key
    var blob = self.dat.createBlobReadStream(opts.key, opts.filename, opts)
    blob.on('error', function(err) {
      return self.error(res, 404, {"error": "Not Found"})
    })
@@ -56,13 +56,13 @@ RestHandler.prototype.blob = function(req, res, opts) {
    var reqUrl = url.parse(req.url, true)
    var qs = reqUrl.query
    var doc = {
      id: opts.id,
      key: opts.key,
      version: qs.version
    }
    self.auth(req, res, function(err) {
      if (err) return self.authError(req, res)
      var id = doc.id
      self.dat.get(id, { version: doc.version }, function(err, existing) {
      var key = doc.key
      self.dat.get(key, { version: doc.version }, function(err, existing) {
        if (existing) {
          if (existing.attachments) existing.attachments = JSON.parse(existing.attachments)
          doc = existing
@@ -196,7 +196,7 @@ RestHandler.prototype.json = function(res, json) {

RestHandler.prototype.get = function(req, res, opts) {
  var self = this
  this.dat.get(opts.id, url.parse(req.url, true).query || {}, function(err, json) {
  this.dat.get(opts.key, url.parse(req.url, true).query || {}, function(err, json) {
    if (err && err.message === 'range not found') return self.error(res, 404, {error: "Not Found"})
    if (err) return self.error(res, 500, err.message)
    if (json === null) return self.error(res, 404, {error: "Not Found"})
@@ -221,7 +221,7 @@ RestHandler.prototype.post = function(req, res) {

RestHandler.prototype.delete = function(req, res, opts) {
  var self = this
  self.dat.delete(opts.id, function(err, stored) {
  self.dat.delete(opts.key, function(err, stored) {
    if (err) return self.error(res, 500, err)
    self.json(res, {deleted: true})
  })
@@ -249,7 +249,7 @@ RestHandler.prototype.bulk = function(req, res) {
RestHandler.prototype.document = function(req, res, opts) {
  var self = this
  if (req.method === "GET") {
    if (!opts.id) return self.hello(req, res)
    if (!opts.key) return self.hello(req, res)
    return this.get(req, res, opts)
  }
  this.auth(req, res, function(err) {

lib/schema.js

@@ -6,7 +6,7 @@ var Schema = function(storage, onReady) {
  this.protobuf = null
  this.columns = []
  this.index = {}
  this.reserved = ['id', 'change', 'version', 'deleted']
  this.reserved = ['key', 'change', 'version', 'deleted']
  this.update(onReady)
}

lib/storage.js

@@ -158,19 +158,19 @@ Database.prototype.put = function (key, val, opts, cb) {
  if (!opts) opts = {}
  
  if (Buffer.isBuffer(val)) { // assume val is a protobuf
    doc = {id: key}
    doc = {key: key}
    if (opts.version) doc.version = opts.version
  } else if (val) {
    // assume val is an Object
    doc = val
    val = undefined
    // key overrides val.id
    if (key) doc.id = key
    // key overrides val.key
    if (key) doc.key = key
  } else {
    throw new Error('put() requires a value')
  }
  
  // at this point doc should be an object with id === key (or falsy to get an auto-gen'd key)
  // at this point doc should be an object with a .key (or falsy to get an auto-gen'd key)
  
  debug('args', [key, val, opts, typeof cb])
  
@@ -188,13 +188,13 @@ Database.prototype.put = function (key, val, opts, cb) {
    if (err) return cb(err)
    
    // TODO implement primary + hash options from writeStream here (see writeStream.writeBatch)
    if (!doc.id) {
    if (!doc.key) {
      isNew = true
      return store()
    }
    
    debug('check', doc.id)
    self.get(doc.id, function(err, existing) {
    debug('check', doc.key)
    self.get(doc.key, function(err, existing) {
      if (err) {
        isNew = true
        return store()
@@ -218,12 +218,12 @@ Database.prototype.put = function (key, val, opts, cb) {
    
    var change = self.change = self.change + 1
    
    var keys = docUtils.rowKeys(self.keys, self.sep, updated.id, updated.version, change, updated._deleted)
    var keys = docUtils.rowKeys(self.keys, self.sep, updated.key, updated.version, change, updated._deleted)
    
    opts.valueEncoding = 'binary'
    
    if (!val) val = self.schema.encode(updated)
    var changeVal = [change, updated.id, updated.version]
    var changeVal = [change, updated.key, updated.version]
    
    if (updated._deleted) {
      changeVal.push(true) // 4th spot in changeVal array is a deleted boolean
@@ -295,7 +295,7 @@ Database.prototype.createReadStream = function(opts) {
    var key = self._key(self.keys.data, currentKey[2] + self.sep + row.value)
    // internal option, not documented, used to implement createKeyStream
    if (opts.keysOnly) {
      var obj = {id: currentKey[2]}
      var obj = {key: currentKey[2]}
      if (currentVal[0]) obj.version = docUtils.unpack(currentVal[0])
      if (currentVal[1] === '1') obj.deleted = true
      stream.push(obj)
@@ -325,14 +325,14 @@ Database.prototype.createReadStream = function(opts) {
  return stream
}

// gets all versions of a id
Database.prototype.createVersionStream = function (id, opts) {
// gets all versions of a key
Database.prototype.createVersionStream = function (key, opts) {
  var self = this
  if (!opts) opts = {}
  if (typeof opts.valueEncoding === 'undefined') opts.valueEncoding = 'binary'
  
  if (!opts.start) opts.start = self._key(self.keys.data, id)
  if (!opts.end) opts.end = self._key(self.keys.data, id + self.sep + self.sep)
  if (!opts.start) opts.start = self._key(self.keys.data, key)
  if (!opts.end) opts.end = self._key(self.keys.data, key + self.sep + self.sep)
  
  var stream = through.obj(write)
  
@@ -435,7 +435,7 @@ Database.prototype.createChangesStream = function (opts) {
  function getWrite(changeRow, end, next) {
    var change = docUtils.decodeChange(changeRow.value)
    var entry = { 
      id: change.id,
      key: change.key,
      change: change.change,
      version: change.version
    }
@@ -444,7 +444,7 @@ Database.prototype.createChangesStream = function (opts) {
      // even if it was deleted we do a get to ensure correct ordering by relying on the mutex
      var getOpts = { version: entry.version }
      pending++
      self.get(entry.id, getOpts, function (e, value) {
      self.get(entry.key, getOpts, function (e, value) {
        if (!entry.deleted) entry.data = value
        getStream.push(entry)
        if (--pending === 0 && ended) {

lib/write-stream.js

@@ -57,7 +57,7 @@ module.exports = function(dat, opts) {
      struct = structs[i]
      var putOpts = {overwrite: opts.overwrite, skipSchemaCheck: true }
      putOpts.version = struct.version
      dat.put(struct.id, struct.data, putOpts, rowDone)
      dat.put(struct.key, struct.data, putOpts, rowDone)
    }
  }
}
@@ -97,7 +97,7 @@ function csvOptions(opts) {
}

function primaryKeyStream(opts, schema) {
  var primary = [].concat(opts.primary || 'id')
  var primary = [].concat(opts.primary || 'key')
  var format = opts.primaryFormat
  var separator = opts.separator || ''

@@ -113,7 +113,7 @@ function primaryKeyStream(opts, schema) {
      var priKey = doc[pri]

      if (format) priKey = format(priKey)
      if (!priKey && pri === 'id') priKey = cuid()
      if (!priKey && pri === 'key') priKey = cuid()
      if (key) key += separator

      key += priKey
@@ -127,6 +127,6 @@ function primaryKeyStream(opts, schema) {
// for mad v8 speed
function KeyStruct(version, key, data) {
  this.version = version
  this.id = key
  this.key = key
  this.data = data
}

test/tests/cli.js

@@ -46,7 +46,7 @@ module.exports.importCSV = function(test, common) {
              if (l !== '') rows.push(JSON.parse(l))
            })
            t.equal(rows.length, 3)
            rows.map(function(r) { t.ok(r.id, 'row has id') })
            rows.map(function(r) { t.ok(r.key, 'row has key') })
            common.destroyTmpDats(function() {
              t.end()
            })