Skip to content

Commit

Permalink
Merge pull request #157 from Level/fixed-native-order
Browse files Browse the repository at this point in the history
Test and document native order
  • Loading branch information
vweevers committed Dec 27, 2018
2 parents 0b1507f + 2ee292d commit 4e1f3ac
Show file tree
Hide file tree
Showing 5 changed files with 227 additions and 7 deletions.
40 changes: 40 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,44 @@ If you desire normalization for keys and values (e.g. to stringify numbers), wra

Another reason you might want to use `encoding-down` is that the structured clone algorithm, while rich in types, can be slower than `JSON.stringify`.

### Sort Order

Unless `level-js` is wrapped with [`encoding-down`][encoding-down], IndexedDB will sort your keys in the following order:

1. number (numeric)
2. date (numeric, by epoch offset)
3. binary (bitwise)
4. string (lexicographic)
5. array (componentwise).

You can take advantage of this fact with `levelup` streams. For example, if your keys are dates, you can select everything greater than a specific date (let's be happy and ignore timezones for a moment):

```js
const db = levelup(leveljs('time-db'))

db.createReadStream({ gt: new Date('2019-01-01') })
.pipe(..)
```

Or if your keys are arrays, you can do things like:

```js
const db = levelup(leveljs('books-db'))

await db.put(['Roald Dahl', 'Charlie and the Chocolate Factory'], {})
await db.put(['Roald Dahl', 'Fantastic Mr Fox'], {})

// Select all books by Roald Dahl
db.createReadStream({ gt: ['Roald Dahl'], lt: ['Roald Dahl', '\xff'] })
.pipe(..)
```

To achieve this on other `abstract-leveldown` implementations, wrap them with [`encoding-down`][encoding-down] and [`charwise`][charwise] (or similar).

#### Known Browser Issues

IE11 and Edge yield incorrect results for `{ gte: '' }` if the database contains any key types other than strings.

### Buffer vs ArrayBuffer

For interoperability it is recommended to use `Buffer` as your binary type. While we recognize that Node.js core modules are moving towards supporting `ArrayBuffer` and views thereof, `Buffer` remains the primary binary type in the Level ecosystem.
Expand Down Expand Up @@ -225,6 +263,8 @@ See the [contribution guide](https://github.com/Level/community/blob/master/CONT

[abstract-leveldown]: https://github.com/Level/abstract-leveldown

[charwise]: https://github.com/dominictarr/charwise

[levelup]: https://github.com/Level/levelup

[leveldown]: https://github.com/Level/leveldown
Expand Down
6 changes: 0 additions & 6 deletions iterator.js
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,6 @@ Iterator.prototype.createKeyRange = function (options) {
var lowerOpen = ltgt.lowerBoundExclusive(options)
var upperOpen = ltgt.upperBoundExclusive(options)

// Temporary workaround for Level/abstract-leveldown#318
if ((Buffer.isBuffer(lower) || typeof lower === 'string') && lower.length === 0) lower = undefined
if ((Buffer.isBuffer(upper) || typeof upper === 'string') && upper.length === 0) upper = undefined
if ((Buffer.isBuffer(lowerOpen) || typeof lowerOpen === 'string') && lowerOpen.length === 0) lowerOpen = undefined
if ((Buffer.isBuffer(upperOpen) || typeof upperOpen === 'string') && upperOpen.length === 0) upperOpen = undefined

if (lower !== undefined && upper !== undefined) {
return IDBKeyRange.bound(lower, upper, lowerOpen, upperOpen)
} else if (lower !== undefined) {
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
]
},
"dependencies": {
"abstract-leveldown": "~6.0.0",
"abstract-leveldown": "~6.0.1",
"immediate": "~3.2.3",
"inherits": "^2.0.3",
"ltgt": "^2.1.2",
Expand Down
1 change: 1 addition & 0 deletions test/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,4 @@ require('./custom-test')(leveljs, test, testCommon)
require('./structured-clone-test')(leveljs, test, testCommon)
require('./key-type-test')(leveljs, test, testCommon)
require('./key-type-illegal-test')(leveljs, test, testCommon)
require('./native-order-test')(leveljs, test, testCommon)
185 changes: 185 additions & 0 deletions test/native-order-test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
'use strict'

var concat = require('level-concat-iterator')

module.exports = function (leveljs, test, testCommon) {
// Type sort order per IndexedDB Second Edition, excluding
// types that aren't supported by all environments.
var basicKeys = [
// Should sort naturally
{ type: 'number', value: '-Infinity', key: -Infinity },
{ type: 'number', value: '2', key: 2 },
{ type: 'number', value: '10', key: 10 },
{ type: 'number', value: '+Infinity', key: Infinity },

// Should sort naturally (by epoch offset)
{ type: 'date', value: 'new Date(2)', key: new Date(2) },
{ type: 'date', value: 'new Date(10)', key: new Date(10) },

// Should sort lexicographically
{ type: 'string', value: '"10"', key: '10' },
{ type: 'string', value: '"2"', key: '2' }
]

makeTest('on basic key types', basicKeys, function (verify) {
// Should be ignored
verify({ gt: undefined })
verify({ gte: undefined })
verify({ lt: undefined })
verify({ lte: undefined })

verify({ gt: -Infinity }, 1)
verify({ gte: -Infinity })
verify({ gt: +Infinity }, 4)
verify({ gte: +Infinity }, 3)

verify({ lt: -Infinity }, 0, 0)
verify({ lte: -Infinity }, 0, 1)
verify({ lt: +Infinity }, 0, 3)
verify({ lte: +Infinity }, 0, 4)

verify({ gt: 10 }, 3)
verify({ gte: 10 }, 2)
verify({ lt: 10 }, 0, 2)
verify({ lte: 10 }, 0, 3)

verify({ gt: new Date(10) }, 6)
verify({ gte: new Date(10) }, 5)
verify({ lt: new Date(10) }, 0, 5)
verify({ lte: new Date(10) }, 0, 6)

// IE 11 and Edge fail this test (yield 0 results), but only when the db
// contains key types other than strings (see strings-only test below).
// verify({ gte: '' }, 6)

verify({ gt: '' }, 6)
verify({ lt: '' }, 0, 6)
verify({ lte: '' }, 0, 6)

verify({ gt: '10' }, 7)
verify({ gte: '10' }, 6)
verify({ lt: '10' }, 0, 6)
verify({ lte: '10' }, 0, 7)

verify({ gt: '2' }, 0, 0)
verify({ gte: '2' }, -1)
verify({ lt: '2' }, 0, -1)
verify({ lte: '2' })
})

makeTest('on string keys only', basicKeys.filter(matchType('string')), function (verify) {
verify({ gt: '' })
verify({ gte: '' })
verify({ lt: '' }, 0, 0)
verify({ lte: '' }, 0, 0)
})

if (leveljs.binaryKeys) {
var binaryKeys = [
// Should sort bitwise
{ type: 'binary', value: 'Uint8Array.from([0, 2])', key: binary([0, 2]) },
{ type: 'binary', value: 'Uint8Array.from([1, 1])', key: binary([1, 1]) }
]

makeTest('on binary keys', basicKeys.concat(binaryKeys), function (verify) {
verify({ gt: binary([]) }, -2)
verify({ gte: binary([]) }, -2)
verify({ lt: binary([]) }, 0, -2)
verify({ lte: binary([]) }, 0, -2)
})
}

if (leveljs.arrayKeys) {
var arrayKeys = [
// Should sort componentwise
{ type: 'array', value: '[100]', key: [100] },
{ type: 'array', value: '["10"]', key: ['10'] },
{ type: 'array', value: '["2"]', key: ['2'] }
]

makeTest('on array keys', basicKeys.concat(arrayKeys), function (verify) {
verify({ gt: [] }, -3)
verify({ gte: [] }, -3)
verify({ lt: [] }, 0, -3)
verify({ lte: [] }, 0, -3)
})
}

if (leveljs.binaryKeys && leveljs.arrayKeys) {
makeTest('on all key types', basicKeys.concat(binaryKeys).concat(arrayKeys))
}

function makeTest (name, input, fn) {
var prefix = 'native order (' + name + '): '
var db

test(prefix + 'open', function (t) {
db = testCommon.factory()
db.open(t.end.bind(t))
})

test(prefix + 'prepare', function (t) {
db.batch(input.map(function (item) {
return { type: 'put', key: item.key, value: item.value }
}), t.end.bind(t))
})

function verify (options, begin, end) {
test(prefix + humanRange(options), function (t) {
t.plan(2)

options.valueAsBuffer = false
concat(db.iterator(options), function (err, result) {
t.ifError(err, 'no concat error')
t.same(result.map(getValue), input.slice(begin, end).map(getValue))
})
})
}

verify({})
if (fn) fn(verify)

test(prefix + 'close', function (t) {
db.close(t.end.bind(t))
})
}
}

function matchType (type) {
return function (item) {
return item.type === type
}
}

function getValue (kv) {
return kv.value
}

// Replacement for TypedArray.from()
function binary (bytes) {
var arr = new Uint8Array(bytes.length)
for (var i = 0; i < bytes.length; i++) arr[i] = bytes[i]
return arr
}

function humanRange (options) {
var a = []

;['gt', 'gte', 'lt', 'lte'].forEach(function (opt) {
if (options.hasOwnProperty(opt)) {
var target = options[opt]

if (typeof target === 'string' || Array.isArray(target)) {
target = JSON.stringify(target)
} else if (Object.prototype.toString.call(target) === '[object Date]') {
target = 'new Date(' + target.valueOf() + ')'
} else if (Object.prototype.toString.call(target) === '[object Uint8Array]') {
target = 'Uint8Array.from([' + target + '])'
}

a.push(opt + ': ' + target)
}
})

return a.length ? a.join(', ') : 'all'
}

0 comments on commit 4e1f3ac

Please sign in to comment.