Skip to content

Commit 9b78443

Browse files
committed
Keep track of iterator end (#56)
Closes #19.
1 parent e119cad commit 9b78443

File tree

7 files changed

+234
-14
lines changed

7 files changed

+234
-14
lines changed

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -721,7 +721,7 @@ A reference to the database that created this iterator.
721721

722722
#### `iterator.count`
723723

724-
Read-only getter that indicates how many keys have been yielded so far (by any method) excluding calls that errored or yielded `undefined`.
724+
Read-only getter that indicates how many entries have been yielded so far (by any method) excluding calls that errored or yielded `undefined`.
725725

726726
#### `iterator.limit`
727727

abstract-iterator.js

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ const kKeys = Symbol('keys')
1717
const kValues = Symbol('values')
1818
const kLimit = Symbol('limit')
1919
const kCount = Symbol('count')
20+
const kEnded = Symbol('ended')
2021

2122
// This class is an internal utility for common functionality between AbstractIterator,
2223
// AbstractKeyIterator and AbstractValueIterator. It's not exported.
@@ -40,6 +41,10 @@ class CommonIterator {
4041
this[kCount] = 0
4142
this[kSignal] = options.signal != null ? options.signal : null
4243

44+
// Ending means reaching the natural end of the data and (unlike closing) that can
45+
// be reset by seek(), unless the limit was reached.
46+
this[kEnded] = false
47+
4348
this.db = db
4449
this.db.attachResource(this)
4550
}
@@ -56,21 +61,25 @@ class CommonIterator {
5661
startWork(this)
5762

5863
try {
59-
if (this[kCount] >= this[kLimit]) {
64+
if (this[kEnded] || this[kCount] >= this[kLimit]) {
65+
this[kEnded] = true
6066
return undefined
6167
}
6268

6369
let item = await this._next()
6470

71+
if (item === undefined) {
72+
this[kEnded] = true
73+
return undefined
74+
}
75+
6576
try {
66-
if (item !== undefined) {
67-
item = this[kDecodeOne](item)
68-
this[kCount]++
69-
}
77+
item = this[kDecodeOne](item)
7078
} catch (err) {
7179
throw new IteratorDecodeError(err)
7280
}
7381

82+
this[kCount]++
7483
return item
7584
} finally {
7685
endWork(this)
@@ -92,10 +101,18 @@ class CommonIterator {
92101
startWork(this)
93102

94103
try {
95-
if (size <= 0) return []
104+
if (this[kEnded] || size <= 0) {
105+
this[kEnded] = true
106+
return []
107+
}
96108

97109
const items = await this._nextv(size, options)
98110

111+
if (items.length === 0) {
112+
this[kEnded] = true
113+
return items
114+
}
115+
99116
try {
100117
this[kDecodeMany](items)
101118
} catch (err) {
@@ -112,10 +129,16 @@ class CommonIterator {
112129
async _nextv (size, options) {
113130
const acc = []
114131

115-
let item
132+
while (acc.length < size) {
133+
const item = await this._next(options)
116134

117-
while (acc.length < size && (item = await this._next(options)) !== undefined) {
118-
acc.push(item)
135+
if (item !== undefined) {
136+
acc.push(item)
137+
} else {
138+
// Must track this here because we're directly calling _next()
139+
this[kEnded] = true
140+
break
141+
}
119142
}
120143

121144
return acc
@@ -126,7 +149,7 @@ class CommonIterator {
126149
startWork(this)
127150

128151
try {
129-
if (this[kCount] >= this[kLimit]) {
152+
if (this[kEnded] || this[kCount] >= this[kLimit]) {
130153
return []
131154
}
132155

@@ -144,6 +167,8 @@ class CommonIterator {
144167
endWork(this)
145168
await destroy(this, err)
146169
} finally {
170+
this[kEnded] = true
171+
147172
if (this[kWorking]) {
148173
endWork(this)
149174
await this.close()
@@ -196,6 +221,9 @@ class CommonIterator {
196221

197222
const mapped = this.db.prefixKey(keyEncoding.encode(target), keyFormat, false)
198223
this._seek(mapped, options)
224+
225+
// If _seek() was successfull, more data may be available.
226+
this[kEnded] = false
199227
}
200228
}
201229

test/iterator-seek-test.js

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,44 @@ exports.seek = function (test, testCommon) {
132132
return db.close()
133133
})
134134

135+
test(`${mode}().seek() can be used to iterate twice`, async function (t) {
136+
const db = testCommon.factory()
137+
await db.batch(testData())
138+
const it = db[mode]()
139+
140+
t.same(await it.nextv(10), [['one', '1'], ['three', '3'], ['two', '2']].map(mapEntry), 'match')
141+
t.same(await it.nextv(10), [], 'end of iterator')
142+
143+
it.seek('one')
144+
145+
t.same(await it.nextv(10), [['one', '1'], ['three', '3'], ['two', '2']].map(mapEntry), 'match again')
146+
t.same(await it.nextv(10), [], 'end of iterator again')
147+
148+
await it.close()
149+
return db.close()
150+
})
151+
152+
test(`${mode}().seek() can be used to iterate twice, within limit`, async function (t) {
153+
const db = testCommon.factory()
154+
await db.batch(testData())
155+
const limit = 4
156+
const it = db[mode]({ limit })
157+
158+
t.same(await it.nextv(10), [['one', '1'], ['three', '3'], ['two', '2']].map(mapEntry), 'match')
159+
t.same(await it.nextv(10), [], 'end of iterator')
160+
161+
it.seek('one')
162+
163+
t.same(await it.nextv(10), [['one', '1']].map(mapEntry), 'limit reached')
164+
t.same(await it.nextv(10), [], 'end of iterator')
165+
166+
it.seek('one')
167+
t.same(await it.nextv(10), [], 'does not reset after limit has been reached')
168+
169+
await it.close()
170+
return db.close()
171+
})
172+
135173
if (testCommon.supports.snapshots) {
136174
for (const reverse of [false, true]) {
137175
for (const deferred of [false, true]) {

test/iterator-test.js

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,16 @@ exports.iterator = function (test, testCommon) {
270270
await it.close()
271271
})
272272

273+
test(`${mode}().nextv() honors limit and size`, async function (t) {
274+
const it = db[mode]({ limit: 2 })
275+
276+
t.same(await it.nextv(1), [['foobatch1', 'bar1']].map(mapEntry))
277+
t.same(await it.nextv(10), [['foobatch2', 'bar2']].map(mapEntry))
278+
t.same(await it.nextv(10), [])
279+
280+
await it.close()
281+
})
282+
273283
test(`${mode}().nextv() honors limit in reverse`, async function (t) {
274284
const it = db[mode]({ limit: 2, reverse: true })
275285

@@ -279,6 +289,16 @@ exports.iterator = function (test, testCommon) {
279289
await it.close()
280290
})
281291

292+
test(`${mode}().nextv() honors limit and size in reverse`, async function (t) {
293+
const it = db[mode]({ limit: 2, reverse: true })
294+
295+
t.same(await it.nextv(1), [['foobatch3', 'bar3']].map(mapEntry))
296+
t.same(await it.nextv(10), [['foobatch2', 'bar2']].map(mapEntry))
297+
t.same(await it.nextv(10), [])
298+
299+
await it.close()
300+
})
301+
282302
test(`${mode}().all()`, async function (t) {
283303
t.same(await db[mode]().all(), [
284304
['foobatch1', 'bar1'],

test/self/abstract-iterator-test.js

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ const testCommon = require('../common')({
1111
})
1212

1313
for (const Ctor of [AbstractIterator, AbstractKeyIterator, AbstractValueIterator]) {
14+
// Note, these tests don't create fully functional iterators, because they're not
15+
// created via db.iterator() and therefore lack the options necessary to decode data.
16+
// Not relevant for these tests.
17+
1418
test(`test ${Ctor.name} extensibility`, function (t) {
1519
const Test = class TestIterator extends Ctor {}
1620
const db = testCommon.factory()
@@ -67,7 +71,7 @@ for (const Ctor of [AbstractIterator, AbstractKeyIterator, AbstractValueIterator
6771
})
6872

6973
test(`${Ctor.name}.nextv() extensibility`, async function (t) {
70-
t.plan(4 * 2)
74+
t.plan(4)
7175

7276
class TestIterator extends Ctor {
7377
async _nextv (size, options) {
@@ -83,7 +87,6 @@ for (const Ctor of [AbstractIterator, AbstractKeyIterator, AbstractValueIterator
8387
await db.open()
8488
const it = new TestIterator(db, {})
8589
await it.nextv(100)
86-
await it.nextv(100, {})
8790
await db.close()
8891
})
8992

test/self/iterator-test.js

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,41 @@ for (const deferred of [false, true]) {
7171
if (deferred) await db.open()
7272
})
7373

74+
test(`${mode}().next() skips _next() if it previously signaled end (deferred: ${deferred}, default implementation: ${def})`, async function (t) {
75+
class MockLevel extends AbstractLevel {
76+
[privateMethod] (options) {
77+
return new MockIterator(this, options)
78+
}
79+
}
80+
81+
let calls = 0
82+
83+
class MockIterator extends Ctor {
84+
async _next () {
85+
if (calls++) return undefined
86+
87+
if (mode === 'iterator' || def) {
88+
return ['a', 'a']
89+
} else {
90+
return 'a'
91+
}
92+
}
93+
}
94+
95+
const db = new MockLevel(utf8Manifest)
96+
if (!deferred) await db.open()
97+
const it = db[publicMethod]()
98+
99+
t.same(await it.next(), mode === 'iterator' ? ['a', 'a'] : 'a')
100+
t.is(calls, 1, 'got one _next() call')
101+
102+
t.is(await it.next(), undefined)
103+
t.is(calls, 2, 'got another _next() call')
104+
105+
t.is(await it.next(), undefined)
106+
t.is(calls, 2, 'not called again')
107+
})
108+
74109
for (const limit of [2, 0]) {
75110
test(`${mode}().next() skips _next() when limit ${limit} is reached (deferred: ${deferred}, default implementation: ${def})`, async function (t) {
76111
class MockLevel extends AbstractLevel {
@@ -184,6 +219,41 @@ for (const deferred of [false, true]) {
184219
})
185220
}
186221

222+
test(`${mode}().nextv() skips _nextv() if it previously signaled end (deferred: ${deferred}, default implementation: ${def})`, async function (t) {
223+
class MockLevel extends AbstractLevel {
224+
[privateMethod] (options) {
225+
return new MockIterator(this, options)
226+
}
227+
}
228+
229+
let calls = 0
230+
231+
class MockIterator extends Ctor {
232+
async _nextv () {
233+
if (calls++) return []
234+
235+
if (mode === 'iterator' || def) {
236+
return [['a', 'a']]
237+
} else {
238+
return ['a']
239+
}
240+
}
241+
}
242+
243+
const db = new MockLevel(utf8Manifest)
244+
if (!deferred) await db.open()
245+
const it = db[publicMethod]()
246+
247+
t.same(await it.nextv(100), [mode === 'iterator' ? ['a', 'a'] : 'a'])
248+
t.is(calls, 1, 'got one _nextv() call')
249+
250+
t.same(await it.nextv(100), [])
251+
t.is(calls, 2, 'got another _nextv() call')
252+
253+
t.same(await it.nextv(100), [])
254+
t.is(calls, 2, 'not called again')
255+
})
256+
187257
test(`${mode}().nextv() reduces size for _nextv() when near limit (deferred: ${deferred}, default implementation: ${def})`, async function (t) {
188258
class MockLevel extends AbstractLevel {
189259
[privateMethod] (options) {
@@ -615,6 +685,38 @@ for (const deferred of [false, true]) {
615685
}
616686
})
617687

688+
test(`${mode}() default nextv() stops when natural end is reached (deferred: ${deferred}, default implementation: ${def})`, async function (t) {
689+
let calls = 0
690+
691+
class MockLevel extends AbstractLevel {
692+
[privateMethod] (options) {
693+
return new MockIterator(this, options)
694+
}
695+
}
696+
697+
class MockIterator extends Ctor {
698+
async _next () {
699+
if (calls++) return undefined
700+
701+
if (mode === 'iterator' || def) {
702+
return ['a', 'a']
703+
} else {
704+
return 'a'
705+
}
706+
}
707+
}
708+
709+
const db = new MockLevel(utf8Manifest)
710+
if (!deferred) await db.open()
711+
const it = await db[publicMethod]()
712+
713+
t.same(await it.nextv(10), [mode === 'iterator' ? ['a', 'a'] : 'a'])
714+
t.is(calls, 2)
715+
716+
t.same(await it.nextv(10), [], 'ended')
717+
t.is(calls, 2, 'not called again')
718+
})
719+
618720
test(`${mode}() has default all() (deferred: ${deferred}, default implementation: ${def})`, async function (t) {
619721
t.plan(8)
620722

@@ -685,6 +787,35 @@ for (const deferred of [false, true]) {
685787
}
686788
})
687789

790+
test(`${mode}() default all() stops when limit is reached (deferred: ${deferred}, default implementation: ${def})`, async function (t) {
791+
t.plan(2)
792+
let calls = 0
793+
794+
class MockLevel extends AbstractLevel {
795+
[privateMethod] (options) {
796+
return new MockIterator(this, options)
797+
}
798+
}
799+
800+
class MockIterator extends Ctor {
801+
async _nextv (size, options) {
802+
calls++
803+
if (mode === 'iterator' || def) {
804+
return [[String(calls), String(calls)]]
805+
} else {
806+
return [String(calls)]
807+
}
808+
}
809+
}
810+
811+
const db = new MockLevel(utf8Manifest)
812+
if (!deferred) await db.open()
813+
814+
const items = await db[publicMethod]({ limit: 2 }).all()
815+
t.is(items.length, 2)
816+
t.is(calls, 2)
817+
})
818+
688819
test(`${mode}() custom all() (deferred: ${deferred}, default implementation: ${def})`, async function (t) {
689820
t.plan(3)
690821

0 commit comments

Comments
 (0)