Merge pull request #8 from keyvhq/revamp+iterator

.github/workflows/test.yml

@@ -44,6 +44,8 @@ jobs:
       run: |
         sudo /etc/init.d/mysql start
         mysql -e 'CREATE DATABASE ${{ env.MYSQL_DB_DATABASE }}' -u${{ env.MYSQL_DB_USER }} -p${{ env.MYSQL_DB_PASSWORD }}
+        mysql -e 'SET GLOBAL max_connections = 300;' -u${{ env.MYSQL_DB_USER }} -p${{ env.MYSQL_DB_PASSWORD }}
+
     - name: Start PostgreSQL on Ubuntu
       run: |
         sudo systemctl start postgresql.service

packages/keyv-mongo/src/index.js

@@ -100,9 +100,12 @@ class KeyvMongo extends EventEmitter {
 				.then(() => undefined));
 	}
 
-	iterator() {
-		return this.connect
-			.then(store => store.find({ key: new RegExp(`^${this.namespace}:`) }));
+	async * iterator() {
+		const iterator = await this.connect
+			.then(store => store.find({ key: new RegExp(`^${this.namespace}:`) }).map(x => {
+				return [x.key, x.value];
+			}));
+		yield * iterator;
 	}
 }
 module.exports = KeyvMongo;

packages/keyv-mongo/test/test.js

@@ -30,3 +30,4 @@ test('Collection option merges into default options if URL is passed', t => {
 		collection: 'foo'
 	});
 });
+

packages/keyv-redis/src/index.js

@@ -64,20 +64,25 @@ class KeyvRedis extends EventEmitter {
 	}
 
 	async * iterator() {
-		const scan = pify(this.keyv.options.store.redis.scan).bind(this.keyv.options.store.redis);
-
+		const scan = pify(this.redis.scan).bind(this.redis);
+		const get = this.redis.mget.bind(this.redis);
 		async function * iterate(curs, pattern) {
 			const [cursor, keys] = await scan(curs, 'MATCH', pattern);
-			for (const key of keys) {
-				yield key;
+			const values = await get(keys);
+			for (const i in keys) {
+				if (Object.prototype.hasOwnProperty.call(keys, i)) {
+					const key = keys[i];
+					const value = values[i];
+					yield [key, value];
+				}
 			}
 
 			if (cursor !== '0') {
 				yield * iterate(cursor, pattern);
 			}
 		}
 
-		yield * iterate(0, `${this.keyv.options.namespace}:*`);
+		yield * iterate(0, `${this.namespace}:*`);
 	}
 }
 

packages/keyv-sql/src/index.js

@@ -5,7 +5,6 @@ class KeyvSql extends EventEmitter {
 	constructor(options) {
 		super();
 		this.ttlSupport = false;
-		this.asyncIteratorSupport = false;
 
 		this.options = Object.assign({
 			table: 'keyv',
@@ -83,7 +82,7 @@ class KeyvSql extends EventEmitter {
 
 			for (const entry of entries) {
 				offset += 1;
-				yield entry.key;
+				yield [entry.key, entry.value];
 			}
 
 			if (offset !== '0') {

packages/keyv-sql/test/test.js

@@ -31,13 +31,3 @@ class TestSqlite extends KeyvSql {
 
 const store = () => new TestSqlite();
 keyvTestSuite(test, Keyv, store);
-
-test('Async Iterator', async t => {
-	const keyv = new Keyv({ store: store() });
-	await keyv.set('foo', 'bar');
-	const iterator = keyv.options.store.iterator();
-	for await (const key of iterator) {
-		console.log(key);
-		t.assert(key, 'foo');
-	}
-});

packages/keyv-test-suite/src/api.js

@@ -28,7 +28,7 @@ const keyvApiTests = (test, Keyv, store) => {
 		const keyv = new Keyv({ store: store() });
 		await keyv.set('foo', 'bar', ttl);
 		t.is(await keyv.get('foo'), 'bar');
-		if (keyv.options.store.ttlSupport === true) {
+		if (keyv.store.ttlSupport === true) {
 			await delay(ttl + 1);
 		} else {
 			tk.freeze(Date.now() + ttl + 1);

packages/keyv-test-suite/src/index.js

@@ -1,9 +1,10 @@
 const keyvApiTests = require('./api.js');
 const keyvValueTests = require('./values.js');
 const keyvNamepsaceTests = require('./namespace.js');
-const keyvOfficialTests = require('./official.js');
+const keyvIteratorTests = require('./iteration.js');
 
 const keyvTestSuite = (test, Keyv, store) => {
+	keyvIteratorTests(test, Keyv, store);
 	keyvApiTests(test, Keyv, store);
 	keyvValueTests(test, Keyv, store);
 	keyvNamepsaceTests(test, Keyv, store);
@@ -13,7 +14,7 @@ Object.assign(keyvTestSuite, {
 	keyvApiTests,
 	keyvValueTests,
 	keyvNamepsaceTests,
-	keyvOfficialTests
+	keyvIteratorTests
 });
 
 module.exports = keyvTestSuite;

packages/keyv-test-suite/src/iteration.js

@@ -0,0 +1,30 @@
+const keyvIteratorTests = (test, Keyv, store) => {
+	test.beforeEach(async () => {
+		const keyv = new Keyv({ store: store() });
+		await keyv.clear();
+	});
+
+	test.serial('.iterator() returns an asyncIterator', t => {
+		const keyv = new Keyv({ store: store() });
+		t.true(typeof keyv.iterator()[Symbol.asyncIterator] === 'function');
+	});
+
+	test.serial('iterator() iterates over all values', async t => {
+		const keyv = new Keyv({ store: store() });
+		const map = new Map(Array.from({ length: 5 }).fill(0).map((x, i) => [String(i), String(i + 10)]));
+		const toResolve = [];
+		for (const [key, value] of map) {
+			toResolve.push(keyv.set(key, value));
+		}
+
+		await Promise.all(toResolve);
+		t.plan(map.size);
+		for await (const [key, value] of keyv.iterator()) {
+			const doesKeyExist = map.has(key);
+			const isValueSame = map.get(key) === value;
+			t.true(doesKeyExist && isValueSame);
+		}
+	});
+};
+
+module.exports = keyvIteratorTests;

packages/keyv/src/index.js

@@ -3,56 +3,67 @@
 const EventEmitter = require('events');
 const JSONB = require('json-buffer');
 
-const loadStore = options => {
-	const adapters = {
-		redis: '@keyvhq/keyv-redis',
-		mongodb: '@keyvhq/keyv-mongo',
-		mongo: '@keyvhq/keyv-mongo',
-		sqlite: '@keyvhq/keyv-sqlite',
-		postgresql: '@keyvhq/keyv-postgres',
-		postgres: '@keyvhq/keyv-postgres',
-		mysql: '@keyvhq/keyv-mysql'
-	};
-	if (options.adapter || options.uri) {
-		const adapter = options.adapter || /^[^:]*/.exec(options.uri)[0];
-		return new (require(adapters[adapter]))(options);
-	}
-
-	return new Map();
-};
-
 class Keyv extends EventEmitter {
-	constructor(uri, options) {
+	constructor(options) {
 		super();
 		this.options = Object.assign(
 			{
 				namespace: 'keyv',
 				serialize: JSONB.stringify,
 				deserialize: JSONB.parse
 			},
-			(typeof uri === 'string') ? { uri } : uri,
 			options
 		);
 
-		if (!this.options.store) {
-			const adapteroptions = Object.assign({}, this.options);
-			this.options.store = loadStore(adapteroptions);
+		this.store = this.options.store;
+
+		if (!this.store) {
+			this.store = new Map();
 		}
 
-		if (typeof this.options.store.on === 'function') {
-			this.options.store.on('error', error => this.emit('error', error));
+		if (typeof this.store.on === 'function') {
+			this.store.on('error', error => this.emit('error', error));
 		}
 
-		this.options.store.namespace = this.options.namespace;
+		this.store.namespace = this.options.namespace;
+
+		const generateIterator = iterator => async function * () {
+			for await (const [key, raw] of iterator) {
+				const data = (typeof raw === 'string') ? this.options.deserialize(raw) : raw;
+				if (!key.includes(this.options.namespace) || typeof data !== 'object') {
+					continue;
+				}
+
+				if (typeof data.expires === 'number' && Date.now() > data.expires) {
+					this.delete(key);
+					continue;
+				}
+
+				yield [this._getKeyUnprefix(key), data.value];
+			}
+		};
+
+		// Attach iterators
+		if (typeof this.store[Symbol.iterator] === 'function' && this.store instanceof Map) {
+			this.iterator = generateIterator(this.store);
+		} else if (typeof this.store.iterator === 'function') {
+			this.iterator = generateIterator(this.store.iterator());
+		} else {
+			this.iteratorSupport = false;
+		}
 	}
 
 	_getKeyPrefix(key) {
 		return `${this.options.namespace}:${key}`;
 	}
 
+	_getKeyUnprefix(key) {
+		return key.split(':').splice(1).join(':');
+	}
+
 	get(key, options) {
 		const keyPrefixed = this._getKeyPrefix(key);
-		const { store } = this.options;
+		const store = this.store;
 		return Promise.resolve()
 			.then(() => store.get(keyPrefixed))
 			.then(data => {
@@ -82,8 +93,7 @@ class Keyv extends EventEmitter {
 			ttl = undefined;
 		}
 
-		const { store } = this.options;
-
+		const store = this.store;
 		return Promise.resolve()
 			.then(() => {
 				const expires = (typeof ttl === 'number') ? (Date.now() + ttl) : null;
@@ -96,16 +106,15 @@ class Keyv extends EventEmitter {
 
 	delete(key) {
 		const keyPrefixed = this._getKeyPrefix(key);
-		const { store } = this.options;
+		const store = this.store;
 		return Promise.resolve()
 			.then(() => store.delete(keyPrefixed));
 	}
 
 	clear() {
-		const { store } = this.options;
+		const store = this.store;
 		return Promise.resolve()
 			.then(() => store.clear());
 	}
 }
-
 module.exports = Keyv;

packages/keyv/test/storage-adapters/mongodb.js

@@ -1,10 +1,7 @@
 const test = require('ava');
 const keyvTestSuite = require('@keyvhq/keyv-test-suite');
-const { keyvOfficialTests } = keyvTestSuite;
 const Keyv = require('this');
 const KeyvMongo = require('@keyvhq/keyv-mongo');
 
-keyvOfficialTests(test, Keyv, 'mongodb://127.0.0.1:27017', 'mongodb://127.0.0.1:1234');
-
 const store = () => new KeyvMongo('mongodb://127.0.0.1:27017');
 keyvTestSuite(test, Keyv, store);

packages/keyv/test/storage-adapters/mysql.js

@@ -1,10 +1,7 @@
 const test = require('ava');
 const keyvTestSuite = require('@keyvhq/keyv-test-suite');
-const { keyvOfficialTests } = keyvTestSuite;
 const Keyv = require('this');
 const KeyvMysql = require('@keyvhq/keyv-mysql');
 
-keyvOfficialTests(test, Keyv, 'mysql://mysql@localhost/keyv_test', 'mysql://foo');
-
 const store = () => new KeyvMysql('mysql://mysql@localhost/keyv_test');
 keyvTestSuite(test, Keyv, store);

packages/keyv/test/storage-adapters/postgresql.js

@@ -1,10 +1,7 @@
 const test = require('ava');
 const keyvTestSuite = require('@keyvhq/keyv-test-suite');
-const { keyvOfficialTests } = keyvTestSuite;
 const Keyv = require('this');
 const KeyvPostgres = require('postgres');
 
-keyvOfficialTests(test, Keyv, 'postgresql://postgres@localhost:5432/keyv_test', 'postgresql://foo');
-
 const store = () => new KeyvPostgres('postgresql://postgres@localhost:5432/keyv_test');
 keyvTestSuite(test, Keyv, store);

packages/keyv/test/storage-adapters/redis.js

@@ -1,10 +1,7 @@
 const test = require('ava');
 const keyvTestSuite = require('@keyvhq/keyv-test-suite');
-const { keyvOfficialTests } = keyvTestSuite;
 const Keyv = require('this');
 const KeyvRedis = require('redis');
 
-keyvOfficialTests(test, Keyv, 'redis://localhost', 'redis://foo');
-
 const store = () => new KeyvRedis('redis://localhost');
 keyvTestSuite(test, Keyv, store);

packages/keyv/test/storage-adapters/sqlite.js

@@ -1,10 +1,7 @@
 const test = require('ava');
 const keyvTestSuite = require('@keyvhq/keyv-test-suite');
-const { keyvOfficialTests } = keyvTestSuite;
 const Keyv = require('this');
 const KeyvSqlite = require('sqlite');
 
-keyvOfficialTests(test, Keyv, 'sqlite://test/testdb.sqlite', 'sqlite://non/existent/database.sqlite');
-
 const store = () => new KeyvSqlite('sqlite://test/testdb.sqlite');
 keyvTestSuite(test, Keyv, store);