From 626e293581d63acfe069d146fc1274ecf889c87c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kat=20March=C3=A1n?= Date: Wed, 1 Mar 2017 18:54:53 -0800 Subject: [PATCH] test(api): filled out a bunch of test cases --- test/content.put-stream.js | 46 +++++- test/content.read.js | 4 +- test/get.js | 315 +++++++++++++++++++++++++++++++++++++ test/index.find.js | 4 + test/index.insert.js | 19 ++- test/index.ls.js | 4 + test/put.js | 158 ++++++++++++++++++- 7 files changed, 539 insertions(+), 11 deletions(-) create mode 100644 test/get.js diff --git a/test/content.put-stream.js b/test/content.put-stream.js index 3c387d3..9db2f5e 100644 --- a/test/content.put-stream.js +++ b/test/content.put-stream.js @@ -77,6 +77,34 @@ test('errors if stream ends with no data', function (t) { }) }) +test('errors if input size does not match expected', function (t) { + t.plan(10) + let dig1 = null + pipe(fromString('abc'), putStream(CACHE, { + size: 5 + }).on('digest', function (d) { + dig1 = d + }), function (err) { + t.ok(err, 'got an error when data smaller than expected') + t.equal(dig1, null, 'no digest returned') + t.equal(err.code, 'EBADSIZE', 'returns useful error code') + t.equal(err.expected, 5, 'error includes expected size') + t.equal(err.found, 3, 'error includes found size') + }) + let dig2 = null + pipe(fromString('abcdefghi'), putStream(CACHE, { + size: 5 + }).on('digest', function (d) { + dig2 = d + }), function (err) { + t.ok(err, 'got an error when data bigger than expected') + t.equal(dig2, null, 'no digest returned') + t.equal(err.code, 'EBADSIZE', 'returns useful error code') + t.equal(err.expected, 5, 'error includes expected size') + t.equal(err.found, 9, 'error includes found size') + }) +}) + test('does not overwrite content if already on disk', function (t) { const CONTENT = 'foobarbaz' const DIGEST = crypto.createHash('sha1').update(CONTENT).digest('hex') @@ -181,7 +209,23 @@ test('cleans up tmp on successful completion', function (t) { }) }) -test('cleans up tmp on error') +test('cleans up tmp on error', function (t) { + const CONTENT = 'foobarbaz' + pipe(fromString(CONTENT), putStream(CACHE, { size: 1 }), function (err) { + t.ok(err, 'got an error') + t.equal(err.code, 'EBADSIZE', 'got expected code') + const tmp = path.join(CACHE, 'tmp') + fs.readdir(tmp, function (err, files) { + if (!err || (err && err.code === 'ENOENT')) { + files = files || [] + t.deepEqual(files, [], 'nothing in the tmp dir!') + t.end() + } else { + throw err + } + }) + }) +}) test('checks the size of stream data if opts.size provided', function (t) { const CONTENT = 'foobarbaz' diff --git a/test/content.read.js b/test/content.read.js index aa41b35..8e87af2 100644 --- a/test/content.read.js +++ b/test/content.read.js @@ -35,7 +35,7 @@ test('readStream: returns a stream with cache content data', function (t) { test('readStream: allows hashAlgorithm configuration', function (t) { const CONTENT = 'foobarbaz' - const HASH = 'sha1' + const HASH = 'sha512' const DIGEST = crypto.createHash(HASH).update(CONTENT).digest('hex') const dir = {} dir[DIGEST] = File(CONTENT) @@ -48,7 +48,7 @@ test('readStream: allows hashAlgorithm configuration', function (t) { let buf = '' stream.on('data', function (data) { buf += data }) stream.on('end', function () { - t.ok(true, 'stream completed successfully, off a sha1') + t.ok(true, 'stream completed successfully, off a sha512') t.equal(CONTENT, buf, 'cache contents read correctly') t.end() }) diff --git a/test/get.js b/test/get.js new file mode 100644 index 0000000..a178b8a --- /dev/null +++ b/test/get.js @@ -0,0 +1,315 @@ +'use strict' + +const Promise = require('bluebird') + +const crypto = require('crypto') +const finished = Promise.promisify(require('mississippi').finished) +const index = require('../lib/entry-index') +const memo = require('../lib/memoization') +const path = require('path') +const rimraf = Promise.promisify(require('rimraf')) +const Tacks = require('tacks') +const test = require('tap').test +const testDir = require('./util/test-dir')(__filename) + +const Dir = Tacks.Dir +const File = Tacks.File + +const CACHE = path.join(testDir, 'cache') +const CONTENT = bufferise('foobarbaz') +const KEY = 'my-test-key' +const ALGO = 'sha512' +const DIGEST = crypto.createHash(ALGO).update(CONTENT).digest('hex') +const METADATA = { foo: 'bar' } + +var get = require('../get') + +function bufferise (string) { + return Buffer.from + ? Buffer.from(string, 'utf8') + : new Buffer(string, 'utf8') +} + +// Simple wrapper util cause this gets WORDY +function streamGet (byDigest) { + const args = [].slice.call(arguments, 1) + let data = [] + let dataLen = 0 + let hashAlgorithm + let digest + let metadata + const stream = ( + byDigest ? get.stream.byDigest : get.stream + ).apply(null, args) + stream.on('data', d => { + data.push(d) + dataLen += d.length + }).on('hashAlgorithm', h => { + hashAlgorithm = h + }).on('digest', d => { + digest = d + }).on('metadata', m => { + metadata = m + }) + return finished(stream).then(() => ({ + data: Buffer.concat(data, dataLen), hashAlgorithm, digest, metadata + })) +} + +test('basic bulk get', t => { + const fixture = new Tacks(Dir({ + 'content': Dir({ + [DIGEST]: File(CONTENT) + }) + })) + fixture.create(CACHE) + return index.insert(CACHE, KEY, DIGEST, { + metadata: METADATA, + hashAlgorithm: ALGO + }).then(() => { + return get(CACHE, KEY) + }).then(res => { + t.deepEqual(res, { + metadata: METADATA, + data: CONTENT, + hashAlgorithm: ALGO, + digest: DIGEST + }, 'bulk key get returned proper data') + }).then(() => { + return get.byDigest(CACHE, DIGEST, {hashAlgorithm: ALGO}) + }).then(res => { + t.deepEqual(res, CONTENT, 'byDigest returned proper data') + }) +}) + +test('basic stream get', t => { + const fixture = new Tacks(Dir({ + 'content': Dir({ + [DIGEST]: File(CONTENT) + }) + })) + fixture.create(CACHE) + return index.insert(CACHE, KEY, DIGEST, { + metadata: METADATA, + hashAlgorithm: ALGO + }).then(() => { + return Promise.join( + streamGet(false, CACHE, KEY), + streamGet(true, CACHE, DIGEST, { hashAlgorithm: ALGO }), + (byKey, byDigest) => { + t.deepEqual(byKey, { + data: CONTENT, + hashAlgorithm: ALGO, + digest: DIGEST, + metadata: METADATA + }, 'got all expected data and fields from key fetch') + t.deepEqual( + byDigest.data, + CONTENT, + 'got correct data from digest fetch' + ) + } + ) + }) +}) + +test('ENOENT if not found', t => { + return get(CACHE, KEY).then(() => { + throw new Error('lookup should fail') + }).catch(err => { + t.ok(err, 'got an error') + t.equal(err.code, 'ENOENT', 'error code is ENOENT') + return get.info(CACHE, KEY) + }).catch(err => { + t.ok(err, 'got an error') + t.equal(err.code, 'ENOENT', 'error code is ENOENT') + }) +}) + +test('get.info index entry lookup', t => { + return index.insert(CACHE, KEY, DIGEST, { + metadata: METADATA, + hashAlgorithm: ALGO + }).then(ENTRY => { + return get.info(CACHE, KEY).then(entry => { + t.deepEqual(entry, ENTRY, 'get.info() returned the right entry') + }) + }) +}) + +test('memoizes data on bulk read', t => { + memo.clearMemoized() + const fixture = new Tacks(Dir({ + 'content': Dir({ + [DIGEST]: File(CONTENT) + }) + })) + fixture.create(CACHE) + return index.insert(CACHE, KEY, DIGEST, { + metadata: METADATA, + hashAlgorithm: ALGO + }).then(ENTRY => { + return get(CACHE, KEY).then(() => { + t.deepEqual(memo.get(CACHE, KEY), null, 'no memoization!') + return get(CACHE, KEY, { memoize: true }) + }).then(res => { + t.deepEqual(res, { + metadata: METADATA, + data: CONTENT, + hashAlgorithm: ALGO, + digest: DIGEST + }, 'usual data returned') + t.deepEqual(memo.get(CACHE, KEY), { + entry: ENTRY, + data: CONTENT + }, 'data inserted into memoization cache') + return rimraf(CACHE) + }).then(() => { + return get(CACHE, KEY) + }).then(res => { + t.deepEqual(res, { + metadata: METADATA, + data: CONTENT, + hashAlgorithm: ALGO, + digest: DIGEST + }, 'memoized data fetched by default') + return get(CACHE, KEY, { memoize: false }).then(() => { + throw new Error('expected get to fail') + }).catch(err => { + t.ok(err, 'got an error from unmemoized get') + t.equal(err.code, 'ENOENT', 'cached content not found') + t.deepEqual(memo.get(CACHE, KEY), { + entry: ENTRY, + data: CONTENT + }, 'data still in memoization cache') + }) + }) + }) +}) + +test('memoizes data on stream read', t => { + memo.clearMemoized() + const fixture = new Tacks(Dir({ + 'content': Dir({ + [DIGEST]: File(CONTENT) + }) + })) + fixture.create(CACHE) + return index.insert(CACHE, KEY, DIGEST, { + metadata: METADATA, + hashAlgorithm: ALGO + }).then(ENTRY => { + return Promise.join( + streamGet(false, CACHE, KEY), + streamGet(true, CACHE, DIGEST, { hashAlgorithm: ALGO }), + () => { + t.deepEqual(memo.get(CACHE, KEY), null, 'no memoization by key!') + t.deepEqual( + memo.get.byDigest(CACHE, DIGEST, ALGO), + null, + 'no memoization by digest!' + ) + } + ).then(() => { + memo.clearMemoized() + return streamGet(true, CACHE, DIGEST, { + memoize: true, + hashAlgorithm: ALGO + }) + }).then(byDigest => { + t.deepEqual(byDigest.data, CONTENT, 'usual data returned from stream') + t.deepEqual(memo.get(CACHE, KEY), null, 'digest fetch = no key entry') + t.deepEqual( + memo.get.byDigest(CACHE, DIGEST, ALGO), + CONTENT, + 'content memoized' + ) + t.deepEqual( + memo.get.byDigest(CACHE, DIGEST, 'sha1'), + null, + 'content memoization filtered by hashAlgo' + ) + t.deepEqual( + memo.get.byDigest('whatev', DIGEST, ALGO), + null, + 'content memoization filtered by cache' + ) + }).then(() => { + memo.clearMemoized() + return streamGet(false, CACHE, KEY, { memoize: true }) + }).then(byKey => { + t.deepEqual(byKey, { + metadata: METADATA, + data: CONTENT, + hashAlgorithm: ALGO, + digest: DIGEST + }, 'usual data returned from key fetch') + t.deepEqual(memo.get(CACHE, KEY), { + entry: ENTRY, + data: CONTENT + }, 'data inserted into memoization cache') + t.deepEqual( + memo.get.byDigest(CACHE, DIGEST, ALGO), + CONTENT, + 'content memoized by digest, too' + ) + t.deepEqual( + memo.get('whatev', KEY), + null, + 'entry memoization filtered by cache' + ) + }).then(() => { + return rimraf(CACHE) + }).then(() => { + return Promise.join( + streamGet(false, CACHE, KEY), + streamGet(true, CACHE, DIGEST, { hashAlgorithm: ALGO }), + (byKey, byDigest) => { + t.deepEqual(byKey, { + metadata: METADATA, + data: CONTENT, + hashAlgorithm: ALGO, + digest: DIGEST + }, 'key fetch fulfilled by memoization cache') + t.deepEqual( + byDigest.data, + CONTENT, + 'digest fetch fulfilled by memoization cache' + ) + } + ) + }).then(() => { + return Promise.join( + streamGet(false, CACHE, KEY, { + memoize: false + }).catch(err => err), + streamGet(true, CACHE, DIGEST, { + hashAlgorithm: ALGO, + memoize: false + }).catch(err => err), + (keyErr, digestErr) => { + t.equal(keyErr.code, 'ENOENT', 'key get memoization bypassed') + t.equal(keyErr.code, 'ENOENT', 'digest get memoization bypassed') + } + ) + }) + }) +}) + +test('get.info uses memoized data', t => { + memo.clearMemoized() + const ENTRY = { + key: KEY, + digest: DIGEST, + hashAlgorithm: ALGO, + time: +(new Date()), + metadata: null + } + memo.put(CACHE, ENTRY, CONTENT) + return get.info(CACHE, KEY).then(info => { + t.deepEqual(info, ENTRY, 'got the entry from memoization cache') + }) +}) + +test('identical hashes with different algorithms do not conflict') +test('throw error if something is really wrong with bucket') diff --git a/test/index.find.js b/test/index.find.js index 9d2aafd..9700757 100644 --- a/test/index.find.js +++ b/test/index.find.js @@ -19,6 +19,7 @@ test('index.find cache hit', function (t) { const entry = { key: 'whatever', digest: 'deadbeef', + hashAlgorithm: 'whatnot', time: 12345, metadata: 'omgsometa' } @@ -100,6 +101,7 @@ test('index.find path-breaking characters', function (t) { key: ';;!registry\nhttps://registry.npmjs.org/back \\ slash@Cool™?', digest: 'deadbeef', time: 12345, + hashAlgorithm: 'whatnot', metadata: 'omgsometa' } const idx = {} @@ -128,6 +130,7 @@ test('index.find extremely long keys', function (t) { key: key, digest: 'deadbeef', time: 12345, + hashAlgorithm: 'whatnot', metadata: 'woo' } const idx = {} @@ -197,6 +200,7 @@ test('index.find hash conflict in same bucket', function (t) { const entry = { key: 'whatever', digest: 'deadbeef', + hashAlgorithm: 'whatnot', time: 12345, metadata: 'yay' } diff --git a/test/index.insert.js b/test/index.insert.js index 500df76..5ae2c71 100644 --- a/test/index.insert.js +++ b/test/index.insert.js @@ -17,21 +17,32 @@ const index = require('../lib/entry-index') const KEY = 'foo' const KEYHASH = index._hashKey(KEY) const DIGEST = 'deadbeef' +const ALGO = 'whatnot' test('basic insertion', function (t) { return index.insert( - CACHE, KEY, DIGEST - ).then(() => { + CACHE, KEY, DIGEST, { metadata: 'foo', hashAlgorithm: ALGO } + ).then(entry => { + t.deepEqual(entry, { + key: KEY, + digest: DIGEST, + hashAlgorithm: ALGO, + path: path.join(CACHE, 'content', DIGEST), + time: entry.time, + metadata: 'foo' + }, 'formatted entry returned') const bucket = path.join(CACHE, 'index', KEYHASH) return fs.readFileAsync(bucket, 'utf8') }).then(data => { t.equal(data[0], '{', 'first entry starts with a {, not \\n') const entry = JSON.parse(data) t.ok(entry.time, 'entry has a timestamp') - delete entry.time t.deepEqual(entry, { key: KEY, - digest: DIGEST + digest: DIGEST, + hashAlgorithm: ALGO, + time: entry.time, + metadata: 'foo' }, 'entry matches what was inserted') }) }) diff --git a/test/index.ls.js b/test/index.ls.js index b2b3e71..fcc134e 100644 --- a/test/index.ls.js +++ b/test/index.ls.js @@ -16,12 +16,14 @@ test('basic listing', function (t) { 'whatever': { key: 'whatever', digest: 'deadbeef', + hashAlgorithm: 'whatnot', time: 12345, metadata: 'omgsometa' }, 'whatnot': { key: 'whatnot', digest: 'bada55', + hashAlgorithm: 'whateva', time: 54321, metadata: null } @@ -44,12 +46,14 @@ test('separate keys in conflicting buckets', function (t) { 'whatever': { key: 'whatever', digest: 'deadbeef', + hashAlgorithm: 'whatnot', time: 12345, metadata: 'omgsometa' }, 'whatev': { key: 'whatev', digest: 'bada55', + hashAlgorithm: 'whateva', time: 54321, metadata: null } diff --git a/test/put.js b/test/put.js index 1633770..e11172f 100644 --- a/test/put.js +++ b/test/put.js @@ -1,7 +1,157 @@ 'use strict' -var test = require('tap').test +const Promise = require('bluebird') -test('stream ends when piped stream finishes') -test('signals error if error writing to cache') -test('adds correct entry to index before finishing') +const crypto = require('crypto') +const fromString = require('./util/from-string') +const fs = Promise.promisifyAll(require('fs')) +const index = require('../lib/entry-index') +const memo = require('../lib/memoization') +const path = require('path') +const pipe = Promise.promisify(require('mississippi').pipe) +const test = require('tap').test +const testDir = require('./util/test-dir')(__filename) + +const CACHE = path.join(testDir, 'cache') +const CONTENT = bufferise('foobarbaz') +const KEY = 'my-test-key' +const ALGO = 'sha1' +const DIGEST = crypto.createHash(ALGO).update(CONTENT).digest('hex') +const METADATA = { foo: 'bar' } +const contentPath = require('../lib/content/path') + +var put = require('../put') + +function bufferise (string) { + return Buffer.from + ? Buffer.from(string, 'utf8') + : new Buffer(string, 'utf8') +} + +test('basic bulk insertion', t => { + return put(CACHE, KEY, CONTENT).then(digest => { + t.equal(digest, DIGEST, 'returned content digest') + const dataPath = contentPath(CACHE, digest) + return fs.readFileAsync(dataPath) + }).then(data => { + t.deepEqual(data, CONTENT, 'content was correctly inserted') + }) +}) + +test('basic stream insertion', t => { + let foundDigest + const src = fromString(CONTENT) + const stream = put.stream(CACHE, KEY).on('digest', function (d) { + foundDigest = d + }) + return pipe(src, stream).then(() => { + t.equal(foundDigest, DIGEST, 'returned digest matches expected') + return fs.readFileAsync(contentPath(CACHE, foundDigest)) + }).then(data => { + t.deepEqual(data, CONTENT, 'contents are identical to inserted content') + }) +}) + +test('adds correct entry to index before finishing', t => { + return put(CACHE, KEY, CONTENT, {metadata: METADATA}).then(() => { + return index.find(CACHE, KEY) + }).then(entry => { + t.ok(entry, 'got an entry') + t.equal(entry.key, KEY, 'entry has the right key') + t.equal(entry.digest, DIGEST, 'entry has the right key') + t.deepEqual(entry.metadata, METADATA, 'metadata also inserted') + }) +}) + +test('optionally memoizes data on bulk insertion', t => { + return put(CACHE, KEY, CONTENT, { + metadata: METADATA, + hashAlgorithm: ALGO, + memoize: true + }).then(digest => { + t.equal(digest, DIGEST, 'digest returned as usual') + return index.find(CACHE, KEY) // index.find is not memoized + }).then(entry => { + t.deepEqual(memo.get(CACHE, KEY), { + data: CONTENT, + entry: entry + }, 'content inserted into memoization cache by key') + t.deepEqual( + memo.get.byDigest(CACHE, DIGEST, ALGO), + CONTENT, + 'content inserted into memoization cache by digest' + ) + }) +}) + +test('optionally memoizes data on stream insertion', t => { + let foundDigest + const src = fromString(CONTENT) + const stream = put.stream(CACHE, KEY, { + hashAlgorithm: ALGO, + metadata: METADATA, + memoize: true + }).on('digest', function (d) { + foundDigest = d + }) + return pipe(src, stream).then(() => { + t.equal(foundDigest, DIGEST, 'digest emitted as usual') + return fs.readFileAsync(contentPath(CACHE, foundDigest)) + }).then(data => { + t.deepEqual(data, CONTENT, 'contents are identical to inserted content') + return index.find(CACHE, KEY) // index.find is not memoized + }).then(entry => { + t.deepEqual(memo.get(CACHE, KEY), { + data: CONTENT, + entry: entry + }, 'content inserted into memoization cache by key') + t.deepEqual( + memo.get.byDigest(CACHE, DIGEST, ALGO), + CONTENT, + 'content inserted into memoization cache by digest' + ) + }) +}) + +test('signals error if error writing to cache', t => { + return Promise.join( + put(CACHE, KEY, CONTENT, { + size: 2 + }).then(() => { + throw new Error('expected error') + }).catch(err => err), + pipe(fromString(CONTENT), put.stream(CACHE, KEY, { + size: 2 + })).then(() => { + throw new Error('expected error') + }).catch(err => err), + (bulkErr, streamErr) => { + t.equal(bulkErr.code, 'EBADSIZE', 'got error from bulk write') + t.equal(streamErr.code, 'EBADSIZE', 'got error from stream write') + } + ) +}) + +test('errors if input stream errors', function (t) { + let foundDigest + const putter = put.stream(CACHE, KEY).on('digest', function (d) { + foundDigest = d + }) + const stream = fromString(false) + return pipe( + stream, putter + ).then(() => { + throw new Error('expected error') + }).catch(err => { + t.ok(err, 'got an error') + t.ok(!foundDigest, 'no digest returned') + t.match( + err.message, + /Invalid non-string/, + 'returns the error from input stream' + ) + return fs.readdirAsync(testDir) + }).then(files => { + t.deepEqual(files, [], 'no files created') + }) +})