From db1e094f948df12c9cd43be320c88febe9d1f497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kat=20March=C3=A1n?= Date: Sun, 4 Nov 2018 20:21:20 -0800 Subject: [PATCH] feat(get): add sync API for reading --- get.js | 49 +++++++++++++++++++ lib/entry-index.js | 107 ++++++++++++++++++++++++++++++++---------- lib/util/fix-owner.js | 46 ++++++++++++++++++ locales/en.js | 2 + locales/es.js | 2 + test/get.js | 18 +++++++ 6 files changed, 199 insertions(+), 25 deletions(-) diff --git a/get.js b/get.js index 7bafe12..e7a3fe1 100644 --- a/get.js +++ b/get.js @@ -63,6 +63,55 @@ function getData (byDigest, cache, key, opts) { }) } +module.exports.sync = function get (cache, key, opts) { + return getDataSync(false, cache, key, opts) +} +module.exports.sync.byDigest = function getByDigest (cache, digest, opts) { + return getDataSync(true, cache, digest, opts) +} +function getDataSync (byDigest, cache, key, opts) { + opts = GetOpts(opts) + const memoized = ( + byDigest + ? memo.get.byDigest(cache, key, opts) + : memo.get(cache, key, opts) + ) + if (memoized && opts.memoize !== false) { + return byDigest ? memoized : { + metadata: memoized.entry.metadata, + data: memoized.data, + integrity: memoized.entry.integrity, + size: memoized.entry.size + } + } + const entry = !byDigest && index.find.sync(cache, key, opts) + if (!entry && !byDigest) { + throw new index.NotFoundError(cache, key) + } + const data = read.sync( + cache, + byDigest ? key : entry.integrity, + { + integrity: opts.integrity, + size: opts.size + } + ) + const res = byDigest + ? data + : { + metadata: entry.metadata, + data: data, + size: entry.size, + integrity: entry.integrity + } + if (opts.memoize && byDigest) { + memo.put.byDigest(cache, key, res, opts) + } else if (opts.memoize) { + memo.put(cache, entry, res.data, opts) + } + return res +} + module.exports.stream = getStream function getStream (cache, key, opts) { opts = GetOpts(opts) diff --git a/lib/entry-index.js b/lib/entry-index.js index 43fa7b9..29a688e 100644 --- a/lib/entry-index.js +++ b/lib/entry-index.js @@ -75,10 +75,36 @@ function insert (cache, key, integrity, opts) { }) } +module.exports.insert.sync = insertSync +function insertSync (cache, key, integrity, opts) { + opts = IndexOpts(opts) + const bucket = bucketPath(cache, key) + const entry = { + key, + integrity: integrity && ssri.stringify(integrity), + time: Date.now(), + size: opts.size, + metadata: opts.metadata + } + fixOwner.mkdirfix.sync(path.dirname(bucket), opts.uid, opts.gid) + const stringified = JSON.stringify(entry) + fs.appendFileSync( + bucket, `\n${hashEntry(stringified)}\t${stringified}` + ) + try { + fixOwner.chownr.sync(bucket, opts.uid, opts.gid) + } catch (err) { + if (err.code !== 'ENOENT') { + throw err + } + } + return formatEntry(cache, entry) +} + module.exports.find = find function find (cache, key) { const bucket = bucketPath(cache, key) - return bucketEntries(cache, bucket).then(entries => { + return bucketEntries(bucket).then(entries => { return entries.reduce((latest, next) => { if (next && next.key === key) { return formatEntry(cache, next) @@ -95,11 +121,36 @@ function find (cache, key) { }) } +module.exports.find.sync = findSync +function findSync (cache, key) { + const bucket = bucketPath(cache, key) + try { + return bucketEntriesSync(bucket).reduce((latest, next) => { + if (next && next.key === key) { + return formatEntry(cache, next) + } else { + return latest + } + }, null) + } catch (err) { + if (err.code === 'ENOENT') { + return null + } else { + throw err + } + } +} + module.exports.delete = del function del (cache, key, opts) { return insert(cache, key, null, opts) } +module.exports.delete.sync = delSync +function delSync (cache, key, opts) { + return insertSync(cache, key, null, opts) +} + module.exports.lsStream = lsStream function lsStream (cache) { const indexDir = bucketDir(cache) @@ -116,7 +167,6 @@ function lsStream (cache) { // "/cachename///*" return readdirOrEmpty(subbucketPath).map(entry => { const getKeyToEntry = bucketEntries( - cache, path.join(subbucketPath, entry) ).reduce((acc, entry) => { acc.set(entry.key, entry) @@ -152,32 +202,39 @@ function ls (cache) { }) } -function bucketEntries (cache, bucket, filter) { +function bucketEntries (bucket, filter) { return readFileAsync( bucket, 'utf8' - ).then(data => { - let entries = [] - data.split('\n').forEach(entry => { - if (!entry) { return } - const pieces = entry.split('\t') - if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) { - // Hash is no good! Corruption or malice? Doesn't matter! - // EJECT EJECT - return - } - let obj - try { - obj = JSON.parse(pieces[1]) - } catch (e) { - // Entry is corrupted! - return - } - if (obj) { - entries.push(obj) - } - }) - return entries + ).then(data => _bucketEntries(data, filter)) +} + +function bucketEntriesSync (bucket, filter) { + const data = fs.readFileSync(bucket, 'utf8') + return _bucketEntries(data, filter) +} + +function _bucketEntries (data, filter) { + let entries = [] + data.split('\n').forEach(entry => { + if (!entry) { return } + const pieces = entry.split('\t') + if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) { + // Hash is no good! Corruption or malice? Doesn't matter! + // EJECT EJECT + return + } + let obj + try { + obj = JSON.parse(pieces[1]) + } catch (e) { + // Entry is corrupted! + return + } + if (obj) { + entries.push(obj) + } }) + return entries } module.exports._bucketDir = bucketDir diff --git a/lib/util/fix-owner.js b/lib/util/fix-owner.js index 7000bff..0c8f9f8 100644 --- a/lib/util/fix-owner.js +++ b/lib/util/fix-owner.js @@ -31,6 +31,34 @@ function fixOwner (filepath, uid, gid) { ) } +module.exports.chownr.sync = fixOwnerSync +function fixOwnerSync (filepath, uid, gid) { + if (!process.getuid) { + // This platform doesn't need ownership fixing + return + } + if (typeof uid !== 'number' && typeof gid !== 'number') { + // There's no permissions override. Nothing to do here. + return + } + if ((typeof uid === 'number' && process.getuid() === uid) && + (typeof gid === 'number' && process.getgid() === gid)) { + // No need to override if it's already what we used. + return + } + try { + chownr.sync( + filepath, + typeof uid === 'number' ? uid : process.getuid(), + typeof gid === 'number' ? gid : process.getgid() + ) + } catch (err) { + if (err.code === 'ENOENT') { + return null + } + } +} + module.exports.mkdirfix = mkdirfix function mkdirfix (p, uid, gid, cb) { return mkdirp(p).then(made => { @@ -42,3 +70,21 @@ function mkdirfix (p, uid, gid, cb) { return fixOwner(p, uid, gid).then(() => null) }) } + +module.exports.mkdirfix.sync = mkdirfixSync +function mkdirfixSync (p, uid, gid) { + try { + const made = mkdirp.sync(p) + if (made) { + fixOwnerSync(made, uid, gid) + return made + } + } catch (err) { + if (err.code === 'EEXIST') { + fixOwnerSync(p, uid, gid) + return null + } else { + throw err + } + } +} diff --git a/locales/en.js b/locales/en.js index 22025cf..cd0702b 100644 --- a/locales/en.js +++ b/locales/en.js @@ -18,6 +18,8 @@ x.ls.stream = cache => ls.stream(cache) x.get = (cache, key, opts) => get(cache, key, opts) x.get.byDigest = (cache, hash, opts) => get.byDigest(cache, hash, opts) +x.get.sync = (cache, key, opts) => get.sync(cache, key, opts) +x.get.sync.byDigest = (cache, key, opts) => get.sync.byDigest(cache, key, opts) x.get.stream = (cache, key, opts) => get.stream(cache, key, opts) x.get.stream.byDigest = (cache, hash, opts) => get.stream.byDigest(cache, hash, opts) x.get.copy = (cache, key, dest, opts) => get.copy(cache, key, dest, opts) diff --git a/locales/es.js b/locales/es.js index 9a27de6..49003cf 100644 --- a/locales/es.js +++ b/locales/es.js @@ -18,6 +18,8 @@ x.ls.flujo = cache => ls.stream(cache) x.saca = (cache, clave, ops) => get(cache, clave, ops) x.saca.porHacheo = (cache, hacheo, ops) => get.byDigest(cache, hacheo, ops) +x.saca.sinc = (cache, clave, ops) => get.sync(cache, clave, ops) +x.saca.sinc.porHacheo = (cache, hacheo, ops) => get.sync.byDigest(cache, hacheo, ops) x.saca.flujo = (cache, clave, ops) => get.stream(cache, clave, ops) x.saca.flujo.porHacheo = (cache, hacheo, ops) => get.stream.byDigest(cache, hacheo, ops) x.sava.copia = (cache, clave, destino, opts) => get.copy(cache, clave, destino, opts) diff --git a/test/get.js b/test/get.js index 6544a6a..58b7e03 100644 --- a/test/get.js +++ b/test/get.js @@ -80,6 +80,24 @@ test('basic bulk get', t => { }) }) +test('basic sync get', t => { + const fixture = new Tacks(CacheContent({ + [INTEGRITY]: CONTENT + })) + fixture.create(CACHE) + index.insert.sync(CACHE, KEY, INTEGRITY, opts()) + const res = get.sync(CACHE, KEY) + t.deepEqual(res, { + metadata: METADATA, + data: CONTENT, + integrity: INTEGRITY, + size: SIZE + }, 'bulk key get returned proper data') + const resByDig = get.sync.byDigest(CACHE, INTEGRITY) + t.deepEqual(resByDig, CONTENT, 'byDigest returned proper data') + t.done() +}) + test('basic stream get', t => { const fixture = new Tacks(CacheContent({ [INTEGRITY]: CONTENT