diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 1d5fce3..d3bce6c 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -22,4 +22,9 @@ jobs: with: node-version: ${{ matrix.node-version }} - run: npm ci - - run: npm test + - name: check if json up to date + run: | + npm run xmltest.json + git add . # make sure line endings are sanitized by git + git diff HEAD --exit-code + - run: npm run test.zip && npm test diff --git a/README.md b/README.md index 007a253..5ccbe89 100644 --- a/README.md +++ b/README.md @@ -101,14 +101,23 @@ All methods have doc comments that include types. - `getContent` - `getEntries` - `load` +- `contentLoader` +- `entriesLoader` - `replaceWithWrappedCodePointAt` - `replaceNonTextChars` - `run` (Feel free to contribute by automating the extraction of the documentation to this or another file.) +### with different zip files + +The API can be used with other zip files by passing relative or absolute file names as arguments: +- `load` (second argument) +- `run` (first argument) + ## Related Resources - The page of the author linking to xmltest.zip: -- THe way I found those testcases since they are part of a bigger testsuite for (Java SAX parsers) +- The way I found those testcases since they are part of a bigger testsuite for (Java SAX parsers) +- The W3C also provides an XML test suite: (the files in `xmltest.zip` are part of this but there is no clear license for the whole package) - The PR that initially led to the creation of this package: diff --git a/cache.js b/cache.js new file mode 100644 index 0000000..e861074 --- /dev/null +++ b/cache.js @@ -0,0 +1,16 @@ +exports.cache = () => { + let map = new Map(); + + return { + clear: () => { + map = new Map(); + }, + delete: (key) => map.delete(key), + get: (key) => map.get(key), + has: (key) => map.has(key), + keys: () => [...map.keys()], + set: (key, value) => { + map.set(key, value); + }, + }; +}; diff --git a/package.json b/package.json index 48cbbd1..6a7204c 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "node": ">=10" }, "files": [ + "cache.js", "README.md", "LICENSE", "xmltest.zip", @@ -18,7 +19,9 @@ "scripts": { "extract": "npx extract-zip xmltest.zip $PWD/data", "test": "jest", - "start": "jest --watch" + "test.zip": "./test.zip.sh", + "start": "npm run test -- --watch", + "xmltest.json": "runex ./xmltest.js > xmltest.json" }, "repository": { "type": "git", diff --git a/test.zip b/test.zip new file mode 100644 index 0000000..c32eb9c Binary files /dev/null and b/test.zip differ diff --git a/test.zip.sh b/test.zip.sh new file mode 100755 index 0000000..6b6610b --- /dev/null +++ b/test.zip.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Creates test.zip file in the same directory + +# clear the folder if it already existed +rm -rf data/folder +# the `data` directory is .gitignored, so we can use it as temp +mkdir -p data/folder +# add one file that has content +echo CONTENT > data/folder/file.ext +# and one empty file +touch data/folder/empty + +# It is important for the tests to have the optional `folder` entry in the zip file. +(cd data && zip ../test.zip folder folder/*) diff --git a/test/cache.test.js b/test/cache.test.js new file mode 100644 index 0000000..b740279 --- /dev/null +++ b/test/cache.test.js @@ -0,0 +1,46 @@ +const { cache } = require("../cache"); + +describe("cache", () => { + test("should have no keys initially", () => { + expect(cache().keys()).toHaveLength(0); + }); + test.each(["key", 0, 1, "", null, NaN, undefined])( + "should store value for key `%s`", + (key) => { + const value = {}; + const it = cache(); + + expect(it.has(key)).toBe(false); + it.set(key, value); + expect(it.has(key)).toBe(true); + expect(it.keys()).toHaveLength(1); + expect(it.get(key)).toBe(value); + } + ); + test.each(["key", 0, 1, "", null, NaN, undefined])( + "should return undefined for key `%s`", + (key) => { + const it = cache(); + + expect(it.has(key)).toBe(false); + expect(it.get(key)).toBeUndefined(); + } + ); + test.each(["key", 0, 1, "", null, NaN, undefined])( + "should delete key for key `%s`", + (key) => { + const it = cache(); + it.set(key, {}); + it.delete(key); + expect(it.has(key)).toBe(false); + expect(it.get(key)).toBeUndefined(); + } + ); + test("should clear the cache", () => { + const it = cache(); + it.set("key", {}); + it.clear(); + expect(it.has("key")).toBe(false); + expect(it.keys()).toHaveLength(0); + }); +}); diff --git a/test/run.test.js b/test/run.test.js new file mode 100644 index 0000000..a019ff3 --- /dev/null +++ b/test/run.test.js @@ -0,0 +1,61 @@ +const entries = require("../xmltest.json"); +const { run, contentLoader, entriesLoader } = require("../xmltest.js"); +const path = require("path"); + +const README_PATH = "xmltest/readme.html"; +const TEST_ZIP_PATH = path.join(__dirname, "..", "test.zip"); + +const TEST_ZIP_ENTRIES = { + "folder/": "", + "folder/file.ext": "file.ext", + "folder/empty": "empty", +}; + +describe("run", () => { + beforeEach(contentLoader.CACHE.clear); + beforeEach(entriesLoader.CACHE.clear); + describe("only filter arguments", () => { + test("should return entries without any arguments", async () => { + // FYI: xmltest.zip doesn't contain any folder entries + expect(await run()).toEqual(entries); + expect(contentLoader.CACHE.keys()).toHaveLength(0); + expect(entriesLoader.CACHE.keys()).toHaveLength(1); + }); + test("should return all (file) keys in entries with first argument 'xmltest'", async () => { + expect(Object.keys(await run("xmltest"))).toEqual(Object.keys(entries)); + expect(contentLoader.CACHE.keys()).toHaveLength(1); + expect(entriesLoader.CACHE.keys()).toHaveLength(0); + }); + test("should return the content of readme.html with first argument 'xmltest/readme.html'", async () => { + expect(await run(README_PATH)).toMatch(/^.*/); + }); + test("should return dict with single key when multiple filters only match one entry", async () => { + const actual = await run(...README_PATH.split("/")); + expect(Object.keys(actual)).toHaveLength(1); + expect(actual[README_PATH]).toMatch(/^.*/); + }); + }); + describe("first argument is path to zip", () => { + test.each(["./test.zip", "../xmltest/test.zip", TEST_ZIP_PATH])( + "should return all entries without any filter arguments %s", + async (pathToZip) => { + expect(await run(pathToZip)).toEqual(TEST_ZIP_ENTRIES); + } + ); + test("should return all file keys in entries with first filter argument 'folder'", async () => { + const actual = await run(TEST_ZIP_PATH, "folder"); + expect(Object.keys(actual)).toEqual( + Object.keys(TEST_ZIP_ENTRIES).filter((entry) => !entry.endsWith("/")) + ); + }); + test("should return the content when first filter argument matches a file", async () => { + expect(await run(TEST_ZIP_PATH, "folder/file.ext")).toBe("CONTENT\n"); + expect(await run(TEST_ZIP_PATH, "folder/empty")).toBe(""); + }); + test("should return dict with single key when multiple filters only match one entry", async () => { + const actual = await run(TEST_ZIP_PATH, "folder", "file"); + expect(Object.keys(actual)).toHaveLength(1); + expect(actual["folder/file.ext"]).toMatch("CONTENT\n"); + }); + }); +}); diff --git a/xmltest.js b/xmltest.js index c8cca42..1a123a4 100644 --- a/xmltest.js +++ b/xmltest.js @@ -1,13 +1,15 @@ -const entries = require('./xmltest.json') const getStream = require('get-stream') const path = require('path') const {promisify} = require('util') const yauzl = require('yauzl') + +const {cache} = require('./cache') // for type definitions const {Entry} = require('yauzl') /** - * @typedef PromiseResolve {function (response: typeof entries)} + * @typedef Entries {Record} + * @typedef PromiseResolve {function (response: Entries)} * @typedef PromiseReject {function (reason: Error)} * @typedef ReadFile {async function (response: Entry): Promise} * @typedef EntryHandler {async function (response: Entry, readFile: ReadFile): Promise} @@ -16,22 +18,18 @@ const {Entry} = require('yauzl') */ /** - * Loads all file content from the zip file and caches it + * Loads all file content from the zip file. * * @param resolve {PromiseResolve} * @param reject {PromiseReject} * @returns {LoaderInstance} */ -const dataLoader = (resolve, reject) => { - if (dataLoader.DATA) { - resolve({...dataLoader.DATA}) - } - /** @type {Partial} */ - const data = {} +const contentLoader = (resolve, reject) => { + /** @type {Entries} */ + const data = {}; const end = () => { - dataLoader.DATA = data - resolve(dataLoader.DATA) + resolve(data) } const entry = async (entry, readFile) => { @@ -44,9 +42,9 @@ const dataLoader = (resolve, reject) => { /** * The module level cache for the zip file content. * - * @type {null | typeof entries} + * @type {null | Entries} */ -dataLoader.DATA = null +contentLoader.CACHE = cache(); /** * Loads the list of files and directories. @@ -61,8 +59,8 @@ dataLoader.DATA = null * @param reject {PromiseReject} * @returns {LoaderInstance} */ -const jsonLoader = (resolve, reject) => { - /** @type {Partial} */ +const entriesLoader = (resolve, reject) => { + /** @type {Entries} */ const data = {} const end = () => { resolve(data) @@ -74,20 +72,28 @@ const jsonLoader = (resolve, reject) => { } return {end, entry} } +entriesLoader.CACHE = cache(); /** - * Uses `loader` to iterate entries in a zipfile using `yauzl`. + * Uses `loader` to iterate entries in a zip file using `yauzl`. + * If `loader.CACHE` is set it is assumed to be an instance of `cache`, + * and is used to store the resolved result. + * If `loader.CACHE.has(location)` is true the zip file is not read again, + * since the cached result is returned. + * Use `loader.CACHE.delete(location)` or `loader.CACHE.clear()` when needed. * - * @see dataLoader - * @see jsonLoader + * @see contentLoader + * @see contentLoader.CACHE + * @see entriesLoader + * @see entriesLoader.CACHE * - * @param loader {Loader} the loader to use (default: `dataLoader`) + * @param loader {Loader} the loader to use (default: `contentLoader`) * @param location {string} absolute path to zip file (default: xmltest.zip) - * @returns {Promise} + * @returns {Promise} */ -const load = async (loader = dataLoader, location = path.join(__dirname, 'xmltest.zip')) => { - if (loader.DATA) { - return {...loader.DATA} +const load = async (loader = contentLoader, location = path.join(__dirname, 'xmltest.zip')) => { + if (loader.CACHE && loader.CACHE.has(location)) { + return {...loader.CACHE.get(location)} } const zipfile = await promisify(yauzl.open)( @@ -95,13 +101,19 @@ const load = async (loader = dataLoader, location = path.join(__dirname, 'xmltes ) const readFile = promisify(zipfile.openReadStream.bind(zipfile)) return new Promise((resolve, reject) => { - const handler = loader(resolve, reject) - zipfile.on('end', handler.end) + const resolver = loader.CACHE + ? (data) => { + loader.CACHE.set(location, data); + resolve(data); + } + : resolve; + const handler = loader(resolver, reject); + zipfile.on('end', handler.end); zipfile.on('entry', async (entry) => { - await handler.entry(entry, readFile) - zipfile.readEntry() - }) - zipfile.readEntry() + await handler.entry(entry, readFile); + zipfile.readEntry(); + }); + zipfile.readEntry(); }) } @@ -216,9 +228,9 @@ const RELATED = { * Filters `data` by applying `filters` to it's keys * * @see combineFilters - * @param data {typeof entries} + * @param data {Entries} * @param filters {(string | RegExp | Predicate)[]} - * @returns {string | Partial} the value + * @returns {string | Entries} the value * if the only filter only results a single entry, * otherwise on object with all keys that match the filter. */ @@ -236,7 +248,7 @@ const getFiltered = (data, filters) => { acc[key] = data[key] return acc }, - /** @type {Partial} */{} + /** @type {Entries} */{} ) } @@ -249,7 +261,7 @@ const getFiltered = (data, filters) => { * @see load * * @param filters {string | RegExp | Predicate} - * @returns {Promise>} the value + * @returns {Promise} the value * if the only filter only results a single entry, * otherwise on object with all keys that match the filter. */ @@ -260,30 +272,45 @@ const getContent = async (...filters) => getFiltered(await load(), filters) * * @see combineFilters * @param filters {string | RegExp | Predicate} - * @returns {string | Partial} the value + * @returns {string | Entries} the value * if the only filter only results a single entry, * otherwise on object with all keys that match the filter. */ -const getEntries = (...filters) => getFiltered(entries, filters) +const getEntries = (...filters) => getFiltered(require('./xmltest.json') + , filters) /** * Makes module executable using `runex`. - * With no arguments: Returns Object structure to store in `xmltest.json` + * If the first argument begins with `/`, `./` or `../` and ends with `.zip`, + * it is removed from the list of filter arguments and used as the path + * to the archive to load. + * + * With no filter arguments: Returns Object structure to store in `xmltest.json` * `npx runex . > xmltest.json` - * With one argument: Returns content string if exact key match, + * With one filter argument: Returns content string if exact key match, * or content dict with filtered keys - * With more arguments: Returns content dict with filtered keys + * With more filter arguments: Returns content dict with filtered keys * * @see getFiltered * @see combineFilters * @see load + * @see https://github.com/karfau/runex * - * @param filters {(string | RegExp | Predicate)[]} - * @returns {Promise>} + * @param filters {string} + * @returns {Promise} */ -const run = async (...filters) => filters.length === 0 - ? getEntries() - : getContent.apply(null, filters) +const run = async (...filters) => { + let file; + + if (filters.length > 0 && /^\.?\.?\/.*\.zip$/.test(filters[0])) { + file = filters.shift(); + } + + return getFiltered( + await load(filters.length === 0 ? entriesLoader : contentLoader, file), + filters + ); +}; const replaceWithWrappedCodePointAt = char => `{!${char.codePointAt(0).toString(16)}!}` @@ -312,13 +339,15 @@ module.exports = { getContent, getEntries, load, + contentLoader, + entriesLoader, replaceNonTextChars, replaceWithWrappedCodePointAt, run } if (require.main === module) { - // if you don't want to use `runex` just "launch" this module/package - module.exports.run().then(console.log) + // if you don't want to use `runex` just "launch" this module/package: + // node xmltest ... + module.exports.run(...process.argv.slice(2)).then(console.log) } -