diff --git a/.eslintignore b/.eslintignore index a261f29..bb44150 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1 +1,2 @@ dist/* +__tests/helpers/output/ diff --git a/.gitignore b/.gitignore index ca6370c..f20766e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ node_modules # Built code dist/ +__tests__/helpers/output/ +__tests__/helpers/large/ # Logs logs diff --git a/__tests__/helpers/templates/also-cool.json b/__tests__/helpers/templates/also-cool.json new file mode 100644 index 0000000..b7d87a0 --- /dev/null +++ b/__tests__/helpers/templates/also-cool.json @@ -0,0 +1 @@ +{ "fullName": "{{ name }}" } diff --git a/__tests__/helpers/templates/cool.md b/__tests__/helpers/templates/cool.md new file mode 100644 index 0000000..cc4599f --- /dev/null +++ b/__tests__/helpers/templates/cool.md @@ -0,0 +1 @@ +# Hello, {{name}}! diff --git a/__tests__/helpers/templates/not-rendered.txt b/__tests__/helpers/templates/not-rendered.txt new file mode 100644 index 0000000..bb4f2f6 --- /dev/null +++ b/__tests__/helpers/templates/not-rendered.txt @@ -0,0 +1 @@ +This {{ name }} is not changed. diff --git a/__tests__/index.ts b/__tests__/index.ts index 40f05e1..5384dba 100644 --- a/__tests__/index.ts +++ b/__tests__/index.ts @@ -1,7 +1,14 @@ import test from 'ava'; import { promises as fs } from 'fs'; +import mkdirp from 'mkdirp'; import path from 'path'; -import { renderString, renderTemplateFile } from '../src'; +import { + renderGlob, + renderString, + renderTemplateFile, + renderToFolder +} from '../src'; +import { limitOpenFiles } from '../src/utils'; test('Data is replaced when given string', t => { // Should return the same without regard of consistent spacing @@ -67,3 +74,91 @@ test('Data is replaced when given file path', async t => { t.is(actual, expected); }); + +test('Renders from a glob', async t => { + const actualFiles: { name: string; contents: string }[] = []; + const expectedFiles = [ + { + name: './__tests__/helpers/templates/also-cool.json', + contents: '{ "fullName": "Bob" }\n' + }, + { + name: './__tests__/helpers/templates/cool.md', + contents: '# Hello, Bob!\n' + } + ]; + + await renderGlob( + './__tests__/helpers/templates/**/*.!(txt)', + { name: 'Bob' }, + (name, contents) => { + actualFiles.push({ name, contents }); + } + ); + + t.is(actualFiles.length, 2); + + if (actualFiles[0].name === expectedFiles[0].name) { + t.deepEqual(actualFiles, expectedFiles); + } else { + t.deepEqual(actualFiles.reverse(), expectedFiles); + } +}); + +test('Can render output to a file', async t => { + const expectedFiles = [ + { + name: './__tests__/helpers/output/also-cool.json', + contents: '{ "fullName": "Kai" }\n' + }, + { + name: './__tests__/helpers/output/cool.md', + contents: '# Hello, Kai!\n' + } + ]; + + await renderToFolder( + './__tests__/helpers/templates/**/*.!(txt)', + './__tests__/helpers/output', + { name: 'Kai' } + ); + + for (const { name, contents } of expectedFiles) { + const actualContents = await fs.readFile(name, { encoding: 'utf-8' }); + t.is(actualContents, contents); + } +}); + +test('Can render a ton of files', async t => { + const expectedFiles = [] as { name: string; contents: string }[]; + + // Pre-test setup + const templateFolder = './__tests__/helpers/large/'; + const outputFolder = `${templateFolder}/output`; + const template = 'Hello, {{ name }}'; + + await mkdirp(templateFolder); + await Promise.all( + Array.from({ length: 50000 }, (_, i) => { + const basename = `${i}.template`; + + expectedFiles.push({ + name: `${outputFolder}/${basename}`, + contents: 'Hello, Test' + }); + + return limitOpenFiles(() => + fs.writeFile(`${templateFolder}/${basename}`, template) + ); + }) + ); + + await renderToFolder(`${templateFolder}/*.template`, outputFolder, { + name: 'Test' + }); + + for (const { name, contents } of expectedFiles) { + const actualContents = await fs.readFile(name, { encoding: 'utf-8' }); + t.is(actualContents, contents); + } +}); diff --git a/package.json b/package.json index 91993f3..c35dcd5 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,8 @@ ], "require": [ "ts-node/register" - ] + ], + "timeout": "30s" }, "browserslist": [ ">0.2%", @@ -50,6 +51,7 @@ "@blakek/deep": "^2.1.1", "glob": "^7.1.6", "meow": "^8.0.0", + "mkdirp": "^1.0.4", "p-limit": "^3.1.0" }, "devDependencies": { @@ -63,6 +65,7 @@ "@rollup/plugin-node-resolve": "^11.0.0", "@rollup/plugin-typescript": "^8.0.0", "@types/glob": "^7.1.3", + "@types/mkdirp": "^1.0.1", "@typescript-eslint/eslint-plugin": "^4.9.0", "@typescript-eslint/parser": "^4.9.0", "amper-scripts": "^1.0.0-1", diff --git a/src/cli.ts b/src/cli.ts index 2ef9ca6..3bc8445 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,17 +1,10 @@ #!/usr/bin/env node -import { promises as fs } from 'fs'; -import _glob from 'glob'; import meow from 'meow'; -import pLimit from 'p-limit'; import path from 'path'; -import { promisify } from 'util'; -import { renderTemplateFile } from '.'; +import { renderToFolder } from '.'; async function main() { - const glob = promisify(_glob); - const limitOpenFiles = pLimit(64); - const cli = meow(` Usage $ template-file @@ -36,21 +29,7 @@ async function main() { const [dataFile, sourceGlob, destination] = cli.input; const data = await import(path.resolve(dataFile)); - function renderToFile(file: string, destination: string) { - return limitOpenFiles(() => - renderTemplateFile(file, data).then(renderedString => - fs.writeFile(destination, renderedString) - ) - ); - } - - glob(sourceGlob) - .then(files => - files.map(file => - renderToFile(file, path.join(destination, path.basename(file))) - ) - ) - .then(fileWriteOperations => Promise.all(fileWriteOperations)); + renderToFolder(sourceGlob, destination, data); } main(); diff --git a/src/index.ts b/src/index.ts index d86dbb6..5afdc31 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,10 @@ import { get } from '@blakek/deep'; import { promises as fs } from 'fs'; +import _glob from 'glob'; +import mkdirp from 'mkdirp'; +import path from 'path'; +import { promisify } from 'util'; +import { limitOpenFiles } from './utils'; interface Data extends Record< @@ -7,6 +12,20 @@ interface Data string | number | Data | (() => string | number | Data) > {} +export async function renderGlob( + sourceGlob: string, + data: Data, + onFileCallback: (filename: string, contents: string) => void +): Promise { + const glob = promisify(_glob); + const files = await glob(sourceGlob); + + for (const file of files) { + const contents = await limitOpenFiles(() => renderTemplateFile(file, data)); + onFileCallback(file, contents); + } +} + export function renderString( template: string, data: Data @@ -35,3 +54,18 @@ export async function renderTemplateFile( const templateString = await fs.readFile(filepath, { encoding: 'utf-8' }); return renderString(templateString, data); } + +export async function renderToFolder( + sourceGlob: string, + destination: string, + data: Data +): Promise { + await mkdirp(destination); + + function writeFile(filename: string, contents: string) { + const fullPath = path.join(destination, path.basename(filename)); + fs.writeFile(fullPath, contents); + } + + return renderGlob(sourceGlob, data, writeFile); +} diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..fc6e7e6 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,5 @@ +import pLimit from 'p-limit'; + +const OPEN_FILE_LIMIT = Number(process.env.TF_FILE_LIMIT) || 1024; + +export const limitOpenFiles = pLimit(OPEN_FILE_LIMIT); diff --git a/yarn.lock b/yarn.lock index 1483147..f2c0255 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1022,6 +1022,13 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.1.tgz#283f669ff76d7b8260df8ab7a4262cc83d988256" integrity sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg== +"@types/mkdirp@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-1.0.1.tgz#0930b948914a78587de35458b86c907b6e98bbf6" + integrity sha512-HkGSK7CGAXncr8Qn/0VqNtExEE+PHMWb+qlR1faHMao7ng6P3tAaoWWBMdva0gL5h4zprjIO89GJOLXsMcDm1Q== + dependencies: + "@types/node" "*" + "@types/node@*": version "14.14.10" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.10.tgz#5958a82e41863cfc71f2307b3748e3491ba03785" @@ -3410,6 +3417,11 @@ minimist@^1.2.0, minimist@^1.2.5: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== +mkdirp@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"