From 74a4b19f8119f4de70045f7f4c806dc6c793e283 Mon Sep 17 00:00:00 2001 From: "Mark S. Everitt" Date: Sat, 4 May 2024 11:57:26 +0100 Subject: [PATCH] Overhaul the execution graph API. --- index.js | 460 +++++++++++++++------------- lib/execution-graph.js | 214 +++++++------ tests/units/execution-graph.test.js | 146 ++++----- 3 files changed, 440 insertions(+), 380 deletions(-) diff --git a/index.js b/index.js index dbd1fe8f..737c75ed 100644 --- a/index.js +++ b/index.js @@ -18,7 +18,7 @@ import loadReplyFiles from './lib/load-reply-files.js'; import buildBacklinks from './lib/build-backlinks.js'; import collateTags from './lib/collate-tags.js'; import getLastCommitTime from './lib/get-last-commit-time.js'; -import ExecutionGraph from './lib/execution-graph.js'; +import ExecutionGraph, { GraphNode, WatchableResult } from './lib/execution-graph.js'; import hashCopy from './lib/hash-copy.js'; import NetlifyHeaders from './lib/netlify-headers.js'; @@ -43,8 +43,9 @@ function renderResources({ noIndex, resources, template, cssPath, baseUrl, backl })); } -function makeWriteEntries({ renderedDependencies, pathFragment }) { - return { +function makeWriteEntries({ name, renderedDependencies, pathFragment }) { + return new GraphNode({ + name, dependencies: ['targetPath', renderedDependencies], action({ [renderedDependencies]: rendered }) { return rendered.map(({ content, filename }) => { @@ -52,16 +53,17 @@ function makeWriteEntries({ renderedDependencies, pathFragment }) { return writeFile(new URL(path, targetPath), content); }); } - }; + }); } -function writeRendered(renderedName, path) { - return { +function writeRendered(name, renderedName, path) { + return new GraphNode({ + name, dependencies: ['targetPath', renderedName], action(config) { return writeFile(new URL(path, targetPath), config[renderedName]); } - }; + }); } async function copyStaticDirectory(sourceDirectory, targetDirectory, allowedFileEndings) { @@ -72,7 +74,7 @@ async function copyStaticDirectory(sourceDirectory, targetDirectory, allowedFile recursive: true }); - return ExecutionGraph.createWatchableResult({ path: sourceDirectory, result: targetDirectory }); + return new WatchableResult({ path: sourceDirectory, result: targetDirectory }); } async function makeDirectory(path) { @@ -84,85 +86,81 @@ async function makeDirectory(path) { return directory; } -function makeDirectoryNode(path, watchPath) { - return { +function makeDirectoryNode(name, path, watchPath) { + return new GraphNode({ + name, dependencies: ['targetPath'], async action() { const result = await makeDirectory(path); - return watchPath ? ExecutionGraph.createWatchableResult({ path: watchPath, result }) : result; + return watchPath ? new WatchableResult({ path: watchPath, result }) : result; } - }; + }); } -// This is where it all kicks off. This function loads posts and templates, -// renders it all to files, and saves them to the public directory. -export async function build({ baseUrl, baseTitle, repoUrl, dev }) { - let watcher = null; - - if (dev) { - async function* makeWatcher() { - for await (const { eventType, filename } of watch('./', { recursive: true })) { - if (filename && (filename.startsWith('content/') || filename.startsWith('src/'))) { - yield { eventType, url: pathToFileURL(pathJoin(import.meta.dirname, filename)) }; - } - } +async function* makeWatcher() { + for await (const { eventType, filename } of watch('./', { recursive: true })) { + if (filename && (filename.startsWith('content/') || filename.startsWith('src/'))) { + yield { eventType, url: pathToFileURL(pathJoin(import.meta.dirname, filename)) }; } - watcher = makeWatcher(); } +} +// This is where it all kicks off. This function loads posts and templates, +// renders it all to files, and saves them to the public directory. +export async function build({ baseUrl, baseTitle, repoUrl, dev }) { + const watcher = dev ? makeWatcher() : null; const graph = new ExecutionGraph({ watcher }); - await graph.addNodes({ - async targetPath() { - await rm(targetPath, { recursive: true, force: true }); - await mkdir(targetPath); - }, - commitTime() { + await graph.addNodes([ + new GraphNode({ + name: 'targetPath', + async action() { + await rm(targetPath, { recursive: true, force: true }); + await mkdir(targetPath); + } + }), + function commitTime() { return getLastCommitTime(contentPath); }, - async templates() { + async function templates() { const path = new URL('templates/', sourcePath); const result = await loadTemplates(path, { baseTitle }); - return ExecutionGraph.createWatchableResult({ path, result }); + return new WatchableResult({ path, result }); }, - async feeds() { + async function feeds() { const feedsPath = new URL('feeds.json', contentPath); const json = await readFile(feedsPath, 'utf8'); - return ExecutionGraph.createWatchableResult({ - path: feedsPath, - result: JSON.parse(json) - }); + return new WatchableResult({ path: feedsPath, result: JSON.parse(json) }); }, - async publications() { + async function publications() { const publicationsPath = new URL('publications.json', contentPath); const json = await readFile(publicationsPath, 'utf8'); - return ExecutionGraph.createWatchableResult({ - path: publicationsPath, - result: JSON.parse(json) - }); + return new WatchableResult({ path: publicationsPath, result: JSON.parse(json) }); }, - specialFiles: { + new GraphNode({ + name: 'specialFiles', dependencies: ['extraCss', 'hashedScripts', 'images'], async action({ extraCss, hashedScripts, images }) { const specialPath = new URL('special/', contentPath); - return ExecutionGraph.createWatchableResult({ + return new WatchableResult({ path: specialPath, result: await loadPostFiles({ path: specialPath, basePath, repoUrl, baseUrl, extraCss, hashedScripts, type: null, images }) }); } - }, - postFiles: { + }), + new GraphNode({ + name: 'postFiles', dependencies: ['extraCss', 'mathStyles', 'codeStyle', 'hashedScripts', 'images'], async action({ extraCss, mathStyles, codeStyle, hashedScripts, images }) { const postsPath = new URL('posts/', contentPath); const joinedStyles = new Map([...extraCss, ...mathStyles, ...codeStyle]); - return ExecutionGraph.createWatchableResult({ + return new WatchableResult({ path: postsPath, result: await loadPostFiles({ path: postsPath, @@ -177,8 +175,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { }) }); } - }, - populateHeaders: { + }), + new GraphNode({ + name: 'populateHeaders', dependencies: ['targetPath', 'postFiles', 'specialFiles'], action({ postFiles, specialFiles }) { const headers = new NetlifyHeaders(); @@ -207,73 +206,61 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { writeFile(new URL('_headers', targetPath), `${headers.generate()}\n`); } - }, - japaneseNotesFiles: { + }), + new GraphNode({ + name: 'japaneseNotesFiles', dependencies: ['hashedScripts', 'images'], async action({ hashedScripts, images }) { const notesPath = new URL('japanese-notes/', contentPath); const type = 'japanese-notes'; - return ExecutionGraph.createWatchableResult({ + return new WatchableResult({ path: notesPath, result: await loadPostFiles({ path: notesPath, basePath, repoUrl, baseUrl, type, hashedScripts, images }) }); } - }, - noteFiles: { + }), + new GraphNode({ + name: 'noteFiles', dependencies: ['images'], async action({ images: imagesDimensions }) { const dir = new URL('notes/', contentPath); - return ExecutionGraph.createWatchableResult({ - path: dir, - result: await loadNoteFiles({ baseUrl, dir, imagesDimensions }) - }); + return new WatchableResult({ path: dir, result: await loadNoteFiles({ baseUrl, dir, imagesDimensions }) }); } - }, - async studySessionFiles() { + }), + async function studySessionFiles() { const dir = new URL('study-sessions/', contentPath); - return ExecutionGraph.createWatchableResult({ - path: dir, - result: await loadStudySessionsFiles({ baseUrl, dir }) - }); + return new WatchableResult({ path: dir, result: await loadStudySessionsFiles({ baseUrl, dir }) }); }, - async linkFiles() { + async function linkFiles() { const dir = new URL('links/', contentPath); - return ExecutionGraph.createWatchableResult({ - path: dir, - result: await loadLinkFiles({ baseUrl, dir }) - }); + return new WatchableResult({ path: dir, result: await loadLinkFiles({ baseUrl, dir }) }); }, - async likeFiles() { + async function likeFiles() { const dir = new URL('likes/', contentPath); - return ExecutionGraph.createWatchableResult({ - path: dir, - result: await loadLikeFiles({ baseUrl, dir }) - }); + return new WatchableResult({ path: dir, result: await loadLikeFiles({ baseUrl, dir }) }); }, - async replyFiles() { + async function replyFiles() { const dir = new URL('replies/', contentPath); - return ExecutionGraph.createWatchableResult({ - path: dir, - result: await loadReplyFiles({ baseUrl, dir }) - }); - }, - stylesDirectory: makeDirectoryNode('styles/'), - blogDirectory: makeDirectoryNode('blog/'), - japaneseNotesDirectory: makeDirectoryNode('japanese-notes/'), - notesDirectory: makeDirectoryNode('notes/'), - studySessionsDirectory: makeDirectoryNode('study-sessions/'), - linksDirectory: makeDirectoryNode('links/'), - likesDirectory: makeDirectoryNode('likes/'), - repliesDirectory: makeDirectoryNode('replies/'), - tagsDirectory: makeDirectoryNode('tags/'), - imagesTarget: makeDirectoryNode('images/', new URL('images/', sourcePath)), - activitypubDocuments: { + return new WatchableResult({ path: dir, result: await loadReplyFiles({ baseUrl, dir }) }); + }, + makeDirectoryNode('stylesDirectory', 'styles/'), + makeDirectoryNode('blogDirectory', 'blog/'), + makeDirectoryNode('japaneseNotesDirectory', 'japanese-notes/'), + makeDirectoryNode('notesDirectory', 'notes/'), + makeDirectoryNode('studySessionsDirectory', 'study-sessions/'), + makeDirectoryNode('linksDirectory', 'links/'), + makeDirectoryNode('likesDirectory', 'likes/'), + makeDirectoryNode('repliesDirectory', 'replies/'), + makeDirectoryNode('tagsDirectory', 'tags/'), + makeDirectoryNode('imagesTarget', 'images/', new URL('images/', sourcePath)), + new GraphNode({ + name: 'activitypubDocuments', dependencies: ['targetPath'], action() { const sourceDirectory = new URL('activitypub/', sourcePath); @@ -281,8 +268,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { return copyStaticDirectory(sourceDirectory, targetDirectory, ['.json']); } - }, - fingerTarget: { + }), + new GraphNode({ + name: 'fingerTarget', dependencies: ['targetPath'], async action() { const wellKnownDirectory = await makeDirectory('.well-known/'); @@ -298,8 +286,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { ] })); } - }, - icons: { + }), + new GraphNode({ + name: 'icons', dependencies: ['targetPath'], action() { const sourceDirectory = new URL('icons/', sourcePath); @@ -307,8 +296,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { return copyStaticDirectory(sourceDirectory, targetDirectory, ['.png', '.svg']); } - }, - img: { + }), + new GraphNode({ + name: 'img', dependencies: ['targetPath'], action() { const sourceDirectory = new URL('img/', sourcePath); @@ -316,8 +306,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { return copyStaticDirectory(sourceDirectory, targetDirectory, ['.jpg', '.jpeg', '.webp', '.avif']); } - }, - scripts: { + }), + new GraphNode({ + name: 'scripts', dependencies: ['targetPath'], action() { const sourceDirectory = new URL('scripts/', contentPath); @@ -325,8 +316,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { return copyStaticDirectory(sourceDirectory, targetDirectory, ['.js']); } - }, - hashedScripts: { + }), + new GraphNode({ + name: 'hashedScripts', dependencies: ['targetPath', 'scripts'], async action({ scripts }) { const path = new URL('scripts/', contentPath); @@ -334,10 +326,11 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { const entries = await Promise.all(items.map(item => hashCopy(targetPath, new URL(item, path), scripts))); const result = Object.fromEntries(entries); - return ExecutionGraph.createWatchableResult({ path, result }); + return new WatchableResult({ path, result }); } - }, - papers: { + }), + new GraphNode({ + name: 'papers', dependencies: ['targetPath'], action() { const sourceDirectory = new URL('papers/', contentPath); @@ -345,8 +338,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { return copyStaticDirectory(sourceDirectory, targetDirectory, ['.pdf']); } - }, - images: { + }), + new GraphNode({ + name: 'images', dependencies: ['imagesTarget'], async action({ imagesTarget }) { const directory = new URL('images/', contentPath); @@ -363,41 +357,39 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { }) )); } - }, - googleSiteVerification: { + }), + new GraphNode({ + name: 'googleSiteVerification', dependencies: ['targetPath'], action() { const name = 'google91826e4f943d9ee9.html'; return writeFile(new URL(name, targetPath), `google-site-verification: ${name}\n`); } - }, - keybaseVerification: { + }), + new GraphNode({ + name: 'keybaseVerification', dependencies: ['targetPath'], async action() { const verificationPath = new URL('keybase.txt', sourcePath); await copyFile(verificationPath, new URL('keybase.txt', targetPath)); - return ExecutionGraph.createWatchableResult({ - path: verificationPath, - result: null - }); + return new WatchableResult({ path: verificationPath, result: null }); } - }, - robotsFile: { + }), + new GraphNode({ + name: 'robotsFile', dependencies: ['targetPath'], async action() { const robotsPath = new URL('robots.txt', sourcePath); await copyFile(robotsPath, new URL('robots.txt', targetPath)); - return ExecutionGraph.createWatchableResult({ - path: robotsPath, - result: null - }); + return new WatchableResult({ path: robotsPath, result: null }); } - }, - css: { + }), + new GraphNode({ + name: 'css', dependencies: ['targetPath'], async action() { const { url, htmlPath } = await generateMainCss({ @@ -405,27 +397,26 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { targetDirectory: targetPath }); - return ExecutionGraph.createWatchableResult({ - path: new URL('css/', sourcePath), - result: { url, cssPath: htmlPath } - }); + return new WatchableResult({ path: new URL('css/', sourcePath), result: { url, cssPath: htmlPath } }); }, onRemove() { return unlink(this.result.url); } - }, - extraCss: { + }), + new GraphNode({ + name: 'extraCss', dependencies: ['stylesDirectory'], async action({ stylesDirectory }) { const cssPath = new URL('styles/', contentPath); - return ExecutionGraph.createWatchableResult({ + return new WatchableResult({ path: cssPath, result: await generateSpecificCss(cssPath, stylesDirectory) }); } - }, - mathsFontName: { + }), + new GraphNode({ + name: 'mathsFontName', dependencies: ['stylesDirectory'], async action({ stylesDirectory }) { const readWoff = await readFile(new URL('node_modules/temml/dist/Temml.woff2', basePath)); @@ -437,16 +428,14 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { await writeFile(path, readWoff); - return ExecutionGraph.createWatchableResult({ - path, - result: { woffName: hashedWoffName, url: path } - }); + return new WatchableResult({ path, result: { woffName: hashedWoffName, url: path } }); }, onRemove() { return unlink(this.result.url); } - }, - mathStyles: { + }), + new GraphNode({ + name: 'mathStyles', dependencies: ['stylesDirectory', 'mathsFontName'], async action({ stylesDirectory, mathsFontName }) { const cssPath = new URL('node_modules/temml/dist/Temml-Local.css', basePath); @@ -459,7 +448,7 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { await writeFile(new URL(hashedCssName, stylesDirectory), updatedCss); - return ExecutionGraph.createWatchableResult({ + return new WatchableResult({ path: cssPath, result: new Map([['/styles/temml.css', `/styles/${hashedCssName}`]]) }); @@ -467,8 +456,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { onRemove() { // todo } - }, - codeStyle: { + }), + new GraphNode({ + name: 'codeStyle', dependencies: ['stylesDirectory'], async action({ stylesDirectory }) { const cssPath = new URL('node_modules/highlight.js/styles/default.css', basePath); @@ -485,74 +475,86 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { onRemove() { // todo } - }, - collatedTags: { + }), + new GraphNode({ + name: 'collatedTags', dependencies: ['css', 'templates', 'postFiles'], action({ postFiles: posts, css: { cssPath }, templates: { tag: template } }) { return collateTags({ posts, cssPath, baseUrl, dev, template }); } - }, - backlinks: { + }), + new GraphNode({ + name: 'backlinks', dependencies: ['japaneseNotesFiles', 'postFiles', 'specialFiles'], action({ japaneseNotesFiles, postFiles, specialFiles }) { return buildBacklinks([...japaneseNotesFiles, ...postFiles, ...specialFiles]); } - }, - renderedShortlinks: { + }), + new GraphNode({ + name: 'renderedShortlinks', dependencies: ['templates', 'postFiles'], action({ postFiles, templates }) { return templates.shortlinks({ name: 'shortlinks', items: postFiles, baseUrl }); } - }, - renderedSpecials: { + }), + new GraphNode({ + name: 'renderedSpecials', dependencies: ['css', 'templates', 'specialFiles', 'backlinks'], action({ specialFiles: resources, backlinks, templates: { blog: template }, css: { cssPath } }) { return renderResources({ resources, backlinks, template, cssPath, baseUrl, dev }); } - }, - renderedPosts: { + }), + new GraphNode({ + name: 'renderedPosts', dependencies: ['css', 'templates', 'postFiles', 'backlinks'], action({ postFiles: resources, backlinks, templates: { blog: template }, css: { cssPath } }) { return renderResources({ resources, backlinks, template, cssPath, baseUrl, dev }); } - }, - renderedJapaneseNotes: { + }), + new GraphNode({ + name: 'renderedJapaneseNotes', dependencies: ['css', 'templates', 'japaneseNotesFiles', 'backlinks'], action({ japaneseNotesFiles: resources, backlinks, templates: { blog: template }, css: { cssPath } }) { return renderResources({ noIndex: true, resources, backlinks, template, cssPath, baseUrl, dev }); } - }, - renderedNotes: { + }), + new GraphNode({ + name: 'renderedNotes', dependencies: ['css', 'templates', 'noteFiles'], action({ noteFiles: resources, templates: { note: template }, css: { cssPath } }) { return renderResources({ noIndex: true, resources, template, cssPath, baseUrl, dev }); } - }, - renderedStudySessions: { + }), + new GraphNode({ + name: 'renderedStudySessions', dependencies: ['css', 'templates', 'studySessionFiles'], action({ studySessionFiles: resources, templates: { 'study-session': template }, css: { cssPath } }) { return renderResources({ noIndex: true, resources, template, cssPath, baseUrl, dev }); } - }, - renderedLinks: { + }), + new GraphNode({ + name: 'renderedLinks', dependencies: ['css', 'templates', 'linkFiles'], action({ linkFiles: resources, templates: { link: template }, css: { cssPath } }) { return renderResources({ resources, template, cssPath, baseUrl, dev }); } - }, - renderedLikes: { + }), + new GraphNode({ + name: 'renderedLikes', dependencies: ['css', 'templates', 'likeFiles'], action({ likeFiles: resources, templates: { like: template }, css: { cssPath } }) { return renderResources({ resources, template, cssPath, baseUrl, dev }); } - }, - renderedReplies: { + }), + new GraphNode({ + name: 'renderedReplies', dependencies: ['css', 'templates', 'replyFiles'], action({ replyFiles: resources, templates: { reply: template }, css: { cssPath } }) { return renderResources({ resources, template, cssPath, baseUrl, dev }); } - }, - renderedBlogIndex: { + }), + new GraphNode({ + name: 'renderedBlogIndex', dependencies: ['css', 'templates', 'postFiles'], action({ postFiles: posts, templates, css: { cssPath } }) { return templates.blogs({ @@ -566,8 +568,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'A collection of my long form articles.' }); } - }, - renderedJapaneseNotesIndex: { + }), + new GraphNode({ + name: 'renderedJapaneseNotesIndex', dependencies: ['css', 'templates', 'japaneseNotesFiles'], action({ japaneseNotesFiles: posts, templates, css: { cssPath } }) { return templates.blogs({ @@ -583,8 +586,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'A collection of notes I\'ve written on the Japanese language, as I study it.' }); } - }, - renderedNotesIndex: { + }), + new GraphNode({ + name: 'renderedNotesIndex', dependencies: ['css', 'templates', 'noteFiles'], action({ noteFiles: notes, templates, css: { cssPath } }) { return templates.notes({ @@ -598,8 +602,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'A collection of my short notes.' }); } - }, - renderedStudySessionsIndex: { + }), + new GraphNode({ + name: 'renderedStudySessionsIndex', dependencies: ['css', 'templates', 'studySessionFiles'], action({ studySessionFiles: studySessions, templates, css: { cssPath } }) { return templates['study-sessions']({ @@ -613,8 +618,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'A collection of my study sessions.' }); } - }, - renderedLinksIndex: { + }), + new GraphNode({ + name: 'renderedLinksIndex', dependencies: ['css', 'templates', 'linkFiles'], action({ linkFiles: links, templates, css: { cssPath } }) { return templates.links({ @@ -627,8 +633,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'A collection of links to articles elsewhere on the web.' }); } - }, - renderedLikesIndex: { + }), + new GraphNode({ + name: 'renderedLikesIndex', dependencies: ['css', 'templates', 'likeFiles'], action({ likeFiles: likes, templates, css: { cssPath } }) { return templates.likes({ @@ -641,8 +648,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'A collection of likes of articles elsewhere on the web.' }); } - }, - renderedRepliesIndex: { + }), + new GraphNode({ + name: 'renderedRepliesIndex', dependencies: ['css', 'templates', 'replyFiles'], action({ replyFiles: replies, templates, css: { cssPath } }) { return templates.replies({ @@ -655,8 +663,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'A collection of the replies to articles elsewhere on the web.' }); } - }, - renderedAbout: { + }), + new GraphNode({ + name: 'renderedAbout', dependencies: ['css', 'templates'], action({ templates, css: { cssPath } }) { return templates.about({ @@ -668,8 +677,9 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'The personal site of Mark Stanley Everitt.' }); } - }, - renderedPublications: { + }), + new GraphNode({ + name: 'renderedPublications', dependencies: ['css', 'templates', 'publications'], action({ templates, css: { cssPath }, publications }) { return templates.publications({ @@ -682,14 +692,16 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'A collection of academic publications I have authored and coauthored.' }); } - }, - renderedFourOhFour: { + }), + new GraphNode({ + name: 'renderedFourOhFour', dependencies: ['css', 'templates'], action({ templates, css: { cssPath } }) { return templates[404]({ cssPath, dev, baseUrl, localUrl: '/404', title: 'Not Found' }); } - }, - renderedWebmentionConfirmation: { + }), + new GraphNode({ + name: 'renderedWebmentionConfirmation', dependencies: ['css', 'templates'], action({ templates, css: { cssPath } }) { return templates.webmention({ @@ -701,15 +713,17 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { description: 'Your mention is confirmed! Please check back later.' }); } - }, - renderedSitemap: { + }), + new GraphNode({ + name: 'renderedSitemap', dependencies: ['templates', 'collatedTags', 'specialFiles', 'postFiles'], action({ templates, collatedTags, specialFiles, postFiles }) { const pages = [...specialFiles, ...postFiles]; return templates.sitemap({ tags: collatedTags, pages, baseUrl }); } - }, - renderedAtomFeeds: { + }), + new GraphNode({ + name: 'renderedAtomFeeds', dependencies: ['templates', 'commitTime', 'postFiles', 'noteFiles', 'linkFiles', 'likeFiles', 'replyFiles'], action({ templates, commitTime, postFiles, noteFiles, linkFiles, likeFiles, replyFiles }) { function descending(a, b) { @@ -726,21 +740,22 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { social: templates.atom({ name: 'social.atom', items: social, baseUrl, updated: commitTime }) }; } - }, - writtenIndex: writeRendered('renderedAbout', 'index.html'), - writtenBlogIndex: writeRendered('renderedBlogIndex', 'blog/index.html'), - writtenJapaneseNotesIndex: writeRendered('renderedJapaneseNotesIndex', 'japanese-notes/index.html'), - writtenNotesIndex: writeRendered('renderedNotesIndex', 'notes/index.html'), - writtenStudySessionsIndex: writeRendered('renderedStudySessionsIndex', 'study-sessions/index.html'), - writtenLinksIndex: writeRendered('renderedLinksIndex', 'links/index.html'), - writtenLikesIndex: writeRendered('renderedLikesIndex', 'likes/index.html'), - writtenRepliesIndex: writeRendered('renderedRepliesIndex', 'replies/index.html'), - writtenPublications: writeRendered('renderedPublications', 'publications.html'), - writtenWebmentionConfirmation: writeRendered('renderedWebmentionConfirmation', 'webmention.html'), - writtenFourOhFour: writeRendered('renderedFourOhFour', '404.html'), - writtenSitemap: writeRendered('renderedSitemap', 'sitemap.txt'), - writtenShortlinks: writeRendered('renderedShortlinks', 'shortlinks.txt'), - writtenAtomFeeds: { + }), + writeRendered('writtenIndex', 'renderedAbout', 'index.html'), + writeRendered('writtenBlogIndex', 'renderedBlogIndex', 'blog/index.html'), + writeRendered('writtenJapaneseNotesIndex', 'renderedJapaneseNotesIndex', 'japanese-notes/index.html'), + writeRendered('writtenNotesIndex', 'renderedNotesIndex', 'notes/index.html'), + writeRendered('writtenStudySessionsIndex', 'renderedStudySessionsIndex', 'study-sessions/index.html'), + writeRendered('writtenLinksIndex', 'renderedLinksIndex', 'links/index.html'), + writeRendered('writtenLikesIndex', 'renderedLikesIndex', 'likes/index.html'), + writeRendered('writtenRepliesIndex', 'renderedRepliesIndex', 'replies/index.html'), + writeRendered('writtenPublications', 'renderedPublications', 'publications.html'), + writeRendered('writtenWebmentionConfirmation', 'renderedWebmentionConfirmation', 'webmention.html'), + writeRendered('writtenFourOhFour', 'renderedFourOhFour', '404.html'), + writeRendered('writtenSitemap', 'renderedSitemap', 'sitemap.txt'), + writeRendered('writtenShortlinks', 'renderedShortlinks', 'shortlinks.txt'), + new GraphNode({ + name: 'writtenAtomFeeds', dependencies: ['targetPath', 'renderedAtomFeeds'], action({ renderedAtomFeeds: { all, posts, social } }) { return Promise.all([ @@ -749,14 +764,16 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { writeFile(new URL('social.atom.xml', targetPath), social) ]); } - }, - writtenOpml: { + }), + new GraphNode({ + name: 'writtenOpml', dependencies: ['targetPath', 'templates', 'feeds'], action({ templates, feeds }) { return writeFile(new URL('feeds.opml', targetPath), templates.feeds({ feeds })); } - }, - writtenBlogroll: { + }), + new GraphNode({ + name: 'writtenBlogroll', dependencies: ['css', 'targetPath', 'templates', 'feeds'], action({ css: { cssPath }, templates, feeds }) { return writeFile(new URL('blogroll.html', targetPath), templates.blogroll({ @@ -768,23 +785,24 @@ export async function build({ baseUrl, baseTitle, repoUrl, dev }) { title: 'Blogroll' })); } - }, - writtenSpecials: makeWriteEntries({ renderedDependencies: 'renderedSpecials', pathFragment: null }), - writtenPosts: makeWriteEntries({ renderedDependencies: 'renderedPosts', pathFragment: 'blog' }), - writtenJapaneseNotes: makeWriteEntries({ renderedDependencies: 'renderedJapaneseNotes', pathFragment: 'japanese-notes' }), - writtenNotes: makeWriteEntries({ renderedDependencies: 'renderedNotes', pathFragment: 'notes' }), - writtenStudySessions: makeWriteEntries({ renderedDependencies: 'renderedStudySessions', pathFragment: 'study-sessions' }), - writtenLinks: makeWriteEntries({ renderedDependencies: 'renderedLinks', pathFragment: 'links' }), - writtenLikes: makeWriteEntries({ renderedDependencies: 'renderedLikes', pathFragment: 'likes' }), - writtenReplies: makeWriteEntries({ renderedDependencies: 'renderedReplies', pathFragment: 'replies' }), - writtenTags: makeWriteEntries({ renderedDependencies: 'collatedTags', pathFragment: 'tags' }), - lastBuild: { + }), + makeWriteEntries({ name: 'writtenSpecials', renderedDependencies: 'renderedSpecials', pathFragment: null }), + makeWriteEntries({ name: 'writtenPosts', renderedDependencies: 'renderedPosts', pathFragment: 'blog' }), + makeWriteEntries({ name: 'writtenJapaneseNotes', renderedDependencies: 'renderedJapaneseNotes', pathFragment: 'japanese-notes' }), + makeWriteEntries({ name: 'writtenNotes', renderedDependencies: 'renderedNotes', pathFragment: 'notes' }), + makeWriteEntries({ name: 'writtenStudySessions', renderedDependencies: 'renderedStudySessions', pathFragment: 'study-sessions' }), + makeWriteEntries({ name: 'writtenLinks', renderedDependencies: 'renderedLinks', pathFragment: 'links' }), + makeWriteEntries({ name: 'writtenLikes', renderedDependencies: 'renderedLikes', pathFragment: 'likes' }), + makeWriteEntries({ name: 'writtenReplies', renderedDependencies: 'renderedReplies', pathFragment: 'replies' }), + makeWriteEntries({ name: 'writtenTags', renderedDependencies: 'collatedTags', pathFragment: 'tags' }), + new GraphNode({ + name: 'lastBuild', dependencies: ['targetPath', 'writtenSitemap'], action() { return writeFile(new URL('last-build.txt', targetPath), `${new Date().toISOString()}\n`); } - } - }); + }) + ]); return graph; } diff --git a/lib/execution-graph.js b/lib/execution-graph.js index 39dbdd90..13eef3db 100644 --- a/lib/execution-graph.js +++ b/lib/execution-graph.js @@ -1,70 +1,107 @@ import { EventEmitter, once } from 'node:events'; -async function waitForDependencies(executionGraph, dependencies) { - const results = {}; - - await Promise.all(dependencies.map(async name => { - let dependecyNode = executionGraph.nodes.get(name); - - if (!dependecyNode || !dependecyNode.done) { - ([dependecyNode] = await once(executionGraph, `done:${name}`)); - } - - results[name] = dependecyNode.result; - })); - - return results; -} - -async function watchForChanges(watcher, graph) { - for await (const { url } of watcher) { - for (const [name, { watchPath }] of graph.nodes) { - if (watchPath && url.pathname.startsWith(watchPath.pathname)) { - console.time(`Build succeeded for ${name}`); // eslint-disable-line no-console - console.log('Rerunning Node:', name); // eslint-disable-line no-console - try { - graph.rerunNode({ name }); - } catch (e) { - console.error('BUILD ERROR:', e.stack); // eslint-disable-line no-console - } - console.timeEnd(`Build succeeded for ${name}`); // eslint-disable-line no-console - } - } - } -} - -class WatchableResult { +export class WatchableResult { + /** @param {{ path: string, result: any }} options */ constructor({ path, result }) { this.path = path; this.result = result; } } +export class GraphNode { + /** + * @param {object} options + * @param {string} options.name + * @param {Function} options.action + * @param {Function?} options.onRemove + * @param {string[]?} options.dependencies + */ + constructor({ name, action, onRemove, dependencies = [] }) { + this.done = false; + this.name = name; + this.action = action; + this.onRemove = onRemove; + this.dependencies = dependencies; + this.result = null; + /** @type {string|null} */ + this.watchPath = null; + } +} + export default class ExecutionGraph extends EventEmitter { + /** + * @param {Object} options + * @param {ExecutionGraph} options.parent + * @param {AsyncGenerator<{eventType: string, url: URL}>|null} options.watcher + */ constructor({ parent, watcher } = {}) { super(); this.parent = parent; + /** @type {Map} */ this.nodes = new Map(); this.setMaxListeners(0); if (watcher) { - watchForChanges(watcher, this); + this.#watchForChanges(watcher, this); + } + } + + /** @param {AsyncGenerator<{eventType: string, url: URL}>} watcher */ + async #watchForChanges(watcher) { + for await (const { url } of watcher) { + for (const [name, { watchPath }] of this.nodes) { + if (watchPath && url.pathname.startsWith(watchPath.pathname)) { + console.time(`Build succeeded for ${name}`); // eslint-disable-line no-console + console.log('Rerunning Node:', name); // eslint-disable-line no-console + try { + this.rerunNode(name); + } catch (e) { + console.error('BUILD ERROR:', e.stack); // eslint-disable-line no-console + } + console.timeEnd(`Build succeeded for ${name}`); // eslint-disable-line no-console + } + } } } - async addNode({ name, action, onRemove, dependencies = [] }) { - const node = { action, onRemove, dependencies, done: false }; + /** @param {string[]} dependencies */ + async #waitForDependencies(dependencies) { + /** @type {Record} */ + const results = {}; + + await Promise.all(dependencies.map(async name => { + let dependecyNode = this.nodes.get(name); - if (dependencies.includes(name)) { + if (!dependecyNode || !dependecyNode.done) { + ([dependecyNode] = await once(this, `done:${name}`)); + } + + results[name] = dependecyNode.result; + })); + + return results; + } + + // /** + // * @param {Object} options + // * @param {string} options.name + // * @param {Function} options.action + // * @param {Function|undefined} options.onRemove + // * @param {string[]} options.dependencies + // */ + /** @param {GraphNode|Function} node */ + async addNode(rawNode) { + const node = rawNode instanceof GraphNode ? rawNode : new GraphNode({ name: rawNode.name, action: rawNode }); + + if (node.dependencies.includes(node.name)) { throw new Error('Node may not depend on itself.'); } - this.nodes.set(name, node); + this.nodes.set(node.name, node); - const dependencyResults = await waitForDependencies(this, dependencies); - - const result = await action(dependencyResults); + const dependencyResults = await this.#waitForDependencies(node.dependencies); + const result = await node.action(dependencyResults); if (result instanceof WatchableResult) { node.result = result.result; @@ -75,33 +112,33 @@ export default class ExecutionGraph extends EventEmitter { node.done = true; - this.emit(`done:${name}`, node); - this.emit('done', name, node); + this.emit(`done:${node.name}`, node); + this.emit('done', node.name, node); return node.result; } - addNodes(nodes) { - const graph = this; - - async function processNode([name, spec]) { - let action; - let onRemove; - let dependencies; + /** @param {(GraphNode|Function)[]} nodes */ + async addNodes(nodes) { + const promises = []; - if (typeof spec === 'function') { - action = spec; - } else { - ({ action, onRemove, dependencies } = spec); - } + /** @type {Record} */ + const results = {}; - return [name, await graph.addNode({ name, action, onRemove, dependencies })]; + for (const node of nodes) { + promises.push( + this.addNode(node).then(r => { + results[node.name] = r; + }) + ); } - return Promise.all(Object.entries(nodes).map(processNode)).then(Object.fromEntries); + await Promise.all(promises); + + return results; } - async removeNode({ name }) { + async removeNode(name) { const node = this.nodes.get(name); if (!node) { @@ -127,8 +164,12 @@ export default class ExecutionGraph extends EventEmitter { this.nodes.delete(name); } - // Inversion! Given a node, which nodes directly depend on it? - getDirectDependents({ name }) { + /** + * Inversion! Given a node, which nodes directly depend on it? + * + * @param {string} name + */ + getDirectDependents(name) { const dependents = []; for (const [nodeName, { dependencies }] of this.nodes) { @@ -140,37 +181,36 @@ export default class ExecutionGraph extends EventEmitter { return dependents; } - // rerunning a node amounts to finding the subtree which depends on it, - // removing its nodes, and re-adding them. - // TODO: This is very clunky. There's probably a more elegant way. - // eslint-disable-next-line max-statements - async rerunNode({ name }) { + /** + * rerunning a node amounts to finding the subtree which depends on it, + * removing its nodes, and re-adding them. + * + * @todo This is very clunky. There's probably a more elegant way. + * + * @param {string} name + */ + async rerunNode(name) { const node = this.nodes.get(name); - const dependents = this.getDirectDependents({ name }); - const collected = new Map([[name, { node, dependents }]]); - let oldSize; - let newSize; + if (!node) { + return; + } - // Collect the tree of nodes dependent on the targeted node. - do { - oldSize = collected.size; + const dependents = this.getDirectDependents(name); + const collected = new Map([[name, { node, dependents }]]); + for (let oldSize = collected.size, newSize = 0; newSize !== oldSize; newSize = oldSize, oldSize = collected.size) { for (const { dependents } of collected.values()) { for (const name of dependents) { if (!collected.has(name)) { const node = this.nodes.get(name); - collected.set(name, { - node, - dependents: this.getDirectDependents({ name }) - }); + collected.set(name, { node, dependents: this.getDirectDependents(name) }); } } } + } - newSize = collected.size; - } while (oldSize !== newSize); - + /** @type {Set} */ const removed = new Set(); // Remove the nodes, starting with leaves. @@ -187,16 +227,18 @@ export default class ExecutionGraph extends EventEmitter { await Promise.all([...collected].map(([, { node }]) => node.onRemove && node.onRemove())); // Re-add the nodes. - const results = await Promise.all( - [...collected].map(([name, { node: { action, onRemove, dependencies } }]) => this.addNode({ name, action, onRemove, dependencies })) + await Promise.all( + [...collected].map(([ + name, + { node: { action, onRemove, dependencies } } + ]) => this.addNode(new GraphNode({ name, action, onRemove, dependencies }))) ); this.emit('rerun'); - - return results; } get results() { + /** @type {Record} */ const results = {}; for (const [key, { result, done }] of this.nodes) { @@ -207,8 +249,4 @@ export default class ExecutionGraph extends EventEmitter { return results; } - - static createWatchableResult({ path, result }) { - return new WatchableResult({ path, result }); - } } diff --git a/tests/units/execution-graph.test.js b/tests/units/execution-graph.test.js index dee04ce0..12530c30 100644 --- a/tests/units/execution-graph.test.js +++ b/tests/units/execution-graph.test.js @@ -2,7 +2,7 @@ import { describe, beforeEach, it } from 'node:test'; import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; import crypto from 'node:crypto'; -import ExecutionGraph from '../../lib/execution-graph.js'; +import ExecutionGraph, { GraphNode, WatchableResult } from '../../lib/execution-graph.js'; import { setTimeout as wait } from 'timers/promises'; describe('execution-graph', () => { @@ -26,7 +26,7 @@ describe('execution-graph', () => { describe('adding a node', () => { it('returns a promise', () => { - assert.ok(graph.addNode({ name: 'a-name', action() {} }) instanceof Promise); + assert.ok(graph.addNode(new GraphNode({ name: 'a-name', action() {} })) instanceof Promise); }); it('resolves the promise after a synchronous action is run when there are no dependencies', async () => { @@ -34,12 +34,12 @@ describe('execution-graph', () => { graph.once('done:a-name', () => order.push('event')); - const promise = graph.addNode({ + const promise = graph.addNode(new GraphNode({ name: 'a-name', action() { order.push('action'); } - }); + })); await promise.then(() => order.push('promise')); @@ -48,12 +48,12 @@ describe('execution-graph', () => { it('resolves to the return value of a synchronous action to the node', async () => { const expected = crypto.randomBytes(8).toString('hex'); - const result = await graph.addNode({ + const result = await graph.addNode(new GraphNode({ name: 'a-name', action() { return expected; } - }); + })); assert.equal(result, expected); }); @@ -63,13 +63,13 @@ describe('execution-graph', () => { graph.once('done:a-name', () => order.push('event')); - const promise = graph.addNode({ + const promise = graph.addNode(new GraphNode({ name: 'a-name', async action() { await wait(100); order.push('action'); } - }); + })); await promise.then(() => order.push('promise')); @@ -78,12 +78,12 @@ describe('execution-graph', () => { it('resolves to the resulution value of an asynchronous action to the node', async () => { const expected = crypto.randomBytes(8).toString('hex'); - const result = await graph.addNode({ + const result = await graph.addNode(new GraphNode({ name: 'a-name', action() { return Promise.resolve(expected); } - }); + })); assert.equal(result, expected); }); @@ -92,27 +92,27 @@ describe('execution-graph', () => { const order = []; const promises = []; - promises.push(graph.addNode({ + promises.push(graph.addNode(new GraphNode({ name: 'a', dependencies: ['b', 'c'], action() { assert.deepEqual(order, ['c', 'b']); } - }).then(() => order.push('a'))); + })).then(() => order.push('a'))); - promises.push(graph.addNode({ + promises.push(graph.addNode(new GraphNode({ name: 'b', action() { return wait(100); } - }).then(() => order.push('b'))); + })).then(() => order.push('b'))); - promises.push(graph.addNode({ + promises.push(graph.addNode(new GraphNode({ name: 'c', action() { return wait(50); } - }).then(() => order.push('c'))); + })).then(() => order.push('c'))); await Promise.all(promises); @@ -120,35 +120,35 @@ describe('execution-graph', () => { }); it('passes results from dependencies into an action', async () => { - graph.addNode({ + graph.addNode(new GraphNode({ name: 'a', action() { return 'result-a'; } - }); + })); - graph.addNode({ + graph.addNode(new GraphNode({ name: 'b', action() { return Promise.resolve('result-b'); } - }); + })); - graph.addNode({ + graph.addNode(new GraphNode({ name: 'c', async action() { await wait(10); return 'result-c'; } - }); + })); - const passedIn = await graph.addNode({ + const passedIn = await graph.addNode(new GraphNode({ name: 'd', dependencies: ['a', 'b', 'c'], action(input) { return input; } - }); + })); assert.deepEqual(passedIn, { a: 'result-a', b: 'result-b', c: 'result-c' }); }); @@ -156,61 +156,65 @@ describe('execution-graph', () => { describe('adding multiple nodes', () => { it('returns a promise', () => { - assert.ok(graph.addNodes({ name: { action() {} } }) instanceof Promise); + assert.ok(graph.addNodes([new GraphNode({ name: 'name', action() {} })]) instanceof Promise); }); it('resolves to a collection of results for the added nodes', async () => { - const results = await graph.addNodes({ - a: { + const results = await graph.addNodes([ + new GraphNode({ + name: 'a', async action() { await wait(10); return 'result-a'; } - }, - b: { + }), + new GraphNode({ + name: 'b', dependencies: ['a'], action({ a }) { return `result-b: result-a is "${a}"`; } - } - }); + }) + ]); assert.deepEqual(results, { a: 'result-a', b: 'result-b: result-a is "result-a"' }); }); it('has a compact API for nodes with no dependencies', async () => { - const results = await graph.addNodes({ - async a() { + const results = await graph.addNodes([ + async function a() { await wait(10); return 'result-a'; }, - b: { + new GraphNode({ + name: 'b', dependencies: ['a'], action({ a }) { return `result-b: result-a is "${a}"`; } - } - }); + }) + ]); assert.deepEqual(results, { a: 'result-a', b: 'result-b: result-a is "result-a"' }); }); it('does not include results of nodes not in the collection added', async () => { - await graph.addNode({ + await graph.addNode(new GraphNode({ name: 'x', action() { return 3; } - }); + })); - const results = await graph.addNodes({ - a: { + const results = await graph.addNodes([ + new GraphNode({ + name: 'a', dependencies: ['x'], action({ x }) { return x ** 2; } - } - }); + }) + ]); assert.deepEqual(results, { a: 9 }); }); @@ -218,36 +222,36 @@ describe('execution-graph', () => { describe('removing nodes', () => { it('removes a node with no dependencies', async () => { - await graph.addNode({ + await graph.addNode(new GraphNode({ name: 'a', action() {} - }); + })); - await graph.removeNode({ name: 'a' }); + await graph.removeNode('a'); assert.equal(graph.nodes.size, 0); }); it('rejects when attempting to remove a node with dependencies', async () => { await Promise.all([ - graph.addNode({ + graph.addNode(new GraphNode({ name: 'a', action() {} - }), - graph.addNode({ + })), + graph.addNode(new GraphNode({ name: 'b', dependencies: ['a'], action() {} - }), - graph.addNode({ + })), + graph.addNode(new GraphNode({ name: 'c', dependencies: ['a'], action() {} - }) + })) ]); assert.rejects( - () => graph.removeNode({ name: 'a' }), + () => graph.removeNode('a'), Error, 'Node a has dependent nodes: b, c' ); @@ -259,7 +263,7 @@ describe('execution-graph', () => { let guard = false; await Promise.all([ - graph.addNode({ + graph.addNode(new GraphNode({ name: 'a', action() {}, onRemove() { @@ -268,10 +272,10 @@ describe('execution-graph', () => { resolve(); }, 500)); } - }) + })) ]); - await graph.removeNode({ name: 'a' }); + await graph.removeNode('a'); assert.equal(guard, true); }); @@ -286,14 +290,14 @@ describe('execution-graph', () => { guard = false; await Promise.all([ - graph.addNode({ + graph.addNode(new GraphNode({ name: 'a', async action() { await wait(10); nodesRerun.push('a'); } - }), - graph.addNode({ + })), + graph.addNode(new GraphNode({ name: 'b', dependencies: ['a'], async action() { @@ -306,38 +310,38 @@ describe('execution-graph', () => { resolve(); }, 500)); } - }), - graph.addNode({ + })), + graph.addNode(new GraphNode({ name: 'c', dependencies: ['b'], async action() { await wait(10); nodesRerun.push('c'); } - }) + })) ]); nodesRerun = []; }); it('reruns leaf nodes', async () => { - await graph.rerunNode({ name: 'c' }); + await graph.rerunNode('c'); assert.deepEqual(nodesRerun, ['c']); }); it('reruns on-leaf nodes and their decendents in order', async () => { - await graph.rerunNode({ name: 'b' }); + await graph.rerunNode('b'); assert.deepEqual(nodesRerun, ['b', 'c']); }); it('runs cleanup on nodes which need it', async () => { - await graph.rerunNode({ name: 'c' }); + await graph.rerunNode('c'); assert.equal(guard, false); - await graph.rerunNode({ name: 'a' }); + await graph.rerunNode('a'); assert.equal(guard, true); }); @@ -345,25 +349,25 @@ describe('execution-graph', () => { describe('getting results', () => { it('gets results of returned and resolved nodes', async () => { - graph.addNode({ + graph.addNode(new GraphNode({ name: 'a', action() { return 1; } - }); - graph.addNode({ + })); + graph.addNode(new GraphNode({ name: 'c', async action() { await wait(200); return '3'; } - }); - await graph.addNode({ + })); + await graph.addNode(new GraphNode({ name: 'b', action() { return Promise.resolve(2); } - }); + })); assert.deepEqual(graph.results, { a: 1, b: 2 }); });