From be8f5d00e91b910ef669fe6018e6db92a8bd4c26 Mon Sep 17 00:00:00 2001 From: Jarrod Overson Date: Tue, 13 Dec 2022 13:58:09 -0500 Subject: [PATCH 1/3] added concept of plugins to dynamically generate configuration --- src/commands/generate.ts | 1 + src/config.ts | 3 ++- src/generate.ts | 17 ++++++++-------- src/plugins.ts | 42 ++++++++++++++++++++++++++++++++++++++ src/process.ts | 44 +++++++++++++++++++++++++++++++++++++++- test/config.test.ts | 15 +++++++------- test/plugin.test.ts | 37 +++++++++++++++++++++++++++++++++ test/test-plugin.ts | 12 +++++++++++ 8 files changed, 153 insertions(+), 18 deletions(-) create mode 100644 src/plugins.ts create mode 100644 test/plugin.test.ts create mode 100644 test/test-plugin.ts diff --git a/src/commands/generate.ts b/src/commands/generate.ts index c8e2009..94bc941 100644 --- a/src/commands/generate.ts +++ b/src/commands/generate.ts @@ -42,6 +42,7 @@ export async function fromConfig(configContents: string) { .split("---") .map((v) => v.trim()) .map((v) => yaml.parse(v) as Configuration); + const outputs: Output[] = []; for (const config of configs) { const o = await process(config); diff --git a/src/config.ts b/src/config.ts index 3d5d17f..39566a6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -5,7 +5,8 @@ export type Config = { [key: string]: unknown }; export interface Configuration { spec: string; config?: Config; - generates: Record; + plugins?: string[]; + generates?: Record; } export interface Target { diff --git a/src/generate.ts b/src/generate.ts index 9ce2319..f32b02b 100644 --- a/src/generate.ts +++ b/src/generate.ts @@ -22,9 +22,10 @@ async function processConfig(config: Configuration): Promise { continue; } - const url = generatorConfig.module.startsWith(".") || - generatorConfig.module.startsWith("/") + const url = generatorConfig.module.startsWith(".") ? new URL("file:///" + path.join(Deno.cwd(), generatorConfig.module)) + : file.startsWith("/") + ? new URL("file:///" + generatorConfig.module) : new URL(generatorConfig.module); const visitorConfig: Config = {}; @@ -41,13 +42,11 @@ async function processConfig(config: Configuration): Promise { visitorConfig["$filename"] = file; log.debug( - `Generating source for '${file}' with generator from ${url} with config\n${ - JSON.stringify( - visitorConfig, - null, - 2, - ) - }`, + `Generating source for '${file}' with generator from ${url} with config\n${JSON.stringify( + visitorConfig, + null, + 2 + )}` ); const generator = await import(url.toString()); diff --git a/src/plugins.ts b/src/plugins.ts new file mode 100644 index 0000000..ad1cbc7 --- /dev/null +++ b/src/plugins.ts @@ -0,0 +1,42 @@ +import * as apex from "https://deno.land/x/apex_core@v0.1.0/mod.ts"; +import * as log from "https://deno.land/std@0.167.0/log/mod.ts"; +import * as path from "https://deno.land/std@0.167.0/path/mod.ts"; +import * as streams from "https://deno.land/std@0.167.0/streams/read_all.ts"; + +import { Configuration } from "./config.ts"; + +export async function processPlugin( + config: Configuration +): Promise { + const apexSource = await Deno.readTextFile(config.spec); + // TODO: implement resolver callback + const doc = apex.parse(apexSource); + + for (const file of config.plugins || []) { + const url = file.startsWith(".") + ? new URL("file:///" + path.join(Deno.cwd(), file)) + : file.startsWith("/") + ? new URL("file:///" + file) + : new URL(file); + + log.debug(`Generating configuration with plugin from ${url}`); + + const plugin = await import(url.toString()); + config = plugin.default(doc, config); + } + + return config; +} + +// Detect piped input +if (!Deno.isatty(Deno.stdin.rid) && import.meta.main) { + const stdinContent = await streams.readAll(Deno.stdin); + const content = new TextDecoder().decode(stdinContent); + try { + const config = JSON.parse(content) as Configuration; + console.log(JSON.stringify(await processPlugin(config))); + } catch (e) { + console.error(e); + throw e; + } +} diff --git a/src/process.ts b/src/process.ts index 33a683a..484b62d 100644 --- a/src/process.ts +++ b/src/process.ts @@ -6,6 +6,8 @@ import { Configuration, Output } from "./config.ts"; import { cliFormatters, sourceFormatters } from "./formatters.ts"; export async function process(config: Configuration): Promise { + config = await processPlugins(config); + // Run the generation process with restricted permissions. const href = new URL("./generate.ts", import.meta.url).href; const p = await Deno.run({ @@ -41,6 +43,43 @@ export async function process(config: Configuration): Promise { return JSON.parse(output) as Output[]; } +async function processPlugins(config: Configuration): Promise { + if (!config.plugins) return config; + + // Run the plugin process with restricted permissions. + const href = new URL("./plugins.ts", import.meta.url).href; + const p = await Deno.run({ + cmd: [ + Deno.execPath(), + "run", + "--allow-read", + "--allow-net=deno.land,raw.githubusercontent.com", + href, + ], + stdout: "piped", + stderr: "piped", + stdin: "piped", + }); + + const input = JSON.stringify(config); + await p.stdin.write(new TextEncoder().encode(input)); + p.stdin.close(); + + const rawOutput = await p.output(); + const rawError = await p.stderrOutput(); + + const { code } = await p.status(); + p.close(); + + if (code !== 0) { + const errorString = new TextDecoder().decode(rawError); + throw new Error(errorString); + } + + const output = new TextDecoder().decode(rawOutput); + return JSON.parse(output) as Configuration; +} + export async function writeOutput(generated: Output): Promise { let source = generated.source; const ext = fileExtension(generated.file).toLowerCase(); @@ -68,7 +107,10 @@ export async function writeOutput(generated: Output): Promise { // Execute additional tooling via "runAfter". if (generated.runAfter) { generated.runAfter.forEach(async (cmdConfig) => { - const joined = cmdConfig.command.trim().split("\n").map((v) => v.trim()) + const joined = cmdConfig.command + .trim() + .split("\n") + .map((v) => v.trim()) .join(" "); const args = joined.split(/\s+/); const cmd = args.shift()!; diff --git a/test/config.test.ts b/test/config.test.ts index da661ab..71bc5a0 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -4,7 +4,6 @@ import * as path from "https://deno.land/std@0.167.0/path/mod.ts"; const __dirname = new URL(".", import.meta.url).pathname; -// Compact form: name and function Deno.test( "process", { permissions: { read: true, net: true, run: true } }, @@ -20,10 +19,12 @@ Deno.test( }); const fixture = await Deno.readTextFile(path.join(__dirname, "fixture.rs")); - assertEquals(generated, [{ - file: "file.rs", - source: fixture, - executable: false, - }]); - }, + assertEquals(generated, [ + { + file: "file.rs", + source: fixture, + executable: false, + }, + ]); + } ); diff --git a/test/plugin.test.ts b/test/plugin.test.ts new file mode 100644 index 0000000..da97944 --- /dev/null +++ b/test/plugin.test.ts @@ -0,0 +1,37 @@ +import { assertEquals } from "https://deno.land/std@0.167.0/testing/asserts.ts"; +import { processPlugin } from "../src/plugins.ts"; +import * as path from "https://deno.land/std@0.167.0/path/mod.ts"; + +const __dirname = new URL(".", import.meta.url).pathname; + +Deno.test( + "plugin", + { permissions: { read: true, net: true, run: true } }, + async () => { + const config = await processPlugin({ + spec: path.join(__dirname, "test.axdl"), + plugins: [path.join(__dirname, "test-plugin.ts")], + }); + + assertEquals(config.generates, { "dynamic.0.file": { module: "this.ts" } }); + } +); + +Deno.test( + "plugin", + { permissions: { read: true, net: true, run: true } }, + async () => { + const config = await processPlugin({ + spec: path.join(__dirname, "test.axdl"), + plugins: [ + path.join(__dirname, "test-plugin.ts"), + path.join(__dirname, "test-plugin.ts"), + ], + }); + + assertEquals(config.generates, { + "dynamic.0.file": { module: "this.ts" }, + "dynamic.1.file": { module: "this.ts" }, + }); + } +); diff --git a/test/test-plugin.ts b/test/test-plugin.ts new file mode 100644 index 0000000..6a86cf6 --- /dev/null +++ b/test/test-plugin.ts @@ -0,0 +1,12 @@ +import { Configuration } from "../src/config.ts"; +import * as apex from "https://deno.land/x/apex_core@v0.1.0/mod.ts"; + +export default function ( + _doc: apex.ast.Document, + config: Configuration +): Configuration { + config.generates ||= {}; + const numFiles = Object.keys(config.generates).length; + config.generates[`dynamic.${numFiles}.file`] = { module: "this.ts" }; + return config; +} From c0c7d3cd9593e23e811641656c61706765f337cb Mon Sep 17 00:00:00 2001 From: Jarrod Overson Date: Fri, 16 Dec 2022 13:28:46 -0500 Subject: [PATCH 2/3] removed duplication, added generator test --- src/generate.ts | 29 +++++++++++++++------ src/plugins.ts | 42 ------------------------------ src/process.ts | 40 ++--------------------------- src/utils.ts | 8 ++++++ test/plugin.test.ts | 58 ++++++++++++++++++++++++++++++++---------- test/test-generator.ts | 15 +++++++++++ test/test-plugin.ts | 17 ++++++++++--- 7 files changed, 105 insertions(+), 104 deletions(-) delete mode 100644 src/plugins.ts create mode 100644 test/test-generator.ts diff --git a/src/generate.ts b/src/generate.ts index f32b02b..3cb947e 100644 --- a/src/generate.ts +++ b/src/generate.ts @@ -5,13 +5,15 @@ import * as path from "https://deno.land/std@0.167.0/path/mod.ts"; import * as streams from "https://deno.land/std@0.167.0/streams/read_all.ts"; import { Config, Configuration, Output } from "./config.ts"; -import { existsSync } from "./utils.ts"; +import { existsSync, makeRelativeUrl } from "./utils.ts"; -async function processConfig(config: Configuration): Promise { +export async function processConfig(config: Configuration): Promise { const apexSource = await Deno.readTextFile(config.spec); // TODO: implement resolver callback const doc = apex.parse(apexSource); + config = await processPlugin(doc, config); + const output: Output[] = []; for (const file in config.generates) { const generatorConfig = config.generates[file]; @@ -21,12 +23,7 @@ async function processConfig(config: Configuration): Promise { //log.info(`Skipping ${file}`); continue; } - - const url = generatorConfig.module.startsWith(".") - ? new URL("file:///" + path.join(Deno.cwd(), generatorConfig.module)) - : file.startsWith("/") - ? new URL("file:///" + generatorConfig.module) - : new URL(generatorConfig.module); + const url = makeRelativeUrl(generatorConfig.module); const visitorConfig: Config = {}; if (config.config) { @@ -67,6 +64,22 @@ async function processConfig(config: Configuration): Promise { return output; } +export async function processPlugin( + doc: apex.ast.Document, + config: Configuration +): Promise { + for (const file of config.plugins || []) { + const url = makeRelativeUrl(file); + + log.debug(`Generating configuration with plugin from ${url}`); + + const plugin = await import(url.toString()); + config = plugin.default(doc, config); + } + + return config; +} + // Detect piped input if (!Deno.isatty(Deno.stdin.rid) && import.meta.main) { const stdinContent = await streams.readAll(Deno.stdin); diff --git a/src/plugins.ts b/src/plugins.ts deleted file mode 100644 index ad1cbc7..0000000 --- a/src/plugins.ts +++ /dev/null @@ -1,42 +0,0 @@ -import * as apex from "https://deno.land/x/apex_core@v0.1.0/mod.ts"; -import * as log from "https://deno.land/std@0.167.0/log/mod.ts"; -import * as path from "https://deno.land/std@0.167.0/path/mod.ts"; -import * as streams from "https://deno.land/std@0.167.0/streams/read_all.ts"; - -import { Configuration } from "./config.ts"; - -export async function processPlugin( - config: Configuration -): Promise { - const apexSource = await Deno.readTextFile(config.spec); - // TODO: implement resolver callback - const doc = apex.parse(apexSource); - - for (const file of config.plugins || []) { - const url = file.startsWith(".") - ? new URL("file:///" + path.join(Deno.cwd(), file)) - : file.startsWith("/") - ? new URL("file:///" + file) - : new URL(file); - - log.debug(`Generating configuration with plugin from ${url}`); - - const plugin = await import(url.toString()); - config = plugin.default(doc, config); - } - - return config; -} - -// Detect piped input -if (!Deno.isatty(Deno.stdin.rid) && import.meta.main) { - const stdinContent = await streams.readAll(Deno.stdin); - const content = new TextDecoder().decode(stdinContent); - try { - const config = JSON.parse(content) as Configuration; - console.log(JSON.stringify(await processPlugin(config))); - } catch (e) { - console.error(e); - throw e; - } -} diff --git a/src/process.ts b/src/process.ts index 484b62d..9bb863d 100644 --- a/src/process.ts +++ b/src/process.ts @@ -6,7 +6,7 @@ import { Configuration, Output } from "./config.ts"; import { cliFormatters, sourceFormatters } from "./formatters.ts"; export async function process(config: Configuration): Promise { - config = await processPlugins(config); + log.debug(`Configuration is: ${JSON.stringify(config, null, 2)}`); // Run the generation process with restricted permissions. const href = new URL("./generate.ts", import.meta.url).href; @@ -40,46 +40,10 @@ export async function process(config: Configuration): Promise { } const output = new TextDecoder().decode(rawOutput); + log.debug(`Generator output: ${output}`); return JSON.parse(output) as Output[]; } -async function processPlugins(config: Configuration): Promise { - if (!config.plugins) return config; - - // Run the plugin process with restricted permissions. - const href = new URL("./plugins.ts", import.meta.url).href; - const p = await Deno.run({ - cmd: [ - Deno.execPath(), - "run", - "--allow-read", - "--allow-net=deno.land,raw.githubusercontent.com", - href, - ], - stdout: "piped", - stderr: "piped", - stdin: "piped", - }); - - const input = JSON.stringify(config); - await p.stdin.write(new TextEncoder().encode(input)); - p.stdin.close(); - - const rawOutput = await p.output(); - const rawError = await p.stderrOutput(); - - const { code } = await p.status(); - p.close(); - - if (code !== 0) { - const errorString = new TextDecoder().decode(rawError); - throw new Error(errorString); - } - - const output = new TextDecoder().decode(rawOutput); - return JSON.parse(output) as Configuration; -} - export async function writeOutput(generated: Output): Promise { let source = generated.source; const ext = fileExtension(generated.file).toLowerCase(); diff --git a/src/utils.ts b/src/utils.ts index e22c74a..8c7ca11 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -84,3 +84,11 @@ export async function mkdirAll(path: string, mode: number) { } } } + +export function makeRelativeUrl(file: string): URL { + return file.startsWith(".") + ? new URL("file:///" + path.join(Deno.cwd(), file)) + : file.startsWith("/") + ? new URL("file:///" + file) + : new URL(file); +} diff --git a/test/plugin.test.ts b/test/plugin.test.ts index da97944..d30c93a 100644 --- a/test/plugin.test.ts +++ b/test/plugin.test.ts @@ -1,19 +1,25 @@ +import * as apex from "https://deno.land/x/apex_core@v0.1.1/mod.ts"; import { assertEquals } from "https://deno.land/std@0.167.0/testing/asserts.ts"; -import { processPlugin } from "../src/plugins.ts"; +import { processPlugin, processConfig } from "../src/generate.ts"; import * as path from "https://deno.land/std@0.167.0/path/mod.ts"; const __dirname = new URL(".", import.meta.url).pathname; +const spec = path.join(__dirname, "test.axdl"); +const plugin = path.join(__dirname, "test-plugin.ts"); +const generator = path.join(__dirname, "test-generator.ts"); Deno.test( "plugin", { permissions: { read: true, net: true, run: true } }, async () => { - const config = await processPlugin({ - spec: path.join(__dirname, "test.axdl"), - plugins: [path.join(__dirname, "test-plugin.ts")], + const apexSource = await Deno.readTextFile(spec); + const doc = apex.parse(apexSource); + const config = await processPlugin(doc, { + spec, + plugins: [plugin], }); - assertEquals(config.generates, { "dynamic.0.file": { module: "this.ts" } }); + assertEquals(config.generates, { "Test.0.file": { module: generator } }); } ); @@ -21,17 +27,43 @@ Deno.test( "plugin", { permissions: { read: true, net: true, run: true } }, async () => { - const config = await processPlugin({ - spec: path.join(__dirname, "test.axdl"), - plugins: [ - path.join(__dirname, "test-plugin.ts"), - path.join(__dirname, "test-plugin.ts"), - ], + const apexSource = await Deno.readTextFile(spec); + const doc = apex.parse(apexSource); + const config = await processPlugin(doc, { + spec, + plugins: [plugin, plugin], }); assertEquals(config.generates, { - "dynamic.0.file": { module: "this.ts" }, - "dynamic.1.file": { module: "this.ts" }, + "Test.0.file": { module: generator }, + "Test.1.file": { module: generator }, }); } ); + +Deno.test( + "generate", + { permissions: { read: true, net: true, run: true } }, + async () => { + const apexSource = await Deno.readTextFile(spec); + const output = await processConfig({ + spec, + plugins: [plugin, plugin], + }); + + assertEquals(output, [ + { + file: "Test.0.file", + source: "startend", + executable: false, + runAfter: undefined, + }, + { + file: "Test.1.file", + source: "startend", + executable: false, + runAfter: undefined, + }, + ]); + } +); diff --git a/test/test-generator.ts b/test/test-generator.ts new file mode 100644 index 0000000..1fa1f50 --- /dev/null +++ b/test/test-generator.ts @@ -0,0 +1,15 @@ +import * as model from "https://deno.land/x/apex_core@v0.1.0/model/mod.ts"; + +type Context = model.Context; + +export default class DefaultVisitor extends model.BaseVisitor { + visitContextBefore(context: Context): void { + super.visitContextBefore(context); + this.write("start"); + } + + visitContextAfter(context: Context): void { + super.visitContextAfter(context); + this.write("end"); + } +} diff --git a/test/test-plugin.ts b/test/test-plugin.ts index 6a86cf6..8ade21b 100644 --- a/test/test-plugin.ts +++ b/test/test-plugin.ts @@ -1,12 +1,23 @@ import { Configuration } from "../src/config.ts"; import * as apex from "https://deno.land/x/apex_core@v0.1.0/mod.ts"; +import * as path from "https://deno.land/std@0.167.0/path/mod.ts"; + +const __dirname = new URL(".", import.meta.url).pathname; +const generator = path.join(__dirname, "test-generator.ts"); export default function ( - _doc: apex.ast.Document, + doc: apex.ast.Document, config: Configuration ): Configuration { config.generates ||= {}; - const numFiles = Object.keys(config.generates).length; - config.generates[`dynamic.${numFiles}.file`] = { module: "this.ts" }; + const interfaces = doc.definitions.filter( + (def) => def.getKind() === apex.ast.Kind.InterfaceDefinition + ) as apex.ast.InterfaceDefinition[]; + const num = Object.keys(config.generates).length; + for (const iface of interfaces) { + config.generates[`${iface.name.value}.${num}.file`] = { + module: generator, + }; + } return config; } From c57a9b35da783343ce5785017a02fd0f34cb0129 Mon Sep 17 00:00:00 2001 From: Jarrod Overson Date: Fri, 16 Dec 2022 13:39:22 -0500 Subject: [PATCH 3/3] updated formatter settings --- .vscode/settings.json | 13 ++++++++++++- src/generate.ts | 14 ++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index aa1c94e..ebd5461 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,15 @@ { "deno.enable": true, - "deno.unstable": true + "deno.unstable": true, + "editor.formatOnSave": true, + "editor.defaultFormatter": "denoland.vscode-deno", + "[javascript]": { + "editor.defaultFormatter": "denoland.vscode-deno" + }, + "[typescript]": { + "editor.defaultFormatter": "denoland.vscode-deno" + }, + "[json]": { + "editor.defaultFormatter": "denoland.vscode-deno" + } } diff --git a/src/generate.ts b/src/generate.ts index 3cb947e..602aa61 100644 --- a/src/generate.ts +++ b/src/generate.ts @@ -39,11 +39,13 @@ export async function processConfig(config: Configuration): Promise { visitorConfig["$filename"] = file; log.debug( - `Generating source for '${file}' with generator from ${url} with config\n${JSON.stringify( - visitorConfig, - null, - 2 - )}` + `Generating source for '${file}' with generator from ${url} with config\n${ + JSON.stringify( + visitorConfig, + null, + 2, + ) + }`, ); const generator = await import(url.toString()); @@ -66,7 +68,7 @@ export async function processConfig(config: Configuration): Promise { export async function processPlugin( doc: apex.ast.Document, - config: Configuration + config: Configuration, ): Promise { for (const file of config.plugins || []) { const url = makeRelativeUrl(file);