diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b7baf616c9..5603b17804 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,8 @@ steps in advance to help us fix any potential bug as fast as possible. you might want to check [this section](#i-have-a-question)). - To see if other users have experienced (and potentially already solved) the same issue you are having, check if there is not already a bug report existing - for your bug or error in the [bug tracker](https://github.com/metatypedev/metatype/issues?q=label%3Abug). + for your bug or error in the + [bug tracker](https://github.com/metatypedev/metatype/issues?q=label%3Abug). - Also make sure to search the internet (including Stack Overflow) to see if users outside the GitHub community have discussed the issue. - Collect information about the bug: @@ -230,3 +231,16 @@ rustflags = [ "-C", "link-arg=-fuse-ld=mold" ] ``` + +#### Local typegraph with Nodejs + +Currently, the `typegraph/sdk/node/dist` project is generated dynamically. +Depending on your package manager, the protocol used may differ. + +```bash +# uses the `file:..` protocol +npm install path/to/typegraph/sdk/node/dist + +# uses the `link:..` protocol (equivalent to `file:..` but for directories only) +pnpm install path/to/typegraph/sdk/node/dist +``` diff --git a/Cargo.lock b/Cargo.lock index 265638600b..9d72ba990b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11107,7 +11107,6 @@ dependencies = [ "serde 1.0.196", "serde_json", "sha2 0.10.8", - "typescript", "wit-bindgen", ] diff --git a/meta-cli/src/typegraph/loader/mod.rs b/meta-cli/src/typegraph/loader/mod.rs index 2499848ffc..3ea2188c0c 100644 --- a/meta-cli/src/typegraph/loader/mod.rs +++ b/meta-cli/src/typegraph/loader/mod.rs @@ -42,10 +42,7 @@ impl LoaderPool { pub fn new(config: Arc, max_parallel_loads: usize) -> Self { Self { config, - postprocessors: vec![ - postprocess::Validator.into(), - postprocess::ReformatScripts.into(), - ], + postprocessors: vec![postprocess::Validator.into()], semaphore: Semaphore::new(max_parallel_loads), } } diff --git a/meta-cli/src/typegraph/postprocess.rs b/meta-cli/src/typegraph/postprocess.rs index 530244e8d9..8e84056e1d 100644 --- a/meta-cli/src/typegraph/postprocess.rs +++ b/meta-cli/src/typegraph/postprocess.rs @@ -11,11 +11,10 @@ use colored::Colorize; use common::archive::archive_entries; use common::typegraph::utils::{map_from_object, object_from_map}; use common::typegraph::validator::validate_typegraph; -use common::typegraph::{Materializer, Typegraph}; +use common::typegraph::Typegraph; use ignore::WalkBuilder; use log::error; use std::path::Path; -use typescript::parser::transform_script; pub trait PostProcessor { fn postprocess(&self, tg: &mut Typegraph, config: &Config) -> Result<()>; @@ -35,14 +34,6 @@ where #[derive(Clone)] pub struct PostProcessorWrapper(Arc>>); -impl PostProcessorWrapper { - pub fn generic( - pp: impl Fn(&mut Typegraph, &Config) -> Result<()> + Sync + Send + 'static, - ) -> Self { - PostProcessorWrapper::from(GenericPostProcessor(pp)) - } -} - impl From for PostProcessorWrapper where T: PostProcessor + Send + Sync + 'static, @@ -112,7 +103,6 @@ fn compress_and_encode(main_path: &Path, tg_path: &Path) -> Result { } pub use deno_rt::DenoModules; -pub use deno_rt::ReformatScripts; pub use prisma_rt::EmbedPrismaMigrations; pub use prisma_rt::EmbeddedPrismaMigrationOptionsPatch; pub use python_rt::PythonModules; @@ -141,44 +131,10 @@ impl PostProcessor for Validator { pub mod deno_rt { use std::fs; - use common::typegraph::runtimes::deno::{FunctionMatData, ModuleMatData}; - use common::typegraph::runtimes::{KnownRuntime, TGRuntime}; - - use common::typegraph::utils::{find_runtimes, get_materializers}; + use common::typegraph::runtimes::deno::ModuleMatData; use super::*; - pub struct ReformatScripts; - - impl From for PostProcessorWrapper { - fn from(_val: ReformatScripts) -> Self { - PostProcessorWrapper::generic(reformat_scripts) - } - } - - fn reformat_materializer_script(mat: &mut Materializer) -> Result<()> { - if mat.name.as_str() == "function" { - let mut mat_data: FunctionMatData = object_from_map(std::mem::take(&mut mat.data))?; - // TODO check variable `_my_lambda` exists and is a function expression/lambda - mat_data.script = transform_script(mat_data.script)?; - mat.data = map_from_object(mat_data)?; - } - Ok(()) - } - - fn reformat_scripts(typegraph: &mut Typegraph, _c: &Config) -> Result<()> { - for rt_idx in find_runtimes(typegraph, |rt| { - matches!(rt, TGRuntime::Known(KnownRuntime::Deno(_))) - }) - .into_iter() - { - for mat_idx in get_materializers(typegraph, rt_idx as u32) { - reformat_materializer_script(&mut typegraph.materializers[mat_idx])?; - } - } - Ok(()) - } - #[derive(Default, Debug, Clone)] pub struct DenoModules { codegen: bool, diff --git a/typegate/engine/bindings.ts b/typegate/engine/bindings.ts index ecb7108a31..da646c12a5 100644 --- a/typegate/engine/bindings.ts +++ b/typegate/engine/bindings.ts @@ -570,3 +570,7 @@ export function archive(a0: ArchiveInp): ArchiveResult { return { Err: { message: err.toString() } }; } } + +export function transformTypescript(a0: string): string { + return Meta.deno.transformTypescript(a0); +} diff --git a/typegate/engine/runtime.d.ts b/typegate/engine/runtime.d.ts index 5fe026aa49..61d8693445 100644 --- a/typegate/engine/runtime.d.ts +++ b/typegate/engine/runtime.d.ts @@ -198,4 +198,8 @@ declare namespace Meta { function registerModule(inp: PythonRegisterInp): string; function unregisterModule(inp: PythonUnregisterInp): string; } + + namespace deno { + function transformTypescript(inp: string): string; + } } diff --git a/typegate/engine/runtime.js b/typegate/engine/runtime.js index c61521a1f4..f55c368fba 100644 --- a/typegate/engine/runtime.js +++ b/typegate/engine/runtime.js @@ -38,6 +38,9 @@ globalThis.Meta = { registerModule: ops.op_register_module, unregisterModule: ops.op_unregister_module, }, + deno: { + transformTypescript: ops.op_deno_transform_typescript, + }, version: ops.op_get_version, typescriptFormatCode: ops.op_typescript_format_code, typegraphValidate: ops.op_typegraph_validate, diff --git a/typegate/engine/src/ext.rs b/typegate/engine/src/ext.rs index 2094e79e93..e57762e079 100644 --- a/typegate/engine/src/ext.rs +++ b/typegate/engine/src/ext.rs @@ -3,7 +3,7 @@ use crate::interlude::*; use crate::{ - runtimes::{prisma, python::python_bindings, temporal, wasmedge}, + runtimes::{deno_rt, prisma, python::python_bindings, temporal, wasmedge}, typegraph, typescript, }; @@ -49,6 +49,7 @@ deno_core::extension!( prisma::op_prisma_reset, prisma::op_unpack, prisma::op_archive, + deno_rt::op_deno_transform_typescript ], esm_entry_point = "ext:tg_metatype_ext/runtime.js", esm = ["runtime.js"], diff --git a/typegate/engine/src/runtimes.rs b/typegate/engine/src/runtimes.rs index 58d2a21103..0556ce9218 100644 --- a/typegate/engine/src/runtimes.rs +++ b/typegate/engine/src/runtimes.rs @@ -1,6 +1,7 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 +pub mod deno_rt; pub mod prisma; pub mod python; pub mod temporal; diff --git a/typegate/engine/src/runtimes/deno_rt.rs b/typegate/engine/src/runtimes/deno_rt.rs new file mode 100644 index 0000000000..66eae5f27f --- /dev/null +++ b/typegate/engine/src/runtimes/deno_rt.rs @@ -0,0 +1,16 @@ +#![allow(clippy::not_unsafe_ptr_arg_deref)] + +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +use crate::interlude::*; + +#[rustfmt::skip] +use deno_core as deno_core; // necessary for re-exported macros to work +use typescript::parser::transform_script; + +#[deno_core::op2] +#[string] +pub fn op_deno_transform_typescript(#[string] script: String) -> Result { + transform_script(script) +} diff --git a/typegate/src/postprocess.ts b/typegate/src/postprocess.ts new file mode 100644 index 0000000000..b83cc3d2e6 --- /dev/null +++ b/typegate/src/postprocess.ts @@ -0,0 +1,34 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { transformTypescript, typescript_format_code } from "native"; +import { nativeResult } from "./utils.ts"; +import { TypeGraphDS } from "./typegraph/mod.ts"; + +abstract class PostProcessor { + /** Postprocess the provided typegraph reference */ + abstract postprocess(tg: TypeGraphDS): void; +} + +class DenoPostProcess extends PostProcessor { + postprocess(tg: TypeGraphDS): void { + for (const mat of tg.materializers) { + if (mat.name == "function") { + const jsScript = transformTypescript(mat.data.script as string); + mat.data.script = nativeResult( + typescript_format_code({ source: jsScript }), + ) + ?.formatted_code!; + } + } + } +} + +export function applyPostProcessors(typegraphRefs: TypeGraphDS[]): void { + const postprocesses = [new DenoPostProcess()] as Array; + for (const postprocess of postprocesses) { + for (const tgJson of typegraphRefs) { + postprocess.postprocess(tgJson); + } + } +} diff --git a/typegate/src/runtimes/deno/deno.ts b/typegate/src/runtimes/deno/deno.ts index 4dbe85a87e..2ea597be09 100644 --- a/typegate/src/runtimes/deno/deno.ts +++ b/typegate/src/runtimes/deno/deno.ts @@ -83,7 +83,6 @@ export class DenoRuntime extends Runtime { for (const mat of materializers) { if (mat.name === "function") { const code = mat.data.script as string; - ops.set(registryCount, { type: "register_func", fnCode: code, diff --git a/typegate/src/runtimes/typegate.ts b/typegate/src/runtimes/typegate.ts index ab0424008a..751670eeea 100644 --- a/typegate/src/runtimes/typegate.ts +++ b/typegate/src/runtimes/typegate.ts @@ -14,6 +14,7 @@ import { closestWord } from "../utils.ts"; import { Type, TypeNode } from "../typegraph/type_node.ts"; import { StringFormat } from "../typegraph/types.ts"; import { mapValues } from "std/collections/map_values.ts"; +import { applyPostProcessors } from "../postprocess.ts"; const logger = getLogger(import.meta); @@ -148,6 +149,8 @@ export class TypeGateRuntime extends Runtime { } const tgJson = await TypeGraph.parseJson(fromString); + applyPostProcessors([tgJson]); + const { engine, response, name } = await this.typegate.pushTypegraph( tgJson, JSON.parse(secrets), diff --git a/typegate/tests/e2e/typegraph/__snapshots__/typegraph_test.ts.snap b/typegate/tests/e2e/typegraph/__snapshots__/typegraph_test.ts.snap index fd0b992384..c7b769c6ba 100644 --- a/typegate/tests/e2e/typegraph/__snapshots__/typegraph_test.ts.snap +++ b/typegate/tests/e2e/typegraph/__snapshots__/typegraph_test.ts.snap @@ -245,7 +245,8 @@ snapshot[`typegraphs creation 1`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=()=>true;" + "script": "var _my_lambda = () => true", + "secrets": [] } }, { @@ -489,7 +490,8 @@ snapshot[`typegraphs creation 2`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=({first,second})=>first*second;" + "script": "var _my_lambda = ({ first, second }) => first * second", + "secrets": [] } } ], @@ -538,7 +540,7 @@ snapshot[`typegraphs creation 2`] = ` `; snapshot[`typegraphs creation 3`] = ` -'[ +\`[ { "\$id": "https://metatype.dev/specs/0.0.3.json", "types": [ @@ -697,7 +699,8 @@ snapshot[`typegraphs creation 3`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=()=>12;" + "script": "var _my_lambda = () => 12", + "secrets": [] } }, { @@ -708,7 +711,8 @@ snapshot[`typegraphs creation 3`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=(_,{context})=>context.provider===\\\\"internal\\\\";" + "script": "var _my_lambda = (_, { context }) => context.provider === 'internal'", + "secrets": [] } }, { @@ -719,7 +723,8 @@ snapshot[`typegraphs creation 3`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=user=>({id:12,user});" + "script": "var _my_lambda = (user) => ({ id: 12, user })", + "secrets": [] } }, { @@ -730,7 +735,8 @@ snapshot[`typegraphs creation 3`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=()=>false;" + "script": "var _my_lambda = () => false", + "secrets": [] } }, { @@ -812,7 +818,7 @@ snapshot[`typegraphs creation 3`] = ` "version": "0.0.3" } } -]' +]\` `; snapshot[`typegraphs creation 4`] = ` @@ -1060,7 +1066,8 @@ snapshot[`typegraphs creation 4`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=()=>true;" + "script": "var _my_lambda = () => true", + "secrets": [] } }, { @@ -1304,7 +1311,8 @@ snapshot[`typegraphs creation 5`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=({first,second})=>first*second;" + "script": "var _my_lambda = ({first, second}) => first * second", + "secrets": [] } } ], @@ -1353,7 +1361,7 @@ snapshot[`typegraphs creation 5`] = ` `; snapshot[`typegraphs creation 6`] = ` -'[ +\`[ { "\$id": "https://metatype.dev/specs/0.0.3.json", "types": [ @@ -1512,7 +1520,8 @@ snapshot[`typegraphs creation 6`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=()=>12;" + "script": "var _my_lambda = () => 12", + "secrets": [] } }, { @@ -1523,7 +1532,8 @@ snapshot[`typegraphs creation 6`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=(_,{context})=>context.provider===\\\\"internal\\\\";" + "script": "var _my_lambda = (_, { context }) => context.provider === 'internal'", + "secrets": [] } }, { @@ -1534,7 +1544,8 @@ snapshot[`typegraphs creation 6`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=user=>({id:12,user});" + "script": "var _my_lambda = (user) => ({ id: 12, user })", + "secrets": [] } }, { @@ -1545,7 +1556,8 @@ snapshot[`typegraphs creation 6`] = ` "idempotent": true }, "data": { - "script": "var _my_lambda=()=>false;" + "script": "var _my_lambda = () => false", + "secrets": [] } }, { @@ -1627,5 +1639,5 @@ snapshot[`typegraphs creation 6`] = ` "version": "0.0.3" } } -]' +]\` `; diff --git a/typegate/tests/e2e/website/website_test.ts b/typegate/tests/e2e/website/website_test.ts index e0de3714ed..b0f2afdf1b 100644 --- a/typegate/tests/e2e/website/website_test.ts +++ b/typegate/tests/e2e/website/website_test.ts @@ -9,6 +9,8 @@ import { Meta } from "../../utils/mod.ts"; import { MetaTest } from "../../utils/test.ts"; import { assertEquals } from "std/assert/assert_equals.ts"; import config from "../../../src/config.ts"; +import { applyPostProcessors } from "../../../src/postprocess.ts"; +import { TypeGraphDS } from "../../../src/typegraph/mod.ts"; export const thisDir = dirname(fromFileUrl(import.meta.url)); @@ -16,6 +18,12 @@ function stripIncomparable(json: string) { return [ // FIXME: python and deno does not produce the same tarball (source: string) => source.replace(/"file:scripts(.)+?"/g, '""'), + (source: string) => { + const tg: TypeGraphDS = JSON.parse(source)?.[0]; + // Required since the typescript format/js convesion Postprocessors are now removed from the cli and sdk + applyPostProcessors([tg]); + return JSON.stringify(tg, null, 2); + }, ].reduce((prev, op) => op(prev), json); } diff --git a/typegraph/core/Cargo.toml b/typegraph/core/Cargo.toml index ee28198907..10a78e5399 100644 --- a/typegraph/core/Cargo.toml +++ b/typegraph/core/Cargo.toml @@ -15,7 +15,6 @@ wit-bindgen = "0.12.0" regex = "1.10.2" indexmap = { version = "2.1.0", features = ["serde"] } common = { path = "../../libs/common" } -typescript = { path = "../../libs/typescript" } indoc = "2.0.4" graphql-parser = "0.4.0" sha2 = "0.10.8" diff --git a/typegraph/core/src/utils/postprocess/deno_rt.rs b/typegraph/core/src/utils/postprocess/deno_rt.rs index ec30e775d5..576fc52891 100644 --- a/typegraph/core/src/utils/postprocess/deno_rt.rs +++ b/typegraph/core/src/utils/postprocess/deno_rt.rs @@ -3,12 +3,11 @@ use crate::utils::fs_host; use common::typegraph::{ - runtimes::deno::{FunctionMatData, ModuleMatData}, + runtimes::deno::ModuleMatData, utils::{map_from_object, object_from_map}, Materializer, Typegraph, }; use std::path::PathBuf; -use typescript::parser::transform_script; use crate::utils::postprocess::{compress_and_encode, PostProcessor}; @@ -17,50 +16,33 @@ pub struct DenoProcessor; impl PostProcessor for DenoProcessor { fn postprocess(self, tg: &mut Typegraph) -> Result<(), String> { for mat in tg.materializers.iter_mut() { - match mat.name.as_str() { - "function" => Self::reformat_materializer_script(mat), - "module" => { - if let Some(main_path) = Self::resolve_module(mat)? { - tg.deps.push(fs_host::make_relative(&main_path)?); - } - Ok(()) + if mat.name.as_str() == "module" { + match Self::resolve_module(mat)? { + Some(dep_path) => tg.deps.push(fs_host::make_relative(&dep_path)?), + None => continue, } - _ => continue, - }?; + } } Ok(()) } } impl DenoProcessor { - pub fn reformat_materializer_script(mat: &mut Materializer) -> Result<(), String> { - if mat.name.as_str() == "function" { - let mut mat_data: FunctionMatData = - object_from_map(std::mem::take(&mut mat.data)).map_err(|e| e.to_string())?; - mat_data.script = transform_script(mat_data.script).map_err(|e| e.to_string())?; - mat.data = map_from_object(mat_data).map_err(|e| e.to_string())?; - } - Ok(()) - } - pub fn resolve_module(mat: &mut Materializer) -> Result, String> { - if mat.name.as_str() == "module" { - let mut mat_data: ModuleMatData = - object_from_map(std::mem::take(&mut mat.data)).map_err(|e| e.to_string())?; - let Some(path) = mat_data.code.strip_prefix("file:").to_owned() else { - return Ok(None); - }; + let mut mat_data: ModuleMatData = + object_from_map(std::mem::take(&mut mat.data)).map_err(|e| e.to_string())?; + let Some(path) = mat_data.code.strip_prefix("file:").to_owned() else { + return Ok(None); + }; - // main_path can be either relative or absolute, - // if relative => make it absolute - // fs::canonicalize wouldn't work in this setup - let main_path = fs_host::make_absolute(&PathBuf::from(path))?; - mat_data.code = compress_and_encode(&main_path)?; + // main_path can be either relative or absolute, + // if relative => make it absolute + // fs::canonicalize wouldn't work in this setup + let main_path = fs_host::make_absolute(&PathBuf::from(path))?; + mat_data.code = compress_and_encode(&main_path)?; - mat.data = map_from_object(mat_data).map_err(|e| e.to_string())?; + mat.data = map_from_object(mat_data).map_err(|e| e.to_string())?; - return Ok(Some(main_path)); - } - Ok(None) + Ok(Some(main_path)) } }