feat: eval v2, blazing fast eval triggered only upon the right changes (

#2164)

* progress

* progress

* progress

* add suggestions to frontend scripts

* all eval

* remove onlyStatic

* handle better conversions

* handle better conversions

* remove variable picker

* fix pen positioning

* fix v1

backend/Cargo.toml

@@ -131,17 +131,18 @@ tokio-util = { version = "^0", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
-deno_fetch = "0.133.0"
-deno_console = "0.109.0"
-deno_url = "0.109.0"
-deno_webidl = "0.109.0"
-deno_web = "0.140.0"
-deno_core = "0.191.0"
-deno_ast = { version = "0.27.2", features = ["transpiling"] }
+deno_fetch = "0.139.0"
+deno_console = "0.115.0"
+deno_url = "0.115.0"
+deno_webidl = "0.115.0"
+deno_web = "0.146.0"
+deno_core = "0.200.0"
+deno_ast = { version = "0.28.0", features = ["transpiling"] }
 async-recursion = "^1"
-swc_common = "0.31.12"
-swc_ecma_parser = "0.136.2"
-swc_ecma_ast = "0.106.2"
+swc_common = "0.31.21"
+swc_ecma_parser = "0.137.15"
+swc_ecma_ast = "0.107.7"
+swc_ecma_visit = "0.93.7"
 base64 = "0.21.0"
 hmac = "0.12.1"
 sha2 = "0.10.6"

backend/parsers/windmill-parser-ts/Cargo.toml

@@ -18,6 +18,7 @@ windmill-parser.workspace = true
 swc_common.workspace = true
 swc_ecma_parser.workspace = true
 swc_ecma_ast.workspace = true
+swc_ecma_visit.workspace = true
 serde_json.workspace = true
 anyhow.workspace = true
 convert_case.workspace = true

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -9,21 +9,78 @@ use regex::Regex;
  */
 // use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
 use serde_json::Value;
+use std::collections::HashSet;
+use swc_ecma_visit::{noop_visit_type, Visit, VisitWith};
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectProperty, Typ};
 
 use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned};
 use swc_ecma_ast::{
     ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, Lit,
-    ModuleDecl, ModuleItem, Number, ObjectLit, ObjectPat, Param, Pat, Str, TsArrayType,
-    TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType,
+    MemberExpr, MemberProp, ModuleDecl, ModuleItem, Number, ObjectLit, ObjectPat, Param, Pat, Str,
+    TsArrayType, TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType,
     TsParenthesizedType, TsPropertySignature, TsType, TsTypeAnn, TsTypeElement, TsTypeLit,
     TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
 };
-use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
+use swc_ecma_parser::{lexer::Lexer, EsConfig, Parser, StringInput, Syntax, TsConfig};
 
 #[cfg(target_arch = "wasm32")]
 use wasm_bindgen::prelude::*;
 
+struct OutputFinder {
+    idents: HashSet<(String, String)>,
+}
+
+impl Visit for OutputFinder {
+    noop_visit_type!();
+
+    fn visit_member_expr(&mut self, m: &MemberExpr) {
+        m.obj.visit_with(self);
+        if let MemberProp::Computed(c) = &m.prop {
+            c.visit_with(self);
+        }
+        match m {
+            MemberExpr { obj, prop: MemberProp::Ident(Ident { sym, .. }), .. } => {
+                match *obj.to_owned() {
+                    Expr::Ident(Ident { sym: sym_i, .. }) => {
+                        self.idents.insert((sym_i.to_string(), sym.to_string()));
+                    }
+                    _ => (),
+                }
+            }
+            _ => (),
+        }
+    }
+}
+
+pub fn parse_expr_for_ids(code: &str) -> anyhow::Result<Vec<(String, String)>> {
+    let cm: Lrc<SourceMap> = Default::default();
+    let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
+    let lexer = Lexer::new(
+        // We want to parse ecmascript
+        Syntax::Es(EsConfig { jsx: false, ..Default::default() }),
+        // EsVersion defaults to es5
+        Default::default(),
+        StringInput::from(&*fm),
+        None,
+    );
+
+    let mut parser = Parser::new_from(lexer);
+
+    let mut err_s = "".to_string();
+    for e in parser.take_errors() {
+        err_s += &e.into_kind().msg().to_string();
+    }
+
+    let expr = parser
+        .parse_module()
+        .map_err(|_| anyhow::anyhow!("Error while parsing code, it is invalid TypeScript"))?;
+
+    let mut visitor = OutputFinder { idents: HashSet::new() };
+    swc_ecma_visit::visit_module(&mut visitor, &expr);
+
+    Ok(visitor.idents.into_iter().collect())
+}
+
 pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainArgSignature> {
     let cm: Lrc<SourceMap> = Default::default();
     let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
@@ -308,8 +365,8 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
 extern "C" {
     pub fn eval(s: &str) -> JsValue;
     pub fn alert(s: &str);
-    // #[wasm_bindgen(js_namespace = console)]
-    // fn log(s: &str);
+    #[wasm_bindgen(js_namespace = console)]
+    fn log(s: &str);
 
 }
 

backend/parsers/windmill-parser-wasm/pkg/package.json

@@ -3,7 +3,7 @@
   "collaborators": [
     "Ruben Fiszel <ruben@windmill.dev>"
   ],
-  "version": "1.147.3",
+  "version": "1.154.3",
   "files": [
     "windmill_parser_wasm_bg.wasm",
     "windmill_parser_wasm.js",
@@ -14,4 +14,4 @@
   "sideEffects": [
     "./snippets/*"
   ]
-}
+}

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts

@@ -9,6 +9,11 @@ export function parse_deno(code: string): string;
 * @param {string} code
 * @returns {string}
 */
+export function parse_outputs(code: string): string;
+/**
+* @param {string} code
+* @returns {string}
+*/
 export function parse_bash(code: string): string;
 /**
 * @param {string} code
@@ -56,6 +61,7 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl
 export interface InitOutput {
   readonly memory: WebAssembly.Memory;
   readonly parse_deno: (a: number, b: number, c: number) => void;
+  readonly parse_outputs: (a: number, b: number, c: number) => void;
   readonly parse_bash: (a: number, b: number, c: number) => void;
   readonly parse_powershell: (a: number, b: number, c: number) => void;
   readonly parse_go: (a: number, b: number, c: number) => void;

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js

@@ -220,6 +220,29 @@ export function parse_deno(code) {
     }
 }
 
+/**
+* @param {string} code
+* @returns {string}
+*/
+export function parse_outputs(code) {
+    let deferred2_0;
+    let deferred2_1;
+    try {
+        const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+        const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.parse_outputs(retptr, ptr0, len0);
+        var r0 = getInt32Memory0()[retptr / 4 + 0];
+        var r1 = getInt32Memory0()[retptr / 4 + 1];
+        deferred2_0 = r0;
+        deferred2_1 = r1;
+        return getStringFromWasm0(r0, r1);
+    } finally {
+        wasm.__wbindgen_add_to_stack_pointer(16);
+        wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+    }
+}
+
 /**
 * @param {string} code
 * @returns {string}
@@ -469,6 +492,10 @@ async function __wbg_load(module, imports) {
 function __wbg_get_imports() {
     const imports = {};
     imports.wbg = {};
+    imports.wbg.__wbg_eval_bfffb337c5ad9d0f = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
         takeObject(arg0);
     };
@@ -520,10 +547,6 @@ function __wbg_get_imports() {
         const ret = new Error(getStringFromWasm0(arg0, arg1));
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_eval_2824c9167fe66e96 = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
         const ret = getObject(arg0) == getObject(arg1);
         return ret;

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts

@@ -2,6 +2,7 @@
 /* eslint-disable */
 export const memory: WebAssembly.Memory;
 export function parse_deno(a: number, b: number, c: number): void;
+export function parse_outputs(a: number, b: number, c: number): void;
 export function parse_bash(a: number, b: number, c: number): void;
 export function parse_powershell(a: number, b: number, c: number): void;
 export function parse_go(a: number, b: number, c: number): void;

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -1,6 +1,7 @@
-use serde_json;
+use serde_json::{self, json};
 use wasm_bindgen::prelude::*;
 use windmill_parser::MainArgSignature;
+use windmill_parser_ts::parse_expr_for_ids;
 
 fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
     if let Ok(r) = r {
@@ -15,6 +16,17 @@ pub fn parse_deno(code: &str) -> String {
     wrap_sig(windmill_parser_ts::parse_deno_signature(code, false))
 }
 
+#[wasm_bindgen]
+pub fn parse_outputs(code: &str) -> String {
+    let parsed = parse_expr_for_ids(code);
+    let r = if let Ok(parsed) = parsed {
+        json!({ "outputs": parsed })
+    } else {
+        json!({"error": parsed.err().unwrap().to_string()})
+    };
+    return serde_json::to_string(&r).unwrap();
+}
+
 #[wasm_bindgen]
 pub fn parse_bash(code: &str) -> String {
     wrap_sig(windmill_parser_bash::parse_bash_sig(code))

backend/parsers/windmill-parser-wasm/tests/wasm.rs

@@ -1,7 +1,7 @@
 use serde_json::json;
 use wasm_bindgen_test::wasm_bindgen_test;
 use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
-use windmill_parser_ts::parse_deno_signature;
+use windmill_parser_ts::{parse_deno_signature, parse_expr_for_ids};
 
 #[wasm_bindgen_test]
 fn test_parse_deno_sig() -> anyhow::Result<()> {
@@ -256,3 +256,22 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
 
     Ok(())
 }
+
+#[wasm_bindgen_test]
+fn test_parse_extract_ident() -> anyhow::Result<()> {
+    let code = "
+    let foo = 3;
+    bar
+    baroof.foob.ar
+    foobar[barfoo.x]
+";
+    assert_eq!(
+        parse_expr_for_ids(code)?,
+        vec![
+            ("baroof".to_string(), "foob".to_string()),
+            ("barfoo".to_string(), "x".to_string())
+        ]
+    );
+
+    Ok(())
+}

backend/windmill-worker/build.rs

@@ -3,6 +3,7 @@ use deno_fetch::FetchPermissions;
 use deno_web::{BlobStore, TimersPermission};
 use std::env;
 use std::path::PathBuf;
+use std::sync::Arc;
 
 pub struct PermissionsContainer;
 
@@ -48,7 +49,10 @@ fn main() {
         deno_webidl::deno_webidl::init_ops_and_esm(),
         deno_url::deno_url::init_ops_and_esm(),
         deno_console::deno_console::init_ops_and_esm(),
-        deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(BlobStore::default(), None),
+        deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(
+            Arc::new(BlobStore::default()),
+            None,
+        ),
         deno_fetch::deno_fetch::init_ops_and_esm::<PermissionsContainer>(Default::default()),
         fetch::init_ops_and_esm(),
     ];
@@ -65,7 +69,7 @@ fn main() {
             startup_snapshot: None,
             extensions: exts,
             compression_cb: None,
-            snapshot_module_load_cb: None,
+            with_runtime_cb: None,
         },
     );
 }

backend/windmill-worker/src/js_eval.rs

@@ -6,14 +6,14 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{cell::RefCell, collections::HashMap, rc::Rc};
+use std::{cell::RefCell, collections::HashMap, rc::Rc, sync::Arc};
 
 use deno_ast::{ParseParams, SourceTextInfo};
 use deno_core::{
     op, serde_v8,
     v8::IsolateHandle,
     v8::{self},
-    Extension, JsRuntime, OpState, RuntimeOptions, Snapshot,
+    Extension, JsRuntime, Op, OpState, RuntimeOptions, Snapshot,
 };
 use deno_fetch::FetchPermissions;
 use deno_web::{BlobStore, TimersPermission};
@@ -88,17 +88,17 @@ pub async fn eval_timeout(
                     // An op for summing an array of numbers
                     // The op-layer automatically deserializes inputs
                     // and serializes the returned Result & value
-                    op_variable::decl(),
-                    op_resource::decl(),
+                    op_variable::DECL,
+                    op_resource::DECL,
                 ])
             }
 
             if by_id.is_some() && authed_client.is_some() {
-                ops.push(op_get_result::decl());
-                ops.push(op_get_id::decl());
+                ops.push(op_get_result::DECL);
+                ops.push(op_get_id::DECL);
             }
 
-            let ext = Extension::builder("js_eval").ops(ops).build();
+            let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() };
             let exts = vec![ext];
             // Use our snapshot to provision our new runtime
             let options = RuntimeOptions {
@@ -444,15 +444,19 @@ pub async fn eval_fetch_timeout(
     timeout(
         std::time::Duration::from_millis(100000),
         tokio::task::spawn_blocking(move || {
-            let ops = vec![op_get_static_args::decl(), op_log::decl()];
-            let ext = Extension::builder("windmill").ops(ops).build();
+            let ops = vec![op_get_static_args::DECL, op_log::DECL];
+            let ext = Extension {
+                  name: "windmill",
+                  ops: ops.into(),
+                  ..Default::default()
+                };
 
             let exts: Vec<Extension> = vec![
                 deno_webidl::deno_webidl::init_ops(),
                 deno_url::deno_url::init_ops(),
                 deno_console::deno_console::init_ops(),
                 deno_web::deno_web::init_ops::<PermissionsContainer>(
-                    BlobStore::default(),
+                    Arc::new(BlobStore::default()),
                     None,
                 ),
                 deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(