From 57cad539457dff7fc273bed5ecaf08bd3dc40d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 26 Oct 2020 14:03:03 +0100 Subject: [PATCH] refactor(cli): rewrite Deno.transpileOnly() to use SWC (#8090) Co-authored-by: Kitson Kelly --- cli/ast.rs | 18 +++-- cli/dts/lib.deno.unstable.d.ts | 3 +- cli/module_graph2.rs | 11 ++- cli/ops/runtime_compiler.rs | 59 ++++++++++++-- cli/tests/compiler_api_test.ts | 9 +-- cli/tsc.rs | 39 +-------- cli/tsc/99_main_compiler.js | 31 -------- cli/tsc_config.rs | 139 ++++++++++++++++++++++++--------- 8 files changed, 178 insertions(+), 131 deletions(-) diff --git a/cli/ast.rs b/cli/ast.rs index 78cafca1be85d..44e5616e704c0 100644 --- a/cli/ast.rs +++ b/cli/ast.rs @@ -225,7 +225,7 @@ impl From for EmitOptions { EmitOptions { check_js: options.check_js, emit_metadata: options.emit_decorator_metadata, - inline_source_map: true, + inline_source_map: options.inline_source_map, jsx_factory: options.jsx_factory, jsx_fragment_factory: options.jsx_fragment_factory, transform_jsx: options.jsx == "react", @@ -356,8 +356,11 @@ impl ParsedModule { /// - `source` - The source code for the module. /// - `media_type` - The media type for the module. /// +// NOTE(bartlomieju): `specifier` has `&str` type instead of +// `&ModuleSpecifier` because runtime compiler APIs don't +// require valid module specifiers pub fn parse( - specifier: &ModuleSpecifier, + specifier: &str, source: &str, media_type: &MediaType, ) -> Result { @@ -505,8 +508,9 @@ mod tests { let source = r#"import * as bar from "./test.ts"; const foo = await import("./foo.ts"); "#; - let parsed_module = parse(&specifier, source, &MediaType::JavaScript) - .expect("could not parse module"); + let parsed_module = + parse(specifier.as_str(), source, &MediaType::JavaScript) + .expect("could not parse module"); let actual = parsed_module.analyze_dependencies(); assert_eq!( actual, @@ -553,7 +557,7 @@ mod tests { } } "#; - let module = parse(&specifier, source, &MediaType::TypeScript) + let module = parse(specifier.as_str(), source, &MediaType::TypeScript) .expect("could not parse module"); let (code, maybe_map) = module .transpile(&EmitOptions::default()) @@ -577,7 +581,7 @@ mod tests { } } "#; - let module = parse(&specifier, source, &MediaType::TSX) + let module = parse(specifier.as_str(), source, &MediaType::TSX) .expect("could not parse module"); let (code, _) = module .transpile(&EmitOptions::default()) @@ -608,7 +612,7 @@ mod tests { } } "#; - let module = parse(&specifier, source, &MediaType::TypeScript) + let module = parse(specifier.as_str(), source, &MediaType::TypeScript) .expect("could not parse module"); let (code, _) = module .transpile(&EmitOptions::default()) diff --git a/cli/dts/lib.deno.unstable.d.ts b/cli/dts/lib.deno.unstable.d.ts index 1c1869827d54f..0bd4c7474ec24 100644 --- a/cli/dts/lib.deno.unstable.d.ts +++ b/cli/dts/lib.deno.unstable.d.ts @@ -480,8 +480,7 @@ declare namespace Deno { * source map. * @param options An option object of options to send to the compiler. This is * a subset of ts.CompilerOptions which can be supported by Deno. - * Many of the options related to type checking and emitting - * type declaration files will have no impact on the output. + * If unsupported option is passed then the API will throw an error. */ export function transpileOnly( sources: Record, diff --git a/cli/module_graph2.rs b/cli/module_graph2.rs index 8ee60eb598243..cc063c6c37255 100644 --- a/cli/module_graph2.rs +++ b/cli/module_graph2.rs @@ -320,7 +320,8 @@ impl Module { /// Parse a module, populating the structure with data retrieved from the /// source of the module. pub fn parse(&mut self) -> Result<(), AnyError> { - let parsed_module = parse(&self.specifier, &self.source, &self.media_type)?; + let parsed_module = + parse(self.specifier.as_str(), &self.source, &self.media_type)?; // parse out any triple slash references for comment in parsed_module.get_leading_comments().iter() { @@ -639,12 +640,13 @@ impl Graph2 { let mut ts_config = TsConfig::new(json!({ "checkJs": false, "emitDecoratorMetadata": false, + "inlineSourceMap": true, "jsx": "react", "jsxFactory": "React.createElement", "jsxFragmentFactory": "React.Fragment", })); let maybe_ignored_options = - ts_config.merge_user_config(options.maybe_config_path)?; + ts_config.merge_tsconfig(options.maybe_config_path)?; let emit_options: EmitOptions = ts_config.into(); let cm = Rc::new(swc_common::SourceMap::new( swc_common::FilePathMapping::empty(), @@ -730,7 +732,7 @@ impl Graph2 { })); } let maybe_ignored_options = - config.merge_user_config(options.maybe_config_path)?; + config.merge_tsconfig(options.maybe_config_path)?; // Short circuit if none of the modules require an emit, or all of the // modules that require an emit have a valid emit. There is also an edge @@ -1187,13 +1189,14 @@ impl Graph2 { let mut ts_config = TsConfig::new(json!({ "checkJs": false, "emitDecoratorMetadata": false, + "inlineSourceMap": true, "jsx": "react", "jsxFactory": "React.createElement", "jsxFragmentFactory": "React.Fragment", })); let maybe_ignored_options = - ts_config.merge_user_config(options.maybe_config_path)?; + ts_config.merge_tsconfig(options.maybe_config_path)?; let emit_options: EmitOptions = ts_config.clone().into(); diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs index b01469fa99b01..5ceb903167d1a 100644 --- a/cli/ops/runtime_compiler.rs +++ b/cli/ops/runtime_compiler.rs @@ -1,12 +1,17 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::ast; +use crate::colors; +use crate::media_type::MediaType; use crate::permissions::Permissions; use crate::tsc::runtime_bundle; use crate::tsc::runtime_compile; -use crate::tsc::runtime_transpile; +use crate::tsc_config; use deno_core::error::AnyError; use deno_core::futures::FutureExt; +use deno_core::serde::Serialize; use deno_core::serde_json; +use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_core::BufVec; use deno_core::OpState; @@ -71,16 +76,58 @@ struct TranspileArgs { options: Option, } +#[derive(Debug, Serialize)] +struct RuntimeTranspileEmit { + source: String, + map: Option, +} + async fn op_transpile( state: Rc>, args: Value, _data: BufVec, ) -> Result { - super::check_unstable2(&state, "Deno.transpile"); + super::check_unstable2(&state, "Deno.transpileOnly"); let args: TranspileArgs = serde_json::from_value(args)?; - let cli_state = super::global_state2(&state); - let program_state = cli_state.clone(); - let result = - runtime_transpile(program_state, &args.sources, &args.options).await?; + + let mut compiler_options = tsc_config::TsConfig::new(json!({ + "checkJs": true, + "emitDecoratorMetadata": false, + "jsx": "react", + "jsxFactory": "React.createElement", + "jsxFragmentFactory": "React.Fragment", + "inlineSourceMap": false, + })); + + let user_options: HashMap = if let Some(options) = args.options + { + serde_json::from_str(&options)? + } else { + HashMap::new() + }; + let maybe_ignored_options = + compiler_options.merge_user_config(&user_options)?; + // TODO(@kitsonk) these really should just be passed back to the caller + if let Some(ignored_options) = maybe_ignored_options { + info!("{}: {}", colors::yellow("warning"), ignored_options); + } + + let emit_options: ast::EmitOptions = compiler_options.into(); + let mut emit_map = HashMap::new(); + + for (specifier, source) in args.sources { + let media_type = MediaType::from(&specifier); + let parsed_module = ast::parse(&specifier, &source, &media_type)?; + let (source, maybe_source_map) = parsed_module.transpile(&emit_options)?; + + emit_map.insert( + specifier.to_string(), + RuntimeTranspileEmit { + source, + map: maybe_source_map, + }, + ); + } + let result = serde_json::to_value(emit_map)?; Ok(result) } diff --git a/cli/tests/compiler_api_test.ts b/cli/tests/compiler_api_test.ts index b4a2f81ef7944..a845e58c6857b 100644 --- a/cli/tests/compiler_api_test.ts +++ b/cli/tests/compiler_api_test.ts @@ -129,17 +129,16 @@ Deno.test({ async fn() { const actual = await Deno.transpileOnly( { - "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, + "foo.ts": `/** This is JSDoc */\nexport enum Foo { Foo, Bar, Baz };\n`, }, { - sourceMap: false, - module: "amd", + removeComments: true, }, ); assert(actual); assertEquals(Object.keys(actual), ["foo.ts"]); - assert(actual["foo.ts"].source.startsWith("define(")); - assert(actual["foo.ts"].map == null); + assert(!actual["foo.ts"].source.includes("This is JSDoc")); + assert(actual["foo.ts"].map); }, }); diff --git a/cli/tsc.rs b/cli/tsc.rs index 796b585feb0af..7b72e8d3656e9 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -725,41 +725,6 @@ pub async fn runtime_bundle( Ok(serde_json::from_str::(&json_str).unwrap()) } -/// This function is used by `Deno.transpileOnly()` API. -pub async fn runtime_transpile( - program_state: Arc, - sources: &HashMap, - maybe_options: &Option, -) -> Result { - let user_options = if let Some(options) = maybe_options { - tsc_config::parse_raw_config(options)? - } else { - json!({}) - }; - - let mut compiler_options = json!({ - "esModuleInterop": true, - "module": "esnext", - "sourceMap": true, - "scriptComments": true, - "target": "esnext", - }); - tsc_config::json_merge(&mut compiler_options, &user_options); - - let req_msg = json!({ - "type": CompilerRequestType::RuntimeTranspile, - "sources": sources, - "compilerOptions": compiler_options, - }) - .to_string(); - - let json_str = - execute_in_tsc(program_state, req_msg).map_err(extract_js_error)?; - let v = serde_json::from_str::(&json_str) - .expect("Error decoding JSON string."); - Ok(v) -} - #[derive(Clone, Debug, PartialEq)] pub struct ImportDesc { pub specifier: String, @@ -793,7 +758,7 @@ pub fn pre_process_file( analyze_dynamic_imports: bool, ) -> Result<(Vec, Vec), AnyError> { let specifier = ModuleSpecifier::resolve_url_or_path(file_name)?; - let module = parse(&specifier, source_code, &media_type)?; + let module = parse(specifier.as_str(), source_code, &media_type)?; let dependency_descriptors = module.analyze_dependencies(); @@ -894,7 +859,6 @@ fn parse_deno_types(comment: &str) -> Option { pub enum CompilerRequestType { RuntimeCompile = 2, RuntimeBundle = 3, - RuntimeTranspile = 4, } impl Serialize for CompilerRequestType { @@ -905,7 +869,6 @@ impl Serialize for CompilerRequestType { let value: i32 = match self { CompilerRequestType::RuntimeCompile => 2 as i32, CompilerRequestType::RuntimeBundle => 3 as i32, - CompilerRequestType::RuntimeTranspile => 4 as i32, }; Serialize::serialize(&value, serializer) } diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index ee03ff6ff9aec..470c1fcee46a6 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -599,7 +599,6 @@ delete Object.prototype.__proto__; const CompilerRequestType = { RuntimeCompile: 2, RuntimeBundle: 3, - RuntimeTranspile: 4, }; function createBundleWriteFile(state) { @@ -999,31 +998,6 @@ delete Object.prototype.__proto__; }; } - function runtimeTranspile(request) { - const result = {}; - const { sources, compilerOptions } = request; - - const parseResult = parseCompilerOptions( - compilerOptions, - ); - const options = parseResult.options; - // TODO(bartlomieju): this options is excluded by `ts.convertCompilerOptionsFromJson` - // however stuff breaks if it's not passed (type_directives_js_main.js, compiler_js_error.ts) - options.allowNonTsExtensions = true; - - for (const [fileName, inputText] of Object.entries(sources)) { - const { outputText: source, sourceMapText: map } = ts.transpileModule( - inputText, - { - fileName, - compilerOptions: options, - }, - ); - result[fileName] = { source, map }; - } - return result; - } - function opCompilerRespond(msg) { core.jsonOpSync("op_compiler_respond", msg); } @@ -1041,11 +1015,6 @@ delete Object.prototype.__proto__; opCompilerRespond(result); break; } - case CompilerRequestType.RuntimeTranspile: { - const result = runtimeTranspile(request); - opCompilerRespond(result); - break; - } default: throw new Error( `!!! unhandled CompilerRequestType: ${request.type} (${ diff --git a/cli/tsc_config.rs b/cli/tsc_config.rs index c4028dea9cfb4..93f269ed355fb 100644 --- a/cli/tsc_config.rs +++ b/cli/tsc_config.rs @@ -2,6 +2,7 @@ use deno_core::error::AnyError; use deno_core::serde_json; +use deno_core::serde_json::json; use deno_core::serde_json::Value; use jsonc_parser::JsonValue; use serde::Deserialize; @@ -20,6 +21,7 @@ use std::str::FromStr; pub struct EmitConfigOptions { pub check_js: bool, pub emit_decorator_metadata: bool, + pub inline_source_map: bool, pub jsx: String, pub jsx_factory: String, pub jsx_fragment_factory: String, @@ -30,77 +32,82 @@ pub struct EmitConfigOptions { #[derive(Debug, Clone, PartialEq)] pub struct IgnoredCompilerOptions { pub items: Vec, - pub path: PathBuf, + pub maybe_path: Option, } impl fmt::Display for IgnoredCompilerOptions { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut codes = self.items.clone(); codes.sort(); - - write!(f, "Unsupported compiler options in \"{}\".\n The following options were ignored:\n {}", self.path.to_string_lossy(), codes.join(", ")) + if let Some(path) = &self.maybe_path { + write!(f, "Unsupported compiler options in \"{}\".\n The following options were ignored:\n {}", path.to_string_lossy(), codes.join(", ")) + } else { + write!(f, "Unsupported compiler options provided.\n The following options were ignored:\n {}", codes.join(", ")) + } } } /// A static slice of all the compiler options that should be ignored that /// either have no effect on the compilation or would cause the emit to not work /// in Deno. -const IGNORED_COMPILER_OPTIONS: [&str; 61] = [ +const IGNORED_COMPILER_OPTIONS: [&str; 10] = [ "allowSyntheticDefaultImports", + "esModuleInterop", + "inlineSourceMap", + "inlineSources", + // TODO(nayeemrmn): Add "isolatedModules" here for 1.6.0. + "module", + "noLib", + "preserveConstEnums", + "reactNamespace", + "sourceMap", + "target", +]; + +const IGNORED_RUNTIME_COMPILER_OPTIONS: [&str; 50] = [ "allowUmdGlobalAccess", "assumeChangesOnlyAffectDirectDependencies", "baseUrl", "build", "composite", "declaration", - "declarationDir", "declarationMap", "diagnostics", "downlevelIteration", "emitBOM", "emitDeclarationOnly", - "esModuleInterop", "extendedDiagnostics", "forceConsistentCasingInFileNames", "generateCpuProfile", "help", "importHelpers", "incremental", - "inlineSourceMap", - "inlineSources", "init", - // TODO(nayeemrmn): Add "isolatedModules" here for 1.6.0. "listEmittedFiles", "listFiles", "mapRoot", "maxNodeModuleJsDepth", - "module", "moduleResolution", "newLine", "noEmit", "noEmitHelpers", "noEmitOnError", - "noLib", "noResolve", "out", "outDir", "outFile", "paths", - "preserveConstEnums", "preserveSymlinks", "preserveWatchOutput", "pretty", - "reactNamespace", "resolveJsonModule", "rootDir", "rootDirs", "showConfig", "skipDefaultLibCheck", "skipLibCheck", - "sourceMap", "sourceRoot", "stripInternal", - "target", "traceResolution", "tsBuildInfoFile", "types", @@ -167,6 +174,34 @@ pub fn parse_raw_config(config_text: &str) -> Result { Ok(jsonc_to_serde(jsonc)) } +fn parse_compiler_options( + compiler_options: &HashMap, + maybe_path: Option, + is_runtime: bool, +) -> Result<(Value, Option), AnyError> { + let mut filtered: HashMap = HashMap::new(); + let mut items: Vec = Vec::new(); + + for (key, value) in compiler_options.iter() { + let key = key.as_str(); + if (!is_runtime && IGNORED_COMPILER_OPTIONS.contains(&key)) + || IGNORED_RUNTIME_COMPILER_OPTIONS.contains(&key) + { + items.push(key.to_string()); + } else { + filtered.insert(key.to_string(), value.to_owned()); + } + } + let value = serde_json::to_value(filtered)?; + let maybe_ignored_options = if !items.is_empty() { + Some(IgnoredCompilerOptions { items, maybe_path }) + } else { + None + }; + + Ok((value, maybe_ignored_options)) +} + /// Take a string of JSONC, parse it and return a serde `Value` of the text. /// The result also contains any options that were ignored. pub fn parse_config( @@ -176,29 +211,12 @@ pub fn parse_config( assert!(!config_text.is_empty()); let jsonc = jsonc_parser::parse_to_value(config_text)?.unwrap(); let config: TSConfigJson = serde_json::from_value(jsonc_to_serde(jsonc))?; - let mut compiler_options: HashMap = HashMap::new(); - let mut items: Vec = Vec::new(); - if let Some(in_compiler_options) = config.compiler_options { - for (key, value) in in_compiler_options.iter() { - if IGNORED_COMPILER_OPTIONS.contains(&key.as_str()) { - items.push(key.to_owned()); - } else { - compiler_options.insert(key.to_owned(), value.to_owned()); - } - } - } - let options_value = serde_json::to_value(compiler_options)?; - let ignored_options = if !items.is_empty() { - Some(IgnoredCompilerOptions { - items, - path: path.to_path_buf(), - }) + if let Some(compiler_options) = config.compiler_options { + parse_compiler_options(&compiler_options, Some(path.to_owned()), false) } else { - None - }; - - Ok((options_value, ignored_options)) + Ok((json!({}), None)) + } } /// A structure for managing the configuration of TypeScript @@ -237,7 +255,7 @@ impl TsConfig { /// /// When there are options ignored out of the file, a warning will be written /// to stderr regarding the options that were ignored. - pub fn merge_user_config( + pub fn merge_tsconfig( &mut self, maybe_path: Option, ) -> Result, AnyError> { @@ -263,6 +281,19 @@ impl TsConfig { Ok(None) } } + + /// Take a map of compiler options, filtering out any that are ignored, then + /// merge it with the current configuration, returning any options that might + /// have been ignored. + pub fn merge_user_config( + &mut self, + user_options: &HashMap, + ) -> Result, AnyError> { + let (value, maybe_ignored_options) = + parse_compiler_options(user_options, None, true)?; + json_merge(&mut self.0, &value); + Ok(maybe_ignored_options) + } } impl Serialize for TsConfig { @@ -321,11 +352,43 @@ mod tests { ignored, Some(IgnoredCompilerOptions { items: vec!["build".to_string()], - path: config_path, + maybe_path: Some(config_path), }), ); } + #[test] + fn test_tsconfig_merge_user_options() { + let mut tsconfig = TsConfig::new(json!({ + "target": "esnext", + "module": "esnext", + })); + let user_options = serde_json::from_value(json!({ + "target": "es6", + "build": true, + "strict": false, + })) + .expect("could not convert to hashmap"); + let maybe_ignored_options = tsconfig + .merge_user_config(&user_options) + .expect("could not merge options"); + assert_eq!( + tsconfig.0, + json!({ + "module": "esnext", + "target": "es6", + "strict": false, + }) + ); + assert_eq!( + maybe_ignored_options, + Some(IgnoredCompilerOptions { + items: vec!["build".to_string()], + maybe_path: None + }) + ); + } + #[test] fn test_parse_raw_config() { let invalid_config_text = r#"{