diff --git a/cli/compiler.rs b/cli/compiler.rs index 3a036a62768c1b..4233262a39b22b 100644 --- a/cli/compiler.rs +++ b/cli/compiler.rs @@ -50,16 +50,22 @@ impl ModuleMetaData { type CompilerConfig = Option<(String, Vec)>; /// Creates the JSON message send to compiler.ts's onmessage. -fn req(root_names: Vec, compiler_config: CompilerConfig) -> Buf { +fn req( + root_names: Vec, + compiler_config: CompilerConfig, + bundle: Option, +) -> Buf { let j = if let Some((config_path, config_data)) = compiler_config { json!({ "rootNames": root_names, + "bundle": bundle, "configPath": config_path, "config": str::from_utf8(&config_data).unwrap(), }) } else { json!({ "rootNames": root_names, + "bundle": bundle, }) }; j.to_string().into_boxed_str().into_boxed_bytes() @@ -82,6 +88,67 @@ pub fn get_compiler_config( } } +pub fn bundle_async( + state: ThreadSafeState, + module_name: String, + out_file: String, +) -> impl Future { + debug!( + "Invoking the compiler to bundle. module_name: {}", + module_name + ); + + let root_names = vec![module_name.clone()]; + let compiler_config = get_compiler_config(&state, "typescript"); + let req_msg = req(root_names, compiler_config, Some(out_file)); + + // Count how many times we start the compiler worker. + state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst); + + let mut worker = Worker::new( + "TS".to_string(), + startup_data::compiler_isolate_init(), + // TODO(ry) Maybe we should use a separate state for the compiler. + // as was done previously. + state.clone(), + ); + js_check(worker.execute("denoMain()")); + js_check(worker.execute("workerMain()")); + js_check(worker.execute("compilerMain()")); + + let resource = worker.state.resource.clone(); + let compiler_rid = resource.rid; + let first_msg_fut = resources::post_message_to_worker(compiler_rid, req_msg) + .then(move |_| worker) + .then(move |result| { + if let Err(err) = result { + // TODO(ry) Need to forward the error instead of exiting. + eprintln!("{}", err.to_string()); + std::process::exit(1); + } + debug!("Sent message to worker"); + let stream_future = + resources::get_message_stream_from_worker(compiler_rid).into_future(); + stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f) + }); + + first_msg_fut.map_err(|_| panic!("not handled")).and_then( + move |maybe_msg: Option| { + debug!("Received message from worker"); + + if let Some(msg) = maybe_msg { + let json_str = std::str::from_utf8(&msg).unwrap(); + debug!("Message: {}", json_str); + if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) { + return Err(diagnostics); + } + } + + Ok(()) + }, + ) +} + pub fn compile_async( state: ThreadSafeState, module_meta_data: &ModuleMetaData, @@ -95,7 +162,7 @@ pub fn compile_async( let root_names = vec![module_name.clone()]; let compiler_config = get_compiler_config(&state, "typescript"); - let req_msg = req(root_names, compiler_config); + let req_msg = req(root_names, compiler_config, None); // Count how many times we start the compiler worker. state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst); @@ -197,7 +264,13 @@ mod tests { maybe_source_map: None, }; - out = compile_sync(ThreadSafeState::mock(), &out).unwrap(); + out = compile_sync( + ThreadSafeState::mock(vec![ + String::from("./deno"), + String::from("hello.js"), + ]), + &out, + ).unwrap(); assert!( out .maybe_output_code @@ -210,8 +283,29 @@ mod tests { #[test] fn test_get_compiler_config_no_flag() { let compiler_type = "typescript"; - let state = ThreadSafeState::mock(); + let state = ThreadSafeState::mock(vec![ + String::from("./deno"), + String::from("hello.js"), + ]); let out = get_compiler_config(&state, compiler_type); assert_eq!(out, None); } + + #[test] + fn test_bundle_async() { + let specifier = "./tests/002_hello.ts"; + use crate::worker; + let module_name = worker::root_specifier_to_url(specifier) + .unwrap() + .to_string(); + + let state = ThreadSafeState::mock(vec![ + String::from("./deno"), + String::from("./tests/002_hello.ts"), + String::from("$deno$/bundle.js"), + ]); + let out = + bundle_async(state, module_name, String::from("$deno$/bundle.js")); + assert_eq!(tokio_util::block_on(out), Ok(())); + } } diff --git a/cli/flags.rs b/cli/flags.rs index a7245eba1bec91..b9a298d28d2c0a 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -154,6 +154,16 @@ To get help on the another subcommands (run in this case): Includes versions of Deno, V8 JavaScript Engine, and the TypeScript compiler.", ), + ).subcommand( + SubCommand::with_name("bundle") + .setting(AppSettings::DisableVersion) + .about("Bundle module and dependnecies into single file") + .long_about( + "Fetch, compile, and output to a single file a module and its dependencies. +" + ) + .arg(Arg::with_name("source_file").takes_value(true).required(true)) + .arg(Arg::with_name("out_file").takes_value(true).required(true)), ).subcommand( SubCommand::with_name("fetch") .setting(AppSettings::DisableVersion) @@ -436,6 +446,7 @@ const PRETTIER_URL: &str = "https://deno.land/std@v0.7.0/prettier/main.ts"; /// There is no "Help" subcommand because it's handled by `clap::App` itself. #[derive(Debug, PartialEq)] pub enum DenoSubcommand { + Bundle, Eval, Fetch, Info, @@ -455,6 +466,13 @@ pub fn flags_from_vec( let mut flags = parse_flags(&matches.clone()); let subcommand = match matches.subcommand() { + ("bundle", Some(bundle_match)) => { + flags.allow_write = true; + let source_file: &str = bundle_match.value_of("source_file").unwrap(); + let out_file: &str = bundle_match.value_of("out_file").unwrap(); + argv.extend(vec![source_file.to_string(), out_file.to_string()]); + DenoSubcommand::Bundle + } ("eval", Some(eval_match)) => { flags.allow_net = true; flags.allow_env = true; @@ -1034,4 +1052,19 @@ mod tests { assert_eq!(subcommand, DenoSubcommand::Run); assert_eq!(argv, svec!["deno", "script.ts"]); } + + #[test] + fn test_flags_from_vec_26() { + let (flags, subcommand, argv) = + flags_from_vec(svec!["deno", "bundle", "source.ts", "bundle.js"]); + assert_eq!( + flags, + DenoFlags { + allow_write: true, + ..DenoFlags::default() + } + ); + assert_eq!(subcommand, DenoSubcommand::Bundle); + assert_eq!(argv, svec!["deno", "source.ts", "bundle.js"]) + } } diff --git a/cli/main.rs b/cli/main.rs index cfce2254b31b69..ad0374af2294b5 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -41,6 +41,7 @@ mod tokio_write; pub mod version; pub mod worker; +use crate::compiler::bundle_async; use crate::errors::RustOrJsError; use crate::progress::Progress; use crate::state::ThreadSafeState; @@ -261,6 +262,26 @@ fn xeval_command(flags: DenoFlags, argv: Vec) { tokio_util::run(main_future); } +fn bundle_command(flags: DenoFlags, argv: Vec) { + let (mut _worker, state) = create_worker_and_state(flags, argv); + + let main_module = state.main_module().unwrap(); + let main_url = root_specifier_to_url(&main_module).unwrap(); + assert!(state.argv.len() >= 3); + let out_file = state.argv[2].clone(); + debug!(">>>>> bundle_async START"); + let bundle_future = bundle_async(state, main_url.to_string(), out_file) + .map_err(|e| { + debug!("diagnostics returned, exiting!"); + eprintln!("\n{}", e.to_string()); + std::process::exit(1); + }).and_then(move |_| { + debug!(">>>>> bundle_async END"); + Ok(()) + }); + tokio_util::run(bundle_future); +} + fn run_repl(flags: DenoFlags, argv: Vec) { let (mut worker, _state) = create_worker_and_state(flags, argv); @@ -322,6 +343,7 @@ fn main() { }); match subcommand { + DenoSubcommand::Bundle => bundle_command(flags, argv), DenoSubcommand::Eval => eval_command(flags, argv), DenoSubcommand::Fetch => fetch_or_info_command(flags, argv, false), DenoSubcommand::Info => fetch_or_info_command(flags, argv, true), diff --git a/cli/state.rs b/cli/state.rs index d7681fc7930ed2..9a8b1cab225f94 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -311,8 +311,7 @@ impl ThreadSafeState { } #[cfg(test)] - pub fn mock() -> ThreadSafeState { - let argv = vec![String::from("./deno"), String::from("hello.js")]; + pub fn mock(argv: Vec) -> ThreadSafeState { ThreadSafeState::new( flags::DenoFlags::default(), argv, @@ -349,5 +348,8 @@ impl ThreadSafeState { #[test] fn thread_safe() { fn f(_: S) {} - f(ThreadSafeState::mock()); + f(ThreadSafeState::mock(vec![ + String::from("./deno"), + String::from("hello.js"), + ])); } diff --git a/cli/worker.rs b/cli/worker.rs index f95826674f8ab9..99470c2a7d6846 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -280,7 +280,10 @@ mod tests { } fn create_test_worker() -> Worker { - let state = ThreadSafeState::mock(); + let state = ThreadSafeState::mock(vec![ + String::from("./deno"), + String::from("hello.js"), + ]); let mut worker = Worker::new("TEST".to_string(), startup_data::deno_isolate_init(), state); js_check(worker.execute("denoMain()")); diff --git a/js/compiler.ts b/js/compiler.ts index 6b0881700d9cae..db59a8bd8c7181 100644 --- a/js/compiler.ts +++ b/js/compiler.ts @@ -1,20 +1,22 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import * as msg from "gen/cli/msg_generated"; +import * as ts from "typescript"; + +import { assetSourceCode } from "./assets"; +import { bold, cyan, yellow } from "./colors"; +import { Console } from "./console"; import { core } from "./core"; import { Diagnostic, fromTypeScriptDiagnostic } from "./diagnostics"; -import * as flatbuffers from "./flatbuffers"; +import { cwd } from "./dir"; import { sendSync } from "./dispatch"; -import { TextDecoder } from "./text_encoding"; -import * as ts from "typescript"; +import * as flatbuffers from "./flatbuffers"; import * as os from "./os"; -import { bold, cyan, yellow } from "./colors"; -import { window } from "./window"; -import { postMessage, workerClose, workerMain } from "./workers"; -import { Console } from "./console"; +import { TextDecoder, TextEncoder } from "./text_encoding"; import { assert, notImplemented } from "./util"; import * as util from "./util"; -import { cwd } from "./dir"; -import { assetSourceCode } from "./assets"; +import { window } from "./window"; +import { postMessage, workerClose, workerMain } from "./workers"; +import { writeFileSync } from "./write_file"; // Startup boilerplate. This is necessary because the compiler has its own // snapshot. (It would be great if we could remove these things or centralize @@ -32,6 +34,7 @@ const OUT_DIR = "$deno$"; /** The format of the work message payload coming from the privileged side */ interface CompilerReq { rootNames: string[]; + bundle?: string; // TODO(ry) add compiler config to this interface. // options: ts.CompilerOptions; configPath?: string; @@ -116,6 +119,7 @@ interface EmitResult { diagnostics?: Diagnostic; } +/** Ops to Rust to resolve and fetch a modules meta data. */ function fetchModuleMetaData( specifier: string, referrer: string @@ -151,7 +155,23 @@ function fetchModuleMetaData( }; } -/** For caching source map and compiled js */ +/** Utility function to turn the number of bytes into a human readable + * unit */ +function humanFileSize(bytes: number): string { + const thresh = 1000; + if (Math.abs(bytes) < thresh) { + return bytes + " B"; + } + const units = ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + let u = -1; + do { + bytes /= thresh; + ++u; + } while (Math.abs(bytes) >= thresh && u < units.length - 1); + return `${bytes.toFixed(1)} ${units[u]}`; +} + +/** Ops to rest for caching source map and compiled js */ function cache(extension: string, moduleId: string, contents: string): void { util.log("compiler.cache", moduleId); const builder = flatbuffers.createBuilder(); @@ -168,6 +188,21 @@ function cache(extension: string, moduleId: string, contents: string): void { assert(baseRes == null); } +const encoder = new TextEncoder(); + +/** Given a fileName and the data, emit the file to the file system. */ +function emitBundle(fileName: string, data: string): void { + // For internal purposes, when trying to emit to `$deno$` just no-op + if (fileName.startsWith("$deno$")) { + console.warn("skipping compiler.emitBundle", fileName); + return; + } + const encodedData = encoder.encode(data); + console.log(`Emitting bundle to "${fileName}"`); + writeFileSync(fileName, encodedData); + console.log(`${humanFileSize(encodedData.length)} emitted.`); +} + /** Returns the TypeScript Extension enum for a given media type. */ function getExtension( fileName: string, @@ -200,6 +235,46 @@ class Host implements ts.CompilerHost { target: ts.ScriptTarget.ESNext }; + private _resolveModule(specifier: string, referrer: string): ModuleMetaData { + // Handle built-in assets specially. + if (specifier.startsWith(ASSETS)) { + const moduleName = specifier.split("/").pop()!; + const assetName = moduleName.includes(".") + ? moduleName + : `${moduleName}.d.ts`; + assert(assetName in assetSourceCode, `No such asset "${assetName}"`); + const sourceCode = assetSourceCode[assetName]; + return { + moduleName, + filename: specifier, + mediaType: msg.MediaType.TypeScript, + sourceCode + }; + } + return fetchModuleMetaData(specifier, referrer); + } + + /* Deno specific APIs */ + + /** Provides the `ts.HostCompiler` interface for Deno. + * + * @param _bundle Set to a string value to configure the host to write out a + * bundle instead of caching individual files. + */ + constructor(private _bundle?: string) { + if (this._bundle) { + // options we need to change when we are generating a bundle + const bundlerOptions: ts.CompilerOptions = { + module: ts.ModuleKind.AMD, + inlineSourceMap: true, + outDir: undefined, + outFile: `${OUT_DIR}/bundle.js`, + sourceMap: false + }; + Object.assign(this._options, bundlerOptions); + } + } + /** Take a configuration string, parse it, and use it to merge with the * compiler's configuration options. The method returns an array of compiler * options which were ignored, or `undefined`. @@ -234,17 +309,32 @@ class Host implements ts.CompilerHost { }; } + /* TypeScript CompilerHost APIs */ + + fileExists(_fileName: string): boolean { + return notImplemented(); + } + + getCanonicalFileName(fileName: string): string { + // console.log("getCanonicalFileName", fileName); + return fileName; + } + getCompilationSettings(): ts.CompilerOptions { util.log("getCompilationSettings()"); return this._options; } - fileExists(_fileName: string): boolean { - return notImplemented(); + getCurrentDirectory(): string { + return ""; } - readFile(_fileName: string): string | undefined { - return notImplemented(); + getDefaultLibFileName(_options: ts.CompilerOptions): string { + return ASSETS + "/lib.deno_runtime.d.ts"; + } + + getNewLine(): string { + return "\n"; } getSourceFile( @@ -266,47 +356,8 @@ class Host implements ts.CompilerHost { ); } - getDefaultLibFileName(_options: ts.CompilerOptions): string { - return ASSETS + "/lib.deno_runtime.d.ts"; - } - - writeFile( - fileName: string, - data: string, - writeByteOrderMark: boolean, - onError?: (message: string) => void, - sourceFiles?: ReadonlyArray - ): void { - util.log("writeFile", fileName); - assert(sourceFiles != null && sourceFiles.length == 1); - const sourceFileName = sourceFiles![0].fileName; - - if (fileName.endsWith(".map")) { - // Source Map - cache(".map", sourceFileName, data); - } else if (fileName.endsWith(".js") || fileName.endsWith(".json")) { - // Compiled JavaScript - cache(".js", sourceFileName, data); - } else { - assert(false, "Trying to cache unhandled file type " + fileName); - } - } - - getCurrentDirectory(): string { - return ""; - } - - getCanonicalFileName(fileName: string): string { - // console.log("getCanonicalFileName", fileName); - return fileName; - } - - useCaseSensitiveFileNames(): boolean { - return true; - } - - getNewLine(): string { - return "\n"; + readFile(_fileName: string): string | undefined { + return notImplemented(); } resolveModuleNames( @@ -335,23 +386,42 @@ class Host implements ts.CompilerHost { ); } - private _resolveModule(specifier: string, referrer: string): ModuleMetaData { - // Handle built-in assets specially. - if (specifier.startsWith(ASSETS)) { - const moduleName = specifier.split("/").pop()!; - const assetName = moduleName.includes(".") - ? moduleName - : `${moduleName}.d.ts`; - assert(assetName in assetSourceCode, `No such asset "${assetName}"`); - const sourceCode = assetSourceCode[assetName]; - return { - moduleName, - filename: specifier, - mediaType: msg.MediaType.TypeScript, - sourceCode - }; + useCaseSensitiveFileNames(): boolean { + return true; + } + + writeFile( + fileName: string, + data: string, + writeByteOrderMark: boolean, + onError?: (message: string) => void, + sourceFiles?: ReadonlyArray + ): void { + util.log("writeFile", fileName); + try { + if (this._bundle) { + emitBundle(this._bundle, data); + } else { + assert(sourceFiles != null && sourceFiles.length == 1); + const sourceFileName = sourceFiles![0].fileName; + + if (fileName.endsWith(".map")) { + // Source Map + cache(".map", sourceFileName, data); + } else if (fileName.endsWith(".js") || fileName.endsWith(".json")) { + // Compiled JavaScript + cache(".js", sourceFileName, data); + } else { + assert(false, "Trying to cache unhandled file type " + fileName); + } + } + } catch (e) { + if (onError) { + onError(String(e)); + } else { + throw e; + } } - return fetchModuleMetaData(specifier, referrer); } } @@ -360,12 +430,12 @@ class Host implements ts.CompilerHost { window.compilerMain = function compilerMain(): void { // workerMain should have already been called since a compiler is a worker. window.onmessage = ({ data }: { data: CompilerReq }): void => { + const { rootNames, configPath, config, bundle } = data; + const host = new Host(bundle); + let emitSkipped = true; let diagnostics: ts.Diagnostic[] | undefined; - const { rootNames, configPath, config } = data; - const host = new Host(); - // if there is a configuration supplied, we need to parse that if (config && config.length && configPath) { const configResult = host.configure(configPath, config); @@ -410,6 +480,9 @@ window.compilerMain = function compilerMain(): void { // We will only proceed with the emit if there are no diagnostics. if (diagnostics && diagnostics.length === 0) { + if (bundle) { + console.log(`Bundling "${bundle}"`); + } const emitResult = program.emit(); emitSkipped = emitResult.emitSkipped; // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned