Add TextEncoder/TextDecoder

Fixes #470

This commit increases exe size
out/release/deno  49M -> 53M

js/assets.ts

@@ -48,6 +48,7 @@ import libGlobalsDts from "/js/lib.globals.d.ts!string";
 // Static definitions
 import typescriptDts from "/third_party/node_modules/typescript/lib/typescript.d.ts!string";
 import typesDts from "/js/types.d.ts!string";
+import textEncodingDts from "/js/text_encoding.d.ts!string";
 // tslint:enable:max-line-length
 
 // prettier-ignore
@@ -92,6 +93,7 @@ export const assetSourceCode: { [key: string]: string } = {
   "lib.globals.d.ts": libGlobalsDts,
 
   // Static definitions
+  "text_encoding.d.ts": textEncodingDts,
   "typescript.d.ts": typescriptDts,
   "types.d.ts": typesDts,
 

js/globals.ts

@@ -1,8 +1,10 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
-
+// tslint:disable-next-line:no-reference
+/// <reference path="./text_encoding.d.ts" />
 import { Console } from "./console";
 import { RawSourceMap } from "./types";
 import * as timers from "./timers";
+import { TextEncoder, TextDecoder } from "text-encoding";
 
 declare global {
   interface Window {
@@ -16,6 +18,11 @@ declare global {
 
   const console: Console;
   const window: Window;
+
+  // tslint:disable:variable-name
+  let TextEncoder: TextEncoder;
+  let TextDecoder: TextDecoder;
+  // tslint:enable:variable-name
 }
 
 // If you use the eval function indirectly, by invoking it via a reference
@@ -49,10 +56,8 @@ window["clearTimeout"] = timers.clearTimer;
 window["clearInterval"] = timers.clearTimer;
 
 window.console = new Console(libdeno.print);
+window.TextEncoder = TextEncoder;
+window.TextDecoder = TextDecoder;
 
 // import { fetch } from "./fetch";
 // window["fetch"] = fetch;
-
-// import { TextEncoder, TextDecoder } from "text-encoding";
-// window["TextEncoder"] = TextEncoder;
-// window["TextDecoder"] = TextDecoder;

js/os.ts

@@ -85,15 +85,34 @@ export function codeCache(
 }
 
 export function readFileSync(filename: string): Uint8Array {
-  assert(false, "Not Implemented");
-  return null;
-  /*
-  const res = pubInternal("os", {
+  /* Ideally we could write
+  const res = send({
     command: fbs.Command.READ_FILE_SYNC,
     readFileSyncFilename: filename
   });
   return res.readFileSyncData;
 	*/
+  const builder = new flatbuffers.Builder();
+  const filename_ = builder.createString(filename);
+  fbs.ReadFileSync.startReadFileSync(builder);
+  fbs.ReadFileSync.addFilename(builder, filename_);
+  const msg = fbs.ReadFileSync.endReadFileSync(builder);
+  fbs.Base.startBase(builder);
+  fbs.Base.addMsg(builder, msg);
+  fbs.Base.addMsgType(builder, fbs.Any.ReadFileSync);
+  builder.finish(fbs.Base.endBase(builder));
+  const resBuf = libdeno.send(builder.asUint8Array());
+  assert(resBuf != null);
+
+  const bb = new flatbuffers.ByteBuffer(new Uint8Array(resBuf));
+  const baseRes = fbs.Base.getRootAsBase(bb);
+  if (fbs.Any.NONE === baseRes.msgType()) {
+    throw Error(baseRes.error());
+  }
+  assert(fbs.Any.ReadFileSyncRes === baseRes.msgType());
+  const res = new fbs.ReadFileSyncRes();
+  assert(baseRes.msg(res) != null);
+  return new Uint8Array(res.dataArray());
 }
 
 export function writeFileSync(

js/text_encoding.d.ts

@@ -0,0 +1,78 @@
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0
+
+THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+MERCHANTABLITY OR NON-INFRINGEMENT.
+
+See the Apache Version 2.0 License for specific language governing permissions
+and limitations under the License.
+***************************************************************************** */
+
+type BufferSource = ArrayBufferView | ArrayBuffer;
+
+interface TextDecodeOptions {
+  stream?: boolean;
+}
+
+interface TextDecoderOptions {
+  fatal?: boolean;
+  ignoreBOM?: boolean;
+}
+
+interface TextDecoder {
+  /**
+   * Returns encoding's name, lowercased.
+   */
+  readonly encoding: string;
+  /**
+   * Returns true if error mode is "fatal", and false
+   * otherwise.
+   */
+  readonly fatal: boolean;
+  /**
+   * Returns true if ignore BOM flag is set, and false otherwise.
+   */
+  readonly ignoreBOM: boolean;
+  /**
+   * Returns the result of running encoding's decoder. The
+   * method can be invoked zero or more times with options's stream set to
+   * true, and then once without options's stream (or set to false), to process
+   * a fragmented stream. If the invocation without options's stream (or set to
+   * false) has no input, it's clearest to omit both arguments.
+   * var string = "", decoder = new TextDecoder(encoding), buffer;
+   * while(buffer = next_chunk()) {
+   * string += decoder.decode(buffer, {stream:true});
+   * }
+   * string += decoder.decode(); // end-of-stream
+   * If the error mode is "fatal" and encoding's decoder returns error, throws a TypeError.
+   */
+  decode(input?: BufferSource, options?: TextDecodeOptions): string;
+}
+
+interface TextEncoder {
+  /**
+   * Returns "utf-8".
+   */
+  readonly encoding: string;
+  /**
+   * Returns the result of running UTF-8's encoder.
+   */
+  encode(input?: string): Uint8Array;
+}
+
+declare module "text-encoding" {
+  export let TextDecoder: {
+    prototype: TextDecoder;
+    new (label?: string, options?: TextDecoderOptions): TextDecoder;
+  };
+
+  export let TextEncoder: {
+    prototype: TextEncoder;
+    new (): TextEncoder;
+  };
+}

package.json

@@ -16,6 +16,7 @@
     "rollup-plugin-typescript2": "^0.16.1",
     "rollup-pluginutils": "^2.3.0",
     "source-map-support": "^0.5.6",
+    "text-encoding": "^0.6.4",
     "tslint": "^5.10.0",
     "tslint-eslint-rules": "^5.3.1",
     "tslint-no-circular-imports": "^0.5.0",

src/handlers.h

@@ -10,9 +10,9 @@ void handle_code_fetch(Deno* d, uint32_t cmd_id, const char* module_specifier,
                        const char* containing_file);
 void handle_code_cache(Deno* d, uint32_t cmd_id, const char* filename,
                        const char* source_code, const char* output_code);
-
 void handle_timer_start(Deno* d, uint32_t cmd_id, uint32_t timer_id,
                         bool interval, uint32_t delay);
 void handle_timer_clear(Deno* d, uint32_t cmd_id, uint32_t timer_id);
+void handle_read_file_sync(Deno* d, uint32_t cmd_id, const char* filename);
 }  // extern "C"
 #endif  // HANDLERS_H_

src/handlers.rs

@@ -2,12 +2,17 @@
 use binding;
 use binding::{deno_buf, deno_set_response, DenoC};
 use flatbuffers;
+use flatbuffers::ByteStringOffset;
+use flatbuffers::LabeledUOffsetT;
 use from_c;
+use fs;
 use futures;
 use futures::sync::oneshot;
 use libc::c_char;
+use mem;
 use msg_generated::deno as msg;
 use std::ffi::CStr;
+use std::path::Path;
 
 // Help. Is there a way to do this without macros?
 // Want: fn str_from_ptr(*const c_char) -> &str
@@ -226,6 +231,60 @@ fn send_timer_ready(d: *const DenoC, timer_id: u32, done: bool) {
   );
 }
 
+// Prototype https://github.com/denoland/deno/blob/golang/os.go#L171-L184
+#[no_mangle]
+pub extern "C" fn handle_read_file_sync(
+  d: *const DenoC,
+  cmd_id: u32,
+  filename: *const c_char,
+) {
+  // let deno = from_c(d);
+  let filename = str_from_ptr!(filename);
+
+  debug!("handle_read_file_sync {}", filename);
+  let result = fs::read_file_sync(Path::new(filename));
+  if result.is_err() {
+    let err = result.unwrap_err();
+    let errmsg = format!("{}", err);
+    reply_error(d, cmd_id, &errmsg);
+    return;
+  }
+
+  // Build the response message. memcpy data into msg.
+  let mut builder = flatbuffers::FlatBufferBuilder::new();
+
+  let vec = result.unwrap();
+  //let data =
+  //  flatbuffers::LabeledUOffsetT::new(builder.push_bytes(vec.as_slice()));
+
+  let data_ = builder.create_byte_vector(vec.as_slice());
+
+  // TODO(ry) This is a hack that can be removed once builder.create_byte_vector works properly.
+  let data = unsafe {
+    mem::transmute::<LabeledUOffsetT<ByteStringOffset>, LabeledUOffsetT<&[i8]>>(
+      data_,
+    )
+  };
+
+  let msg = msg::CreateReadFileSyncRes(
+    &mut builder,
+    &msg::ReadFileSyncResArgs {
+      data,
+      ..Default::default()
+    },
+  );
+  builder.finish(msg);
+  set_response_base(
+    d,
+    &mut builder,
+    &msg::BaseArgs {
+      msg: Some(msg.union()),
+      msg_type: msg::Any::ReadFileSyncRes,
+      ..Default::default()
+    },
+  );
+}
+
 // TODO(ry) Use Deno instead of DenoC as first arg.
 fn remove_timer(d: *const DenoC, timer_id: u32) {
   let deno = from_c(d);

src/reply.cc

@@ -95,6 +95,13 @@ void deno_handle_msg_from_js(Deno* d, deno_buf buf) {
       break;
     }
 
+    case deno::Any_ReadFileSync: {
+      auto msg = base->msg_as_ReadFileSync();
+      auto filename = msg->filename()->c_str();
+      handle_read_file_sync(d, cmd_id, filename);
+      break;
+    }
+
     case deno::Any_NONE:
       CHECK(false && "Got message with msg_type == Any_NONE");
       break;

tests/read_file_sync.ts

@@ -0,0 +1,18 @@
+// TODO(ry) Once unit_tests.js lands (#448) this file should be removed
+// and replaced with a faster version like was done in the prototype.
+// https://github.com/denoland/deno/blob/golang/tests.ts#L34-L45
+import * as deno from "deno";
+
+const data = deno.readFileSync("package.json");
+if (!data.byteLength) {
+  throw Error(
+    `Expected positive value for data.byteLength ${data.byteLength}`
+  );
+}
+const decoder = new TextDecoder("utf-8");
+const json = decoder.decode(data);
+const pkg = JSON.parse(json);
+if (pkg['devDependencies'] == null) {
+  throw Error("Expected a positive number of devDependencies");
+}
+console.log("ok");

tests/read_file_sync.ts.out

@@ -0,0 +1 @@
+ok