add a new compiler command line switch for generating utf-8 BOM in the output

src/compiler/commandLineParser.ts

@@ -23,6 +23,10 @@ module ts {
             name: "diagnostics",
             type: "boolean",
         },
+        {
+            name: "emitBOM", 
+            type: "boolean"
+        },
         {
             name: "help",
             shortName: "h",

src/compiler/emitter.ts

@@ -157,8 +157,8 @@ module ts {
             return text.substring(skipTrivia(text, node.pos), node.end);
         }
 
-        function writeFile(filename: string, data: string) {
-            compilerHost.writeFile(filename, data, hostErrorMessage => {
+        function writeFile(filename: string, data: string, writeByteOrderMark: boolean) {
+            compilerHost.writeFile(filename, data, writeByteOrderMark, hostErrorMessage => {
                 diagnostics.push(createCompilerDiagnostic(Diagnostics.Could_not_write_file_0_Colon_1, filename, hostErrorMessage));
             });
         }
@@ -426,7 +426,7 @@ module ts {
                     sourceMapNameIndices.pop();
                 };
 
-                function writeJavaScriptAndSourceMapFile(emitOutput: string) {
+                function writeJavaScriptAndSourceMapFile(emitOutput: string, writeByteOrderMark: boolean) {
                     // Write source map file
                     encodeLastRecordedSourceMapSpan();
                     writeFile(sourceMapData.sourceMapFilePath, JSON.stringify({
@@ -436,11 +436,11 @@ module ts {
                         sources: sourceMapData.sourceMapSources,
                         names: sourceMapData.sourceMapNames,
                         mappings: sourceMapData.sourceMapMappings
-                    }));
+                    }), /*writeByteOrderMark*/ false);
                     sourceMapDataList.push(sourceMapData);
 
                     // Write sourcemap url to the js file and write the js file
-                    writeJavaScriptFile(emitOutput + "//# sourceMappingURL=" + sourceMapData.jsSourceMappingURL);
+                    writeJavaScriptFile(emitOutput + "//# sourceMappingURL=" + sourceMapData.jsSourceMappingURL, writeByteOrderMark);
                 }
 
                 // Initialize source map data
@@ -513,8 +513,8 @@ module ts {
                 scopeEmitEnd = recordScopeNameEnd;
             }
 
-            function writeJavaScriptFile(emitOutput: string) {
-                writeFile(jsFilePath, emitOutput);
+            function writeJavaScriptFile(emitOutput: string, writeByteOrderMark: boolean) {
+                writeFile(jsFilePath, emitOutput, writeByteOrderMark);
             }
 
             function emitTokenText(tokenKind: SyntaxKind, startPos: number, emitFn?: () => void) {
@@ -1854,7 +1854,7 @@ module ts {
             }
 
             writeLine();
-            writeEmittedFiles(writer.getText());
+            writeEmittedFiles(writer.getText(), /*writeByteOrderMark*/ compilerOptions.emitBOM);
         }
 
         function emitDeclarations(jsFilePath: string, root?: SourceFile) {
@@ -2448,7 +2448,7 @@ module ts {
             // TODO(shkamat): Should we not write any declaration file if any of them can produce error, 
             // or should we just not write this file like we are doing now
             if (!reportedDeclarationError) {
-                writeFile(getModuleNameFromFilename(jsFilePath) + ".d.ts", referencePathsOutput + writer.getText());
+                writeFile(getModuleNameFromFilename(jsFilePath) + ".d.ts", referencePathsOutput + writer.getText(), compilerOptions.emitBOM);
             }
         }
 

src/compiler/parser.ts

@@ -417,7 +417,7 @@ module ts {
         nodeIsNestedInLabel(label: Identifier, requireIterationStatement: boolean, stopAtFunctionBoundary: boolean): ControlBlockContext;
     }
 
-    export function createSourceFile(filename: string, sourceText: string, languageVersion: ScriptTarget, byteOrderMark: ByteOrderMark, version: number = 0, isOpen: boolean = false): SourceFile {
+    export function createSourceFile(filename: string, sourceText: string, languageVersion: ScriptTarget, version: number = 0, isOpen: boolean = false): SourceFile {
         var file: SourceFile;
         var scanner: Scanner;
         var token: SyntaxKind;
@@ -3535,7 +3535,6 @@ module ts {
         file.nodeCount = nodeCount;
         file.identifierCount = identifierCount;
         file.version = version;
-        file.byteOrderMark = byteOrderMark;
         file.isOpen = isOpen;
         file.languageVersion = languageVersion;
         return file;

src/compiler/sys.ts

@@ -6,7 +6,7 @@ interface System {
     useCaseSensitiveFileNames: boolean;
     write(s: string): void;
     readFile(fileName: string, encoding?: string): string;
-    writeFile(fileName: string, data: string): void;
+    writeFile(fileName: string, data: string, writeByteOrderMark?: boolean): void;
     watchFile?(fileName: string, callback: (fileName: string) => void): FileWatcher;
     resolvePath(path: string): string;
     fileExists(path: string): boolean;
@@ -75,15 +75,21 @@ var sys: System = (function () {
             }
         }
 
-        function writeFile(fileName: string, data: string): void {
+        function writeFile(fileName: string, data: string, writeByteOrderMark?: boolean): void {
             fileStream.Open();
             binaryStream.Open();
             try {
                 // Write characters in UTF-8 encoding
                 fileStream.Charset = "utf-8";
                 fileStream.WriteText(data);
-                // Skip byte order mark and copy remaining data to binary stream
-                fileStream.Position = 3;
+                // If we don't want the BOM, then skip it by setting the starting location to 3 (size of BOM).
+                // If not, start from position 0, as the BOM will be added automatically when charset==utf8.
+                if (writeByteOrderMark) {
+                    fileStream.Position = 0;
+                }
+                else {
+                    fileStream.Position = 3;
+                }
                 fileStream.CopyTo(binaryStream);
                 binaryStream.SaveToFile(fileName, 2 /*overwrite*/);
             }
@@ -175,7 +181,12 @@ var sys: System = (function () {
             return buffer.toString("utf8");
         }
 
-        function writeFile(fileName: string, data: string): void {
+        function writeFile(fileName: string, data: string, writeByteOrderMark?: boolean): void {
+            // If a BOM is required, emit one
+            if (writeByteOrderMark) {
+                data = '\uFEFF' + data;
+            }
+
             _fs.writeFileSync(fileName, data, "utf8");
         }
 

src/compiler/tc.ts

@@ -142,10 +142,10 @@ module ts {
                 }
                 text = "";
             }
-            return text !== undefined ? createSourceFile(filename, text, languageVersion, ByteOrderMark.None) : undefined;
+            return text !== undefined ? createSourceFile(filename, text, languageVersion) : undefined;
         }
 
-        function writeFile(fileName: string, data: string, onError?: (message: string) => void) {
+        function writeFile(fileName: string, data: string, writeByteOrderMark: boolean, onError?: (message: string) => void) {
 
             function directoryExists(directoryPath: string): boolean {
                 if (hasProperty(existingDirectories, directoryPath)) {
@@ -168,7 +168,7 @@ module ts {
 
             try {
                 ensureDirectoriesExist(getDirectoryPath(normalizePath(fileName)));
-                sys.writeFile(fileName, data);
+                sys.writeFile(fileName, data, writeByteOrderMark);
             }
             catch (e) {
                 if (onError) onError(e.message);

src/compiler/types.ts

@@ -525,7 +525,6 @@ module ts {
         nodeCount: number;
         identifierCount: number;
         symbolCount: number;
-        byteOrderMark: ByteOrderMark;
         isOpen: boolean;
         version: number;
         languageVersion: ScriptTarget;
@@ -940,6 +939,7 @@ module ts {
         codepage?: number;
         declaration?: boolean;
         diagnostics?: boolean;
+        emitBOM?: boolean;
         help?: boolean;
         locale?: string;
         mapRoot?: string;
@@ -1131,17 +1131,10 @@ module ts {
         getSourceFile(filename: string, languageVersion: ScriptTarget, onError?: (message: string) => void): SourceFile;
         getDefaultLibFilename(): string;
         getCancellationToken? (): CancellationToken;
-        writeFile(filename: string, data: string, onError?: (message: string) => void): void;
+        writeFile(filename: string, data: string, writeByteOrderMark: boolean, onError?: (message: string) => void): void;
         getCurrentDirectory(): string;
         getCanonicalFileName(fileName: string): string;
         useCaseSensitiveFileNames(): boolean;
         getNewLine(): string;
     }
-
-    export enum ByteOrderMark {
-        None = 0,
-        Utf8 = 1,
-        Utf16BigEndian = 2,
-        Utf16LittleEndian = 3,
-    }
 }

src/harness/compilerRunner.ts

@@ -118,6 +118,10 @@ class CompilerBaselineRunner extends RunnerBase {
                 }
             });
 
+            function getByteOrderMarkText(file: Harness.Compiler.GeneratedFile): string {
+                return file.writeByteOrderMark ? "\u00EF\u00BB\u00BF" : "";
+            }
+
             function getErrorBaseline(toBeCompiled: { unitName: string; content: string }[],
                 otherFiles: { unitName: string; content: string }[],
                 result: Harness.Compiler.CompilerResult
@@ -282,6 +286,7 @@ class CompilerBaselineRunner extends RunnerBase {
                         var jsCode = '';
                         for (var i = 0; i < result.files.length; i++) {
                             jsCode += '//// [' + Harness.Path.getFileName(result.files[i].fileName) + ']\r\n';
+                            jsCode += getByteOrderMarkText(result.files[i]);
                             jsCode += result.files[i].code;
                             // Re-enable this if we want to do another comparison of old vs new compiler baselines
                             // jsCode += SyntacticCleaner.clean(result.files[i].code);
@@ -291,6 +296,7 @@ class CompilerBaselineRunner extends RunnerBase {
                             jsCode += '\r\n\r\n';
                             for (var i = 0; i < result.files.length; i++) {
                                 jsCode += '//// [' + Harness.Path.getFileName(result.declFilesCode[i].fileName) + ']\r\n';
+                                jsCode += getByteOrderMarkText(result.declFilesCode[i]);
                                 jsCode += result.declFilesCode[i].code;
                             }
                         }
@@ -320,6 +326,7 @@ class CompilerBaselineRunner extends RunnerBase {
                         var sourceMapCode = '';
                         for (var i = 0; i < result.sourceMaps.length; i++) {
                             sourceMapCode += '//// [' + Harness.Path.getFileName(result.sourceMaps[i].fileName) + ']\r\n';
+                            sourceMapCode += getByteOrderMarkText(result.sourceMaps[i]);
                             sourceMapCode += result.sourceMaps[i].code;
                         }
 

src/harness/fourslash.ts

@@ -1895,12 +1895,12 @@ module FourSlash {
         var result = '';
         var fourslashFilename = 'fourslash.ts';
         var tsFn = 'tests/cases/fourslash/' + fourslashFilename;
-        fourslashSourceFile = fourslashSourceFile || ts.createSourceFile(tsFn, Harness.IO.readFile(tsFn), ts.ScriptTarget.ES5, ts.ByteOrderMark.None, /*version*/ 0, /*isOpen*/ false);
-        libdtsSourceFile = libdtsSourceFile || ts.createSourceFile('lib.d.ts', Harness.Compiler.libTextMinimal, ts.ScriptTarget.ES3, ts.ByteOrderMark.None, /*version*/ 0, /*isOpen*/ false);
+        fourslashSourceFile = fourslashSourceFile || ts.createSourceFile(tsFn, Harness.IO.readFile(tsFn), ts.ScriptTarget.ES5, /*version*/ 0, /*isOpen*/ false);
+        libdtsSourceFile = libdtsSourceFile || ts.createSourceFile('lib.d.ts', Harness.Compiler.libTextMinimal, ts.ScriptTarget.ES3, /*version*/ 0, /*isOpen*/ false);
 
         var files: { [filename: string]: ts.SourceFile; } = {};
         files[fourslashFilename] = fourslashSourceFile;
-        files[fileName] = ts.createSourceFile(fileName, content, ts.ScriptTarget.ES5, ts.ByteOrderMark.None, /*version*/ 0, /*isOpen*/ false);
+        files[fileName] = ts.createSourceFile(fileName, content, ts.ScriptTarget.ES5, /*version*/ 0, /*isOpen*/ false);
         files['lib.d.ts'] = libdtsSourceFile;
 
         var host = Harness.Compiler.createCompilerHost(files, (fn, contents) => result = contents);

src/harness/harness.ts

@@ -534,7 +534,7 @@ module Harness {
         export var libText = IO.readFile(libFolder + "lib.d.ts");
         export var libTextMinimal = IO.readFile('bin/lib.core.d.ts');
 
-        export function createCompilerHost(filemap: { [filename: string]: ts.SourceFile; }, writeFile: (fn: string, contents: string) => void): ts.CompilerHost {
+        export function createCompilerHost(filemap: { [filename: string]: ts.SourceFile; }, writeFile: (fn: string, contents: string, writeByteOrderMark:boolean) => void): ts.CompilerHost {
             return {
                 getCurrentDirectory: sys.getCurrentDirectory,
                 getCancellationToken: (): any => undefined,
@@ -544,7 +544,7 @@ module Harness {
                     } else {
                         var lib = 'lib.d.ts';
                         if (fn.substr(fn.length - lib.length) === lib) {
-                            return filemap[fn] = ts.createSourceFile('lib.d.ts', libTextMinimal, languageVersion, ts.ByteOrderMark.None);
+                            return filemap[fn] = ts.createSourceFile('lib.d.ts', libTextMinimal, languageVersion);
                         }
                         // Don't throw here -- the compiler might be looking for a test that actually doesn't exist as part of the TC
                         return null;
@@ -712,6 +712,10 @@ module Harness {
                             // Not supported yet
                             break;
 
+                        case 'emitbom':
+                            options.emitBOM = !!setting.value;
+                            break;
+
                         default:
                             throw new Error('Unsupported compiler setting ' + setting.flag);
                     }
@@ -720,18 +724,15 @@ module Harness {
                 var filemap: { [name: string]: ts.SourceFile; } = {};
                 var register = (file: { unitName: string; content: string; }) => {
                     var filename = Path.switchToForwardSlashes(file.unitName);
-                    filemap[filename] = ts.createSourceFile(filename, file.content, options.target, ts.ByteOrderMark.None);
+                    filemap[filename] = ts.createSourceFile(filename, file.content, options.target);
                 };
                 inputFiles.forEach(register);
                 otherFiles.forEach(register);
 
-                var fileOutputs: {
-                    fileName: string;
-                    file: string;
-                }[] = [];
+                var fileOutputs: GeneratedFile[] = [];
 
                 var programFiles = inputFiles.map(file => file.unitName);
-                var program = ts.createProgram(programFiles, options, createCompilerHost(filemap, (fn, contents) => fileOutputs.push({ fileName: fn, file: contents })));
+                var program = ts.createProgram(programFiles, options, createCompilerHost(filemap, (fn, contents, writeByteOrderMark) => fileOutputs.push({ fileName: fn, code: contents, writeByteOrderMark: writeByteOrderMark })));
 
                 var hadParseErrors = program.getDiagnostics().length > 0;
 
@@ -810,6 +811,7 @@ module Harness {
         export interface GeneratedFile {
             fileName: string;
             code: string;
+            writeByteOrderMark: boolean;
         }
 
         function stringEndsWith(str: string, end: string) {
@@ -837,19 +839,18 @@ module Harness {
             public sourceMapRecord: string;
 
             /** @param fileResults an array of strings for the fileName and an ITextWriter with its code */
-            constructor(fileResults: { fileName: string; file: string; }[], errors: MinimalDiagnostic[], sourceMapRecordLines: string[]) {
+            constructor(fileResults: GeneratedFile[], errors: MinimalDiagnostic[], sourceMapRecordLines: string[]) {
                 var lines: string[] = [];
 
                 fileResults.forEach(emittedFile => {
-                    var fileObj = { fileName: emittedFile.fileName, code: emittedFile.file };
                     if (isDTS(emittedFile.fileName)) {
                         // .d.ts file, add to declFiles emit
-                        this.declFilesCode.push(fileObj);
+                        this.declFilesCode.push(emittedFile);
                     } else if (isJS(emittedFile.fileName)) {
                         // .js file, add to files
-                        this.files.push(fileObj);
+                        this.files.push(emittedFile);
                     } else if (isJSMap(emittedFile.fileName)) {
-                        this.sourceMaps.push(fileObj);
+                        this.sourceMaps.push(emittedFile);
                     } else {
                         throw new Error('Unrecognized file extension for file ' + emittedFile.fileName);
                     }
@@ -896,7 +897,7 @@ module Harness {
         var optionRegex = /^[\/]{2}\s*@(\w+)\s*:\s*(\S*)/gm;  // multiple matches on multiple lines
 
         // List of allowed metadata names
-        var fileMetadataNames = ["filename", "comments", "declaration", "module", "nolib", "sourcemap", "target", "out", "outDir", "noimplicitany", "noresolve", "newline", "newlines"];
+        var fileMetadataNames = ["filename", "comments", "declaration", "module", "nolib", "sourcemap", "target", "out", "outDir", "noimplicitany", "noresolve", "newline", "newlines", "emitbom"];
 
         function extractCompilerSettings(content: string): CompilerSetting[] {