Add tests for sourcemaps from multiple files

src/files/BrsFile.spec.ts

@@ -9,7 +9,6 @@ import { IntegerType } from '../types/IntegerType';
 import { StringType } from '../types/StringType';
 import { BrsFile } from './BrsFile';
 import { SourceMapConsumer } from 'source-map';
-import { Lexer } from '../lexer/Lexer';
 import { TokenKind } from '../lexer/TokenKind';
 import { DiagnosticMessages } from '../DiagnosticMessages';
 import util, { standardizePath as s } from '../util';
@@ -2550,37 +2549,110 @@ describe('BrsFile', () => {
             expect(location.column).eql(4);
         });
 
-        it('computes correct locations for sourcemap', async () => {
-            let source = `function abc(name)\n    firstName = name\nend function`;
-            let tokens = Lexer.scan(source).tokens
+        describe('sourcemap validation', () => {
+            it('computes correct source and position in sourcemap', async () => {
+                program.options.sourceMap = true;
+
+                const file = program.setFile<BrsFile>('source/main.bs', `function abc(name)\n    firstName = name\nend function`);
+                let i = 0;
                 //remove newlines and EOF
-                .filter(x => x.kind !== TokenKind.Eof && x.kind !== TokenKind.Newline);
+                //set each token to a different file
+                for (const token of file.parser.tokens) {
+                    token.location.uri = util.pathToUri(s`${rootDir}/source/file${i++}.bs`);
+                }
 
-            program.options.sourceMap = true;
-            let result = await testTranspile(source, source, 'none');
-            //load the source map
-            await SourceMapConsumer.with(result.map.toString(), null, (consumer) => {
-                let tokenResult = tokens.map(token => ({
-                    kind: token.kind,
-                    start: token.location?.range.start
-                }));
-                let sourcemapResult = tokens.map(token => {
-                    let originalPosition = consumer.originalPositionFor({
-                        //convert token 0-based line to source-map 1-based line for the lookup
-                        line: token.location?.range.start.line + 1,
-                        column: token.location?.range.start.character
+                const result = await program.getTranspiledFileContents(file.srcPath);
+                const tokens = file.parser.tokens.filter(x => x.kind !== TokenKind.Eof && x.kind !== TokenKind.Newline);
+
+                //load the source map
+                await SourceMapConsumer.with(result.map.toString(), null, (consumer) => {
+                    let sourcemapResult = tokens.map(token => {
+                        let originalPosition = consumer.originalPositionFor({
+                            //convert token 0-based line to source-map 1-based line for the lookup
+                            line: token.location?.range.start.line + 1,
+                            column: token.location?.range.start.character
+                        });
+                        return {
+                            kind: token.kind,
+                            start: Position.create(
+                                //convert source-map 1-based line to token 0-based line
+                                originalPosition.line! - 1,
+                                originalPosition.column!
+                            ),
+                            source: originalPosition.source
+                        };
                     });
-                    return {
-                        kind: token.kind,
-                        start: Position.create(
-                            //convert source-map 1-based line to token 0-based line
-                            originalPosition.line! - 1,
-                            originalPosition.column!
-                        )
-                    };
+                    expect(sourcemapResult).to.eql(
+                        tokens.map(token => ({
+                            kind: token.kind,
+                            start: token.location?.range.start,
+                            source: util.uriToPath(token.location.uri)
+                        }))
+                    );
                 });
-                expect(sourcemapResult).to.eql(tokenResult);
             });
+
+            it('supports merging AST from one file into another', async () => {
+                program.options.sourceMap = true;
+
+                const alpha = program.setFile<BrsFile>('source/alpha.bs', `
+                    function alpha()
+                        print "alpha"
+                    end function
+                `);
+                const beta = program.setFile<BrsFile>('source/beta.bs', `
+                    function beta()
+                        print "beta"
+                    end function
+                `);
+                //merge alpha into beta
+                beta.ast.statements.push(alpha.ast.statements[0]);
+
+                const result = await program.getTranspiledFileContents(beta.srcPath);
+                expect(result.code).to.eql(undent`
+                    function beta()
+                        print "beta"
+                    end function
+
+                    function alpha()
+                        print "alpha"
+                    end function
+                    '//# sourceMappingURL=./beta.brs.map
+                `);
+
+                //load the source map
+                await SourceMapConsumer.with(result.map.toString(), null, (consumer) => {
+
+                    //prin|t "beta"
+                    doTest(1, 8, s`${rootDir}/source/beta.bs`, 2, 24);
+                    //prin|t "alpha"
+                    doTest(5, 8, s`${rootDir}/source/alpha.bs`, 2, 24);
+
+                    function doTest(destLine: number, destChar: number, srcPath: string, sourceLine: number, sourceChar: number) {
+                        let originalPosition = consumer.originalPositionFor({
+                            //convert token 0-based line to source-map 1-based line for the lookup
+                            line: destLine + 1,
+                            column: destChar
+                        });
+                        expect({
+                            start: Position.create(
+                                //convert source-map 1-based line to token 0-based line
+                                originalPosition.line! - 1,
+                                originalPosition.column!
+                            ),
+                            source: originalPosition.source
+                        }).to.eql({
+                            start: Position.create(
+                                //convert source-map 1-based line to token 0-based line
+                                sourceLine,
+                                sourceChar
+                            ),
+                            source: srcPath
+                        });
+                    }
+                });
+            });
+
         });
 
         it('handles empty if block', async () => {

src/files/BrsFile.ts

@@ -360,7 +360,8 @@ export class BrsFile implements BscFile {
             //tokenize the input file
             let lexer = this.program.logger.time('debug', ['lexer.lex', chalk.green(this.srcPath)], () => {
                 return Lexer.scan(fileContents, {
-                    includeWhitespace: false
+                    includeWhitespace: false,
+                    srcPath: this.srcPath
                 });
             });
 

src/lexer/Lexer.ts

@@ -67,6 +67,11 @@ export class Lexer {
      */
     private leadingTrivia: Token[] = [];
 
+    /**
+     * URI of the file being scanned (if available)
+     */
+    private uri?: string;
+
     /**
      * A convenience function, equivalent to `new Lexer().scan(toScan)`, that converts a string
      * containing BrightScript code to an array of `Token` objects that will later be used to build
@@ -99,6 +104,7 @@ export class Lexer {
         this.columnEnd = 0;
         this.tokens = [];
         this.diagnostics = [];
+        this.uri = util.pathToUri(options?.srcPath);
         while (!this.isAtEnd()) {
             this.scanToken();
         }
@@ -108,7 +114,7 @@ export class Lexer {
             isReserved: false,
             text: '',
             location: this.options.trackLocations
-                ? util.createLocation(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1, this.options.srcPath)
+                ? util.createLocation(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1, this.uri)
                 : undefined,
             leadingWhitespace: this.leadingWhitespace,
             leadingTrivia: this.leadingTrivia
@@ -1109,7 +1115,7 @@ export class Lexer {
      */
     private locationOf(): Location {
         if (this.options.trackLocations) {
-            return util.createLocation(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd, this.options?.srcPath);
+            return util.createLocation(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd, this.uri);
         } else {
             return undefined;
         }

src/parser/TranspileState.ts

@@ -65,6 +65,21 @@ export class TranspileState {
 
     public newline = '\n';
 
+    private getSource(locatable: RangeLike) {
+        let srcPath = (locatable as { location: Location })?.location?.uri ?? (locatable as Location).uri;
+        if (srcPath) {
+            //if a sourceRoot is specified, use that instead of the rootDir
+            if (this.options.sourceRoot) {
+                srcPath = srcPath.replace(
+                    this.options.rootDir,
+                    this.options.sourceRoot
+                );
+            }
+        } else {
+            return this.srcPath;
+        }
+    }
+
     /**
      * Shorthand for creating a new source node
      */
@@ -75,7 +90,7 @@ export class TranspileState {
             range ? range.start.line + 1 : null,
             //range and SourceNode character are both 0-based, so no conversion necessary
             range ? range.start.character : null,
-            this.srcPath,
+            this.getSource(locatable),
             code
         );
     }
@@ -91,7 +106,7 @@ export class TranspileState {
             token.location?.range ? token.location.range.start.line + 1 : null,
             //range and SourceNode character are both 0-based, so no conversion necessary
             token.location?.range ? token.location.range.start.character : null,
-            this.srcPath,
+            this.getSource(token),
             token.text
         );
     }
@@ -131,7 +146,6 @@ export class TranspileState {
         return leadingCommentsSourceNodes;
     }
 
-
     /**
      * Create a SourceNode from a token, accounting for missing range and multi-line text
      * Adds all leading trivia for the token
@@ -162,7 +176,7 @@ export class TranspileState {
                         token.location.range.start.line + i + 1,
                         //SourceNode column is 0-based, and this starts at the beginning of the line
                         0,
-                        this.srcPath,
+                        this.getSource(token),
                         lines[i]
                     )
                 );

src/util.ts

@@ -744,14 +744,37 @@ export class Util {
         return subject;
     }
 
+    /**
+     * Does the string appear to be a uri (i.e. does it start with `file:`)
+     */
+    private isUriLike(filePath: string) {
+        return filePath?.indexOf('file:') === 0;// eslint-disable-line @typescript-eslint/prefer-string-starts-ends-with
+    }
+
+    /**
+     * Given a file path, convert it to a URI string
+     */
+    public pathToUri(filePath: string) {
+        if (!filePath) {
+            return filePath;
+        } else if (this.isUriLike(filePath)) {
+            return filePath;
+        } else {
+            return URI.file(filePath).toString();
+        }
+    }
 
     /**
      * Given a URI, convert that to a regular fs path
      */
     public uriToPath(uri: string) {
+        //if this doesn't look like a URI, then assume it's already a path
+        if (this.isUriLike(uri) === false) {
+            return uri;
+        }
         let parsedPath = URI.parse(uri).fsPath;
 
-        //Uri annoyingly coverts all drive letters to lower case...so this will bring back whatever case it came in as
+        //Uri annoyingly converts all drive letters to lower case...so this will bring back whatever case it came in as
         let match = /\/\/\/([a-z]:)/i.exec(uri);
         if (match) {
             let originalDriveCasing = match[1];
@@ -808,13 +831,6 @@ export class Util {
         return true;
     }
 
-    /**
-     * Given a file path, convert it to a URI string
-     */
-    public pathToUri(filePath: string) {
-        return URI.file(filePath).toString();
-    }
-
     /**
      * Get the outDir from options, taking into account cwd and absolute outFile paths
      */
@@ -1084,7 +1100,7 @@ export class Util {
      */
     public createLocationFromRange(uri: string, range: Range): Location {
         return {
-            uri: uri,
+            uri: util.pathToUri(uri),
             range: range
         };
     }
@@ -1094,7 +1110,7 @@ export class Util {
      */
     public createLocation(startLine: number, startCharacter: number, endLine: number, endCharacter: number, uri?: string): Location {
         return {
-            uri: uri,
+            uri: util.pathToUri(uri),
             range: {
                 start: {
                     line: startLine,