diff --git a/src/bscPlugin/semanticTokens/BrsFileSemanticTokensProcessor.spec.ts b/src/bscPlugin/semanticTokens/BrsFileSemanticTokensProcessor.spec.ts index 8b7b8b096..a2ee183bc 100644 --- a/src/bscPlugin/semanticTokens/BrsFileSemanticTokensProcessor.spec.ts +++ b/src/bscPlugin/semanticTokens/BrsFileSemanticTokensProcessor.spec.ts @@ -19,7 +19,23 @@ describe('BrsFileSemanticTokensProcessor', () => { program.dispose(); }); - function expectSemanticTokens(file: BscFile, tokens: SemanticToken[], validateDiagnostics = true) { + /** + * Ensure the specified tokens are present in the full list + */ + function expectSemanticTokensIncludes(file: BscFile, expected: Array, validateDiagnostics = true) { + const result = getSemanticTokenResults(file, expected, validateDiagnostics); + expect(result.actual).to.include.members(result.expected); + } + + /** + * Ensure that the full list of tokens exactly equals the expected list + */ + function expectSemanticTokens(file: BscFile, expected: Array, validateDiagnostics = true) { + const result = getSemanticTokenResults(file, expected, validateDiagnostics); + expect(result.actual).to.eql(result.expected); + } + + function getSemanticTokenResults(file: BscFile, expected: Array, validateDiagnostics = true) { program.validate(); if (validateDiagnostics) { expectZeroDiagnostics(program); @@ -29,21 +45,30 @@ describe('BrsFileSemanticTokensProcessor', () => { ); //sort modifiers - for (const collection of [result, tokens]) { - for (const token of collection) { + for (let collection of [result, expected]) { + for (let i = 0; i < collection.length; i++) { + if (Array.isArray(collection[i])) { + const parts = collection[i]; + collection[i] = { + tokenType: parts[0], + range: util.createRange(parts[1], parts[2], parts[3], parts[4]), + tokenModifiers: parts[5] ?? [] + }; + } + let token = collection[i] as SemanticToken; token.tokenModifiers ??= []; token.tokenModifiers.sort(); } } - expect( - result - ).to.eql( - util.sortByRange( - tokens - ) - ); - return result; + function stringify(token: SemanticToken) { + return `${token.tokenType}|${util.rangeToString(token.range)}|${token.tokenModifiers?.join(',')}`; + } + + return { + actual: result.map(x => stringify(x)), + expected: (expected as SemanticToken[]).map(x => stringify(x)) + }; } it('matches each namespace section for class', () => { @@ -60,16 +85,14 @@ describe('BrsFileSemanticTokensProcessor', () => { end class end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(3, 34, 3, 43), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(3, 44, 3, 50), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(3, 51, 3, 56), - tokenType: SemanticTokenTypes.class - }]); + expectSemanticTokensIncludes(file, [ + //m.alien = new |Humanoids|.Aliens.Alien() + [SemanticTokenTypes.namespace, 3, 34, 3, 43], + //m.alien = new Humanoids.|Aliens|.Alien() + [SemanticTokenTypes.namespace, 3, 44, 3, 50], + //m.alien = new Humanoids.Aliens.|Alien|() + [SemanticTokenTypes.class, 3, 51, 3, 56] + ]); }); it('matches each namespace section for namespaced function calls', () => { @@ -82,16 +105,28 @@ describe('BrsFileSemanticTokensProcessor', () => { end function end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 16, 2, 25), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 26, 2, 32), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 33, 2, 39), - tokenType: SemanticTokenTypes.function - }]); + expectSemanticTokensIncludes(file, [ + // |Humanoids|.Aliens.Invade("earth") + [SemanticTokenTypes.namespace, 2, 16, 2, 25], + // Humanoids.|Aliens|.Invade("earth") + [SemanticTokenTypes.namespace, 2, 26, 2, 32], + // Humanoids.Aliens.|Invade|("earth") + [SemanticTokenTypes.function, 2, 33, 2, 39] + ]); + }); + + it('matches class name in assignment statement', () => { + const file = program.setFile('source/main.bs', ` + sub test() + ctor = Person + end sub + class Person + end class + `); + expectSemanticTokensIncludes(file, [ + // ctor = |Person| + [SemanticTokenTypes.class, 2, 23, 2, 29] + ]); }); it('matches namespace-relative parts', () => { @@ -105,15 +140,12 @@ describe('BrsFileSemanticTokensProcessor', () => { namespace alpha.lineHeight end namespace `); - expectSemanticTokens(file, [{ + expectSemanticTokensIncludes(file, [ //|lineHeight| = 1 - range: util.createRange(3, 20, 3, 30), - tokenType: SemanticTokenTypes.namespace - }, { + [SemanticTokenTypes.variable, 3, 20, 3, 30], //print |lineHeight| - range: util.createRange(4, 26, 4, 36), - tokenType: SemanticTokenTypes.namespace - }], false); + [SemanticTokenTypes.variable, 4, 26, 4, 36] + ], false); }); it('matches namespace-relative parts in parameters', () => { @@ -125,14 +157,13 @@ describe('BrsFileSemanticTokensProcessor', () => { namespace alpha.lineHeight end namespace `); - expectSemanticTokens(file, [{ + expectSemanticTokensIncludes(file, [ //sub test(|lineHeight| as integer) - range: util.createRange(2, 25, 2, 35), - tokenType: SemanticTokenTypes.namespace - }], false); + [SemanticTokenTypes.parameter, 2, 25, 2, 35] + ], false); }); - it('matches namespace-relative parts in parameters', () => { + it('matches parameters variable names', () => { const file = program.setFile('source/main.bs', ` namespace designSystem function getIcon(image = "" as string, size = -1 as float) as object @@ -142,11 +173,12 @@ describe('BrsFileSemanticTokensProcessor', () => { namespace designSystem.size end namespace `); - expectSemanticTokens(file, [{ - //sub test(|lineHeight| as integer) - range: util.createRange(2, 55, 2, 59), - tokenType: SemanticTokenTypes.namespace - }], false); + expectSemanticTokensIncludes(file, [ + // function getIcon(|image| = "" as string, size = -1 as float) as object + [SemanticTokenTypes.parameter, 2, 33, 2, 38], + // function getIcon(image = "" as string, |size| = -1 as float) as object + [SemanticTokenTypes.parameter, 2, 55, 2, 59] + ], false); }); it('matches each namespace section for namespaced function assignment', () => { @@ -159,16 +191,14 @@ describe('BrsFileSemanticTokensProcessor', () => { end function end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 25, 2, 34), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 35, 2, 41), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 42, 2, 48), - tokenType: SemanticTokenTypes.function - }]); + expectSemanticTokensIncludes(file, [ + // action = |Humanoids|.Aliens.Invade + [SemanticTokenTypes.namespace, 2, 25, 2, 34], + // action = Humanoids.|Aliens|.Invade + [SemanticTokenTypes.namespace, 2, 35, 2, 41], + // action = Humanoids.Aliens.|Invade| + [SemanticTokenTypes.function, 2, 42, 2, 48] + ]); }); it('matches each namespace section for namespaced function as function parameter', () => { @@ -181,20 +211,16 @@ describe('BrsFileSemanticTokensProcessor', () => { end function end namespace `); - expectSemanticTokens(file, [{ - //`type` function call - range: util.createRange(2, 29, 2, 33), - tokenType: SemanticTokenTypes.function - }, { //Humanoids - range: util.createRange(2, 34, 2, 43), - tokenType: SemanticTokenTypes.namespace - }, { //Aliens - range: util.createRange(2, 44, 2, 50), - tokenType: SemanticTokenTypes.namespace - }, { //Invade - range: util.createRange(2, 51, 2, 57), - tokenType: SemanticTokenTypes.function - }]); + expectSemanticTokensIncludes(file, [ + // actionName = |type|(Humanoids.Aliens.Invade) + [SemanticTokenTypes.function, 2, 29, 2, 33], + // actionName = type(|Humanoids|.Aliens.Invade) + [SemanticTokenTypes.namespace, 2, 34, 2, 43], + // actionName = type(Humanoids.|Aliens|.Invade) + [SemanticTokenTypes.namespace, 2, 44, 2, 50], + // actionName = type(Humanoids.Aliens.|Invade|) + [SemanticTokenTypes.function, 2, 51, 2, 57] + ]); }); it('matches each namespace section for namespaced function in print statement', () => { @@ -207,43 +233,27 @@ describe('BrsFileSemanticTokensProcessor', () => { end function end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 22, 2, 31), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 32, 2, 38), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 39, 2, 45), - tokenType: SemanticTokenTypes.function - }]); + expectSemanticTokensIncludes(file, [ + // print |Humanoids|.Aliens.Invade + [SemanticTokenTypes.namespace, 2, 22, 2, 31], + // print Humanoids.|Aliens|.Invade + [SemanticTokenTypes.namespace, 2, 32, 2, 38], + // print Humanoids.Aliens.|Invade| + [SemanticTokenTypes.function, 2, 39, 2, 45] + ]); }); - it('matches each namespace section for enums', () => { + it('matches each namespace section for namespace declaration', () => { const file = program.setFile('source/main.bs', ` - sub main() - print Earthlings.Species.Human.Male - end sub - namespace Earthlings.Species - enum Human - Male - Female - end enum + namespace Sentients.Humanoids end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 22, 2, 32), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 33, 2, 40), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 41, 2, 46), - tokenType: SemanticTokenTypes.enum - }, { - range: util.createRange(2, 47, 2, 51), - tokenType: SemanticTokenTypes.enumMember - }]); + expectSemanticTokens(file, [ + // namespace |Sentients|.Humanoids + [SemanticTokenTypes.namespace, 1, 22, 1, 31], + // namespace Sentients.|Humanoids| + [SemanticTokenTypes.namespace, 1, 32, 1, 41] + ]); }); it('matches each namespace section for enum', () => { @@ -259,19 +269,16 @@ describe('BrsFileSemanticTokensProcessor', () => { end enum end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 22, 2, 31), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 32, 2, 41), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 42, 2, 54), - tokenType: SemanticTokenTypes.enum - }, { - range: util.createRange(2, 55, 2, 60), - tokenType: SemanticTokenTypes.enumMember - }]); + expectSemanticTokensIncludes(file, [ + // print |Sentients|.Humanoids.HumanoidType.Cylon + [SemanticTokenTypes.namespace, 2, 22, 2, 31], + // print Sentients.|Humanoids|.HumanoidType.Cylon + [SemanticTokenTypes.namespace, 2, 32, 2, 41], + // print Sentients.Humanoids.|HumanoidType|.Cylon + [SemanticTokenTypes.enum, 2, 42, 2, 54], + // print Sentients.Humanoids.HumanoidType.|Cylon| + [SemanticTokenTypes.enumMember, 2, 55, 2, 60] + ]); }); it('matches enums in if statements', () => { @@ -289,16 +296,14 @@ describe('BrsFileSemanticTokensProcessor', () => { end enum end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 19, 2, 28), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 29, 2, 41), - tokenType: SemanticTokenTypes.enum - }, { - range: util.createRange(2, 42, 2, 47), - tokenType: SemanticTokenTypes.enumMember - }]); + expectSemanticTokensIncludes(file, [ + // if |Humanoids|.HumanoidType.Cylon = "Cylon" then + [SemanticTokenTypes.namespace, 2, 19, 2, 28], + // if Humanoids.|HumanoidType|.Cylon = "Cylon" then + [SemanticTokenTypes.enum, 2, 29, 2, 41], + // if Humanoids.HumanoidType.|Cylon| = "Cylon" then + [SemanticTokenTypes.enumMember, 2, 42, 2, 47] + ]); }); it('matches enum with invalid member name', () => { @@ -314,13 +319,12 @@ describe('BrsFileSemanticTokensProcessor', () => { end enum end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 22, 2, 31), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 32, 2, 44), - tokenType: SemanticTokenTypes.enum - }]); + expectSemanticTokensIncludes(file, [ + // print |Humanoids|.HumanoidType.INVALID_VALUE 'bs:disable-line + [SemanticTokenTypes.namespace, 2, 22, 2, 31], + // print Humanoids.|HumanoidType|.INVALID_VALUE 'bs:disable-line + [SemanticTokenTypes.enum, 2, 32, 2, 44] + ]); }); it('matches class with invalid stuff after it', () => { @@ -334,16 +338,39 @@ describe('BrsFileSemanticTokensProcessor', () => { end class end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 30, 2, 39), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 40, 2, 46), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 47, 2, 52), - tokenType: SemanticTokenTypes.class - }]); + expectSemanticTokensIncludes(file, [ + // m.alien = new |Humanoids|.Aliens.Alien.NOT_A_CLASS() 'bs:disable-line + [SemanticTokenTypes.namespace, 2, 30, 2, 39], + // m.alien = new Humanoids.|Aliens|.Alien.NOT_A_CLASS() 'bs:disable-line + [SemanticTokenTypes.namespace, 2, 40, 2, 46], + // m.alien = new Humanoids.Aliens.|Alien|.NOT_A_CLASS() 'bs:disable-line + [SemanticTokenTypes.class, 2, 47, 2, 52] + ]); + }); + + it('matches aliases', () => { + program.setFile('source/alpha.bs', ` + namespace alpha + sub test() + end sub + end namespace + `); + const file = program.setFile('source/main.bs', ` + alias alpha2 = alpha + sub main() + print alpha2.test() + end sub + `); + expectSemanticTokensIncludes(file, [ + // alias |alpha2| = alpha + [SemanticTokenTypes.namespace, 1, 18, 1, 24], + // alias alpha2 = |alpha| + [SemanticTokenTypes.namespace, 1, 27, 1, 32], + // print |alpha2|.test() + [SemanticTokenTypes.namespace, 3, 22, 3, 28], + // print alpha2.|test|() + [SemanticTokenTypes.function, 3, 29, 3, 33] + ]); }); it('matches consts', () => { @@ -357,26 +384,22 @@ describe('BrsFileSemanticTokensProcessor', () => { const FIRST_NAME = "bob" end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 22, 2, 29), - tokenType: SemanticTokenTypes.variable, - tokenModifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] - }, { - range: util.createRange(3, 22, 3, 26), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(3, 27, 3, 37), - tokenType: SemanticTokenTypes.variable, - tokenModifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] - }, { - range: util.createRange(5, 18, 5, 25), - tokenType: SemanticTokenTypes.variable, - tokenModifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] - }, { - range: util.createRange(7, 22, 7, 32), - tokenType: SemanticTokenTypes.variable, - tokenModifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] - }]); + expectSemanticTokens(file, [ + // sub |init|() + [SemanticTokenTypes.function, 1, 16, 1, 20], + // print |API_URL| + [SemanticTokenTypes.variable, 2, 22, 2, 29, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]], + // print |info|.FIRST_NAME + [SemanticTokenTypes.namespace, 3, 22, 3, 26], + // print info.|FIRST_NAME| + [SemanticTokenTypes.variable, 3, 27, 3, 37, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]], + // const |API_URL| = "some_url" + [SemanticTokenTypes.variable, 5, 18, 5, 25, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]], + // namespace |info| + [SemanticTokenTypes.namespace, 6, 22, 6, 26], + // const |FIRST_NAME| = "bob" + [SemanticTokenTypes.variable, 7, 22, 7, 32, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]] + ]); }); it('matches consts in assignment expressions', () => { @@ -392,40 +415,30 @@ describe('BrsFileSemanticTokensProcessor', () => { const API_URL = "url" `); expectSemanticTokens(file, [ + // sub |main|() + [SemanticTokenTypes.function, 1, 16, 1, 20], + // |value| = "" + [SemanticTokenTypes.variable, 2, 16, 2, 21], + // |value| += constants.API_KEY + [SemanticTokenTypes.variable, 3, 16, 3, 21], // value += |constants|.API_KEY - { - range: util.createRange(3, 25, 3, 34), - tokenType: SemanticTokenTypes.namespace - }, + [SemanticTokenTypes.namespace, 3, 25, 3, 34], // value += constants.|API_KEY| - { - range: util.createRange(3, 35, 3, 42), - tokenType: SemanticTokenTypes.variable, - tokenModifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] - }, + [SemanticTokenTypes.variable, 3, 35, 3, 42, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]], + // |value| += API_URL + [SemanticTokenTypes.variable, 4, 16, 4, 21], // value += |API_URL| - { - range: util.createRange(4, 25, 4, 32), - tokenType: SemanticTokenTypes.variable, - tokenModifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] - }, + [SemanticTokenTypes.variable, 4, 25, 4, 32, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]], + // namespace |constants| + [SemanticTokenTypes.namespace, 6, 22, 6, 31], // const |API_KEY| = "test" - { - range: util.createRange(7, 22, 7, 29), - tokenType: SemanticTokenTypes.variable, - tokenModifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] - }, + [SemanticTokenTypes.variable, 7, 22, 7, 29, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]], //const |API_URL| = "url" - { - range: util.createRange(9, 18, 9, 25), - tokenType: SemanticTokenTypes.variable, - tokenModifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] - } + [SemanticTokenTypes.variable, 9, 18, 9, 25, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]] ]); }); - - it('matches native interfaces', () => { + it('matches new statement of non-class', () => { const file = program.setFile('source/main.bs', ` sub init() m.alien = new Humanoids.Aliens.Alien.NOT_A_CLASS() 'bs:disable-line @@ -436,16 +449,70 @@ describe('BrsFileSemanticTokensProcessor', () => { end class end namespace `); - expectSemanticTokens(file, [{ - range: util.createRange(2, 30, 2, 39), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 40, 2, 46), - tokenType: SemanticTokenTypes.namespace - }, { - range: util.createRange(2, 47, 2, 52), - tokenType: SemanticTokenTypes.class - }]); + expectSemanticTokens(file, [ + // sub |init|() + [SemanticTokenTypes.function, 1, 16, 1, 20], + // |m|.alien = new Humanoids.Aliens.Alien.NOT_A_CLASS() 'bs:disable-line + [SemanticTokenTypes.variable, 2, 16, 2, 17], + // m.alien = new |Humanoids|.Aliens.Alien.NOT_A_CLASS() 'bs:disable-line + [SemanticTokenTypes.namespace, 2, 30, 2, 39], + // m.alien = new Humanoids.|Aliens|.Alien.NOT_A_CLASS() 'bs:disable-line + [SemanticTokenTypes.namespace, 2, 40, 2, 46], + // m.alien = new Humanoids.Aliens.|Alien|.NOT_A_CLASS() 'bs:disable-line + [SemanticTokenTypes.class, 2, 47, 2, 52], + // namespace |Humanoids|.Aliens + [SemanticTokenTypes.namespace, 5, 22, 5, 31], + // namespace Humanoids.|Aliens| + [SemanticTokenTypes.namespace, 5, 32, 5, 38], + // class |Alien| + [SemanticTokenTypes.class, 6, 22, 6, 27] + ]); + }); + + it('matches interface names', () => { + const file = program.setFile('source/main.bs', ` + interface Human + name as string + end interface + + namespace Humanoids + interface Alien + name as string + end interface + end namespace + `); + expectSemanticTokensIncludes(file, [ + // interface |Human| + [SemanticTokenTypes.interface, 1, 22, 1, 27], + // interface |Alien| + [SemanticTokenTypes.interface, 6, 26, 6, 31] + ]); }); + it('works for `new` statement', () => { + const file = program.setFile('source/main.bs', ` + class Person + sub speak() + end sub + end class + sub test() + dude = new Person() + dude.speak() + end sub + `); + expectSemanticTokens(file, [ + // class |Person| + [SemanticTokenTypes.class, 1, 18, 1, 24], + // sub |test|() + [SemanticTokenTypes.function, 5, 16, 5, 20], + // |dude| = new Person() + [SemanticTokenTypes.variable, 6, 16, 6, 20], + // dude = new |Person|() + [SemanticTokenTypes.class, 6, 27, 6, 33], + // |dude|.speak() + [SemanticTokenTypes.variable, 7, 16, 7, 20], + // dude.|speak|() + [SemanticTokenTypes.method, 7, 21, 7, 26] + ]); + }); }); diff --git a/src/bscPlugin/semanticTokens/BrsFileSemanticTokensProcessor.ts b/src/bscPlugin/semanticTokens/BrsFileSemanticTokensProcessor.ts index 601722de3..765f67553 100644 --- a/src/bscPlugin/semanticTokens/BrsFileSemanticTokensProcessor.ts +++ b/src/bscPlugin/semanticTokens/BrsFileSemanticTokensProcessor.ts @@ -1,15 +1,14 @@ -import type { Range } from 'vscode-languageserver-protocol'; import { SemanticTokenModifiers } from 'vscode-languageserver-protocol'; import { SemanticTokenTypes } from 'vscode-languageserver-protocol'; -import { isCallExpression, isCallableType, isClassType, isComponentType, isConstStatement, isEnumMemberType, isEnumType, isInterfaceType, isNamespaceStatement, isNamespaceType, isNativeType, isNewExpression } from '../../astUtils/reflection'; +import { isCallableType, isClassType, isComponentType, isConstStatement, isDottedGetExpression, isEnumMemberType, isEnumType, isFunctionExpression, isFunctionStatement, isInterfaceType, isNamespaceType, isVariableExpression } from '../../astUtils/reflection'; import type { BrsFile } from '../../files/BrsFile'; -import type { ExtraSymbolData, OnGetSemanticTokensEvent } from '../../interfaces'; -import type { Locatable } from '../../lexer/Token'; -import { ParseMode } from '../../parser/Parser'; -import type { NamespaceStatement } from '../../parser/Statement'; +import type { ExtraSymbolData, OnGetSemanticTokensEvent, SemanticToken, TypeChainEntry } from '../../interfaces'; +import type { Locatable, Token } from '../../lexer/Token'; import util from '../../util'; import { SymbolTypeFlag } from '../../SymbolTypeFlag'; import type { BscType } from '../../types/BscType'; +import { WalkMode, createVisitor } from '../../astUtils/visitors'; +import type { AstNode } from '../../parser/AstNode'; export class BrsFileSemanticTokensProcessor { public constructor( @@ -19,155 +18,128 @@ export class BrsFileSemanticTokensProcessor { } public process() { - this.handleClasses(); - this.handleConstDeclarations(); - this.iterateNodes(); - } + const scope = this.event.scopes[0]; + this.result.clear(); + scope.linkSymbolTable(); - private handleConstDeclarations() { - // eslint-disable-next-line @typescript-eslint/dot-notation - for (const stmt of this.event.file['_cachedLookups'].constStatements) { - this.addToken(stmt.tokens.name, SemanticTokenTypes.variable, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]); - } - } + this.event.file.ast.walk(createVisitor({ + VariableExpression: (node) => { + this.tryAddToken(node, node.tokens.name); + }, + AssignmentStatement: (node) => { + this.addToken(node.tokens.name, SemanticTokenTypes.variable); + }, + DottedGetExpression: (node) => { + this.tryAddToken(node, node.tokens.name); + }, + ConstStatement: (node) => { + this.tryAddToken(node, node.tokens.name); + }, + AliasStatement: (node) => { + this.tryAddToken(node, node.tokens.name); + }, + ClassStatement: (node) => { + this.tryAddToken(node, node.tokens.name); + }, + InterfaceStatement: (node) => { + this.tryAddToken(node, node.tokens.name); + }, + EnumStatement: (node) => { + this.tryAddToken(node, node.tokens.name); + }, + FunctionStatement: (node) => { + this.tryAddToken(node, node.tokens.name); + }, + FunctionParameterExpression: (node) => { + this.addToken(node.tokens.name, SemanticTokenTypes.parameter); + } + }), { + walkMode: WalkMode.visitAllRecursive + }); - private handleClasses() { + scope.unlinkSymbolTable(); - const classes = [] as Array<{ className: string; namespaceName: string; range: Range }>; + //add all tokens to the event + this.event.semanticTokens.push( + ...this.result.values() + ); + } - //classes used in function param types - // eslint-disable-next-line @typescript-eslint/dot-notation - for (const func of this.event.file['_cachedLookups'].functionExpressions) { - for (const param of func.parameters) { - if (isClassType(param.getType({ flags: SymbolTypeFlag.typetime }))) { - const namespace = param.findAncestor(isNamespaceStatement); - classes.push({ - className: util.getAllDottedGetParts(param.typeExpression.expression).map(x => x.text).join('.'), - namespaceName: namespace?.getName(ParseMode.BrighterScript), - range: param.typeExpression.range - }); - } - } - } + private result = new Map(); - for (const cls of classes) { - if ( - cls.className.length > 0 && - //only highlight classes that are in scope - this.event.scopes.some(x => x.hasClass(cls.className, cls.namespaceName)) - ) { - const tokens = util.splitGetRange('.', cls.className, cls.range); - this.addTokens(tokens.reverse(), SemanticTokenTypes.class, SemanticTokenTypes.namespace); - } - } - } /** - * Add tokens for each locatable item in the list. - * Each locatable is paired with a token type. If there are more locatables than token types, all remaining locatables are given the final token type + * Add the given token and node IF we have a resolvable type */ - private addTokens(locatables: Locatable[], ...semanticTokenTypes: SemanticTokenTypes[]) { - for (let i = 0; i < locatables.length; i++) { - const locatable = locatables[i]; - //skip items that don't have a location - if (locatable?.range) { - this.addToken( - locatables[i], - //use the type at the index, or the last type if missing - semanticTokenTypes[i] ?? semanticTokenTypes[semanticTokenTypes.length - 1] - ); + private tryAddToken(node: AstNode, token: Token) { + const extraData = {} as ExtraSymbolData; + const chain = [] as TypeChainEntry[]; + // eslint-disable-next-line no-bitwise + const symbolType = node.getType({ flags: SymbolTypeFlag.runtime, data: extraData, typeChain: chain }); + if (symbolType?.isResolvable()) { + let info = this.getSemanticTokenInfo(node, symbolType, extraData); + if (info) { + this.addToken(token, info.type, info.modifiers); } } } private addToken(locatable: Locatable, type: SemanticTokenTypes, modifiers: SemanticTokenModifiers[] = []) { - this.event.semanticTokens.push({ + //only keep a single token per range. Last-in wins + this.result.set(util.rangeToString(locatable.range), { range: locatable.range, tokenType: type, tokenModifiers: modifiers }); } - private iterateNodes() { - const scope = this.event.scopes[0]; - - //if this file has no scopes, there's nothing else we can do about this - if (!scope) { - return; - } - scope.linkSymbolTable(); - const nodes = [ - // eslint-disable-next-line @typescript-eslint/dot-notation - ...this.event.file['_cachedLookups'].expressions, - //make a new VariableExpression to wrap the name. This is a hack, we could probably do it better - // eslint-disable-next-line @typescript-eslint/dot-notation - ...this.event.file['_cachedLookups'].assignmentStatements, - // eslint-disable-next-line @typescript-eslint/dot-notation - ...this.event.file['_cachedLookups'].functionExpressions.map(x => x.parameters).flat() - ]; - - for (let node of nodes) { - //lift the callee from call expressions to handle namespaced function calls - if (isCallExpression(node)) { - node = node.callee; - } else if (isNewExpression(node)) { - node = node.call.callee; + private getSemanticTokenInfo(node: AstNode, type: BscType, extraData: ExtraSymbolData): { type: SemanticTokenTypes; modifiers?: SemanticTokenModifiers[] } { + if (isConstStatement(extraData?.definingNode)) { + return { type: SemanticTokenTypes.variable, modifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] }; + // non-instances of classes should be colored like classes + } else if (isClassType(type) && extraData.isInstance !== true) { + return { type: SemanticTokenTypes.class }; + //function statements and expressions + } else if (isCallableType(type)) { + //if the typetime type is a class, then color this like a class + const typetimeType = node.getType({ flags: SymbolTypeFlag.typetime }); + if (isClassType(typetimeType)) { + return { type: SemanticTokenTypes.class }; } - const containingNamespaceNameLower = node.findAncestor(isNamespaceStatement)?.getName(ParseMode.BrighterScript).toLowerCase(); - const tokens = util.getAllDottedGetParts(node); - const processedNames: string[] = []; - for (const token of tokens ?? []) { - processedNames.push(token.text?.toLowerCase()); - const entityName = processedNames.join('.'); - - if (scope.getEnumMemberFileLink(entityName, containingNamespaceNameLower)) { - this.addToken(token, SemanticTokenTypes.enumMember); - } else if (scope.getEnum(entityName, containingNamespaceNameLower)) { - this.addToken(token, SemanticTokenTypes.enum); - } else if (scope.getClass(entityName, containingNamespaceNameLower)) { - this.addToken(token, SemanticTokenTypes.class); - } else if (scope.getInterface(entityName, containingNamespaceNameLower)) { - this.addToken(token, SemanticTokenTypes.interface); - } else if (scope.getCallableByName(entityName)) { - this.addToken(token, SemanticTokenTypes.function); - } else if (scope.getNamespace(entityName, containingNamespaceNameLower)) { - this.addToken(token, SemanticTokenTypes.namespace); - } else if (scope.getConstFileLink(entityName, containingNamespaceNameLower)) { - this.addToken(token, SemanticTokenTypes.variable, [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static]); - } else { - const extraData = {}; - const symbolType = scope.symbolTable.getSymbolType(token.text, { flags: SymbolTypeFlag.typetime, data: extraData }); - if (symbolType?.isResolvable()) { - this.addToken(token, this.getSemanticTokenTypeFromType(symbolType, extraData, !!containingNamespaceNameLower)); - } - } + //if this is a function statement or expression, treat it as a function + if (isFunctionExpression(node) || isFunctionStatement(node)) { + return { type: SemanticTokenTypes.function }; } - } - scope.unlinkSymbolTable(); - } + if ( + //if this is a standalone function + isVariableExpression(node) || + //if this is a dottedGet, and the LHS is a namespace, treat it as a function. + (isDottedGetExpression(node) && isNamespaceType(node.obj.getType({ flags: SymbolTypeFlag.typetime }))) + ) { + return { type: SemanticTokenTypes.function }; - // TODO: We can use the actual symbol tables to find methods and member fields. - private getSemanticTokenTypeFromType(type: BscType, extraData: ExtraSymbolData, areMembers = false) { - if (isConstStatement(extraData?.definingNode)) { - return SemanticTokenTypes.variable; - } else if (isClassType(type)) { - return SemanticTokenTypes.class; - } else if (isCallableType(type)) { - return areMembers ? SemanticTokenTypes.method : SemanticTokenTypes.function; + //all others should be treated as methods + } else { + return { type: SemanticTokenTypes.method }; + } } else if (isInterfaceType(type)) { - return SemanticTokenTypes.interface; + return { type: SemanticTokenTypes.interface }; } else if (isComponentType(type)) { - return SemanticTokenTypes.class; + return { type: SemanticTokenTypes.class }; } else if (isEnumType(type)) { - return SemanticTokenTypes.enum; + return { type: SemanticTokenTypes.enum }; } else if (isEnumMemberType(type)) { - return SemanticTokenTypes.enumMember; + return { type: SemanticTokenTypes.enumMember }; } else if (isNamespaceType(type)) { - return SemanticTokenTypes.namespace; - } else if (isNativeType(type)) { - return SemanticTokenTypes.type; + return { type: SemanticTokenTypes.namespace }; + //this is separate from the checks above because we want to resolve alias lookups before turning this variable into a const + } else if (isConstStatement(node)) { + return { type: SemanticTokenTypes.variable, modifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.static] }; + } else if (isVariableExpression(node)) { + return { type: SemanticTokenTypes.variable }; + } else { + //we don't know what it is...return undefined to prevent creating a semantic token } - return areMembers ? SemanticTokenTypes.property : SemanticTokenTypes.variable; } } diff --git a/src/interfaces.ts b/src/interfaces.ts index 295901fee..de9304ed8 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -916,7 +916,11 @@ export interface ExtraSymbolData { */ isAlias?: boolean; /** - * is this symbol an instance of the type + * Is this symbol an instance of the type. + * + * `true` means `true`, and `false` or `undefined` means `false`, + * + * so check for `=== true` or `!== true` */ isInstance?: boolean; } diff --git a/src/parser/Expression.ts b/src/parser/Expression.ts index d63d52331..f79adccf0 100644 --- a/src/parser/Expression.ts +++ b/src/parser/Expression.ts @@ -1216,7 +1216,7 @@ export class VariableExpression extends Expression { } } - options.typeChain?.push(new TypeChainEntry({ name: nameKey, type: resultType, data: options.data, astNode: this })); + options?.typeChain?.push(new TypeChainEntry({ name: nameKey, type: resultType, data: options.data, astNode: this })); return resultType; } diff --git a/src/types/ReferenceType.ts b/src/types/ReferenceType.ts index 1ed2546f4..55ad699db 100644 --- a/src/types/ReferenceType.ts +++ b/src/types/ReferenceType.ts @@ -113,7 +113,7 @@ export class ReferenceType extends BscType { if (resolvedType) { return resolvedType.getMemberType(memberName, options); } - const refLookUp = `${memberName.toLowerCase()}-${options.flags}`; + const refLookUp = `${memberName.toLowerCase()}-${options?.flags}`; let memberTypeReference = this.memberTypeReferences.get(refLookUp); if (memberTypeReference) { return memberTypeReference; diff --git a/src/util.ts b/src/util.ts index d24a20594..751f6ff96 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2187,7 +2187,7 @@ export class Util { public isClassUsedAsFunction(potentialClassType: BscType, expression: Expression, options: GetTypeOptions) { // eslint-disable-next-line no-bitwise - if (options.flags & SymbolTypeFlag.runtime && + if ((options?.flags ?? 0) & SymbolTypeFlag.runtime && isClassType(potentialClassType) && !options.isExistenceTest && potentialClassType.name.toLowerCase() === this.getAllDottedGetPartsAsString(expression).toLowerCase() &&