From 24f4a8f24c08dd25686afc4cfb78be2e0045e844 Mon Sep 17 00:00:00 2001 From: Snehil Shah Date: Fri, 31 May 2024 21:36:45 +0530 Subject: [PATCH] feat: add syntax highlighting in the REPL PR-URL: #2254 Resolves: #2072 --------- Signed-off-by: Snehil Shah Reviewed-by: Athan Reines Reviewed-by: Philipp Burckhardt --- .../@stdlib/repl/lib/ansi_colors.js | 58 +++ .../@stdlib/repl/lib/complete_expression.js | 2 +- lib/node_modules/@stdlib/repl/lib/main.js | 8 + .../@stdlib/repl/lib/resolve_globals.js | 2 +- .../@stdlib/repl/lib/resolve_local_scopes.js | 354 ++++++++--------- .../@stdlib/repl/lib/syntax_highlighter.js | 168 ++++++++ lib/node_modules/@stdlib/repl/lib/themes.js | 60 +++ .../@stdlib/repl/lib/tokenizer.js | 359 ++++++++++++++++++ 8 files changed, 834 insertions(+), 177 deletions(-) create mode 100644 lib/node_modules/@stdlib/repl/lib/ansi_colors.js create mode 100644 lib/node_modules/@stdlib/repl/lib/syntax_highlighter.js create mode 100644 lib/node_modules/@stdlib/repl/lib/themes.js create mode 100644 lib/node_modules/@stdlib/repl/lib/tokenizer.js diff --git a/lib/node_modules/@stdlib/repl/lib/ansi_colors.js b/lib/node_modules/@stdlib/repl/lib/ansi_colors.js new file mode 100644 index 00000000000..1df33d1bea4 --- /dev/null +++ b/lib/node_modules/@stdlib/repl/lib/ansi_colors.js @@ -0,0 +1,58 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2024 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +// MAIN // + +/** +* Table mapping generic color names to their ANSI color codes. +* +* @private +* @name ANSI +* @type {Object} +*/ +var ANSI = { + // Original colors: + 'black': '\u001b[30m', + 'red': '\u001b[31m', + 'green': '\u001b[32m', + 'yellow': '\u001b[33m', + 'blue': '\u001b[34m', + 'magenta': '\u001b[35m', + 'cyan': '\u001b[36m', + 'white': '\u001b[37m', + + // Bright colors: + 'brightBlack': '\u001b[90m', + 'brightRed': '\u001b[91m', + 'brightGreen': '\u001b[92m', + 'brightYellow': '\u001b[93m', + 'brightBlue': '\u001b[94m', + 'brightMagenta': '\u001b[95m', + 'brightCyan': '\u001b[96m', + 'brightWhite': '\u001b[97m', + + // Reset colors: + 'reset': '\u001b[0m' +}; + + +// EXPORTS // + +module.exports = ANSI; diff --git a/lib/node_modules/@stdlib/repl/lib/complete_expression.js b/lib/node_modules/@stdlib/repl/lib/complete_expression.js index 64b664344fa..769d7dd17bf 100644 --- a/lib/node_modules/@stdlib/repl/lib/complete_expression.js +++ b/lib/node_modules/@stdlib/repl/lib/complete_expression.js @@ -74,7 +74,7 @@ function complete( out, context, expression ) { ast = parse( expression, AOPTS ); debug( 'Resolving local scopes within the AST.' ); - ast = resolveLocalScopes( ast ); + resolveLocalScopes( ast ); // Get the last program top-level AST "node": debug( 'Number of statements: %d', ast.body.length ); diff --git a/lib/node_modules/@stdlib/repl/lib/main.js b/lib/node_modules/@stdlib/repl/lib/main.js index b5afa1583a1..d812cf445ce 100644 --- a/lib/node_modules/@stdlib/repl/lib/main.js +++ b/lib/node_modules/@stdlib/repl/lib/main.js @@ -63,6 +63,7 @@ var processLine = require( './process_line.js' ); var completerFactory = require( './completer.js' ); var PreviewCompleter = require( './completer_preview.js' ); var AutoCloser = require( './auto_close_pairs.js' ); +var SyntaxHighlighter = require( './syntax_highlighter.js' ); var ALIAS_OVERRIDES = require( './alias_overrides.js' ); var SETTINGS = require( './settings.js' ); var SETTINGS_VALIDATORS = require( './settings_validators.js' ); @@ -270,6 +271,9 @@ function REPL( options ) { // Initialize a preview completer: setNonEnumerableReadOnly( this, '_previewCompleter', new PreviewCompleter( this._rli, this._completer, this._ostream, this._settings.completionPreviews ) ); + // Initialize a syntax-highlighter: + setNonEnumerableReadOnly( this, '_syntaxHighlighter', new SyntaxHighlighter( this, this._ostream ) ); + // Cache a reference to the private readline interface `ttyWrite` to allow calling the method when wanting default behavior: setNonEnumerableReadOnly( this, '_ttyWrite', this._rli._ttyWrite ); @@ -334,6 +338,7 @@ function REPL( options ) { */ function onKeypress( data, key ) { var autoClosed; + if ( key && key.name === 'tab' ) { return; } @@ -343,6 +348,9 @@ function REPL( options ) { if ( autoClosed ) { self._previewCompleter.clear(); } + if ( self._isTTY ) { + self._syntaxHighlighter.onKeypress(); + } self._previewCompleter.onKeypress( data, key ); } diff --git a/lib/node_modules/@stdlib/repl/lib/resolve_globals.js b/lib/node_modules/@stdlib/repl/lib/resolve_globals.js index aaab1d774db..b97f8419f3f 100644 --- a/lib/node_modules/@stdlib/repl/lib/resolve_globals.js +++ b/lib/node_modules/@stdlib/repl/lib/resolve_globals.js @@ -76,7 +76,7 @@ function resolveGlobals( ast ) { globals = []; // Resolve local scopes: - ast = resolveLocalScopes( ast ); + resolveLocalScopes( ast ); // Define callbacks for relevant AST nodes: visitors = { diff --git a/lib/node_modules/@stdlib/repl/lib/resolve_local_scopes.js b/lib/node_modules/@stdlib/repl/lib/resolve_local_scopes.js index 459adde0643..7ec3cdaff12 100644 --- a/lib/node_modules/@stdlib/repl/lib/resolve_local_scopes.js +++ b/lib/node_modules/@stdlib/repl/lib/resolve_local_scopes.js @@ -27,217 +27,221 @@ var isScope = require( './is_scope.js' ); var isBlockScope = require( './is_block_scope.js' ); -// VARIABLES // - -var VISITORS = { - 'VariableDeclaration': VariableDeclaration, - 'FunctionDeclaration': FunctionDeclaration, - 'Function': declareFunction, - 'ClassDeclaration': ClassDeclaration, - 'TryStatement': TryStatement, - 'ImportDefaultSpecifier': ModuleImportSpecifier, - 'ImportSpecifier': ModuleImportSpecifier, - 'ImportNamespaceSpecifier': ModuleImportSpecifier -}; - - -// FUNCTIONS // +// MAIN // /** -* Resolves identifiers arising from a function declaration, such as function parameters and the function name. +* Resolves local scopes within an abstract syntax tree (AST). +* +* ## Notes +* +* - This function modifies the provided AST by adding a `locals` property to select AST nodes and returns all resolved `locals`. * * @private -* @param {Node} node - function declaration AST node +* @param {AST} ast - abstract syntax tree (AST) +* @throws {TypeError} must provide a program AST node +* @returns {Array} array of `Identifier` nodes of all resolved `locals` in the AST */ -function declareFunction( node ) { - var i; +function resolveScopes( ast ) { + var declarations; + var VISITORS; - node.locals = node.locals || []; - for ( i = 0; i < node.params.length; i++ ) { - declarationPattern( node.params[ i ], node ); - } - if ( node.id ) { - appendUnique( node.locals, node.id.name ); + if ( ast.type !== 'Program' ) { + throw new TypeError( 'invalid argument. Must provide a program AST node.' ); } -} - -/** -* Resolves identifiers based on the variable declaration pattern. -* -* @private -* @param {Node} node - AST node -* @param {Node} parent - parent AST node -* @throws {Error} unrecognized pattern type -*/ -function declarationPattern( node, parent ) { - var i; - switch ( node.type ) { // eslint-disable-line padded-blocks + VISITORS = { + 'VariableDeclaration': VariableDeclaration, + 'FunctionDeclaration': FunctionDeclaration, + 'Function': declareFunction, + 'ClassDeclaration': ClassDeclaration, + 'TryStatement': TryStatement, + 'ImportDefaultSpecifier': ModuleImportSpecifier, + 'ImportSpecifier': ModuleImportSpecifier, + 'ImportNamespaceSpecifier': ModuleImportSpecifier + }; + declarations = []; + ast.locals = []; + walk( ast, VISITORS ); + return declarations; + + /** + * Resolves identifiers arising from a function declaration, such as function parameters and the function name. + * + * @private + * @param {Node} node - function declaration AST node + */ + function declareFunction( node ) { + var i; + + node.locals = node.locals || []; + for ( i = 0; i < node.params.length; i++ ) { + declarationPattern( node.params[ i ], node ); + } + if ( node.id ) { + appendUnique( node.locals, node.id.name ); + appendUnique( declarations, node.id ); + } + } - // Actual identifier name: - case 'Identifier': - appendUnique( parent.locals, node.name ); - break; + /** + * Resolves identifiers based on the variable declaration pattern. + * + * @private + * @param {Node} node - AST node + * @param {Node} parent - parent AST node + * @throws {Error} unrecognized pattern type + */ + function declarationPattern( node, parent ) { + var i; + + switch ( node.type ) { // eslint-disable-line padded-blocks + + // Actual identifier name: + case 'Identifier': + appendUnique( parent.locals, node.name ); + appendUnique( declarations, node ); + break; - // `var { a, b } = { 'a': 10, 'b': 20 }` || `var { ...o } = {}` - case 'ObjectPattern': - for ( i = 0; i < node.properties.length; i++ ) { - declarationPattern( node.properties[ i ].value || node.properties[ i ].argument, parent ); // eslint-disable-line max-len - } - break; + // `var { a, b } = { 'a': 10, 'b': 20 }` || `var { ...o } = {}` + case 'ObjectPattern': + for ( i = 0; i < node.properties.length; i++ ) { + declarationPattern( node.properties[ i ].value || node.properties[ i ].argument, parent ); // eslint-disable-line max-len + } + break; - // `var [ x, y ] = [ 10, 20 ]` - case 'ArrayPattern': - for ( i = 0; i < node.elements.length; i++ ) { - if ( node.elements[ i ] ) { - declarationPattern( node.elements[ i ], parent ); + // `var [ x, y ] = [ 10, 20 ]` + case 'ArrayPattern': + for ( i = 0; i < node.elements.length; i++ ) { + if ( node.elements[ i ] ) { + declarationPattern( node.elements[ i ], parent ); + } } - } - break; + break; - // `var [ x, y, ...z ] = [ 10, 20, 30, 40, 50 ]` || `var [ ...z ] = []` - case 'RestElement': - declarationPattern( node.argument, parent ); - break; + // `var [ x, y, ...z ] = [ 10, 20, 30, 40, 50 ]` || `var [ ...z ] = []` + case 'RestElement': + declarationPattern( node.argument, parent ); + break; - // `var [ x=5, y=7] = [ 1 ]` - case 'AssignmentPattern': - declarationPattern( node.left, parent ); - break; + // `var [ x=5, y=7] = [ 1 ]` + case 'AssignmentPattern': + declarationPattern( node.left, parent ); + break; - default: - throw new Error( format( 'internal error. Unrecognized pattern type: `%s`.', node.type ) ); + default: + throw new Error( format( 'internal error. Unrecognized pattern type: `%s`.', node.type ) ); + } } -} -/** -* Callback invoked upon encountering a `VariableDeclaration` AST node. -* -* @private -* @param {Node} node - AST node -* @param {Array} parents - array of parent AST nodes -*/ -function VariableDeclaration( node, parents ) { - var parent; - var i; + /** + * Callback invoked upon encountering a `VariableDeclaration` AST node. + * + * @private + * @param {Node} node - AST node + * @param {Array} parents - array of parent AST nodes + */ + function VariableDeclaration( node, parents ) { + var parent; + var i; + + // Case: `var x` + if ( node.kind === 'var' ) { + for ( i = parents.length-1; i >= 0; i-- ) { + if ( isScope( parents[ i ] ) ) { + parent = parents[ i ]; + break; + } + } + } + // Case: `let x` || `const x` + else { + for ( i = parents.length-1; i >= 0; i-- ) { + if ( isBlockScope( parents[ i ] ) ) { + parent = parents[ i ]; + break; + } + } + } + parent.locals = parent.locals || []; + for ( i = 0; i < node.declarations.length; i++ ) { + declarationPattern( node.declarations[ i ].id, parent ); + } + } - // Case: `var x` - if ( node.kind === 'var' ) { - for ( i = parents.length-1; i >= 0; i-- ) { + /** + * Callback invoked upon encountering a `FunctionDeclaration` AST node. + * + * @private + * @param {Node} node - AST node + * @param {Array} parents - array of parent AST nodes + */ + function FunctionDeclaration( node, parents ) { + var parent; + var i; + for ( i = parents.length-2; i >= 0; i-- ) { if ( isScope( parents[ i ] ) ) { parent = parents[ i ]; break; } } + parent.locals = parent.locals || []; + if ( node.id ) { + appendUnique( parent.locals, node.id.name ); + appendUnique( declarations, node.id ); + } + declareFunction( node ); } - // Case: `let x` || `const x` - else { - for ( i = parents.length-1; i >= 0; i-- ) { + + /** + * Callback invoked upon encountering a `ClassDeclaration` AST node. + * + * @private + * @param {Node} node - AST node + * @param {Array} parents - array of parent AST nodes + */ + function ClassDeclaration( node, parents ) { + var parent; + var i; + for ( i = parents.length-2; i >= 0; i-- ) { if ( isBlockScope( parents[ i ] ) ) { parent = parents[ i ]; break; } } - } - parent.locals = parent.locals || []; - for ( i = 0; i < node.declarations.length; i++ ) { - declarationPattern( node.declarations[ i ].id, parent ); - } -} - -/** -* Callback invoked upon encountering a `FunctionDeclaration` AST node. -* -* @private -* @param {Node} node - AST node -* @param {Array} parents - array of parent AST nodes -*/ -function FunctionDeclaration( node, parents ) { - var parent; - var i; - for ( i = parents.length-2; i >= 0; i-- ) { - if ( isScope( parents[ i ] ) ) { - parent = parents[ i ]; - break; + parent.locals = parent.locals || []; + if ( node.id ) { + appendUnique( parent.locals, node.id.name ); + appendUnique( declarations, node.id ); } } - parent.locals = parent.locals || []; - if ( node.id ) { - appendUnique( parent.locals, node.id.name ); - } - declareFunction( node ); -} -/** -* Callback invoked upon encountering a `ClassDeclaration` AST node. -* -* @private -* @param {Node} node - AST node -* @param {Array} parents - array of parent AST nodes -*/ -function ClassDeclaration( node, parents ) { - var parent; - var i; - for ( i = parents.length-2; i >= 0; i-- ) { - if ( isBlockScope( parents[ i ] ) ) { - parent = parents[ i ]; - break; + /** + * Callback invoked upon encountering a `TryStatement` AST node. + * + * @private + * @param {Node} node - AST node + * @param {Array} parents - array of parent AST nodes + */ + function TryStatement( node ) { + if ( node.handler ) { + node.handler.locals = node.handler.locals || []; + appendUnique( node.handler.locals, node.handler.param.name ); + appendUnique( declarations, node.handler.param ); } } - parent.locals = parent.locals || []; - if ( node.id ) { - appendUnique( parent.locals, node.id.name ); - } -} - -/** -* Callback invoked upon encountering a `TryStatement` AST node. -* -* @private -* @param {Node} node - AST node -* @param {Array} parents - array of parent AST nodes -*/ -function TryStatement( node ) { - if ( node.handler ) { - node.handler.locals = node.handler.locals || []; - appendUnique( node.handler.locals, node.handler.param.name ); - } -} -/** -* Callback invoked upon encountering a module `import` specifier AST node. -* -* @private -* @param {Node} node - AST node -* @param {Array} parents - array of parent AST nodes -*/ -function ModuleImportSpecifier( node, parents ) { - parents[ 0 ].locals = parents[ 0 ].locals || []; - appendUnique( parents[ 0 ].locals, node.local.name ); -} - - -// MAIN // - -/** -* Resolves local scopes within an abstract syntax tree (AST). -* -* ## Notes -* -* - This function modifies the provided AST by adding a `locals` property to select AST nodes. -* -* @private -* @param {AST} ast - abstract syntax tree (AST) -* @throws {TypeError} must provide a program AST node -* @returns {AST} input AST -*/ -function resolveScopes( ast ) { - if ( ast.type !== 'Program' ) { - throw new TypeError( 'invalid argument. Must provide a program AST node.' ); + /** + * Callback invoked upon encountering a module `import` specifier AST node. + * + * @private + * @param {Node} node - AST node + * @param {Array} parents - array of parent AST nodes + */ + function ModuleImportSpecifier( node, parents ) { + parents[ 0 ].locals = parents[ 0 ].locals || []; + appendUnique( parents[ 0 ].locals, node.local.name ); + appendUnique( declarations, node.local ); } - ast.locals = []; - walk( ast, VISITORS ); - return ast; } diff --git a/lib/node_modules/@stdlib/repl/lib/syntax_highlighter.js b/lib/node_modules/@stdlib/repl/lib/syntax_highlighter.js new file mode 100644 index 00000000000..4cba7313d55 --- /dev/null +++ b/lib/node_modules/@stdlib/repl/lib/syntax_highlighter.js @@ -0,0 +1,168 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2024 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/* eslint-disable no-restricted-syntax, no-invalid-this, no-underscore-dangle */ + +'use strict'; + +// MODULES // + +var readline = require( 'readline' ); +var logger = require( 'debug' ); +var setNonEnumerableReadOnly = require( '@stdlib/utils/define-nonenumerable-read-only-property' ); +var tokenizer = require( './tokenizer.js' ); +var THEMES = require( './themes.js' ); +var ANSI = require( './ansi_colors.js' ); + + +// VARIABLES // + +var debug = logger( 'repl:syntax-highlighter' ); + + +// FUNCTIONS // + +/** +* Compare function for sorting tokens in ascending order of their indices. +* +* @private +* @param {Object} a - first token +* @param {Object} b - second token +* @returns {boolean} boolean indicating if the first token is greater +*/ +function tokenComparator( a, b ) { + return a.start - b.start; +} + + +// MAIN // + +/** +* Constructor for creating a syntax-highlighter. +* +* @private +* @constructor +* @param {REPL} repl - REPL instance +* @param {WritableStream} ostream - writable stream +* @returns {SyntaxHighlighter} syntax-highlighter instance +*/ +function SyntaxHighlighter( repl, ostream ) { + if ( !( this instanceof SyntaxHighlighter ) ) { + return new SyntaxHighlighter( repl, ostream ); + } + debug( 'Creating a new syntax-highlighter' ); + + // Cache a reference to the provided REPL instance: + this._repl = repl; + + // Cache a reference to the provided readline interface: + this._rli = repl._rli; + + // Cache a reference to the output writable stream: + this._ostream = ostream; + + // Initialize a buffer containing the current line to validate line changes: + this._line = ''; + + return this; +} + +/** +* Highlights the input line. +* +* @private +* @name _highlightLine +* @memberof SyntaxHighlighter.prototype +* @type {Function} +* @param {str} line - input line +* @param {Array} tokens - array of tokens in the line +* @returns {str} highlighted line +*/ +setNonEnumerableReadOnly( SyntaxHighlighter.prototype, '_highlightLine', function highlightLine( line, tokens ) { + var highlightedLine = ''; + var resetCode = ANSI[ 'reset' ]; + var colorCode; + var colors = THEMES[ 'default' ]; + var offset = 0; + var token; + var i; + + // Sort and traverse the tokens... + tokens.sort( tokenComparator ); + for ( i = 0; i < tokens.length; i++ ) { + token = tokens[ i ]; + colorCode = ANSI[ colors[ token.type ] ]; + + // Highlight token if it's color exists in the theme... + if ( colorCode ) { + highlightedLine += line.slice( offset, token.start ); // add text before token + highlightedLine += colorCode; // insert colorCode + highlightedLine += line.slice( token.start, token.end ); // add token + highlightedLine += resetCode; // reset color + offset = token.end; + } + } + highlightedLine += line.slice( offset ); // add remaining text + + return highlightedLine; +}); + +/** +* Callback for handling a "keypress" event. +* +* @name onKeypress +* @memberof SyntaxHighlighter.prototype +* @type {Function} +* @param {string} data - input data +* @param {(Object|void)} key - key object +* @returns {void} +*/ +setNonEnumerableReadOnly( SyntaxHighlighter.prototype, 'onKeypress', function onKeypress() { + var highlightedLine; + var tokens; + + if ( !this._rli.line || this._line === this._rli.line ) { + debug( 'Empty line or no change detected. Skipping highlighting...' ); + return; + } + this._line = this._rli.line; + + // Tokenize: + debug( 'Tokenizing line: %s', this._line ); + tokens = tokenizer( this._line, this._repl._context ); + if ( !tokens ) { + debug( 'No tokens found. Skipping highlighting...' ); + return; + } + + // Highlight: + debug( '%d tokens found. Highlighting...', tokens.length ); + highlightedLine = this._highlightLine( this._line, tokens ); + + // Replace: + debug( 'Replacing current line with the highlighted line...' ); + readline.moveCursor( this._ostream, -1 * this._rli.cursor, 0 ); + readline.clearLine( this._ostream, 1 ); + this._ostream.write( highlightedLine ); + readline.moveCursor( this._ostream, this._rli.cursor - this._line.length, 0 ); // eslint-disable-line max-len +}); + + +// EXPORTS // + +module.exports = SyntaxHighlighter; diff --git a/lib/node_modules/@stdlib/repl/lib/themes.js b/lib/node_modules/@stdlib/repl/lib/themes.js new file mode 100644 index 00000000000..5840be5cb19 --- /dev/null +++ b/lib/node_modules/@stdlib/repl/lib/themes.js @@ -0,0 +1,60 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2024 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +// MAIN // + +/** +* Object containing themes where each theme is an table mapping tokens to their respective colors. +* +* @private +* @name THEMES +* @type {Object} +*/ +var THEMES = { + 'default': { + // Keywords: + 'control': 'magenta', + 'keyword': 'blue', + 'specialIdentifier': 'cyan', + + // Literals: + 'string': 'brightYellow', + 'number': 'brightGreen', + 'literal': 'brightBlue', + 'regexp': 'red', + + // Identifiers: + 'command': 'brightMagenta', + 'function': 'yellow', + 'object': 'brightCyan', + 'variable': null, + 'name': null, + + // Others: + 'comment': 'brightBlack', + 'punctuation': null, + 'operator': null + } +}; + + +// EXPORTS // + +module.exports = THEMES; diff --git a/lib/node_modules/@stdlib/repl/lib/tokenizer.js b/lib/node_modules/@stdlib/repl/lib/tokenizer.js new file mode 100644 index 00000000000..65c9fb85212 --- /dev/null +++ b/lib/node_modules/@stdlib/repl/lib/tokenizer.js @@ -0,0 +1,359 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2024 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/* eslint-disable max-lines-per-function */ + +'use strict'; + +// MODULES // + +var parse = require( 'acorn-loose' ).parse; +var walk = require( 'acorn-walk' ); +var linkedList = require( '@stdlib/utils/linked-list' ); +var contains = require( '@stdlib/array/base/assert/contains' ); +var resolveLocalScopes = require( './resolve_local_scopes.js' ); +var resolveLocalScope = require( './resolve_local_scope.js' ); +var commands = require( './commands.js' ); + + +// VARIABLES // + +var COMMANDS = commands(); +var isControlKeyword = contains.factory( [ 'if', 'else', 'switch', 'case', 'catch', 'finally', 'try', 'return', 'break', 'continue' ] ); +var isSpecialIdentifier = contains.factory( [ 'this', 'super' ] ); +var isReservedLiteral = contains.factory( [ 'null', 'true', 'false' ] ); +var isUnrecognizedKeyword = contains.factory( [ 'async', 'await', 'let' ] ); +var isStringTokenType = contains.factory( [ 'string', 'template' ] ); +var isLiteralType = contains.factory( [ 'string', 'boolean', 'number' ] ); +var isReservedName = contains.factory( [ 'undefined', 'async', 'await', 'let' ] ); +var RE_PUNCTUATION = /(\{|\}|\[|\]|\(|\)|,|;|:|\.|\?|\?\.|=>|\.\.\.|`|\${)/; + + +// MAIN // + +/** +* Tokenizes the input line based on ECMAScript specification. +* +* @private +* @name tokenizer +* @type {Function} +* @param {string} line - input line +* @param {Object} context - REPL context +* @returns {Array} array of tokens +*/ +function tokenizer( line, context ) { + var declarations; + var VISITORS; + var tokens = []; + var ast; + var i; + + // Parse the given line into tokens & comments... + ast = parse( line, { + 'ecmaVersion': 'latest', + 'onToken': onToken, + 'onComment': onComment + }); + + // Resolve variable declarations from the given line as tokens... + declarations = resolveLocalScopes( ast ); + for ( i = 0; i < declarations.length; i++ ) { + // If declaration cannot be resolved as an identifier, push it as a `name` token: + if ( !resolveIdentifier( declarations[ i ] ) ) { + tokens.push({ + 'value': declarations[ i ].name, + 'type': 'name', + 'start': declarations[ i ].start, + 'end': declarations[ i ].end + }); + } + } + + // Resolve identifiers from the given line as tokens... + VISITORS = { + 'Identifier': resolveIdentifier, + 'MemberExpression': resolveMemberExpression + }; + walk.simple( ast, VISITORS ); + + return tokens; + + /** + * Callback invoked upon encountering a `Token` when parsing. + * + * @private + * @param {Object} token - token object + */ + function onToken( token ) { + // Ignore placeholders & EOF tokens: + if ( token.start === token.end ) { + return; + } + if ( token.type.isLoop || isControlKeyword( token.type.keyword ) ) { + // Control flow keywords - `for`, `while`, `do`, `if`, `else` etc: + token.type = 'control'; + tokens.push( token ); + return; + } + if ( isSpecialIdentifier( token.type.keyword ) ) { + // Special identifiers - `this`, `super`: + token.type = 'specialIdentifier'; + tokens.push( token ); + return; + } + if ( isReservedLiteral( token.type.keyword ) ) { + // Built-in literals - `true`, `false`, `null`, `undefined`: + token.type = 'literal'; + tokens.push( token ); + return; + } + if ( token.type.keyword ) { + // Keywords - `function`, `import`, `var`, `const`, `let` etc.: + token.type = 'keyword'; + tokens.push( token ); + return; + } + if ( isStringTokenType( token.type.label ) ) { + // Strings and template string literals: + token.type = 'string'; + tokens.push( token ); + return; + } + if ( token.type.label === 'regexp' ) { + // Regex expressions: + token.type = 'regexp'; + tokens.push( token ); + return; + } + if ( token.type.label === 'num' ) { + // Numeric literals: + token.type = 'number'; + tokens.push( token ); + return; + } + if ( token.type.binop || token.type.prefix || token.type.postfix || token.type.isAssign ) { // eslint-disable-line max-len + // Operator symbols - `+`, `=`, `++` etc: + token.type = 'operator'; + tokens.push( token ); + return; + } + if ( RE_PUNCTUATION.test( token.type.label ) ) { + // Punctuation symbols - `,`, `(`, `;` etc: + token.value = token.type.label; + token.type = 'punctuation'; + tokens.push( token ); + } + } + + /** + * Callback invoked upon encountering a `Comment` when parsing. + * + * @private + * @param {boolean} block - boolean indicating whether a comment is a block comment + * @param {string} text - comment value + * @param {number} start - start index + * @param {number} end - end index + */ + function onComment( block, text, start, end ) { + if ( block ) { + // TODO: add support for highlighting multi-line comments. + return; + } + tokens.push({ + 'type': 'comment', + 'start': start, + 'end': end + }); + } + + /** + * Resolves an `Identifier` node. + * + * @private + * @param {Object} node - AST node + * @returns {boolean} boolean indicating whether the `Identifier` was resolved + */ + function resolveIdentifier( node ) { + var identifier; + var command; + var i; + + // Ignore placeholder nodes: + if ( node.start === node.end ) { + return true; + } + // If node is an unrecognized `literal`, push it as a token: + if ( node.name === 'undefined' ) { + tokens.push({ + 'value': node.name, + 'type': 'literal', + 'start': node.start, + 'end': node.end + }); + return true; + } + // If node is an unrecognized `keyword`, push it as a token: + if ( isUnrecognizedKeyword( node.name ) ) { + tokens.push({ + 'value': node.name, + 'type': 'keyword', + 'start': node.start, + 'end': node.end + }); + return true; + } + // If identifier is defined in the local scope, assume and treat it like a `variable` and push it as a token... + if ( contains( resolveLocalScope( ast, node ), node.name ) ) { + tokens.push({ + 'value': node.name, + 'type': 'variable', + 'start': node.start, + 'end': node.end + }); + return true; + } + // If identifier is a REPL command, push it as a token... + for ( i = 0; i < COMMANDS.length; i++ ) { + command = COMMANDS[ i ]; + if ( node.name === command[ 0 ] ) { + tokens.push( { + 'value': node.name, + 'type': 'command', + 'start': node.start, + 'end': node.end + }); + return true; + } + } + // If identifier is in global context, push it as a token... + identifier = context[ node.name ]; + if ( identifier ) { + if ( isLiteralType( typeof identifier ) ) { + tokens.push( { + 'value': node.name, + 'type': 'variable', + 'start': node.start, + 'end': node.end + }); + } else { + tokens.push( { + 'value': node.name, + 'type': typeof identifier, + 'start': node.start, + 'end': node.end + }); + } + return true; + } + return false; + } + + /** + * Resolves a `MemberExpression` node. + * + * @private + * @param {Object} node - AST node + */ + function resolveMemberExpression( node ) { + var properties = linkedList(); + var property; + var locals; + var obj; + + // Ignore placeholder nodes: + if ( node.start === node.end ) { + return; + } + + // Resolve members of the `MemberExpression`: + resolveMembers( node ); + + // Check if the object identifier is valid... + property = properties.first(); + if ( !property ) { + return; + } + // Ignore if the object identifier is a reserved name: + if ( isReservedName( property.value.name ) ) { + return; + } + // If object identifier exists in the local scope, don't resolve from global scope... + locals = resolveLocalScope( ast, property.value ); + if ( contains( locals, property.value.name ) ) { + return; + } + + // Enter object's namespace: + obj = context; + properties = properties.iterator(); + obj = obj[ properties.next().value.name ]; + if ( !obj ) { + // Object not defined in context: + return; + } + // Fetch properties from context: + property = properties.next(); + while ( !property.done ) { + obj = obj[ property.value.name ]; + if ( !obj ) { + // Property not found in context: + break; + } + // Push token if property exists in context: + if ( isLiteralType( typeof obj ) ) { + tokens.push({ + 'value': property.value.name, + 'type': 'variable', + 'start': property.value.start, + 'end': property.value.end + }); + } else { + tokens.push({ + 'value': property.value.name, + 'type': typeof obj, + 'start': property.value.start, + 'end': property.value.end + }); + } + property = properties.next(); + } + + /** + * Resolves members of the `MemberExpression` node. + * + * @private + * @param {Object} node - node to be traversed + */ + function resolveMembers( node ) { + if ( node.object.type === 'Identifier' ) { + // Reached a resolvable MemberExpression (a.b in a.b.c.d), save property(b) & object(a) and exit: + properties.unshift( node.property ); + properties.unshift( node.object ); + } else if ( node.object.type === 'MemberExpression' ) { + // Found node with another MemberExpression as object, save property and traverse it: + properties.unshift( node.property ); + resolveMembers( node.object ); + } + } + } +} + + +// EXPORTS // + +module.exports = tokenizer;