Skip to content

Commit

Permalink
Move to the scanner
Browse files Browse the repository at this point in the history
  • Loading branch information
@jakebailey
jakebailey committed Feb 22, 2023
1 parent 481a7dd commit 170042c
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 17 deletions.
25 changes: 11 additions & 14 deletions src/compiler/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1532,8 +1532,6 @@ namespace Parser {
// Note: any errors at the end of the file that do not precede a regular node, should get
// attached to the EOF token.
var parseErrorBeforeNextFinishedNode = false;

var skipJSDoc = false;
/* eslint-enable no-var */

export function parseSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, syntaxCursor: IncrementalParser.SyntaxCursor | undefined, setParentNodes = false, scriptKind?: ScriptKind, setExternalModuleIndicatorOverride?: (file: SourceFile) => void, skipJSDoc = false): SourceFile {
Expand Down Expand Up @@ -1672,7 +1670,6 @@ namespace Parser {
syntaxCursor = _syntaxCursor;
scriptKind = _scriptKind;
languageVariant = getLanguageVariant(_scriptKind);
skipJSDoc = _skipJSDoc;

parseDiagnostics = [];
parsingContext = 0;
Expand Down Expand Up @@ -1701,13 +1698,15 @@ namespace Parser {
scanner.setOnError(scanError);
scanner.setScriptTarget(languageVersion);
scanner.setLanguageVariant(languageVariant);
scanner.setSkipJSDoc(_skipJSDoc && !!(contextFlags & NodeFlags.JavaScriptFile));
}

function clearState() {
// Clear out the text the scanner is pointing at, so it doesn't keep anything alive unnecessarily.
scanner.clearCommentDirectives();
scanner.setText("");
scanner.setOnError(undefined);
scanner.setSkipJSDoc(false);

// Clear any data. We don't want to accidentally hold onto it for too long.
sourceText = undefined!;
Expand All @@ -1722,7 +1721,6 @@ namespace Parser {
identifiers = undefined!;
notParenthesizedArrow = undefined;
topLevel = true;
skipJSDoc = false;
}

function parseSourceFileWorker(languageVersion: ScriptTarget, setParentNodes: boolean, scriptKind: ScriptKind, setExternalModuleIndicator: (file: SourceFile) => void): SourceFile {
Expand All @@ -1739,6 +1737,8 @@ namespace Parser {
const statements = parseList(ParsingContext.SourceElements, parseStatement);
Debug.assert(token() === SyntaxKind.EndOfFileToken);
const endOfFileToken = addJSDocComment(parseTokenNode<EndOfFileToken>());
// TODO(jakebailey): this should really be the following, but why isn't the flag set?
// const endOfFileToken = withJSDoc(parseTokenNode<EndOfFileToken>(), hasPrecedingJSDocComment());

const sourceFile = createSourceFile(fileName, languageVersion, scriptKind, isDeclarationFile, statements, endOfFileToken, sourceFlags, setExternalModuleIndicator);

Expand Down Expand Up @@ -1770,18 +1770,15 @@ namespace Parser {
return hasJSDoc ? addJSDocComment(node) : node;
}

const seeLink = /@(?:see|link)/;
function shouldParseJSDoc<T extends HasJSDoc>(node: T, comment: ts.CommentRange) {
if (!skipJSDoc) return true;
if (node.flags & NodeFlags.JavaScriptFile) return true;
if (seeLink.test(sourceText.slice(comment.pos, comment.end))) return true;
return undefined;
}

let hasDeprecatedTag = false;
/**
* Adds a JSDoc comment to a node, if any exist.
*
* Avoid calling this directly; use {@see withJSDoc} instead.
*/
function addJSDocComment<T extends HasJSDoc>(node: T): T {
Debug.assert(!node.jsDoc); // Should only be called once per node
const jsDoc = mapDefined(getJSDocCommentRanges(node, sourceText), comment => shouldParseJSDoc(node, comment) && JSDocParser.parseJSDocComment(node, comment.pos, comment.end - comment.pos));
const jsDoc = mapDefined(getJSDocCommentRanges(node, sourceText), comment => JSDocParser.parseJSDocComment(node, comment.pos, comment.end - comment.pos));
if (jsDoc.length) node.jsDoc = jsDoc;
if (hasDeprecatedTag) {
hasDeprecatedTag = false;
Expand Down Expand Up @@ -5072,7 +5069,7 @@ namespace Parser {
const equalsGreaterThanToken = parseExpectedToken(SyntaxKind.EqualsGreaterThanToken);
const body = parseArrowFunctionExpressionBody(/*isAsync*/ !!asyncModifier, allowReturnTypeInArrowFunction);
const node = factory.createArrowFunction(asyncModifier, /*typeParameters*/ undefined, parameters, /*type*/ undefined, equalsGreaterThanToken, body);
return addJSDocComment(finishNode(node, pos));
return finishNode(node, pos);
}

function tryParseParenthesizedArrowFunctionExpression(allowReturnTypeInArrowFunction: boolean): Expression | undefined {
Expand Down
22 changes: 19 additions & 3 deletions src/compiler/scanner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ export interface Scanner {
// callback returns something truthy, then the scanner state is not rolled back. The result
// of invoking the callback is returned from this function.
tryScan<T>(callback: () => T): T;
/** @internal */
setSkipJSDoc(skip: boolean): void;
}

/** @internal */
Expand Down Expand Up @@ -325,6 +327,11 @@ const commentDirectiveRegExSingleLine = /^\/\/\/?\s*@(ts-expect-error|ts-ignore)
*/
const commentDirectiveRegExMultiLine = /^(?:\/|\*)*\s*@(ts-expect-error|ts-ignore)/;

/**
* Test for whether a comment contains a JSDoc tag needed by the checker when run in tsc.
*/
const semanticJSDocTagRegEx = /@(?:see|link)/i;

function lookupInUnicodeMap(code: number, map: readonly number[]): boolean {
// Bail out quickly if it couldn't possibly be in the map.
if (code < map[0]) {
Expand Down Expand Up @@ -984,6 +991,8 @@ export function createScanner(languageVersion: ScriptTarget,
var commentDirectives: CommentDirective[] | undefined;
var inJSDocType = 0;

var skipJSDoc = false;

setText(text, start, length);

var scanner: Scanner = {
Expand Down Expand Up @@ -1030,6 +1039,7 @@ export function createScanner(languageVersion: ScriptTarget,
tryScan,
lookAhead,
scanRange,
setSkipJSDoc,
};
/* eslint-enable no-var */

Expand Down Expand Up @@ -1823,9 +1833,7 @@ export function createScanner(languageVersion: ScriptTarget,
// Multi-line comment
if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) {
pos += 2;
if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash) {
tokenFlags |= TokenFlags.PrecedingJSDocComment;
}
const isJSDoc = text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash;

let commentClosed = false;
let lastLineStart = tokenPos;
Expand All @@ -1846,6 +1854,10 @@ export function createScanner(languageVersion: ScriptTarget,
}
}

if (isJSDoc && (!skipJSDoc || semanticJSDocTagRegEx.test(text.slice(tokenPos, pos)))) {
tokenFlags |= TokenFlags.PrecedingJSDocComment;
}

commentDirectives = appendIfCommentDirective(commentDirectives, text.slice(lastLineStart, pos), commentDirectiveRegExMultiLine, lastLineStart);

if (!commentClosed) {
Expand Down Expand Up @@ -2621,6 +2633,10 @@ export function createScanner(languageVersion: ScriptTarget,
languageVariant = variant;
}

function setSkipJSDoc(skip: boolean) {
skipJSDoc = skip;
}

function setTextPos(textPos: number) {
Debug.assert(textPos >= 0);
pos = textPos;
Expand Down

0 comments on commit 170042c

Please sign in to comment.