From b085855bb21c3a82c7ff1656b5f7dc6acdb03726 Mon Sep 17 00:00:00 2001 From: ota-meshi Date: Sat, 9 Jan 2021 22:39:56 +0900 Subject: [PATCH] Improve performance The performance has been improved by eliminating one parses step. And change excludes yaml-unist-parser from the dependencies. --- README.md | 2 +- package.json | 7 +- src/context.ts | 169 ++ src/convert.ts | 1453 ++++++++++------- src/errors.ts | 29 + src/index.ts | 7 +- src/parser.ts | 106 +- src/tags.ts | 136 ++ src/utils.ts | 46 +- src/yaml.ts | 70 + tests/fixtures/parser/ast/flow01-output.json | 4 +- .../parser/ast/pair-in-flow-seq02-input.yaml | 5 + .../parser/ast/pair-in-flow-seq02-output.json | 285 ++++ .../parser/ast/pair-in-flow-seq02-value.json | 8 + .../parser/yaml-test-suite/2LFX-output.json | 12 +- .../parser/yaml-test-suite/6LVF-output.json | 12 +- .../parser/yaml-test-suite/BEC7-output.json | 12 +- tests/src/parser/parser.ts | 155 +- 18 files changed, 1732 insertions(+), 786 deletions(-) create mode 100644 src/context.ts create mode 100644 src/errors.ts create mode 100644 src/tags.ts create mode 100644 src/yaml.ts create mode 100644 tests/fixtures/parser/ast/pair-in-flow-seq02-input.yaml create mode 100644 tests/fixtures/parser/ast/pair-in-flow-seq02-output.json create mode 100644 tests/fixtures/parser/ast/pair-in-flow-seq02-value.json diff --git a/README.md b/README.md index 71e8230..7939805 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A YAML parser that produces output [compatible with ESLint](https://eslint.org/docs/developer-guide/working-with-custom-parsers#all-nodes). -This parser is backed by excellent [yaml](https://github.com/eemeli/yaml) and [yaml-unist-parser](https://github.com/ikatyang/yaml-unist-parser) packages. +*This parser is backed by excellent [yaml](https://github.com/eemeli/yaml) package and it is heavily inspired by [yaml-unist-parser](https://github.com/ikatyang/yaml-unist-parser) package.* [![NPM license](https://img.shields.io/npm/l/yaml-eslint-parser.svg)](https://www.npmjs.com/package/yaml-eslint-parser) [![NPM version](https://img.shields.io/npm/v/yaml-eslint-parser.svg)](https://www.npmjs.com/package/yaml-eslint-parser) diff --git a/package.json b/package.json index acf970e..ba79e72 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "eslint-fix": "npm run lint -- --fix", "test": "mocha --require ts-node/register \"tests/src/**/*.ts\" --reporter dot --timeout 60000", "cover": "nyc --reporter=lcov npm run test", - "debug": "mocha --require ts-node/register --inspect \"tests/src/**/*.ts\" --reporter dot", + "debug": "mocha --require ts-node/register/transpile-only --inspect \"tests/src/**/*.ts\" --reporter dot", "preversion": "npm run lint && npm test", "update-fixtures": "ts-node ./tools/update-fixtures.ts" }, @@ -35,13 +35,14 @@ "homepage": "https://github.com/ota-meshi/yaml-eslint-parser#readme", "dependencies": { "eslint-visitor-keys": "^1.3.0", - "yaml": "^1.10.0", - "yaml-unist-parser": "^1.3.1" + "lodash": "^4.17.20", + "yaml": "^1.10.0" }, "devDependencies": { "@ota-meshi/eslint-plugin": "^0.0.6", "@types/eslint": "^7.2.0", "@types/eslint-visitor-keys": "^1.0.0", + "@types/lodash": "^4.14.167", "@types/mocha": "^7.0.2", "@types/node": "^14.0.13", "@typescript-eslint/eslint-plugin": "^4.9.1", diff --git a/src/context.ts b/src/context.ts new file mode 100644 index 0000000..5fb44a0 --- /dev/null +++ b/src/context.ts @@ -0,0 +1,169 @@ +import type { + Comment, + Locations, + Position, + Range, + Token, + YAMLProgram, +} from "./ast" +import type { ASTNode } from "./yaml" +import lodash from "lodash" +import { traverseNodes } from "./traverse" + +type CSTRangeData = { + start: number + end: number +} +export class Context { + public readonly code: string + + public readonly tokens: Token[] = [] + + public readonly comments: Comment[] = [] + + public hasCR = false + + private readonly locs: LinesAndColumns + + private readonly locsMap = new Map() + + private readonly crs: number[] + + public constructor(origCode: string) { + const len = origCode.length + const lineStartIndices = [0] + const crs: number[] = [] + let code = "" + for (let index = 0; index < len; ) { + const c = origCode[index++] + if (c === "\r") { + const next = origCode[index++] || "" + if (next === "\n") { + code += next + crs.push(index - 2) + } else { + code += `\n${next}` + } + lineStartIndices.push(code.length) + } else { + code += c + if (c === "\n") { + lineStartIndices.push(code.length) + } + } + } + this.code = code + this.locs = new LinesAndColumns(lineStartIndices) + this.hasCR = Boolean(crs.length) + this.crs = crs + } + + public remapCR(ast: YAMLProgram): void { + const cache: Record = {} + const remapIndex = (index: number): number => { + let result = cache[index] + if (result != null) { + return result + } + result = index + for (const cr of this.crs) { + if (cr < result) { + result++ + } else { + break + } + } + return (cache[index] = result) + } + // eslint-disable-next-line func-style -- ignore + const remapRange = (range: [number, number]): [number, number] => { + return [remapIndex(range[0]), remapIndex(range[1])] + } + + traverseNodes(ast, { + enterNode(node) { + node.range = remapRange(node.range) + }, + leaveNode() { + // ignore + }, + }) + for (const token of ast.tokens) { + token.range = remapRange(token.range) + } + for (const comment of ast.comments) { + comment.range = remapRange(comment.range) + } + } + + public getLocFromIndex(index: number): { line: number; column: number } { + let loc = this.locsMap.get(index) + if (!loc) { + loc = this.locs.getLocFromIndex(index) + this.locsMap.set(index, loc) + } + return { + line: loc.line, + column: loc.column, + } + } + + /** + * Get the location information of the given node. + * @param node The node. + */ + public getConvertLocation(node: { range: Range } | ASTNode): Locations { + const [start, end] = node.range! + + return { + range: [start, end], + loc: { + start: this.getLocFromIndex(start), + end: this.getLocFromIndex(end), + }, + } + } + + /** + * Get the location information of the given CSTRange. + * @param node The node. + */ + public getConvertLocationFromCSTRange( + range: CSTRangeData | undefined | null, + ): Locations { + return this.getConvertLocation({ range: [range!.start, range!.end] }) + } + + public addComment(comment: Comment): void { + this.comments.push(comment) + } + + /** + * Add token to tokens + */ + public addToken(type: Token["type"], range: Range): Token { + const token = { + type, + value: this.code.slice(...range), + ...this.getConvertLocation({ range }), + } + this.tokens.push(token) + return token + } +} + +class LinesAndColumns { + private readonly lineStartIndices: number[] + + public constructor(lineStartIndices: number[]) { + this.lineStartIndices = lineStartIndices + } + + public getLocFromIndex(index: number) { + const lineNumber = lodash.sortedLastIndex(this.lineStartIndices, index) + return { + line: lineNumber, + column: index - this.lineStartIndices[lineNumber - 1], + } + } +} diff --git a/src/convert.ts b/src/convert.ts index f9baa0d..c8b977b 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -1,31 +1,3 @@ -import yaml from "yaml" -import type { - Root, - Document, - DocumentHead, - Directive, - DocumentBody, - ContentNode, - Mapping, - MappingItem, - MappingKey, - MappingValue, - Plain, - FlowMapping, - FlowMappingItem, - Sequence, - SequenceItem, - FlowSequence, - FlowSequenceItem, - QuoteDouble, - QuoteSingle, - BlockLiteral, - BlockFolded, - Alias, - Anchor, - Tag, - YamlUnistNode, -} from "yaml-unist-parser" import type { Range, Locations, @@ -51,90 +23,82 @@ import type { YAMLWithMeta, YAMLSequence, } from "./ast" -import { isFalse, isTrue, isNull } from "./utils" +import type { + ASTDocument, + CSTBlankLine, + CSTComment, + CSTDirective, + CSTNode, + ASTContentNode, + ASTBlockMap, + ASTPair, + ASTFlowMap, + ASTBlockSeq, + ASTFlowSeq, + ASTPlainValue, + ASTQuoteDouble, + ASTQuoteSingle, + ASTBlockLiteral, + ASTBlockFolded, + ASTAlias, + CSTSeqItem, +} from "./yaml" +import { Type, PairType } from "./yaml" +import { ParseError } from "./errors" +import type { Context } from "./context" +import { tagResolvers } from "./tags" + +const CHOMPING_MAP = { + CLIP: "clip", + STRIP: "strip", + KEEP: "keep", +} as const /** - * Convert yaml-unist-parser root to YAMLProgram + * Convert yaml root to YAMLProgram */ -export function convertRoot(node: Root, code: string): YAMLProgram { - const comments = node.comments.map((n) => { - const c: Comment = { - type: "Block", - value: n.value, - ...getConvertLocation(n), - } - return c - }) - let stripCommentCode = "" - let startIndex = 0 - for (const comment of comments) { - stripCommentCode += code.slice(startIndex, comment.range[0]) - stripCommentCode += code.slice(...comment.range).replace(/\S/gu, " ") - startIndex = comment.range[1] - } - stripCommentCode += code.slice(startIndex) - - const tokens: Token[] = [] +export function convertRoot( + documents: ASTDocument[], + ctx: Context, +): YAMLProgram { const ast: YAMLProgram = { type: "Program", body: [], - comments, + comments: ctx.comments, sourceType: "module", - tokens, + tokens: ctx.tokens, parent: null, - ...getConvertLocation(node), + ...ctx.getConvertLocation({ range: [0, ctx.code.length] }), } - for (const n of node.children) { - ast.body.push(convertDocument(n, tokens, stripCommentCode, ast)) + let startIndex = 0 + for (const n of documents) { + const doc = convertDocument(n, ctx, ast, startIndex) + ast.body.push(doc) + startIndex = doc.range[1] } - const useRanges = sort(tokens).map((t) => t.range) + const useRanges = sort([...ctx.tokens, ...ctx.comments]).map((t) => t.range) let range = useRanges.shift() - - let line = 1 - let column = 0 - const len = stripCommentCode.length - for (let index = 0; index < len; index++) { - const c = stripCommentCode[index] - if (c === "\n") { - line++ - column = 0 - continue + for (let index = 0; index < ctx.code.length; index++) { + while (range && range[1] <= index) { + range = useRanges.shift() } - if (range) { - while (range && range[1] <= index) { - range = useRanges.shift() - } - if (range && range[0] <= index) { - column++ - continue - } + if (range && range[0] <= index) { + index = range[1] - 1 + continue } - + const c = ctx.code[index] if (isPunctuator(c)) { - addToken( - tokens, - "Punctuator", - { - range: [index, index + 1], - loc: { - start: { - line, - column, - }, - end: { - line, - column: column + 1, - }, - }, - }, - stripCommentCode, - ) + // console.log("*** REM TOKEN ***") + ctx.addToken("Punctuator", [index, index + 1]) + } else if (c.trim()) { + // console.log("*** REM TOKEN ***") + // unknown + ctx.addToken("Identifier", [index, index + 1]) } - - column++ } - sort(tokens) + sort(ctx.comments) + sort(ctx.tokens) return ast /** @@ -149,21 +113,39 @@ export function convertRoot(node: Root, code: string): YAMLProgram { c === "}" || c === "[" || c === "]" || + // c === "?" ) } } /** - * Convert yaml-unist-parser Document to YAMLDocument + * Convert YAML.Document to YAMLDocument */ function convertDocument( - node: Document, - tokens: Token[], - code: string, + node: ASTDocument, + ctx: Context, parent: YAMLProgram, + startIndex: number, ): YAMLDocument { - const loc = getConvertLocation(node) + const cst = node.cstNode! + if (cst.error) { + const range = cst.range || cst.valueRange! + const loc = ctx.getLocFromIndex(range.start) + throw new ParseError( + cst.error.message, + range.start, + loc.line, + loc.column, + ) + } + for (const error of node.errors) { + throw error + } + + const loc = ctx.getConvertLocation({ + range: [skipSpaces(ctx.code, startIndex), node.range![1]], + }) const ast: YAMLDocument = { type: "YAMLDocument", directives: [], @@ -172,158 +154,149 @@ function convertDocument( anchors: {}, ...loc, } - ast.directives.push( - ...convertDocumentHead(node.children[0], tokens, code, ast), - ) - ast.content = convertDocumentBody(node.children[1], tokens, code, ast) + ast.directives.push(...convertDocumentHead(node, ctx, ast)) // Marker - if (code[loc.range[1] - 1] === ".") { - const range: Range = [loc.range[1] - 3, loc.range[1]] - addToken( - tokens, - "Marker", - { - range, - loc: { - start: { - line: loc.loc.end.line, - column: loc.loc.end.column - 3, - }, - end: clone(loc.loc.end), - }, - }, - code, - ) + // @ts-expect-error -- missing types? + const directivesEndMarker = cst.directivesEndMarker + if (directivesEndMarker) { + const range: Range = [ + directivesEndMarker.start, + directivesEndMarker.end, + ] + ctx.addToken("Marker", range) + } + + ast.content = convertDocumentBody(node, ctx, ast) + + // Marker + // @ts-expect-error -- missing types? + const documentEndMarker = cst.documentEndMarker + if (documentEndMarker) { + const range: Range = [documentEndMarker.start, documentEndMarker.end] + const markerToken = ctx.addToken("Marker", range) + ast.range[1] = markerToken.range[1] + ast.loc.end = clone(markerToken.loc.end) } return ast } /** - * Convert yaml-unist-parser DocumentHead to YAMLDirective[] + * Convert YAML.Document.Parsed to YAMLDirective[] */ function* convertDocumentHead( - node: DocumentHead, - tokens: Token[], - code: string, + node: ASTDocument, + ctx: Context, parent: YAMLDocument, ): IterableIterator { - for (const n of node.children) { - yield convertDirective(n, tokens, code, parent) - } - const loc = getConvertLocation(node) - - // Marker - if (code[loc.range[1] - 1] === "-") { - const range: Range = [loc.range[1] - 3, loc.range[1]] - addToken( - tokens, - "Marker", - { - range, - loc: { - start: { - line: loc.loc.end.line, - column: loc.loc.end.column - 3, - }, - end: clone(loc.loc.end), - }, - }, - code, - ) + const cst = node.cstNode! + for (const n of cst.directives) { + if (processComment(n, ctx)) { + yield convertDirective(n, ctx, parent) + } } } /** - * Convert yaml-unist-parser Directive to YAMLDirective + * Convert CSTDirective to YAMLDirective */ function convertDirective( - node: Directive, - tokens: Token[], - code: string, + node: CSTDirective, + ctx: Context, parent: YAMLDocument, ): YAMLDirective { - const loc = getConvertLocation(node) - const value = code.slice(...loc.range) + extractComment(node, ctx) + const loc = ctx.getConvertLocation({ + range: [ + node.range!.start, + lastSkipSpaces(ctx.code, node.range!.start, node.valueRange!.end), + ], + }) + const value = ctx.code.slice(...loc.range) const ast: YAMLDirective = { type: "YAMLDirective", value, parent, ...loc, } - addToken(tokens, "Directive", clone(loc), code) + ctx.addToken("Directive", loc.range) return ast } /** - * Convert yaml-unist-parser DocumentBody to YAMLContent + * Convert Document body to YAMLContent */ function convertDocumentBody( - node: DocumentBody, - tokens: Token[], - code: string, + node: ASTDocument, + ctx: Context, parent: YAMLDocument, ): YAMLContent | YAMLWithMeta | null { - const contentNode = node.children[0] - return contentNode - ? convertContentNode(contentNode, tokens, code, parent, parent) - : null + let ast: YAMLContent | YAMLWithMeta | null = null + for (const content of node.cstNode!.contents) { + if (processComment(content, ctx) && !ast) { + ast = convertContentNode( + node.contents as ASTContentNode, + ctx, + parent, + parent, + ) + } + } + return ast } /** - * Convert yaml-unist-parser ContentNode to YAMLContent + * Convert ContentNode to YAMLContent */ function convertContentNode( - node: ContentNode, - tokens: Token[], - code: string, + node: ASTContentNode, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLContent | YAMLWithMeta { - if (node.type === "mapping") { - return convertMapping(node, tokens, code, parent, doc) + if (node.type === Type.MAP) { + return convertMapping(node, ctx, parent, doc) } - if (node.type === "flowMapping") { - return convertFlowMapping(node, tokens, code, parent, doc) + if (node.type === Type.FLOW_MAP) { + return convertFlowMapping(node, ctx, parent, doc) } - if (node.type === "sequence") { - return convertSequence(node, tokens, code, parent, doc) + if (node.type === Type.SEQ) { + return convertSequence(node, ctx, parent, doc) } - if (node.type === "flowSequence") { - return convertFlowSequence(node, tokens, code, parent, doc) + if (node.type === Type.FLOW_SEQ) { + return convertFlowSequence(node, ctx, parent, doc) } - if (node.type === "plain") { - return convertPlain(node, tokens, code, parent, doc) + if (node.type === Type.PLAIN) { + return convertPlain(node, ctx, parent, doc) } - if (node.type === "quoteDouble") { - return convertQuoteDouble(node, tokens, code, parent, doc) + if (node.type === Type.QUOTE_DOUBLE) { + return convertQuoteDouble(node, ctx, parent, doc) } - if (node.type === "quoteSingle") { - return convertQuoteSingle(node, tokens, code, parent, doc) + if (node.type === Type.QUOTE_SINGLE) { + return convertQuoteSingle(node, ctx, parent, doc) } - if (node.type === "blockLiteral") { - return convertBlockLiteral(node, tokens, code, parent, doc) + if (node.type === Type.BLOCK_LITERAL) { + return convertBlockLiteral(node, ctx, parent, doc) } - if (node.type === "blockFolded") { - return convertBlockFolded(node, tokens, code, parent, doc) + if (node.type === Type.BLOCK_FOLDED) { + return convertBlockFolded(node, ctx, parent, doc) } - if (node.type === "alias") { - return convertAlias(node, tokens, code, parent, doc) + if (node.type === Type.ALIAS) { + return convertAlias(node, ctx, parent, doc) } throw new Error(`Unsupported node: ${(node as any).type}`) } /** - * Convert yaml-unist-parser Mapping to YAMLBlockMapping + * Convert Map to YAMLBlockMapping */ function convertMapping( - node: Mapping, - tokens: Token[], - code: string, + node: ASTBlockMap, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLSequence, doc: YAMLDocument, ): YAMLBlockMapping | YAMLWithMeta { - const loc = getConvertLocation(node) + const loc = ctx.getConvertLocationFromCSTRange(node.cstNode!.valueRange) const ast: YAMLBlockMapping = { type: "YAMLMapping", style: "block", @@ -331,8 +304,17 @@ function convertMapping( parent, ...loc, } - for (const n of node.children) { - ast.pairs.push(convertMappingItem(n, tokens, code, ast, doc)) + const cstPairRanges = processCSTItems(node, ctx) + node.items.forEach((n, index) => { + ast.pairs.push( + convertMappingItem(n, cstPairRanges[index], ctx, ast, doc), + ) + }) + const first = ast.pairs[0] + if (first && ast.range[0] !== first.range[0]) { + // adjust location + ast.range[0] = first.range[0] + ast.loc.start = clone(first.loc.start) } const last = ast.pairs[ast.pairs.length - 1] if (last && ast.range[1] !== last.range[1]) { @@ -340,20 +322,19 @@ function convertMapping( ast.range[1] = last.range[1] ast.loc.end = clone(last.loc.end) } - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser FlowMapping to YAMLFlowMapping + * Convert FlowMap to YAMLFlowMapping */ function convertFlowMapping( - node: FlowMapping, - tokens: Token[], - code: string, + node: ASTFlowMap, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLFlowMapping | YAMLWithMeta { - const loc = getConvertLocation(node) + const loc = ctx.getConvertLocationFromCSTRange(node.cstNode!.valueRange) const ast: YAMLFlowMapping = { type: "YAMLMapping", style: "flow", @@ -361,23 +342,26 @@ function convertFlowMapping( parent, ...loc, } - for (const n of node.children) { - ast.pairs.push(convertMappingItem(n, tokens, code, ast, doc)) - } - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + const cstPairRanges = processCSTItems(node, ctx) + node.items.forEach((n, index) => { + ast.pairs.push( + convertMappingItem(n, cstPairRanges[index], ctx, ast, doc), + ) + }) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser MappingItem to YAMLPair + * Convert Pair to YAMLPair */ function convertMappingItem( - node: MappingItem | FlowMappingItem, - tokens: Token[], - code: string, + node: ASTPair, + cstPairRanges: CSTPairRanges, + ctx: Context, parent: YAMLBlockMapping | YAMLFlowMapping, doc: YAMLDocument, ): YAMLPair { - const loc = getConvertLocation(node) + const loc = ctx.getConvertLocation({ range: cstPairRanges.range }) const ast: YAMLPair = { type: "YAMLPair", key: null, @@ -385,59 +369,71 @@ function convertMappingItem( parent, ...loc, } - ast.key = convertMappingKey(node.children[0], tokens, code, ast, doc) - ast.value = convertMappingValue(node.children[1], tokens, code, ast, doc) - if (ast.value && ast.range[1] !== ast.value.range[1]) { - // adjust location - ast.range[1] = ast.value.range[1] - ast.loc.end = clone(ast.value.loc.end) + ast.key = convertMappingKey(node.key, ctx, ast, doc) + ast.value = convertMappingValue(node.value, ctx, ast, doc) + if (ast.value) { + if (ast.range[1] !== ast.value.range[1]) { + // adjust location + ast.range[1] = ast.value.range[1] + ast.loc.end = clone(ast.value.loc.end) + } + } else if (ast.key) { + if (cstPairRanges.value == null && ast.range[1] !== ast.key.range[1]) { + // adjust location + ast.range[1] = ast.key.range[1] + ast.loc.end = clone(ast.key.loc.end) + } + } + if (ast.key) { + if (ast.key.range[0] < ast.range[0]) { + // adjust location + ast.range[0] = ast.key.range[0] + ast.loc.start = clone(ast.key.loc.start) + } } return ast } /** - * Convert yaml-unist-parser MappingKey to YAMLContent + * Convert MapKey to YAMLContent */ function convertMappingKey( - node: MappingKey, - tokens: Token[], - code: string, + node: ASTContentNode | null, + ctx: Context, parent: YAMLPair, doc: YAMLDocument, ): YAMLContent | YAMLWithMeta | null { - if (node.children.length) { - return convertContentNode(node.children[0], tokens, code, parent, doc) + if (node) { + return convertContentNode(node, ctx, parent, doc) } return null } /** - * Convert yaml-unist-parser MappingValue to YAMLContent + * Convert MapValue to YAMLContent */ function convertMappingValue( - node: MappingValue, - tokens: Token[], - code: string, + node: ASTContentNode | null, + ctx: Context, parent: YAMLPair, doc: YAMLDocument, ): YAMLContent | YAMLWithMeta | null { - if (node.children.length) { - return convertContentNode(node.children[0], tokens, code, parent, doc) + if (node) { + return convertContentNode(node, ctx, parent, doc) } return null } /** - * Convert yaml-unist-parser Sequence to YAMLBlockSequence + * Convert BlockSeq to YAMLBlockSequence */ function convertSequence( - node: Sequence, - tokens: Token[], - code: string, + node: ASTBlockSeq, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLBlockSequence | YAMLWithMeta { - const loc = getConvertLocation(node) + const loc = ctx.getConvertLocationFromCSTRange(node.cstNode!.valueRange) const ast: YAMLBlockSequence = { type: "YAMLSequence", style: "block", @@ -445,29 +441,46 @@ function convertSequence( parent, ...loc, } - for (const n of node.children) { - ast.entries.push(...convertSequenceItem(n, tokens, code, ast, doc)) + const cstSeqItems: CSTSeqItem[] = [] + for (const n of node.cstNode!.items) { + if (n.type === Type.SEQ_ITEM) { + ctx.addToken("Punctuator", [n.range!.start, n.range!.start + 1]) + extractComment(n, ctx) + cstSeqItems.push(n) + continue + } + processComment(n, ctx) } + node.items.forEach((n, index) => { + ast.entries.push( + ...convertSequenceItem( + n as ASTContentNode | ASTPair | null, + cstSeqItems[index], + ctx, + ast, + doc, + ), + ) + }) const last = ast.entries[ast.entries.length - 1] if (last && ast.range[1] !== last.range[1]) { // adjust location ast.range[1] = last.range[1] ast.loc.end = clone(last.loc.end) } - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser FlowSequence to YAMLFlowSequence + * Convert FlowSeq to YAMLFlowSequence */ function convertFlowSequence( - node: FlowSequence, - tokens: Token[], - code: string, + node: ASTFlowSeq, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLFlowSequence | YAMLWithMeta { - const loc = getConvertLocation(node) + const loc = ctx.getConvertLocationFromCSTRange(node.cstNode!.valueRange) const ast: YAMLFlowSequence = { type: "YAMLSequence", style: "flow", @@ -475,113 +488,146 @@ function convertFlowSequence( parent, ...loc, } - for (const n of node.children) { - if (n.type === "flowSequenceItem") { - ast.entries.push( - ...convertFlowSequenceItem(n, tokens, code, ast, doc), - ) - } - if (n.type === "flowMappingItem") { + + const cstPairRanges = processCSTItems(node, ctx) + node.items.forEach((n, index) => { + if (n.type === PairType.PAIR || n.type === PairType.MERGE_PAIR) { + const p = n as ASTPair + const cstPairRange = cstPairRanges[index] const map: YAMLBlockMapping = { type: "YAMLMapping", style: "block", pairs: [], parent, - ...getConvertLocation(n), + ...ctx.getConvertLocation({ range: cstPairRange.range }), } - const pair = convertMappingItem(n, tokens, code, map, doc) + const pair = convertMappingItem(p, cstPairRange, ctx, map, doc) map.pairs.push(pair) + if (pair && map.range[1] !== pair.range[1]) { + // adjust location + map.range[1] = pair.range[1] + map.loc.end = clone(pair.loc.end) + } ast.entries.push(map) + } else { + ast.entries.push( + ...convertFlowSequenceItem( + n as ASTContentNode | null, + ctx, + ast, + doc, + ), + ) } - } - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + }) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser SequenceItem to YAMLContent + * Convert SeqItem to YAMLContent */ function* convertSequenceItem( - node: SequenceItem, - tokens: Token[], - code: string, + node: ASTContentNode | ASTPair | null, + cst: CSTSeqItem, + ctx: Context, parent: YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): IterableIterator { - if (node.children.length) { - yield convertContentNode(node.children[0], tokens, code, parent, doc) + if (node) { + if (node.type === PairType.PAIR || node.type === PairType.MERGE_PAIR) { + const cstRange = cst.node!.range! + const range: Range = [cstRange.start, cstRange.end] + const map: YAMLBlockMapping = { + type: "YAMLMapping", + style: "block", + pairs: [], + parent, + ...ctx.getConvertLocation({ range }), + } + // TODO collect : token + const pair = convertMappingItem( + node, + { range } as CSTPairRanges, + ctx, + map, + doc, + ) + map.pairs.push(pair) + if (pair && map.range[1] !== pair.range[1]) { + // adjust location + map.range[1] = pair.range[1] + map.loc.end = clone(pair.loc.end) + } + yield map + } else { + yield convertContentNode(node as ASTContentNode, ctx, parent, doc) + } } else { yield null } } /** - * Convert yaml-unist-parser FlowSequenceItem to YAMLContent + * Convert FlowSeqItem to YAMLContent */ function* convertFlowSequenceItem( - node: FlowSequenceItem, - tokens: Token[], - code: string, + node: ASTContentNode | null, + ctx: Context, parent: YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): IterableIterator { - if (node.children.length) { - yield convertContentNode(node.children[0], tokens, code, parent, doc) + if (node) { + yield convertContentNode(node, ctx, parent, doc) } } /** - * Convert yaml-unist-parser Plain to YAMLPlainScalar + * Convert PlainValue to YAMLPlainScalar */ function convertPlain( - node: Plain, - tokens: Token[], - code: string, + node: ASTPlainValue, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLPlainScalar | YAMLWithMeta { - const loc = getConvertLocation(node) + const valueRange = node.cstNode!.valueRange! + + const loc = ctx.getConvertLocation({ + range: [ + valueRange.start, + lastSkipSpaces(ctx.code, valueRange.start, valueRange.end), + ], + }) if (loc.range[0] < loc.range[1]) { - const strValue = node.value - let value: string | number | boolean | null + const strValue = node.cstNode!.strValue! + const value = parseValueFromText(strValue) - if (isTrue(strValue)) { - value = true - } else if (isFalse(strValue)) { - value = false - } else if (isNull(strValue)) { - value = null - } else if (needParse(strValue)) { - value = yaml.parse(strValue) || strValue - } else { - value = strValue - } const ast: YAMLPlainScalar = { type: "YAMLScalar", style: "plain", strValue, value, - raw: code.slice(...loc.range), + raw: ctx.code.slice(...loc.range), parent, ...loc, } const type = typeof value if (type === "boolean") { - addToken(tokens, "Boolean", clone(loc), code) + ctx.addToken("Boolean", loc.range) } else if (type === "number" && isFinite(Number(value))) { - addToken(tokens, "Numeric", clone(loc), code) + ctx.addToken("Numeric", loc.range) } else if (value === null) { - addToken(tokens, "Null", clone(loc), code) + ctx.addToken("Null", loc.range) } else { - addToken(tokens, "Identifier", clone(loc), code) + ctx.addToken("Identifier", loc.range) } - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, loc) + return convertAnchorAndTag(node, ctx, parent, ast, doc, loc) } return convertAnchorAndTag( node, - tokens, - code, + ctx, parent, null, doc, @@ -589,310 +635,220 @@ function convertPlain( ) /** - * Checks if the given string needs to be parsed + * Parse value from text */ - function needParse(str: string) { - return ( - // oct - /^0o([0-7]+)$/u.test(str) || - // int - /^[-+]?[0-9]+$/u.test(str) || - // hex - /^0x([0-9a-fA-F]+)$/u.test(str) || - // nan - /^(?:[-+]?\.inf|(\.nan))$/iu.test(str) || - // exp - /^[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)[eE][-+]?[0-9]+$/u.test( - str, - ) || - // float - /^[-+]?(?:\.([0-9]+)|[0-9]+\.([0-9]*))$/u.test(str) - // date - // || /^\d{4}-\d{2}-\d{2}/u.test(str) - ) + function parseValueFromText(str: string): string | number | boolean | null { + for (const tagResolver of tagResolvers) { + if (tagResolver.test(str)) { + return tagResolver.resolve(str) + } + } + return str } } /** - * Convert yaml-unist-parser QuoteDouble to YAMLDoubleQuotedScalar + * Convert QuoteDouble to YAMLDoubleQuotedScalar */ function convertQuoteDouble( - node: QuoteDouble, - tokens: Token[], - code: string, + node: ASTQuoteDouble, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLDoubleQuotedScalar | YAMLWithMeta { - const loc = getConvertLocation(node) - const strValue = node.value + const loc = ctx.getConvertLocationFromCSTRange(node.cstNode!.valueRange) + + const cst = node.cstNode! + const strValue = + typeof cst.strValue === "object" && cst.strValue + ? cst.strValue.str + : cst.strValue || "" const ast: YAMLDoubleQuotedScalar = { type: "YAMLScalar", style: "double-quoted", strValue, value: strValue, - raw: code.slice(...loc.range), + raw: ctx.code.slice(...loc.range), parent, ...loc, } - addToken(tokens, "String", clone(loc), code) - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + ctx.addToken("String", loc.range) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser QuoteSingle to YAMLSingleQuotedScalar + * Convert QuoteSingle to YAMLSingleQuotedScalar */ function convertQuoteSingle( - node: QuoteSingle, - tokens: Token[], - code: string, + node: ASTQuoteSingle, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLSingleQuotedScalar | YAMLWithMeta { - const loc = getConvertLocation(node) - const strValue = node.value + const loc = ctx.getConvertLocationFromCSTRange(node.cstNode!.valueRange) + const cst = node.cstNode! + const strValue = + typeof cst.strValue === "object" && cst.strValue + ? cst.strValue.str + : cst.strValue || "" const ast: YAMLSingleQuotedScalar = { type: "YAMLScalar", style: "single-quoted", strValue, value: strValue, - raw: code.slice(...loc.range), + raw: ctx.code.slice(...loc.range), parent, ...loc, } - addToken(tokens, "String", clone(loc), code) - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + ctx.addToken("String", loc.range) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser BlockLiteral to YAMLBlockLiteral + * Convert BlockLiteral to YAMLBlockLiteral */ function convertBlockLiteral( - node: BlockLiteral, - tokens: Token[], - code: string, + node: ASTBlockLiteral, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLBlockLiteralScalar | YAMLWithMeta { - const loc = getConvertLocation(node) - const value = node.value + const cst = node.cstNode! + const loc = ctx.getConvertLocation({ + range: [cst.header.start, cst.valueRange!.end], + }) + const value = cst.strValue || "" + const ast: YAMLBlockLiteralScalar = { type: "YAMLScalar", style: "literal", - chomping: node.chomping, - indent: node.indent, + chomping: CHOMPING_MAP[cst.chomping], + indent: getBlockIndent(node), value, parent, ...loc, } - const text = code.slice(...loc.range) - if (text.startsWith("|")) { - let line = loc.loc.start.line - let column = loc.loc.start.column + 1 - const offset = loc.range[0] - let index = 1 - while (index < text.length) { - const c = text[index] - if (!c.trim()) { - break - } - column++ - index++ - } - const punctuatorLoc: Locations = { - range: [offset, offset + index], - loc: { - start: clone(loc.loc.start), - end: { - line, - column, - }, - }, - } - addToken(tokens, "Punctuator", punctuatorLoc, code) - let lineFeed = false - while (index < text.length) { - const c = text[index] - if (c.trim()) { - break - } - if (c === "\n") { - if (lineFeed) { - break - } - line++ - column = 0 - lineFeed = true - } else { - column++ - } - index++ - } - const tokenLoc: Locations = { - range: [offset + index, loc.range[1]], - loc: { - start: { - line, - column, - }, - end: clone(loc.loc.end), - }, - } - if (tokenLoc.range[0] < tokenLoc.range[1]) { - addToken(tokens, "BlockLiteral", tokenLoc, code) - } - } else { - // ?? - addToken(tokens, "BlockLiteral", clone(loc), code) + const punctuatorRange: Range = [cst.header.start, cst.header.end] + ctx.addToken("Punctuator", punctuatorRange) + const text = ctx.code.slice(cst.valueRange!.start, cst.valueRange!.end) + const offset = /^[^\S\r\n]*/.exec(text)![0].length + const tokenRange: Range = [ + cst.valueRange!.start + offset, + cst.valueRange!.end, + ] + if (tokenRange[0] < tokenRange[1]) { + ctx.addToken("BlockLiteral", tokenRange) } - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser BlockFolded to YAMLBlockFolded + * Convert BlockFolded to YAMLBlockFolded */ function convertBlockFolded( - node: BlockFolded, - tokens: Token[], - code: string, + node: ASTBlockFolded, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLBlockFoldedScalar | YAMLWithMeta { - const loc = getConvertLocation(node) - const value = node.value + const cst = node.cstNode! + const loc = ctx.getConvertLocation({ + range: [cst.header.start, cst.valueRange!.end], + }) + const value = cst.strValue || "" const ast: YAMLBlockFoldedScalar = { type: "YAMLScalar", style: "folded", - chomping: node.chomping, - indent: node.indent, + chomping: CHOMPING_MAP[cst.chomping], + indent: getBlockIndent(node), value, parent, ...loc, } - const text = code.slice(...loc.range) - if (text.startsWith(">")) { - let line = loc.loc.start.line - let column = loc.loc.start.column + 1 - const offset = loc.range[0] - let index = 1 - while (index < text.length) { - const c = text[index] - if (!c.trim()) { - break - } - column++ - index++ - } - const punctuatorLoc: Locations = { - range: [offset, offset + index], - loc: { - start: clone(loc.loc.start), - end: { - line, - column, - }, - }, - } - addToken(tokens, "Punctuator", punctuatorLoc, code) - let lineFeed = false - while (index < text.length) { - const c = text[index] - if (c.trim()) { - break - } - if (c === "\n") { - if (lineFeed) { - break - } - line++ - column = 0 - lineFeed = true - } else { - column++ - } - index++ - } - const tokenLoc: Locations = { - range: [offset + index, loc.range[1]], - loc: { - start: { - line, - column, - }, - end: clone(loc.loc.end), - }, - } - if (tokenLoc.range[0] < tokenLoc.range[1]) { - addToken(tokens, "BlockFolded", tokenLoc, code) - } - } else { - // ?? - addToken(tokens, "BlockFolded", clone(loc), code) + const punctuatorRange: Range = [cst.header.start, cst.header.end] + ctx.addToken("Punctuator", punctuatorRange) + + const text = ctx.code.slice(cst.valueRange!.start, cst.valueRange!.end) + const offset = /^[^\S\r\n]*/.exec(text)![0].length + const tokenRange: Range = [ + cst.valueRange!.start + offset, + cst.valueRange!.end, + ] + if (tokenRange[0] < tokenRange[1]) { + ctx.addToken("BlockFolded", tokenRange) } - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser Alias to YAMLAlias + * Get block indent from given block + */ +function getBlockIndent(node: ASTBlockLiteral | ASTBlockFolded) { + const cst = node.cstNode! + const numLength = cst.header.end - cst.header.start - 1 + return numLength - (cst.chomping === "CLIP" ? 0 : 1) + ? cst.blockIndent + : null +} + +/** + * Convert Alias to YAMLAlias */ function convertAlias( - node: Alias, - tokens: Token[], - code: string, + node: ASTAlias, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLBlockSequence | YAMLFlowSequence, doc: YAMLDocument, ): YAMLAlias | YAMLWithMeta { - const loc = getConvertLocation(node) - const value = node.value + const cst = node.cstNode! + const range = cst.range! + const valueRange = cst.valueRange! + const nodeRange: Range = [range.start, valueRange.end] + + if (range.start === valueRange.start) { + // adjust + nodeRange[0]-- + } + const loc = ctx.getConvertLocation({ + range: nodeRange, + }) const ast: YAMLAlias = { type: "YAMLAlias", - name: value, + name: cst.rawValue, parent, ...loc, } - const text = code.slice(...loc.range) - if (text.startsWith("*")) { - const punctuatorLoc: Locations = { - range: [loc.range[0], loc.range[0] + 1], - loc: { - start: clone(loc.loc.start), - end: { - line: loc.loc.start.line, - column: loc.loc.start.column + 1, - }, - }, - } - addToken(tokens, "Punctuator", punctuatorLoc, code) - const tokenLoc: Locations = { - range: [punctuatorLoc.range[1], loc.range[1]], - loc: { - start: clone(punctuatorLoc.loc.end), - end: clone(loc.loc.end), - }, - } - if (tokenLoc.range[0] < tokenLoc.range[1]) { - addToken(tokens, "Identifier", tokenLoc, code) - } - } else { - // ?? - addToken(tokens, "Identifier", clone(loc), code) + const starIndex = nodeRange[0] + ctx.addToken("Punctuator", [starIndex, starIndex + 1]) + const tokenRange: Range = [valueRange.start, valueRange.end] + if (tokenRange[0] < tokenRange[1]) { + ctx.addToken("Identifier", tokenRange) } - return convertAnchorAndTag(node, tokens, code, parent, ast, doc, ast) + return convertAnchorAndTag(node, ctx, parent, ast, doc, ast) } /** - * Convert yaml-unist-parser Anchor and Tag + * Convert Anchor and Tag */ function convertAnchorAndTag( - node: ContentNode, - tokens: Token[], - code: string, + node: ASTContentNode, + ctx: Context, parent: YAMLDocument | YAMLPair | YAMLSequence, value: V | null, doc: YAMLDocument, valueLoc: Locations, ): YAMLWithMeta | V { - if (node.anchor || node.tag) { - const ast: YAMLWithMeta = { + const cst = node.cstNode! + let meta: YAMLWithMeta | null = null + + /** + * Get YAMLWithMeta + */ + function getMetaAst(): YAMLWithMeta { + if (meta) { + return meta + } + meta = { type: "YAMLWithMeta", anchor: null, tag: null, @@ -902,182 +858,391 @@ function convertAnchorAndTag( loc: clone(valueLoc.loc), } if (value) { - value.parent = ast + value.parent = meta } + return meta + } - if (node.anchor) { - const anchor = convertAnchor(node.anchor, tokens, code, ast, doc) + for (const range of cst.props) { + const startChar = ctx.code[range.start] + if (startChar === "&") { + const ast = getMetaAst() + const anchor = convertAnchor( + [range.start, range.end], + cst.anchor!, + ctx, + ast, + doc, + ) ast.anchor = anchor - ast.range[0] = anchor.range[0] - ast.loc.start = clone(anchor.loc.start) - } - if (node.tag) { - const tag = convertTag(node.tag, tokens, code, ast) + if (anchor.range[0] < ast.range[0]) { + ast.range[0] = anchor.range[0] + ast.loc.start = clone(anchor.loc.start) + } + } else if (startChar === "!") { + const ast = getMetaAst() + const tag = convertTag( + [range.start, range.end], + node.tag!, + ctx, + ast, + ) ast.tag = tag if (tag.range[0] < ast.range[0]) { ast.range[0] = tag.range[0] ast.loc.start = clone(tag.loc.start) } + } else if (startChar === "#") { + const comment: Comment = { + type: "Block", + value: ctx.code.slice(range.start + 1, range.end), + ...ctx.getConvertLocationFromCSTRange(range), + } + ctx.addComment(comment) } - return ast } - - return value as any + return meta || (value as never) } /** - * Convert yaml-unist-parser Anchor to YAMLAnchor + * Convert anchor to YAMLAnchor */ function convertAnchor( - node: Anchor, - tokens: Token[], - code: string, + range: Range, + name: string, + ctx: Context, parent: YAMLWithMeta, doc: YAMLDocument, ): YAMLAnchor { - const loc = getConvertLocation(node) - const value = node.value + const loc = ctx.getConvertLocation({ range }) const ast: YAMLAnchor = { type: "YAMLAnchor", - name: value, + name, parent, ...loc, } - const anchors = doc.anchors[value] || (doc.anchors[value] = []) + const anchors = doc.anchors[name] || (doc.anchors[name] = []) anchors.push(ast) - const text = code.slice(...loc.range) - if (text.startsWith("&")) { - const punctuatorLoc: Locations = { - range: [loc.range[0], loc.range[0] + 1], - loc: { - start: clone(loc.loc.start), - end: { - line: loc.loc.start.line, - column: loc.loc.start.column + 1, - }, - }, - } - addToken(tokens, "Punctuator", punctuatorLoc, code) - const tokenLoc: Locations = { - range: [punctuatorLoc.range[1], loc.range[1]], - loc: { - start: clone(punctuatorLoc.loc.end), - end: clone(loc.loc.end), - }, - } - if (tokenLoc.range[0] < tokenLoc.range[1]) { - addToken(tokens, "Identifier", tokenLoc, code) - } - } else { - // ?? - addToken(tokens, "Identifier", clone(loc), code) + const punctuatorRange: Range = [loc.range[0], loc.range[0] + 1] + ctx.addToken("Punctuator", punctuatorRange) + const tokenRange: Range = [punctuatorRange[1], loc.range[1]] + if (tokenRange[0] < tokenRange[1]) { + ctx.addToken("Identifier", tokenRange) } return ast } /** - * Convert yaml-unist-parser Anchor to YAMLTag + * Convert tag to YAMLTag */ function convertTag( - node: Tag, - tokens: Token[], - code: string, + range: Range, + tag: string, + ctx: Context, parent: YAMLWithMeta, ): YAMLTag { - const loc = getConvertLocation(node) - const value = node.value + const loc = ctx.getConvertLocation({ range }) const ast: YAMLTag = { type: "YAMLTag", - tag: value, + tag, parent, ...loc, } - const text = code.slice(...loc.range) - if (text.startsWith("!")) { - const offset = text.startsWith("!!") ? 2 : 1 - const punctuatorLoc: Locations = { - range: [loc.range[0], loc.range[0] + offset], - loc: { - start: clone(loc.loc.start), - end: { - line: loc.loc.start.line, - column: loc.loc.start.column + offset, - }, - }, - } - addToken(tokens, "Punctuator", punctuatorLoc, code) - const tokenLoc: Locations = { - range: [punctuatorLoc.range[1], loc.range[1]], - loc: { - start: clone(punctuatorLoc.loc.end), - end: clone(loc.loc.end), - }, - } - if (tokenLoc.range[0] < tokenLoc.range[1]) { - addToken(tokens, "Identifier", tokenLoc, code) - } - } else { - // ?? - addToken(tokens, "Identifier", clone(loc), code) + const text = ctx.code.slice(...loc.range) + const offset = text.startsWith("!!") ? 2 : 1 + const punctuatorRange: Range = [loc.range[0], loc.range[0] + offset] + ctx.addToken("Punctuator", punctuatorRange) + const tokenRange: Range = [punctuatorRange[1], loc.range[1]] + if (tokenRange[0] < tokenRange[1]) { + ctx.addToken("Identifier", tokenRange) } return ast } /** - * Get the location information of the given node. - * @param node The node. + * Process comments */ -function getConvertLocation(node: YamlUnistNode): Locations { - const { start, end } = node.position - - return { - range: [start.offset, end.offset], - loc: { - start: { - line: start.line, - column: start.column - 1, - }, - end: { - line: end.line, - column: end.column - 1, - }, - }, +function processComment( + node: CSTBlankLine | CSTComment | N, + ctx: Context, +): node is N { + if (node.type === Type.BLANK_LINE) { + return false } + if (node.type === Type.COMMENT) { + const comment: Comment = { + type: "Block", + value: node.comment, + ...ctx.getConvertLocationFromCSTRange(node.range), + } + ctx.addComment(comment) + return false + } + return true } /** - * clone the location. + * Extract comments from props */ -function clone(loc: T): T { - if (typeof loc !== "object") { - return loc - } - if (Array.isArray(loc)) { - return (loc as any).map(clone) - } - const n: any = {} - for (const key in loc) { - n[key] = clone(loc[key]) +function extractComment(cst: CSTNode, ctx: Context): void { + for (const range of cst.props) { + const startChar = ctx.code[range.start] + if (startChar === "#") { + const comment: Comment = { + type: "Block", + value: ctx.code.slice(range.start + 1, range.end), + ...ctx.getConvertLocationFromCSTRange(range), + } + ctx.addComment(comment) + } } - return n } +type CSTPairRanges = + | { + key: Range + value: Range + range: Range + } + | { + key: null + value: Range + range: Range + } + | { + key: Range + value: null + range: Range + } + /** - * Add token to tokens + * Process CST items */ -function addToken( - tokens: Token[], - type: Token["type"], - loc: Locations, - code: string, -) { - tokens.push({ - type, - value: code.slice(...loc.range), - ...loc, - }) +function processCSTItems( + node: ASTBlockMap | ASTFlowMap | ASTFlowSeq, + ctx: Context, +): CSTPairRanges[] { + const parsed = [...parseCSTItems(node.cstNode!.items)] + + return parsed + + type CSTItem = Required< + ASTBlockMap | ASTFlowMap | ASTFlowSeq + >["cstNode"]["items"][number] + type CSTPairItem = Exclude + + /* eslint-disable complexity -- ignore */ + /** + * Parse for cst items + */ + function* parseCSTItems( + /* eslint-enable complexity -- ignore */ + items: CSTItem[], + ): IterableIterator { + // eslint-disable-next-line no-shadow -- bug? + const enum PairDataState { + empty, + // ? + keyMark, + // KEY + // ? KEY + key, + // KEY : + // ? KEY : + // ? : + // : + valueMark, + } + let data: { + key: CSTPairItem[] + value: CSTPairItem[] + state: PairDataState + } = { + key: [], + value: [], + state: PairDataState.empty, + } + + for (const cstItem of items) { + if ("char" in cstItem) { + ctx.addToken("Punctuator", [cstItem.offset, cstItem.offset + 1]) + if ( + cstItem.char === "[" || + cstItem.char === "]" || + cstItem.char === "{" || + cstItem.char === "}" + ) { + continue + } + if (cstItem.char === ",") { + if (data.state !== PairDataState.empty) { + yield parseGroup(data) + } + data = { key: [], value: [], state: PairDataState.empty } + continue + } + if (cstItem.char === "?") { + if (data.state !== PairDataState.empty) { + yield parseGroup(data) + data = { + key: [cstItem], + value: [], + state: PairDataState.keyMark, + } + } else { + data.key.push(cstItem) + data.state = PairDataState.keyMark + } + continue + } else if (cstItem.char === ":") { + if ( + data.state === PairDataState.empty || + data.state === PairDataState.keyMark || + data.state === PairDataState.key + ) { + data.value.push(cstItem) + data.state = PairDataState.valueMark + } else { + yield parseGroup(data) + data = { + key: [], + value: [cstItem], + state: PairDataState.valueMark, + } + } + continue + } + } else if (!processComment(cstItem, ctx)) { + continue + } else { + if (cstItem.type === Type.MAP_VALUE) { + ctx.addToken("Punctuator", [ + cstItem.range!.start, + cstItem.range!.start + 1, + ]) + extractComment(cstItem, ctx) + if ( + data.state === PairDataState.empty || + data.state === PairDataState.keyMark || + data.state === PairDataState.key + ) { + data.value.push(cstItem) + yield parseGroup(data) + } else { + yield parseGroup(data) + yield parseGroup({ key: [], value: [cstItem] }) + } + data = { key: [], value: [], state: PairDataState.empty } + continue + } else if (cstItem.type === Type.MAP_KEY) { + ctx.addToken("Punctuator", [ + cstItem.range!.start, + cstItem.range!.start + 1, + ]) + extractComment(cstItem, ctx) + if (data.state !== PairDataState.empty) { + yield parseGroup(data) + data = { + key: [cstItem], + value: [], + state: PairDataState.key, + } + } else { + data.key.push(cstItem) + data.state = PairDataState.key + } + continue + } else { + if ( + data.state === PairDataState.empty || + data.state === PairDataState.keyMark + ) { + data.key.push(cstItem) + data.state = PairDataState.key + continue + } + if (data.state === PairDataState.key) { + yield parseGroup(data) + data = { + key: [cstItem], + value: [], + state: PairDataState.key, + } + continue + } + if (data.state === PairDataState.valueMark) { + data.value.push(cstItem) + yield parseGroup(data) + data = { + key: [], + value: [], + state: PairDataState.empty, + } + continue + } + } + } + } + if (data.state !== PairDataState.empty) { + yield parseGroup(data) + } + } + + /** + * Parse for cst item group + */ + function parseGroup(data: { + key: CSTPairItem[] + value: CSTPairItem[] + }): CSTPairRanges { + if (data.key.length && data.value.length) { + const key = itemsToRange(data.key) + const value = itemsToRange(data.value) + return { + key, + value, + range: [key[0], value[1]], + } + } + if (data.key.length) { + const key = itemsToRange(data.key) + return { + key, + value: null, + range: key, + } + } + if (data.value.length) { + const value = itemsToRange(data.value) + return { + key: null, + value, + range: value, + } + } + throw new Error("Unexpected state") + } + + /** get range */ + function itemsToRange(items: CSTPairItem[]): Range { + const first = itemToRange(items[0]) + if (items.length === 1) { + return first + } + const last = itemToRange(items[items.length - 1]) + return [first[0], last[1]] + } + + /** get range */ + function itemToRange(item: CSTPairItem): Range { + if ("char" in item) { + return [item.offset, item.offset + 1] + } + const range = item.range || item.valueRange! + return [range.start, range.end] + } } /** @@ -1100,3 +1265,45 @@ function sort(tokens: (Token | Comment)[]) { return 0 }) } + +/** + * clone the location. + */ +function clone(loc: T): T { + if (typeof loc !== "object") { + return loc + } + if (Array.isArray(loc)) { + return (loc as any).map(clone) + } + const n: any = {} + for (const key in loc) { + n[key] = clone(loc[key]) + } + return n +} + +/** + * Gets the first index with whitespace skipped. + */ +function skipSpaces(str: string, startIndex: number) { + const len = str.length + for (let index = startIndex; index < len; index++) { + if (str[index].trim()) { + return index + } + } + return len +} + +/** + * Gets the last index with whitespace skipped. + */ +function lastSkipSpaces(str: string, startIndex: number, endIndex: number) { + for (let index = endIndex - 1; index >= startIndex; index--) { + if (str[index].trim()) { + return index + 1 + } + } + return startIndex +} diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..8ef298c --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,29 @@ +/** + * YAML parse errors. + */ +export class ParseError extends SyntaxError { + public index: number + + public lineNumber: number + + public column: number + + /** + * Initialize this ParseError instance. + * @param message The error message. + * @param offset The offset number of this error. + * @param line The line number of this error. + * @param column The column number of this error. + */ + public constructor( + message: string, + offset: number, + line: number, + column: number, + ) { + super(message) + this.index = offset + this.lineNumber = line + this.column = column + } +} diff --git a/src/index.ts b/src/index.ts index 11da787..d182d23 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,9 @@ -import { parseForESLint, ParseError } from "./parser" +import { parseForESLint } from "./parser" import type * as AST from "./ast" import { traverseNodes } from "./traverse" import { getStaticYAMLValue } from "./utils" import { KEYS } from "./visitor-keys" +import { ParseError } from "./errors" export { AST, ParseError } @@ -18,6 +19,6 @@ export { traverseNodes, getStaticYAMLValue } /** * Parse YAML source code */ -export function parseYAML(code: string, _options?: any): AST.YAMLProgram { - return parseForESLint(code).ast +export function parseYAML(code: string, options?: any): AST.YAMLProgram { + return parseForESLint(code, options).ast } diff --git a/src/parser.ts b/src/parser.ts index 1746a77..a32d289 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,9 +1,10 @@ -import type { YAMLSyntaxError } from "yaml-unist-parser" -import { parse as parseYaml } from "yaml-unist-parser" +import { parseAllDocuments } from "yaml" import type { SourceCode } from "eslint" import { KEYS } from "./visitor-keys" import { convertRoot } from "./convert" import type { YAMLProgram } from "./ast" +import { ParseError } from "./errors" +import { Context } from "./context" /** * Parse source code */ @@ -16,8 +17,16 @@ export function parseForESLint( services: { isYAML: boolean } } { try { - const rootNode = parseYaml(code) - const ast = convertRoot(rootNode, code) + const ctx = new Context(code) + const docs = parseAllDocuments(ctx.code, { + merge: false, + keepCstNodes: true, + }) + const ast = convertRoot(docs, ctx) + + if (ctx.hasCR) { + ctx.remapCR(ast) + } return { ast, @@ -28,13 +37,28 @@ export function parseForESLint( } } catch (err) { if (isYAMLSyntaxError(err)) { + let message = err.message + const atIndex = message.lastIndexOf(" at") + if (atIndex >= 0) { + message = message.slice(0, atIndex) + } throw new ParseError( - err.message, - err.position.start.offset, - err.position.start.line, - err.position.start.column - 1, + message, + err.range.start, + err.linePos.start.line, + err.linePos.start.col - 1, ) } + if (isYAMLSyntaxErrorForV1(err)) { + const message = err.message + throw new ParseError( + message, + err.source.range.start, + err.source.rangeAsLinePos.start.line, + err.source.rangeAsLinePos.start.col - 1, + ) + } + throw err } } @@ -42,46 +66,42 @@ export function parseForESLint( /** * Type guard for YAMLSyntaxError. */ -function isYAMLSyntaxError(error: any): error is YAMLSyntaxError { +function isYAMLSyntaxError( + error: any, +): error is { + message: string + range: { start: number } + linePos: { start: { line: number; col: number } } +} { return ( - typeof error.position === "object" && - typeof error.position.start === "object" && - typeof error.position.end === "object" && - typeof error.position.start.line === "number" && - typeof error.position.start.column === "number" && - typeof error.position.start.offset === "number" && - typeof error.position.end.line === "number" && - typeof error.position.end.column === "number" && - typeof error.position.end.offset === "number" + error.linePos && + typeof error.linePos === "object" && + error.linePos.start && + typeof error.linePos.start === "object" && + typeof error.linePos.start.line === "number" && + typeof error.linePos.start.col === "number" ) } /** - * YAML parse errors. + * Type guard for YAMLSyntaxError (yaml@1.10). */ -export class ParseError extends SyntaxError { - public index: number - - public lineNumber: number - - public column: number - - /** - * Initialize this ParseError instance. - * @param message The error message. - * @param offset The offset number of this error. - * @param line The line number of this error. - * @param column The column number of this error. - */ - public constructor( - message: string, - offset: number, - line: number, - column: number, - ) { - super(message) - this.index = offset - this.lineNumber = line - this.column = column +function isYAMLSyntaxErrorForV1( + error: any, +): error is { + message: string + source: { + range: { start: number } + rangeAsLinePos: { start: { line: number; col: number } } } +} { + return ( + error.source && + error.source.rangeAsLinePos && + typeof error.source.rangeAsLinePos === "object" && + error.source.rangeAsLinePos.start && + typeof error.source.rangeAsLinePos.start === "object" && + typeof error.source.rangeAsLinePos.start.line === "number" && + typeof error.source.rangeAsLinePos.start.col === "number" + ) } diff --git a/src/tags.ts b/src/tags.ts new file mode 100644 index 0000000..4ca1893 --- /dev/null +++ b/src/tags.ts @@ -0,0 +1,136 @@ +export type TagResolver = { + tag: string + test: (str: string) => boolean + resolve: (str: string) => T +} + +export const NULL: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:null", + test(str) { + return ( + !str || // empty + // see https://yaml.org/spec/1.2/spec.html#id2805071 + str === "null" || + str === "Null" || + str === "NULL" || + str === "~" + ) + }, + resolve() { + return null + }, +} +export const TRUE: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:bool", + test(str) { + // see https://yaml.org/spec/1.2/spec.html#id2805071 + return str === "true" || str === "True" || str === "TRUE" + }, + resolve() { + return true + }, +} +export const FALSE: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:bool", + test(str) { + // see https://yaml.org/spec/1.2/spec.html#id2805071 + return str === "false" || str === "False" || str === "FALSE" + }, + resolve() { + return false + }, +} +export const INT: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:int", + test(str) { + // see https://yaml.org/spec/1.2/spec.html#id2805071 + return /^[-+]?[0-9]+$/u.test(str) + }, + resolve(str) { + return parseInt(str, 10) + }, +} +export const INT_BASE8: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:int", + test(str) { + // see https://yaml.org/spec/1.2/spec.html#id2805071 + return /^0o[0-7]+$/u.test(str) + }, + resolve(str) { + return parseInt(str.slice(2), 8) + }, +} +export const INT_BASE16: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:int", + test(str) { + // see https://yaml.org/spec/1.2/spec.html#id2805071 + return /^0x[0-9a-fA-F]+$/u.test(str) + }, + resolve(str) { + return parseInt(str.slice(2), 16) + }, +} +export const FLOAT: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:float", + test(str) { + // see https://yaml.org/spec/1.2/spec.html#id2805071 + return /^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$/u.test( + str, + ) + }, + resolve(str) { + return parseFloat(str) + }, +} +export const INFINITY: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:float", + test(str) { + // see https://yaml.org/spec/1.2/spec.html#id2805071 + return /^[-+]?(\.inf |\.Inf|\.INF)$/u.test(str) + }, + resolve(str) { + return str.startsWith("-") ? -Infinity : Infinity + }, +} +export const NAN: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:float", + test(str) { + // see https://yaml.org/spec/1.2/spec.html#id2805071 + return str === "NaN" || str === "nan" || str === "NAN" + }, + resolve() { + return NaN + }, +} +export const STR: TagResolver = { + // see https://yaml.org/spec/1.2/spec.html#id2803311 + tag: "tag:yaml.org,2002:str", + test() { + return true + }, + resolve(str) { + return str + }, +} + +export const tagResolvers = [ + NULL, + TRUE, + FALSE, + INT, + INT_BASE8, + INT_BASE16, + FLOAT, + INFINITY, + NAN, + STR, +] diff --git a/src/utils.ts b/src/utils.ts index 4811ac2..797b386 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,4 +1,4 @@ -import yaml from "yaml" +import { parseDocument } from "yaml" import type { YAMLProgram, YAMLContent, @@ -12,6 +12,7 @@ import type { YAMLWithMeta, YAMLTag, } from "./ast" +import { tagResolvers } from "./tags" type YAMLContentValue = | string @@ -172,45 +173,12 @@ function findAnchor(node: YAMLAlias): YAMLAnchor | null { * Get tagged value */ function getTaggedValue(tag: YAMLTag, text: string, str: string) { - if (tag.tag === "tag:yaml.org,2002:str") { - return str - } else if (tag.tag === "tag:yaml.org,2002:int") { - if (/^(?:[1-9]\d*|0)$/u.test(str)) { - return parseInt(str, 10) - } - } else if (tag.tag === "tag:yaml.org,2002:bool") { - if (isTrue(str)) { - return true - } - if (isFalse(str)) { - return false - } - } else if (tag.tag === "tag:yaml.org,2002:null") { - if (isNull(str) || str === "") { - return null + for (const tagResolver of tagResolvers) { + if (tagResolver.tag === tag.tag && tagResolver.test(str)) { + return tagResolver.resolve(str) } } const tagText = tag.tag.startsWith("!") ? tag.tag : `!<${tag.tag}>` - return yaml.parseDocument(`${tagText} ${text}`).toJSON() -} - -/** - * Checks if the given string is true - */ -export function isTrue(str: string): boolean { - return str === "true" || str === "True" || str === "TRUE" -} - -/** - * Checks if the given string is false - */ -export function isFalse(str: string): boolean { - return str === "false" || str === "False" || str === "FALSE" -} - -/** - * Checks if the given string is null - */ -export function isNull(str: string): boolean { - return str === "null" || str === "Null" || str === "NULL" || str === "~" + const value = parseDocument(`${tagText} ${text}`).toJSON() + return value } diff --git a/src/yaml.ts b/src/yaml.ts new file mode 100644 index 0000000..10e1cf1 --- /dev/null +++ b/src/yaml.ts @@ -0,0 +1,70 @@ +import type { CST, AST } from "yaml" +import type YAML from "yaml" +import type { Alias } from "yaml/types" +import { Pair } from "yaml/types" +export { Type } from "yaml/util" +// eslint-disable-next-line @typescript-eslint/naming-convention -- ignore +export const PairType = Pair.Type + +export type CSTDirective = CST.Directive +export type CSTDocument = CST.Document +export type CSTAlias = CST.Alias +export type CSTBlockValue = CST.BlockValue +export type CSTPlainValue = CST.PlainValue +export type CSTQuoteValue = CST.QuoteValue +export type CSTMap = CST.Map +export type CSTSeq = CST.Seq +export type CSTFlowMap = CST.FlowMap +export type CSTFlowSeq = CST.FlowSeq +export type CSTMapKey = CST.MapKey +export type CSTMapValue = CST.MapValue +export type CSTSeqItem = CST.SeqItem + +export type CSTNode = + | CSTDirective + | CSTDocument + | CSTContentNode + | CSTMapItem + | CSTSeqItem + +export type CSTMapItem = CSTMapKey | CSTMapValue + +export type CSTContentNode = + | CSTAlias + | CSTScalar + | CSTMap + | CSTSeq + | CSTFlowMap + | CSTFlowSeq +export type CSTScalar = CSTBlockValue | CSTPlainValue | CSTQuoteValue + +export type CSTBlankLine = CST.BlankLine +export type CSTComment = CST.Comment +export type CSTFlowChar = CST.FlowChar +export type CSTRange = CST.Range + +export type ASTNode = ASTDocument | ASTContentNode +export type ASTDocument = YAML.Document.Parsed +export type ASTContentNode = + | ASTBlockMap + | ASTFlowMap + | ASTBlockSeq + | ASTFlowSeq + | ASTPlainValue + | ASTQuoteDouble + | ASTQuoteSingle + | ASTBlockLiteral + | ASTBlockFolded + | ASTAlias +export type ASTBlockMap = AST.BlockMap +export type ASTFlowMap = AST.FlowMap +export type ASTBlockSeq = AST.BlockSeq +export type ASTFlowSeq = AST.FlowSeq +export type ASTPlainValue = AST.PlainValue +export type ASTQuoteDouble = AST.QuoteDouble +export type ASTQuoteSingle = AST.QuoteSingle +export type ASTBlockLiteral = AST.BlockLiteral +export type ASTBlockFolded = AST.BlockFolded +export type ASTAlias = Alias + +export type ASTPair = Pair diff --git a/tests/fixtures/parser/ast/flow01-output.json b/tests/fixtures/parser/ast/flow01-output.json index 0e4f806..5065029 100644 --- a/tests/fixtures/parser/ast/flow01-output.json +++ b/tests/fixtures/parser/ast/flow01-output.json @@ -132,7 +132,7 @@ "type": "YAMLScalar", "style": "plain", "strValue": "0", - "value": "0", + "value": 0, "raw": "0", "range": [ 18, @@ -581,7 +581,7 @@ } }, { - "type": "Identifier", + "type": "Numeric", "value": "0", "range": [ 18, diff --git a/tests/fixtures/parser/ast/pair-in-flow-seq02-input.yaml b/tests/fixtures/parser/ast/pair-in-flow-seq02-input.yaml new file mode 100644 index 0000000..63ccd3f --- /dev/null +++ b/tests/fixtures/parser/ast/pair-in-flow-seq02-input.yaml @@ -0,0 +1,5 @@ +[ + ? + ? : b +] +... diff --git a/tests/fixtures/parser/ast/pair-in-flow-seq02-output.json b/tests/fixtures/parser/ast/pair-in-flow-seq02-output.json new file mode 100644 index 0000000..f9bf4d5 --- /dev/null +++ b/tests/fixtures/parser/ast/pair-in-flow-seq02-output.json @@ -0,0 +1,285 @@ +{ + "type": "Program", + "body": [ + { + "type": "YAMLDocument", + "directives": [], + "content": { + "type": "YAMLSequence", + "style": "flow", + "entries": [ + { + "type": "YAMLMapping", + "style": "block", + "pairs": [ + { + "type": "YAMLPair", + "key": null, + "value": null, + "range": [ + 4, + 5 + ], + "loc": { + "start": { + "line": 2, + "column": 2 + }, + "end": { + "line": 2, + "column": 3 + } + } + } + ], + "range": [ + 4, + 5 + ], + "loc": { + "start": { + "line": 2, + "column": 2 + }, + "end": { + "line": 2, + "column": 3 + } + } + }, + { + "type": "YAMLMapping", + "style": "block", + "pairs": [ + { + "type": "YAMLPair", + "key": null, + "value": { + "type": "YAMLScalar", + "style": "plain", + "strValue": "b", + "value": "b", + "raw": "b", + "range": [ + 13, + 14 + ], + "loc": { + "start": { + "line": 3, + "column": 6 + }, + "end": { + "line": 3, + "column": 7 + } + } + }, + "range": [ + 9, + 14 + ], + "loc": { + "start": { + "line": 3, + "column": 2 + }, + "end": { + "line": 3, + "column": 7 + } + } + } + ], + "range": [ + 9, + 14 + ], + "loc": { + "start": { + "line": 3, + "column": 2 + }, + "end": { + "line": 3, + "column": 7 + } + } + } + ], + "range": [ + 0, + 16 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 4, + "column": 1 + } + } + }, + "range": [ + 0, + 20 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 5, + "column": 3 + } + } + } + ], + "comments": [], + "sourceType": "module", + "tokens": [ + { + "type": "Punctuator", + "value": "[", + "range": [ + 0, + 1 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 1 + } + } + }, + { + "type": "Punctuator", + "value": "?", + "range": [ + 4, + 5 + ], + "loc": { + "start": { + "line": 2, + "column": 2 + }, + "end": { + "line": 2, + "column": 3 + } + } + }, + { + "type": "Punctuator", + "value": "?", + "range": [ + 9, + 10 + ], + "loc": { + "start": { + "line": 3, + "column": 2 + }, + "end": { + "line": 3, + "column": 3 + } + } + }, + { + "type": "Punctuator", + "value": ":", + "range": [ + 11, + 12 + ], + "loc": { + "start": { + "line": 3, + "column": 4 + }, + "end": { + "line": 3, + "column": 5 + } + } + }, + { + "type": "Identifier", + "value": "b", + "range": [ + 13, + 14 + ], + "loc": { + "start": { + "line": 3, + "column": 6 + }, + "end": { + "line": 3, + "column": 7 + } + } + }, + { + "type": "Punctuator", + "value": "]", + "range": [ + 15, + 16 + ], + "loc": { + "start": { + "line": 4, + "column": 0 + }, + "end": { + "line": 4, + "column": 1 + } + } + }, + { + "type": "Marker", + "value": "...", + "range": [ + 17, + 20 + ], + "loc": { + "start": { + "line": 5, + "column": 0 + }, + "end": { + "line": 5, + "column": 3 + } + } + } + ], + "range": [ + 0, + 21 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 6, + "column": 0 + } + } +} \ No newline at end of file diff --git a/tests/fixtures/parser/ast/pair-in-flow-seq02-value.json b/tests/fixtures/parser/ast/pair-in-flow-seq02-value.json new file mode 100644 index 0000000..f777455 --- /dev/null +++ b/tests/fixtures/parser/ast/pair-in-flow-seq02-value.json @@ -0,0 +1,8 @@ +[ + { + "null": null + }, + { + "null": "b" + } +] \ No newline at end of file diff --git a/tests/fixtures/parser/yaml-test-suite/2LFX-output.json b/tests/fixtures/parser/yaml-test-suite/2LFX-output.json index dfa93d5..28cad01 100644 --- a/tests/fixtures/parser/yaml-test-suite/2LFX-output.json +++ b/tests/fixtures/parser/yaml-test-suite/2LFX-output.json @@ -6,10 +6,10 @@ "directives": [ { "type": "YAMLDirective", - "value": "%FOO bar baz ", + "value": "%FOO bar baz", "range": [ 0, - 33 + 13 ], "loc": { "start": { @@ -18,7 +18,7 @@ }, "end": { "line": 1, - "column": 33 + "column": 13 } } } @@ -102,10 +102,10 @@ "tokens": [ { "type": "Directive", - "value": "%FOO bar baz ", + "value": "%FOO bar baz", "range": [ 0, - 33 + 13 ], "loc": { "start": { @@ -114,7 +114,7 @@ }, "end": { "line": 1, - "column": 33 + "column": 13 } } }, diff --git a/tests/fixtures/parser/yaml-test-suite/6LVF-output.json b/tests/fixtures/parser/yaml-test-suite/6LVF-output.json index de149da..7c4ba19 100644 --- a/tests/fixtures/parser/yaml-test-suite/6LVF-output.json +++ b/tests/fixtures/parser/yaml-test-suite/6LVF-output.json @@ -6,10 +6,10 @@ "directives": [ { "type": "YAMLDirective", - "value": "%FOO bar baz ", + "value": "%FOO bar baz", "range": [ 0, - 33 + 13 ], "loc": { "start": { @@ -18,7 +18,7 @@ }, "end": { "line": 1, - "column": 33 + "column": 13 } } } @@ -102,10 +102,10 @@ "tokens": [ { "type": "Directive", - "value": "%FOO bar baz ", + "value": "%FOO bar baz", "range": [ 0, - 33 + 13 ], "loc": { "start": { @@ -114,7 +114,7 @@ }, "end": { "line": 1, - "column": 33 + "column": 13 } } }, diff --git a/tests/fixtures/parser/yaml-test-suite/BEC7-output.json b/tests/fixtures/parser/yaml-test-suite/BEC7-output.json index 4c2b71f..7e3d699 100644 --- a/tests/fixtures/parser/yaml-test-suite/BEC7-output.json +++ b/tests/fixtures/parser/yaml-test-suite/BEC7-output.json @@ -6,10 +6,10 @@ "directives": [ { "type": "YAMLDirective", - "value": "%YAML 1.3 ", + "value": "%YAML 1.3", "range": [ 0, - 27 + 9 ], "loc": { "start": { @@ -18,7 +18,7 @@ }, "end": { "line": 1, - "column": 27 + "column": 9 } } } @@ -102,10 +102,10 @@ "tokens": [ { "type": "Directive", - "value": "%YAML 1.3 ", + "value": "%YAML 1.3", "range": [ 0, - 27 + 9 ], "loc": { "start": { @@ -114,7 +114,7 @@ }, "end": { "line": 1, - "column": 27 + "column": 9 } } }, diff --git a/tests/src/parser/parser.ts b/tests/src/parser/parser.ts index c990218..2431eb4 100644 --- a/tests/src/parser/parser.ts +++ b/tests/src/parser/parser.ts @@ -31,15 +31,15 @@ function replacer(key: string, value: any) { return value } -function parse(code: string) { - return parseYAML(code) +function parse(code: string, filePath: string) { + return parseYAML(code, { filePath }) } describe("Check for AST.", () => { for (const filename of fs .readdirSync(AST_FIXTURE_ROOT) .filter((f) => f.endsWith("input.yaml"))) { - it(filename, () => { + describe(filename, () => { const inputFileName = path.join(AST_FIXTURE_ROOT, filename) const outputFileName = inputFileName.replace( /input\.yaml$/u, @@ -51,22 +51,38 @@ describe("Check for AST.", () => { ) const input = fs.readFileSync(inputFileName, "utf8") - const ast = parse(input) - const astJson = JSON.stringify(ast, replacer, 2) - const output = fs.readFileSync(outputFileName, "utf8") - assert.strictEqual(astJson, output) + let ast: any - // check tokens - checkTokens(ast, input) + it("most to generate the expected AST.", () => { + ast = parse(input, inputFileName) + const astJson = JSON.stringify(ast, replacer, 2) + const output = fs.readFileSync(outputFileName, "utf8") + assert.strictEqual(astJson, output) + }) - checkLoc(ast, inputFileName, input) + it("location must be correct.", () => { + // check tokens + checkTokens(ast, input) - // check getStaticYAMLValue - const value = fs.readFileSync(valueFileName, "utf8") - assert.strictEqual( - JSON.stringify(getStaticYAMLValue(ast), null, 2), - value, - ) + checkLoc(ast, inputFileName, input) + }) + + it("return value of getStaticYAMLValue must be correct.", () => { + // check getStaticYAMLValue + const value = fs.readFileSync(valueFileName, "utf8") + assert.strictEqual( + JSON.stringify(getStaticYAMLValue(ast), null, 2), + value, + ) + }) + + it("even if Win, it must be correct.", () => { + const inputForWin = input.replace(/\n/g, "\r\n") + // check + const astForWin = parse(inputForWin, inputFileName) + // check tokens + checkTokens(astForWin, inputForWin) + }) }) } }) @@ -75,7 +91,7 @@ describe("yaml-test-suite.", () => { for (const filename of fs .readdirSync(SUITE_FIXTURE_ROOT) .filter((f) => f.endsWith("input.yaml"))) { - it(filename, () => { + describe(filename, () => { const inputFileName = path.join(SUITE_FIXTURE_ROOT, filename) const outputFileName = inputFileName.replace( /input\.yaml$/u, @@ -89,49 +105,70 @@ describe("yaml-test-suite.", () => { const input = fs.readFileSync(inputFileName, "utf8") const output = fs.readFileSync(outputFileName, "utf8") - let ast - try { - ast = parse(input) - } catch (e) { - if ( - typeof e.lineNumber === "number" && - typeof e.column === "number" - ) { - assert.strictEqual( - `${e.message}@line:${e.lineNumber},column:${e.column}`, - output, - ) - return + let ast: any + it("most to generate the expected AST.", () => { + try { + ast = parse(input, inputFileName) + } catch (e) { + if ( + typeof e.lineNumber === "number" && + typeof e.column === "number" + ) { + assert.strictEqual( + `${e.message}@line:${e.lineNumber},column:${e.column}`, + output, + ) + return + } + throw e } - throw e - } - const astJson = JSON.stringify(ast, replacer, 2) - assert.strictEqual(astJson, output) + const astJson = JSON.stringify(ast, replacer, 2) + assert.strictEqual(astJson, output) + }) - // check tokens - checkTokens(ast, input) + it("location must be correct.", () => { + if (!ast) return - // check keys - traverseNodes(ast, { - enterNode(node) { - const allKeys = KEYS[node.type] - for (const key of getKeys(node, {})) { - assert.ok(allKeys.includes(key), `missing '${key}' key`) - } - }, - leaveNode() { - // noop - }, + // check tokens + checkTokens(ast, input) + + // check keys + traverseNodes(ast, { + enterNode(node) { + const allKeys = KEYS[node.type] + for (const key of getKeys(node, {})) { + assert.ok( + allKeys.includes(key), + `missing '${key}' key`, + ) + } + }, + leaveNode() { + // noop + }, + }) + + checkLoc(ast, inputFileName, input) }) - checkLoc(ast, inputFileName, input) + it("return value of getStaticYAMLValue must be correct.", () => { + if (!ast) return + // check getStaticYAMLValue + const value = fs.readFileSync(valueFileName, "utf8") + assert.strictEqual( + JSON.stringify(getStaticYAMLValue(ast), null, 2), + value, + ) + }) - // check getStaticYAMLValue - const value = fs.readFileSync(valueFileName, "utf8") - assert.strictEqual( - JSON.stringify(getStaticYAMLValue(ast), null, 2), - value, - ) + it("even if Win, it must be correct.", () => { + if (!ast) return + const inputForWin = input.replace(/\n/g, "\r\n") + // check + const astForWin = parse(inputForWin, inputFileName) + // check tokens + checkTokens(astForWin, inputForWin) + }) }) } }) @@ -148,6 +185,16 @@ function checkTokens(ast: YAMLProgram, input: string) { .join("") .replace(/\s/gu, ""), ) + + // check loc + for (const token of allTokens) { + const value = token.type === "Block" ? `#${token.value}` : token.value + + assert.strictEqual( + value, + input.slice(...token.range).replace(/\r\n/g, "\n"), + ) + } } function checkLoc(ast: YAMLProgram, fileName: string, code: string) {