From 6e824c377990526582e0f3d984c49fd6a5525f87 Mon Sep 17 00:00:00 2001 From: Aquariuslt Date: Tue, 19 Sep 2017 00:21:27 +0800 Subject: [PATCH] [Lexer] Refactor Tokens definitions. --- src/lexer/index.ts | 1 + src/parser/index.ts | 2 + src/shared/{token.d.ts => token.ts} | 6 ++- src/shared/token.types.ts | 18 +++++++ src/shared/tokens.d.ts | 74 -------------------------- src/shared/tokens.ts | 75 +++++++++++++++++++++++++++ test/unit/specs/lexer-heading.spec.js | 72 +++++++++++++++---------- test/unit/specs/lexer-list.spec.js | 16 ++---- 8 files changed, 148 insertions(+), 116 deletions(-) rename src/shared/{token.d.ts => token.ts} (79%) create mode 100644 src/shared/token.types.ts delete mode 100644 src/shared/tokens.d.ts create mode 100644 src/shared/tokens.ts diff --git a/src/lexer/index.ts b/src/lexer/index.ts index f1e4188..ae17caf 100644 --- a/src/lexer/index.ts +++ b/src/lexer/index.ts @@ -1,4 +1,5 @@ import * as marked from 'marked'; +import {TokensList} from '../shared/token'; export default class Lexer { diff --git a/src/parser/index.ts b/src/parser/index.ts index 044ff8d..e0021a2 100644 --- a/src/parser/index.ts +++ b/src/parser/index.ts @@ -1,3 +1,5 @@ +import {TokensList} from '../shared/token'; + export default class Parser { constructor(src: TokensList, options?: ParserOptions) { diff --git a/src/shared/token.d.ts b/src/shared/token.ts similarity index 79% rename from src/shared/token.d.ts rename to src/shared/token.ts index 3dc6416..565531b 100644 --- a/src/shared/token.d.ts +++ b/src/shared/token.ts @@ -1,4 +1,6 @@ -declare type Token = +import * as Tokens from 'shared/tokens'; + +export type Token = Tokens.Space | Tokens.Code | Tokens.Heading @@ -16,7 +18,7 @@ declare type Token = | Tokens.Text; -declare type TokensList = Token[] & { +export type TokensList = Token[] & { links: { [key: string]: { href: string; title: string; } } diff --git a/src/shared/token.types.ts b/src/shared/token.types.ts new file mode 100644 index 0000000..a4dbde3 --- /dev/null +++ b/src/shared/token.types.ts @@ -0,0 +1,18 @@ +export default class TokenTypes { + SPACE = 'space'; + CODE = 'code'; + HEADING = 'heading'; + TABLE = 'table'; + HR = 'hr'; + BLOCKQUOTE_START = 'blockquote_start'; + BLOCKQUOTE_END = 'blockquote_end'; + LIST_START = 'list_start'; + LOOSE_ITEM_START = 'loose_item_start'; + LIST_ITEM_START = 'list_item_start'; + LIST_ITEM_END = 'list_item_end'; + LIST_END = 'list_end'; + PARAGRAPH = 'paragraph'; + HTML = 'html'; + TEXT = 'text'; + +} diff --git a/src/shared/tokens.d.ts b/src/shared/tokens.d.ts deleted file mode 100644 index 2d0e6fb..0000000 --- a/src/shared/tokens.d.ts +++ /dev/null @@ -1,74 +0,0 @@ -declare namespace Tokens { - interface Space { - type: 'space'; - } - - interface Code { - type: 'code'; - lang?: string; - text: string; - } - - interface Heading { - type: 'heading'; - depth: number; - text: string; - } - - interface Table { - type: 'table'; - header: string[]; - align: Array<'center' | 'left' | 'right' | null>; - cells: string[][]; - } - - interface Hr { - type: 'hr'; - } - - interface BlockquoteStart { - type: 'blockquote_start'; - } - - interface BlockquoteEnd { - type: 'blockquote_end'; - } - - interface ListStart { - type: 'list_start'; - ordered: boolean; - } - - interface LooseItemStart { - type: 'loose_item_start'; - } - - interface ListItemStart { - type: 'list_item_start'; - } - - interface ListItemEnd { - type: 'list_item_end'; - } - - interface ListEnd { - type: 'list_end'; - } - - interface Paragraph { - type: 'paragraph'; - pre?: boolean; - text: string; - } - - interface HTML { - type: 'html'; - pre: boolean; - text: string; - } - - interface Text { - type: 'text'; - text: string; - } -} diff --git a/src/shared/tokens.ts b/src/shared/tokens.ts new file mode 100644 index 0000000..6c15b58 --- /dev/null +++ b/src/shared/tokens.ts @@ -0,0 +1,75 @@ + +export interface Space { + type: 'space'; +} + +export interface Code { + type: 'code'; + lang?: string; + text: string; +} + +export interface Heading { + type: 'heading'; + depth: number; + text: string; +} + +export interface Table { + type: 'table'; + header: string[]; + align: Array<'center' | 'left' | 'right' | null>; + cells: string[][]; +} + +export interface Hr { + type: 'hr'; +} + +export interface BlockquoteStart { + type: 'blockquote_start'; +} + +export interface BlockquoteEnd { + type: 'blockquote_end'; +} + +export interface ListStart { + type: 'list_start'; + ordered: boolean; +} + +export interface LooseItemStart { + type: 'loose_item_start'; +} + +export interface ListItemStart { + type: 'list_item_start'; +} + +export interface ListItemEnd { + type: 'list_item_end'; +} + +export interface ListEnd { + type: 'list_end'; +} + +export interface Paragraph { + type: 'paragraph'; + pre?: boolean; + text: string; +} + +export interface HTML { + type: 'html'; + pre: boolean; + text: string; +} + +export interface Text { + type: 'text'; + text: string; +} + + diff --git a/test/unit/specs/lexer-heading.spec.js b/test/unit/specs/lexer-heading.spec.js index 4d47e77..94718ba 100644 --- a/test/unit/specs/lexer-heading.spec.js +++ b/test/unit/specs/lexer-heading.spec.js @@ -4,16 +4,32 @@ import Lexer from '@/lexer'; /** * Lexer().lex testing * Test Cases from http://wowubuntu.com/markdown/ example + * * */ describe('lexer:headings', () => { + /** + * @example + * ``` + * new Lexer().lex('# Hello Title') + * ``` + * + * @output + * [ + * { + * type:'heading', + * depth:1, + * text: 'Hello Title' + * } + * ] + * */ it('should lex h1 heading tokens', () => { const mdString = '# Hello Title'; let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 1)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(1); }); it('should lex h2 heading tokens', () => { @@ -21,9 +37,9 @@ describe('lexer:headings', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 2)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(2); }); it('should lex h3 heading tokens', () => { @@ -31,9 +47,9 @@ describe('lexer:headings', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 3)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(3); }); it('should lex h4 heading tokens', () => { @@ -41,9 +57,9 @@ describe('lexer:headings', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 4)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(4); }); it('should lex h5 heading tokens', () => { @@ -51,9 +67,9 @@ describe('lexer:headings', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 5)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(5); }); it('should lex h6 heading tokens', () => { @@ -61,9 +77,9 @@ describe('lexer:headings', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 6)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(6); }); /** @@ -74,7 +90,7 @@ describe('lexer:headings', () => { * This is an H1 * ============= * ` - * @output + * @html *

This is an H1

* */ it('should lex h1 heading tokens from setext mode', () => { @@ -82,9 +98,9 @@ describe('lexer:headings', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 1)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(1); }); /** @@ -103,9 +119,9 @@ describe('lexer:headings', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 2)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(2); }); /** @@ -123,8 +139,8 @@ describe('lexer:headings', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 1)).to.equal(true); - expect(_.isEqual(_.head(tokens).type, 'heading')).to.equal(true); - expect(_.isEqual(_.head(tokens).depth, 1)).to.equal(true); + expect(tokens.length).to.equal(1); + expect(_.head(tokens).type).to.equal('heading'); + expect(_.head(tokens).depth).to.equal(1); }); }); diff --git a/test/unit/specs/lexer-list.spec.js b/test/unit/specs/lexer-list.spec.js index 4c4b919..5ca9147 100644 --- a/test/unit/specs/lexer-list.spec.js +++ b/test/unit/specs/lexer-list.spec.js @@ -10,9 +10,9 @@ describe('lexer:lists', () => { let lexer = new Lexer(); let tokens = lexer.lex(mdString); - expect(_.isEqual(tokens.length, 5)).to.eq(true); - expect(_.isEqual(_.head(tokens).ordered, false)).to.eq(true); - expect(_.isEqual(_.head(tokens).type,'list_start')).to.eq(true); + expect(tokens.length).to.eq(5); + expect(_.head(tokens).ordered).to.eq(false); + expect(_.head(tokens).type).to.eq('list_start'); }); it('should be lex * as unordered list', () => { @@ -31,13 +31,5 @@ describe('lexer:lists', () => { it('should be lex - as unordered list', () => { }); - - it('', () => { - }); - - it('', () => { - }); - - it('', () => { - }); + });