From 693f13e2bf35596390d756a6a2ec6ffc880cf064 Mon Sep 17 00:00:00 2001 From: kazuya kawaguchi Date: Wed, 5 Aug 2020 10:37:34 +0900 Subject: [PATCH 1/2] fix modulo case closes #79 --- src/message/parser.ts | 2 +- src/message/tokenizer.ts | 60 +- .../__snapshots__/compiler.test.ts.snap | 33 ++ .../__snapshots__/tokenizer.test.ts.snap | 38 +- test/message/compiler.test.ts | 15 + .../parser/__snapshots__/modulo.test.ts.snap | 561 ++++++++++++++++++ test/message/parser/modulo.test.ts | 181 ++++++ test/message/tokenizer/linked.test.ts | 51 ++ test/message/tokenizer/named.test.ts | 180 ++++-- test/message/tokenizer/plural.test.ts | 46 ++ test/message/tokenizer/text.test.ts | 57 ++ 11 files changed, 1112 insertions(+), 112 deletions(-) create mode 100644 test/message/parser/__snapshots__/modulo.test.ts.snap create mode 100644 test/message/parser/modulo.test.ts diff --git a/src/message/parser.ts b/src/message/parser.ts index 37ebd3230..2a2e7a324 100644 --- a/src/message/parser.ts +++ b/src/message/parser.ts @@ -1,7 +1,7 @@ import { Position, createLocation, SourceLocation } from './location' import { ParserOptions } from './options' import { createCompileError, CompileErrorCodes } from './errors' -import { Tokenizer, createTokenizer, TokenTypes } from './tokenizer' +import { Tokenizer, createTokenizer, TokenTypes, Token } from './tokenizer' export const enum NodeTypes { Resource, // 0 diff --git a/src/message/tokenizer.ts b/src/message/tokenizer.ts index 08e29a5f7..8ef34c082 100644 --- a/src/message/tokenizer.ts +++ b/src/message/tokenizer.ts @@ -62,6 +62,7 @@ export interface TokenizeContext { lastEndLoc: Position braceNest: number inLinked: boolean + text: string } export interface Tokenizer { @@ -95,7 +96,8 @@ export function createTokenizer( lastStartLoc: _initLoc, lastEndLoc: _initLoc, braceNest: 0, - inLinked: false + inLinked: false, + text: '' } const context = (): TokenizeContext => _context @@ -341,32 +343,33 @@ export function createTokenizer( return ret } - function isTextStart(scnr: Scanner): boolean { - const fn = (hasSpace = false): boolean => { + function isTextStart(scnr: Scanner, reset = true): boolean { + const fn = (hasSpace = false, prev = '', detectModulo = false): boolean => { const ch = scnr.currentPeek() - if ( - ch === TokenChars.BraceLeft || - ch === TokenChars.BraceRight || - ch === TokenChars.Modulo || - ch === TokenChars.LinkedAlias || - !ch - ) { - return hasSpace + if (ch === TokenChars.BraceLeft) { + return prev === TokenChars.Modulo ? false : hasSpace + } else if (ch === TokenChars.LinkedAlias || !ch) { + return prev === TokenChars.Modulo ? true : hasSpace + } else if (ch === TokenChars.Modulo) { + scnr.peek() + return fn(hasSpace, TokenChars.Modulo, true) } else if (ch === TokenChars.Pipe) { - return false + return prev === TokenChars.Modulo || detectModulo + ? true + : !(prev === SPACE || prev === NEW_LINE) } else if (ch === SPACE) { scnr.peek() - return fn(true) + return fn(true, SPACE, detectModulo) } else if (ch === NEW_LINE) { scnr.peek() - return fn(true) + return fn(true, NEW_LINE, detectModulo) } else { return true } } const ret = fn() - scnr.resetPeek() + reset && scnr.resetPeek() return ret } @@ -439,13 +442,26 @@ export function createTokenizer( if ( ch === TokenChars.BraceLeft || ch === TokenChars.BraceRight || - ch === TokenChars.Modulo || ch === TokenChars.LinkedAlias || - ch === EOF + !ch ) { return buf + } else if (ch === TokenChars.Modulo) { + if (isTextStart(scnr)) { + buf += ch + scnr.next() + return fn(buf) + } else { + return buf + } + } else if (ch === TokenChars.Pipe) { + return buf } else if (ch === SPACE || ch === NEW_LINE) { - if (isPluralStart(scnr)) { + if (isTextStart(scnr)) { + buf += ch + scnr.next() + return fn(buf) + } else if (isPluralStart(scnr)) { return buf } else { buf += ch @@ -920,10 +936,6 @@ export function createTokenizer( case TokenChars.LinkedAlias: return readTokenInLinked(scnr, context) || getEndToken(context) - case TokenChars.Modulo: - scnr.next() - return getToken(context, TokenTypes.Modulo, TokenChars.Modulo) - default: if (isPluralStart(scnr)) { token = getToken(context, TokenTypes.Pipe, readPlural(scnr)) @@ -937,6 +949,10 @@ export function createTokenizer( return getToken(context, TokenTypes.Text, readText(scnr)) } + if (ch === TokenChars.Modulo) { + scnr.next() + return getToken(context, TokenTypes.Modulo, TokenChars.Modulo) + } break } diff --git a/test/message/__snapshots__/compiler.test.ts.snap b/test/message/__snapshots__/compiler.test.ts.snap index fffce2f35..1208b9b6d 100644 --- a/test/message/__snapshots__/compiler.test.ts.snap +++ b/test/message/__snapshots__/compiler.test.ts.snap @@ -52,6 +52,39 @@ Object { } `; +exports[`edge cases %: code 1`] = ` +"function __msg__ (ctx) { + const { normalize: _normalize } = ctx + return _normalize([ + \\"%\\" + ]) +}" +`; + +exports[`edge cases hi %s !: code 1`] = ` +"function __msg__ (ctx) { + const { normalize: _normalize } = ctx + return _normalize([ + \\"hi %s !\\" + ]) +}" +`; + +exports[`edge cases no apples %| one apple % | too much apples : code 1`] = ` +"function __msg__ (ctx) { + const { normalize: _normalize, pluralIndex: _pluralIndex, pluralRule: _pluralRule, orgPluralRule: _orgPluralRule } = ctx + return [ + _normalize([ + \\"no apples %\\" + ]), _normalize([ + \\"one apple %\\" + ]), _normalize([ + \\"too much apples \\" + ]) + ][_pluralRule(_pluralIndex, 3, _orgPluralRule)] +}" +`; + exports[`warnHtmlMessage default: code 1`] = ` "function __msg__ (ctx) { const { normalize: _normalize } = ctx diff --git a/test/message/__snapshots__/tokenizer.test.ts.snap b/test/message/__snapshots__/tokenizer.test.ts.snap index b8e420fe6..267f8e9da 100644 --- a/test/message/__snapshots__/tokenizer.test.ts.snap +++ b/test/message/__snapshots__/tokenizer.test.ts.snap @@ -2033,38 +2033,6 @@ world", exports[`token analysis: "hi %name" tokens 1`] = ` Array [ - Object { - "loc": Object { - "end": Object { - "column": 4, - "line": 1, - "offset": 3, - }, - "start": Object { - "column": 1, - "line": 1, - "offset": 0, - }, - }, - "type": 0, - "value": "hi ", - }, - Object { - "loc": Object { - "end": Object { - "column": 5, - "line": 1, - "offset": 4, - }, - "start": Object { - "column": 4, - "line": 1, - "offset": 3, - }, - }, - "type": 4, - "value": "%", - }, Object { "loc": Object { "end": Object { @@ -2073,13 +2041,13 @@ Array [ "offset": 8, }, "start": Object { - "column": 5, + "column": 1, "line": 1, - "offset": 4, + "offset": 0, }, }, "type": 0, - "value": "name", + "value": "hi %name", }, Object { "loc": Object { diff --git a/test/message/compiler.test.ts b/test/message/compiler.test.ts index feae49607..013192df4 100644 --- a/test/message/compiler.test.ts +++ b/test/message/compiler.test.ts @@ -46,6 +46,21 @@ describe('edge cases', () => { }) expect(code.toString()).toMatchSnapshot('code') }) + + test(`hi %s !`, () => { + const code = compile(`hi %s !`) + expect(code.toString()).toMatchSnapshot('code') + }) + + test(`%`, () => { + const code = compile(`%`) + expect(code.toString()).toMatchSnapshot('code') + }) + + test(`no apples %| one apple % | too much apples `, () => { + const code = compile(`no apples %| one apple % | too much apples `) + expect(code.toString()).toMatchSnapshot('code') + }) }) /* eslint-enable @typescript-eslint/no-empty-function, no-irregular-whitespace */ diff --git a/test/message/parser/__snapshots__/modulo.test.ts.snap b/test/message/parser/__snapshots__/modulo.test.ts.snap new file mode 100644 index 000000000..0697f637d --- /dev/null +++ b/test/message/parser/__snapshots__/modulo.test.ts.snap @@ -0,0 +1,561 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`hi % {name}! 1`] = ` +Object { + "body": Object { + "end": 12, + "items": Array [ + Object { + "end": 5, + "loc": Object { + "end": Object { + "column": 6, + "line": 1, + "offset": 5, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 3, + "value": "hi % ", + }, + Object { + "end": 11, + "key": "name", + "loc": Object { + "end": Object { + "column": 12, + "line": 1, + "offset": 11, + }, + "start": Object { + "column": 6, + "line": 1, + "offset": 5, + }, + }, + "start": 5, + "type": 4, + }, + Object { + "end": 12, + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 12, + "line": 1, + "offset": 11, + }, + }, + "start": 11, + "type": 3, + "value": "!", + }, + ], + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 2, + }, + "end": 12, + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 0, +} +`; + +exports[`hi %% name ! 1`] = ` +Object { + "body": Object { + "end": 12, + "items": Array [ + Object { + "end": 12, + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 3, + "value": "hi %% name !", + }, + ], + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 2, + }, + "end": 12, + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 0, +} +`; + +exports[`hi %@:name ! 1`] = ` +Object { + "body": Object { + "end": 12, + "items": Array [ + Object { + "end": 4, + "loc": Object { + "end": Object { + "column": 5, + "line": 1, + "offset": 4, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 3, + "value": "hi %", + }, + Object { + "end": 10, + "key": Object { + "end": 10, + "loc": Object { + "end": Object { + "column": 11, + "line": 1, + "offset": 10, + }, + "start": Object { + "column": 7, + "line": 1, + "offset": 6, + }, + }, + "start": 6, + "type": 7, + "value": "name", + }, + "loc": Object { + "end": Object { + "column": 11, + "line": 1, + "offset": 10, + }, + "start": Object { + "column": 5, + "line": 1, + "offset": 4, + }, + }, + "start": 4, + "type": 6, + }, + Object { + "end": 12, + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 11, + "line": 1, + "offset": 10, + }, + }, + "start": 10, + "type": 3, + "value": " !", + }, + ], + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 2, + }, + "end": 12, + "loc": Object { + "end": Object { + "column": 13, + "line": 1, + "offset": 12, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 0, +} +`; + +exports[`hi %{name}! 1`] = ` +Object { + "body": Object { + "end": 11, + "items": Array [ + Object { + "end": 3, + "loc": Object { + "end": Object { + "column": 4, + "line": 1, + "offset": 3, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 3, + "value": "hi ", + }, + Object { + "end": 10, + "key": "name", + "loc": Object { + "end": Object { + "column": 11, + "line": 1, + "offset": 10, + }, + "start": Object { + "column": 5, + "line": 1, + "offset": 4, + }, + }, + "start": 4, + "type": 4, + }, + Object { + "end": 11, + "loc": Object { + "end": Object { + "column": 12, + "line": 1, + "offset": 11, + }, + "start": Object { + "column": 11, + "line": 1, + "offset": 10, + }, + }, + "start": 10, + "type": 3, + "value": "!", + }, + ], + "loc": Object { + "end": Object { + "column": 12, + "line": 1, + "offset": 11, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 2, + }, + "end": 11, + "loc": Object { + "end": Object { + "column": 12, + "line": 1, + "offset": 11, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 0, +} +`; + +exports[`hi %d %s ! 1`] = ` +Object { + "body": Object { + "end": 10, + "items": Array [ + Object { + "end": 10, + "loc": Object { + "end": Object { + "column": 11, + "line": 1, + "offset": 10, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 3, + "value": "hi %d %s !", + }, + ], + "loc": Object { + "end": Object { + "column": 11, + "line": 1, + "offset": 10, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 2, + }, + "end": 10, + "loc": Object { + "end": Object { + "column": 11, + "line": 1, + "offset": 10, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 0, +} +`; + +exports[`no apples %| one apple % | too much apples 1`] = ` +Object { + "body": Object { + "cases": Array [ + Object { + "end": 0, + "items": Array [ + Object { + "end": 11, + "loc": Object { + "end": Object { + "column": 12, + "line": 1, + "offset": 11, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 3, + "value": "no apples %", + }, + ], + "loc": Object { + "end": Object { + "column": 12, + "line": 1, + "offset": 11, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 2, + }, + Object { + "end": 13, + "items": Array [ + Object { + "end": 24, + "loc": Object { + "end": Object { + "column": 25, + "line": 1, + "offset": 24, + }, + "start": Object { + "column": 14, + "line": 1, + "offset": 13, + }, + }, + "start": 13, + "type": 3, + "value": "one apple %", + }, + ], + "loc": Object { + "end": Object { + "column": 25, + "line": 1, + "offset": 24, + }, + "start": Object { + "column": 14, + "line": 1, + "offset": 13, + }, + }, + "start": 13, + "type": 2, + }, + Object { + "end": 45, + "items": Array [ + Object { + "end": 45, + "loc": Object { + "end": Object { + "column": 46, + "line": 1, + "offset": 45, + }, + "start": Object { + "column": 29, + "line": 1, + "offset": 28, + }, + }, + "start": 28, + "type": 3, + "value": "too much apples ", + }, + ], + "loc": Object { + "end": Object { + "column": 46, + "line": 1, + "offset": 45, + }, + "start": Object { + "column": 29, + "line": 1, + "offset": 28, + }, + }, + "start": 28, + "type": 2, + }, + ], + "end": 45, + "loc": Object { + "end": Object { + "column": 46, + "line": 1, + "offset": 45, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 1, + }, + "end": 45, + "loc": Object { + "end": Object { + "column": 46, + "line": 1, + "offset": 45, + }, + "start": Object { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "start": 0, + "type": 0, +} +`; diff --git a/test/message/parser/modulo.test.ts b/test/message/parser/modulo.test.ts new file mode 100644 index 000000000..676904f03 --- /dev/null +++ b/test/message/parser/modulo.test.ts @@ -0,0 +1,181 @@ +import { + createParser, + NodeTypes, + PluralNode, + MessageNode +} from '../../../src/message/parser' + +let spy +beforeEach(() => { + spy = jest.fn() +}) + +test('hi %{name}!', () => { + const text = `hi %{name}!` + const parser = createParser({ onError: spy }) + const ast = parser.parse(text) + + expect(ast).toMatchSnapshot() + expect(spy).not.toHaveBeenCalled() + + expect(ast.type).toEqual(NodeTypes.Resource) + expect(ast.body.type).toEqual(NodeTypes.Message) + const message = ast.body as MessageNode + expect(message.items).toHaveLength(3) + expect(message.items).toMatchObject([ + { + type: NodeTypes.Text, + value: 'hi ' + }, + { + type: NodeTypes.Named, + key: 'name' + }, + { + type: NodeTypes.Text, + value: '!' + } + ]) +}) + +test('hi % {name}!', () => { + const text = `hi % {name}!` + const parser = createParser({ onError: spy }) + const ast = parser.parse(text) + + expect(ast).toMatchSnapshot() + expect(spy).not.toHaveBeenCalled() + + expect(ast.type).toEqual(NodeTypes.Resource) + expect(ast.body.type).toEqual(NodeTypes.Message) + const message = ast.body as MessageNode + expect(message.items).toHaveLength(3) + expect(message.items).toMatchObject([ + { + type: NodeTypes.Text, + value: 'hi % ' + }, + { + type: NodeTypes.Named, + key: 'name' + }, + { + type: NodeTypes.Text, + value: '!' + } + ]) +}) + +test('hi %@:name !', () => { + const text = `hi %@:name !` + const parser = createParser({ onError: spy }) + const ast = parser.parse(text) + + expect(ast).toMatchSnapshot() + expect(spy).not.toHaveBeenCalled() + + expect(ast.type).toEqual(NodeTypes.Resource) + expect(ast.body.type).toEqual(NodeTypes.Message) + const message = ast.body as MessageNode + expect(message.items).toHaveLength(3) + expect(message.items).toMatchObject([ + { + type: NodeTypes.Text, + value: 'hi %' + }, + { + type: NodeTypes.Linked, + key: { + type: NodeTypes.LinkedKey, + value: 'name' + } + }, + { + type: NodeTypes.Text, + value: ' !' + } + ]) +}) + +test('hi %d %s !', () => { + const text = `hi %d %s !` + const parser = createParser({ onError: spy }) + const ast = parser.parse(text) + + expect(ast).toMatchSnapshot() + expect(spy).not.toHaveBeenCalled() + + expect(ast.type).toEqual(NodeTypes.Resource) + expect(ast.body.type).toEqual(NodeTypes.Message) + const message = ast.body as MessageNode + expect(message.items).toHaveLength(1) + expect(message.items).toMatchObject([ + { + type: NodeTypes.Text, + value: 'hi %d %s !' + } + ]) +}) + +test('hi %% name !', () => { + const text = `hi %% name !` + const parser = createParser({ onError: spy }) + const ast = parser.parse(text) + + expect(ast).toMatchSnapshot() + expect(spy).not.toHaveBeenCalled() + + expect(ast.type).toEqual(NodeTypes.Resource) + expect(ast.body.type).toEqual(NodeTypes.Message) + const message = ast.body as MessageNode + expect(message.items).toHaveLength(1) + expect(message.items).toMatchObject([ + { + type: NodeTypes.Text, + value: 'hi %% name !' + } + ]) +}) + +test('no apples %| one apple % | too much apples ', () => { + const text = `no apples %| one apple % | too much apples ` + const parser = createParser({ onError: spy }) + const ast = parser.parse(text) + + expect(ast).toMatchSnapshot() + expect(spy).not.toHaveBeenCalled() + + expect(ast.type).toEqual(NodeTypes.Resource) + expect(ast.body.type).toEqual(NodeTypes.Plural) + const plural = ast.body as PluralNode + expect(plural.cases).toHaveLength(3) + expect(plural.cases).toMatchObject([ + { + type: NodeTypes.Message, + items: [ + { + type: NodeTypes.Text, + value: 'no apples %' + } + ] + }, + { + type: NodeTypes.Message, + items: [ + { + type: NodeTypes.Text, + value: 'one apple %' + } + ] + }, + { + type: NodeTypes.Message, + items: [ + { + type: NodeTypes.Text, + value: 'too much apples ' + } + ] + } + ]) +}) diff --git a/test/message/tokenizer/linked.test.ts b/test/message/tokenizer/linked.test.ts index ab81cd5d8..e3dffd152 100644 --- a/test/message/tokenizer/linked.test.ts +++ b/test/message/tokenizer/linked.test.ts @@ -306,6 +306,57 @@ test('dot in refer key', () => { }) }) +test('with modulo', () => { + const tokenizer = createTokenizer('hi %@:name !') + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: 'hi %', + loc: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 5, offset: 4 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.LinkedAlias, + value: '@', + loc: { + start: { line: 1, column: 5, offset: 4 }, + end: { line: 1, column: 6, offset: 5 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.LinkedDelimiter, + value: ':', + loc: { + start: { line: 1, column: 6, offset: 5 }, + end: { line: 1, column: 7, offset: 6 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.LinkedKey, + value: 'name', + loc: { + start: { line: 1, column: 7, offset: 6 }, + end: { line: 1, column: 11, offset: 10 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: ' !', + loc: { + start: { line: 1, column: 11, offset: 10 }, + end: { line: 1, column: 13, offset: 12 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.EOF, + loc: { + start: { line: 1, column: 13, offset: 12 }, + end: { line: 1, column: 13, offset: 12 } + } + }) +}) + test('multiple', () => { const tokenizer = createTokenizer('hi @:{name} @:{0}!') expect(tokenizer.nextToken()).toEqual({ diff --git a/test/message/tokenizer/named.test.ts b/test/message/tokenizer/named.test.ts index 5c4d743a8..dc45947bd 100644 --- a/test/message/tokenizer/named.test.ts +++ b/test/message/tokenizer/named.test.ts @@ -312,62 +312,134 @@ test('new line', () => { }) }) -test('with modulo', () => { - const tokenizer = createTokenizer('hi %{name} !') - expect(tokenizer.nextToken()).toEqual({ - type: TokenTypes.Text, - value: 'hi ', - loc: { - start: { line: 1, column: 1, offset: 0 }, - end: { line: 1, column: 4, offset: 3 } - } - }) - expect(tokenizer.nextToken()).toEqual({ - type: TokenTypes.Modulo, - value: '%', - loc: { - start: { line: 1, column: 4, offset: 3 }, - end: { line: 1, column: 5, offset: 4 } - } - }) - expect(tokenizer.nextToken()).toEqual({ - type: TokenTypes.BraceLeft, - value: '{', - loc: { - start: { line: 1, column: 5, offset: 4 }, - end: { line: 1, column: 6, offset: 5 } - } - }) - expect(tokenizer.nextToken()).toEqual({ - type: TokenTypes.Named, - value: 'name', - loc: { - start: { line: 1, column: 6, offset: 5 }, - end: { line: 1, column: 10, offset: 9 } - } - }) - expect(tokenizer.nextToken()).toEqual({ - type: TokenTypes.BraceRight, - value: '}', - loc: { - start: { line: 1, column: 10, offset: 9 }, - end: { line: 1, column: 11, offset: 10 } - } +describe('modulo cases', () => { + test('basic named: hi %{name} !', () => { + const tokenizer = createTokenizer('hi %{name} !') + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: 'hi ', + loc: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 4, offset: 3 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Modulo, + value: '%', + loc: { + start: { line: 1, column: 4, offset: 3 }, + end: { line: 1, column: 5, offset: 4 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.BraceLeft, + value: '{', + loc: { + start: { line: 1, column: 5, offset: 4 }, + end: { line: 1, column: 6, offset: 5 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Named, + value: 'name', + loc: { + start: { line: 1, column: 6, offset: 5 }, + end: { line: 1, column: 10, offset: 9 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.BraceRight, + value: '}', + loc: { + start: { line: 1, column: 10, offset: 9 }, + end: { line: 1, column: 11, offset: 10 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: ' !', + loc: { + start: { line: 1, column: 11, offset: 10 }, + end: { line: 1, column: 13, offset: 12 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.EOF, + loc: { + start: { line: 1, column: 13, offset: 12 }, + end: { line: 1, column: 13, offset: 12 } + } + }) }) - expect(tokenizer.nextToken()).toEqual({ - type: TokenTypes.Text, - value: ' !', - loc: { - start: { line: 1, column: 11, offset: 10 }, - end: { line: 1, column: 13, offset: 12 } - } + + test('not modulo named: hi % {name} !', () => { + const tokenizer = createTokenizer('hi % {name} !') + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: 'hi % ', + loc: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 6, offset: 5 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.BraceLeft, + value: '{', + loc: { + start: { line: 1, column: 6, offset: 5 }, + end: { line: 1, column: 7, offset: 6 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Named, + value: 'name', + loc: { + start: { line: 1, column: 7, offset: 6 }, + end: { line: 1, column: 11, offset: 10 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.BraceRight, + value: '}', + loc: { + start: { line: 1, column: 11, offset: 10 }, + end: { line: 1, column: 12, offset: 11 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: ' !', + loc: { + start: { line: 1, column: 12, offset: 11 }, + end: { line: 1, column: 14, offset: 13 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.EOF, + loc: { + start: { line: 1, column: 14, offset: 13 }, + end: { line: 1, column: 14, offset: 13 } + } + }) }) - expect(tokenizer.nextToken()).toEqual({ - type: TokenTypes.EOF, - loc: { - start: { line: 1, column: 13, offset: 12 }, - end: { line: 1, column: 13, offset: 12 } - } + + test('other placeholder sytnax: hi %s !', () => { + const tokenizer = createTokenizer('hi %s !') + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: 'hi %s !', + loc: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 8, offset: 7 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.EOF, + loc: { + start: { line: 1, column: 8, offset: 7 }, + end: { line: 1, column: 8, offset: 7 } + } + }) }) }) diff --git a/test/message/tokenizer/plural.test.ts b/test/message/tokenizer/plural.test.ts index 56ef0b119..255155acf 100644 --- a/test/message/tokenizer/plural.test.ts +++ b/test/message/tokenizer/plural.test.ts @@ -99,6 +99,52 @@ test('multi lines', () => { }) }) +test('include modulo', () => { + const tokenizer = createTokenizer( + 'no apples %| one apple % | too much apples ' + ) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: 'no apples %', + loc: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 12, offset: 11 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Pipe, + value: '|', + loc: { + start: { line: 1, column: 12, offset: 11 }, + end: { line: 1, column: 14, offset: 13 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: 'one apple %', + loc: { + start: { line: 1, column: 14, offset: 13 }, + end: { line: 1, column: 25, offset: 24 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Pipe, + value: '|', + loc: { + start: { line: 1, column: 25, offset: 24 }, + end: { line: 1, column: 29, offset: 28 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: 'too much apples ', + loc: { + start: { line: 1, column: 29, offset: 28 }, + end: { line: 1, column: 46, offset: 45 } + } + }) +}) + test('complex', () => { const tokenizer = createTokenizer( `@.lower:{'no apples'} | {1} apple | {count}　apples` // eslint-disable-line no-irregular-whitespace diff --git a/test/message/tokenizer/text.test.ts b/test/message/tokenizer/text.test.ts index 52e2bb0cd..dd24f129d 100644 --- a/test/message/tokenizer/text.test.ts +++ b/test/message/tokenizer/text.test.ts @@ -49,6 +49,63 @@ test('empty', () => { }) }) +test('modulo', () => { + const tokenizer = createTokenizer('% foo') + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: '% foo', + loc: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 6, offset: 5 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.EOF, + loc: { + start: { line: 1, column: 6, offset: 5 }, + end: { line: 1, column: 6, offset: 5 } + } + }) +}) + +test('double modulo', () => { + const tokenizer = createTokenizer('%% foo') + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: '%% foo', + loc: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 7, offset: 6 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.EOF, + loc: { + start: { line: 1, column: 7, offset: 6 }, + end: { line: 1, column: 7, offset: 6 } + } + }) +}) + +test('modulo only', () => { + const tokenizer = createTokenizer('%') + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.Text, + value: '%', + loc: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 2, offset: 1 } + } + }) + expect(tokenizer.nextToken()).toEqual({ + type: TokenTypes.EOF, + loc: { + start: { line: 1, column: 2, offset: 1 }, + end: { line: 1, column: 2, offset: 1 } + } + }) +}) + test('spaces', () => { const tokenizer = createTokenizer(' hello world ') expect(tokenizer.nextToken()).toEqual({ From f3f51cd9329994bbd87bd0e4e1f7f190b4f04131 Mon Sep 17 00:00:00 2001 From: kazuya kawaguchi Date: Wed, 5 Aug 2020 10:41:13 +0900 Subject: [PATCH 2/2] remove --- src/message/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/message/parser.ts b/src/message/parser.ts index 2a2e7a324..37ebd3230 100644 --- a/src/message/parser.ts +++ b/src/message/parser.ts @@ -1,7 +1,7 @@ import { Position, createLocation, SourceLocation } from './location' import { ParserOptions } from './options' import { createCompileError, CompileErrorCodes } from './errors' -import { Tokenizer, createTokenizer, TokenTypes, Token } from './tokenizer' +import { Tokenizer, createTokenizer, TokenTypes } from './tokenizer' export const enum NodeTypes { Resource, // 0