Refactor messageformat-parser, moving from PEG.js to moo

.eslintignore

@@ -6,7 +6,8 @@ package-lock.json
 /packages/webpack-loader-example/dist/bundle.js
 /packages/messageformat/lib/*
 /packages/messageformat/messageformat.*
-/packages/parser/parser.js
+/packages/parser/lib/*
+/packages/rollup-plugin/lib/*
 /packages/runtime/esm/*
 /packages/runtime/lib/*
 /packages/website/docs/

.eslintrc.yaml

@@ -8,7 +8,7 @@ parserOptions:
 plugins:
   - prettier
 extends:
-  - "eslint:recommended"
+  - 'eslint:recommended'
   - prettier
 
 rules:

.prettierignore

@@ -5,7 +5,9 @@ package-lock.json
 /packages/webpack-loader-example/dist/bundle.js
 /packages/messageformat/lib/
 /packages/messageformat/messageformat.*
-/packages/parser/parser.js
+/packages/parser/lib/
+/packages/rollup-plugin/lib/
+/packages/runtime/esm/
 /packages/runtime/lib/
 /packages/website/docs/
 /test/browser/dist/

jest.config.js

@@ -12,6 +12,7 @@ module.exports = {
     '^messageformat$': '<rootDir>/packages/messageformat/src/messageformat.ts',
     '^messageformat/compile-module$':
       '<rootDir>/packages/messageformat/src/compile-module.ts',
+    '^messageformat-parser$': '<rootDir>/packages/parser/src/parser.ts',
     '^messageformat-runtime$': '<rootDir>/packages/runtime/src/runtime.ts',
     '^messageformat-runtime/lib/(.*)$': '<rootDir>/packages/runtime/src/$1.ts'
   },

package.json

@@ -26,7 +26,6 @@
     "test:chrome": "npm run pretest:browsers && mocha-selenium-bridge -b chrome test/browser/test.html",
     "test:firefox": "npm run pretest:browsers && mocha-selenium-bridge -b firefox test/browser/test.html",
     "test:serve": "npm run pretest:browsers && serve",
-    "pretest": "lerna run build --scope messageformat-parser",
     "test": "jest"
   },
   "prettier": {
@@ -64,6 +63,7 @@
     "@rollup/plugin-typescript": "^5.0.2",
     "@types/chai": "^4.2.12",
     "@types/jest": "^26.0.13",
+    "@types/moo": "^0.5.3",
     "babel-eslint": "^10.0.2",
     "babel-jest": "^26.3.0",
     "babel-loader": "^8.0.6",
@@ -81,7 +81,6 @@
     "memfs": "^3.2.0",
     "mocha": "^8.1.3",
     "mocha-selenium-bridge": "^0.2.0",
-    "pegjs": "^0.10.0",
     "prettier": "^2.1.1",
     "rimraf": "^3.0.2",
     "rollup": "^2.26.10",

packages/messageformat/compile-module.d.ts

@@ -1,2 +1,2 @@
-export * from './lib/compile-module'
-export { default } from './lib/compile-module'
+export * from './lib/compile-module';
+export { default } from './lib/compile-module';

packages/messageformat/rollup.config.js

@@ -35,9 +35,12 @@ const browserBundle = {
   },
   plugins: [
     resolve(),
-    typescript({ target: 'ES5' }),
-    babel({ presets: [['@babel/preset-env', { targets: browserTargets }]] }),
-    commonjs()
+    commonjs(),
+    typescript({
+      downlevelIteration: true,
+      target: 'ES5'
+    }),
+    babel({ presets: [['@babel/preset-env', { targets: browserTargets }]] })
   ]
 };
 

packages/messageformat/src/compile-module.ts

@@ -3,7 +3,7 @@ import Compiler, { RuntimeMap, StringStructure } from './compiler';
 import MessageFormat, { MessageFunction } from './messageformat';
 import { PluralObject } from './plurals';
 
-export { MessageFunction }
+export { MessageFunction };
 export type MessageModule<T> = T extends string
   ? MessageFunction
   : {

packages/messageformat/src/compiler.ts

@@ -8,9 +8,8 @@ import {
 } from 'messageformat-number-skeleton';
 import {
   parse,
-  FunctionArg,
+  Function,
   Octothorpe,
-  Plural,
   Select,
   Token
 } from 'messageformat-parser';
@@ -95,12 +94,15 @@ export default class Compiler {
     return `function(d) { ${reqArgs}return ${this.concatenate(r, true)}; }`;
   }
 
-  cases(token: Plural | Select, pluralToken: Plural | null) {
+  cases(token: Select, pluralToken: Select | null) {
     let needOther = true;
     const r = token.cases.map(({ key, tokens }) => {
       if (key === 'other') needOther = false;
       const s = tokens.map(tok => this.token(tok, pluralToken));
-      return `${property(null, key)}: ${this.concatenate(s, false)}`;
+      return `${property(null, key.replace(/^=/, ''))}: ${this.concatenate(
+        s,
+        false
+      )}`;
     });
     if (needOther) {
       const { type } = token;
@@ -122,8 +124,8 @@ export default class Compiler {
       : tokens.join(' + ') || '""';
   }
 
-  token(token: Token | Octothorpe, pluralToken: Plural | null) {
-    if (typeof token == 'string') return JSON.stringify(token);
+  token(token: Token | Octothorpe, pluralToken: Select | null) {
+    if (token.type == 'content') return JSON.stringify(token.value);
 
     const { id, lc } = this.plural;
     let args: (number | string)[], fn: string;
@@ -146,14 +148,14 @@ export default class Compiler {
 
       case 'selectordinal':
         fn = 'plural';
-        args.push(token.offset || 0, id, this.cases(token, token), 1);
+        args.push(token.pluralOffset || 0, id, this.cases(token, token), 1);
         this.setLocale(id, true);
         this.setRuntimeFn('plural');
         break;
 
       case 'plural':
         fn = 'plural';
-        args.push(token.offset || 0, id, this.cases(token, token));
+        args.push(token.pluralOffset || 0, id, this.cases(token, token));
         this.setLocale(id, false);
         this.setRuntimeFn('plural');
         break;
@@ -171,9 +173,7 @@ export default class Compiler {
             if (token.param) {
               if (pluralToken && this.options.strictNumberSign)
                 pluralToken = null;
-              const s = token.param.tokens.map(tok =>
-                this.token(tok, pluralToken)
-              );
+              const s = token.param.map(tok => this.token(tok, pluralToken));
               args.push('(' + (s.join(' + ') || '""') + ').trim()');
               if (token.key === 'number')
                 args.push(JSON.stringify(this.options.currency));
@@ -188,7 +188,7 @@ export default class Compiler {
         args = [
           JSON.stringify(this.plural.locale),
           property('d', pluralToken.arg),
-          pluralToken.offset || '0'
+          pluralToken.pluralOffset || 0
         ];
         if (this.options.strictNumberSign) {
           fn = 'strictNumber';
@@ -265,15 +265,19 @@ export default class Compiler {
   }
 
   setDateFormatter(
-    { param }: FunctionArg,
+    { param }: Function,
     args: (number | string)[],
-    plural: Plural | null
+    plural: Select | null
   ) {
     const { locale } = this.plural;
 
-    const argStyle = param && param.tokens.length === 1 && param.tokens[0];
-    if (typeof argStyle === 'string' && /^\s*::/.test(argStyle)) {
-      const argSkeletonText = argStyle.trim().substr(2);
+    const argStyle = param && param.length === 1 && param[0];
+    if (
+      argStyle &&
+      argStyle.type === 'content' &&
+      /^\s*::/.test(argStyle.value)
+    ) {
+      const argSkeletonText = argStyle.value.trim().substr(2);
       const key = identifier(`date_${locale}_${argSkeletonText}`, true);
       if (!this.runtimeIncludes(key, 'formatter')) {
         const fmt: RuntimeEntry = getDateFormatter(locale, argSkeletonText);
@@ -287,19 +291,19 @@ export default class Compiler {
     }
 
     args.push(JSON.stringify(locale));
-    if (param) {
+    if (param && param.length > 0) {
       if (plural && this.options.strictNumberSign) plural = null;
-      const s = param.tokens.map(tok => this.token(tok, plural));
+      const s = param.map(tok => this.token(tok, plural));
       args.push('(' + (s.join(' + ') || '""') + ').trim()');
     }
     this.setFormatter('date');
     return 'date';
   }
 
   setNumberFormatter(
-    { param }: FunctionArg,
+    { param }: Function,
     args: (number | string)[],
-    plural: Plural | null
+    plural: Select | null
   ) {
     const { locale } = this.plural;
 
@@ -312,8 +316,8 @@ export default class Compiler {
     }
 
     args.push(JSON.stringify(locale));
-    if (param.tokens.length === 1 && typeof param.tokens[0] === 'string') {
-      const fmtArg = param.tokens[0].trim();
+    if (param.length === 1 && param[0].type === 'content') {
+      const fmtArg = param[0].value.trim();
 
       switch (fmtArg) {
         case 'currency':
@@ -350,7 +354,7 @@ export default class Compiler {
     }
 
     if (plural && this.options.strictNumberSign) plural = null;
-    const s = param.tokens.map(tok => this.token(tok, plural));
+    const s = param.map(tok => this.token(tok, plural));
     args.push('(' + (s.join(' + ') || '""') + ').trim()');
     args.push(JSON.stringify(this.options.currency));
     this.setFormatter('numberFmt');

packages/messageformat/src/messageformat.test.ts

@@ -96,19 +96,25 @@ describe('compile() errors', () => {
   test('Invalid plural key', () => {
     const mf = new MessageFormat('en');
     const src = '{X, plural, foo{a}}';
-    expect(() => mf.compile(src)).toThrow(/Invalid key `foo`/);
+    expect(() => mf.compile(src)).toThrow(
+      'The plural case foo is not valid in this locale'
+    );
   });
 
   test('Invalid selectordinal key', () => {
     const mf = new MessageFormat('en');
-    const src = '{X, plural, foo{a}}';
-    expect(() => mf.compile(src)).toThrow(/Invalid key `foo`/);
+    const src = '{X, selectordinal, foo{a}}';
+    expect(() => mf.compile(src)).toThrow(
+      'The selectordinal case foo is not valid in this locale'
+    );
   });
 
   test('Invalid plural key for locale', () => {
     const mf = new MessageFormat('en');
     const src = '{X, plural, zero{none} one{one} other{some: #}}';
-    expect(() => mf.compile(src)).toThrow(/Invalid key `zero`/);
+    expect(() => mf.compile(src)).toThrow(
+      'The plural case zero is not valid in this locale'
+    );
   });
 
   test('Undefined formatting function', () => {

packages/parser/.gitignore

@@ -1 +1,2 @@
-parser.js
+/lib/
+/parser.js