build: imporve ESM support

- Use thin ESM wrapper for CJS output instead of double tsc compilation.
- Replace webpack with esbuild
- Add bundled versions of ESM build.

fixes #1383

examples/implementation_languages/impl_lang_spec.js

@@ -12,7 +12,10 @@ function createSanityTest(languageName, parseJson) {
 
 describe("The ability to use Chevrotain using different implementation languages", () => {
   createSanityTest("ECMAScript 5", require("./ecma5/ecma5_json"))
-  createSanityTest("ECMAScript 6/2015", require("./ecma6/ecma6_json"))
+  createSanityTest(
+    "ECMAScript 6/2015",
+    require("./modern_ecmascript/ecma6_json")
+  )
   createSanityTest(
     "TypeScript",
     require("./typescript/typescript_json").parseJson

examples/implementation_languages/impl_lang_spec.mjs

@@ -0,0 +1,12 @@
+import { parseJson }  from "./modern_ecmascript/modern_ecmascript_json.mjs"
+import assert from "assert"
+
+describe("The ability to use Chevrotain using modern ECMAScript", () => {
+  it("works with ESM", () => {
+    const inputText = '{ "arr": [1,2,3], "obj": {"num":666}}'
+    const lexAndParseResult = parseJson(inputText)
+
+    assert.equal(lexAndParseResult.lexErrors.length, 0)
+    assert.equal(lexAndParseResult.parseErrors.length, 0)
+  })
+})

examples/implementation_languages/ecma6/ecma6_json.js → examples/implementation_languages/modern_ecmascript/modern_ecmascript_json.mjs

@@ -1,5 +1,5 @@
 "use strict"
-const { createToken, Lexer, CstParser } = require("chevrotain")
+import { createToken, Lexer, CstParser } from "chevrotain"
 
 // ----------------- lexer -----------------
 const True = createToken({ name: "True", pattern: /true/ })
@@ -114,7 +114,7 @@ class JsonParserES6 extends CstParser {
 // reuse the same parser instance.
 const parser = new JsonParserES6()
 
-module.exports = function (text) {
+export function parseJson (text) {
   const lexResult = JsonLexer.tokenize(text)
   // setting a new input will RESET the parser instance's state.
   parser.input = lexResult.tokens

examples/implementation_languages/package.json

@@ -5,7 +5,9 @@
     "build": "npm-run-all build:ts build:coffee",
     "build:ts": "tsc ./typescript/typescript_json.ts --types \" \"",
     "build:coffee": "coffee -c ./coffeescript/coffeescript_json.coffee",
-    "test": "mocha \"*spec.js\""
+    "test": "npm-run-all test:*",
+    "test:cjs": "mocha \"*spec.js\"",
+    "test:esm": "mocha \"*spec.mjs\""
   },
   "dependencies": {
     "chevrotain": "^7.1.2"

packages/chevrotain/package.json

@@ -23,8 +23,7 @@
     "name": "Shahar Soel"
   },
   "files": [
-    "lib_esm/src/**/*.js",
-    "lib_esm/src/**/*.js.map",
+    "lib_esm/**/*.js",
     "lib/src/**/*.js",
     "lib/src/**/*.js.map",
     "lib/chevrotain.d.ts",
@@ -36,36 +35,36 @@
     "diagrams/**/*.*",
     "CHANGELOG.md"
   ],
-  "main": "lib/src/api.js",
-  "module": "lib_esm/src/api.js",
+  "main": "./lib/src/api.js",
+  "exports": {
+    "require": "./lib/src/api.js",
+    "import": "./lib_esm/api_esm.mjs"
+  },
   "repository": {
     "type": "git",
     "url": "git://github.com/Chevrotain/chevrotain.git"
   },
   "homepage": "https://chevrotain.io/docs/",
   "scripts": {
     "---------- CI FLOWS --------": "",
-    "build": "npm-run-all clean compile build:esm dts api-site:build bundle",
-    "build:esm": "npm-run-all clean:esm compile:esm",
-    "test": "npm-run-all test:esm compile:def coverage check-coverage",
-    "test:esm": "mocha \"./lib_esm/test/**/*spec.js\" --require esm",
+    "build": "npm-run-all clean compile dts api-site:build bundle",
+    "test": "npm-run-all compile:def coverage check-coverage",
     "version": "tsc ./src/version.ts --outDir lib/src && node ./scripts/version-update.js",
     "postversion": "npm-run-all website:build website:upload api-site:build api-site:upload",
     "---------- DEV FLOWS --------": "",
     "watch": "tsc -w",
     "unit-tests": "mocha \"./lib/test/**/*spec.js\"",
     "quick-build": "tsc && yarn run bundle",
     "---------- BUILD STEPS --------": "",
-    "clean": "shx rm -rf coverage dev lib",
-    "clean:esm": "shx rm -rf lib_esm",
-    "compile": "tsc && node ./scripts/fix-coverage-report.js",
-    "compile:esm": "tsc --project tsconfig.esm.json",
+    "clean": "shx rm -rf coverage dev lib lib_esm",
+    "compile": "tsc && node ./scripts/fix-coverage-report.js && npm-run-all gen-esm-wrapper",
     "compile:watch": "tsc -w",
     "compile:def": "npm-run-all compile-def-api compile-def-modules compile-def-namespace",
+    "gen-esm-wrapper": "gen-esm-wrapper . ./lib_esm/api_esm.mjs",
     "dts": "node scripts/process-docs.js",
     "coverage": "nyc mocha \"./lib/test/**/*spec.js\"",
     "check-coverage": "nyc check-coverage --lines 100 --branches 100 --statements 100 --functions 100",
-    "bundle": "npm-run-all bundle:regular bundle:min",
+    "bundle": "npm-run-all bundle:*",
     "api-site:build": "typedoc api.d.ts --out dev/docs --excludeExternals --excludePrivate",
     "api-site:upload": "./scripts/api-site-upload.sh",
     "website:dev": "vuepress dev docs",
@@ -77,8 +76,10 @@
     "compile-def-modules": "tsc --noImplicitAny test_integration/definitions/es6_modules.ts --outDir dev/garbage --lib \"es2015\"",
     "compile-def-namespace": "tsc --noImplicitAny test_integration/definitions/namespaces.ts --module none --outDir dev/garbage --lib \"es2015\"",
     "---------- BUNDLING --------": "",
-    "bundle:regular": "webpack --config webpack_release.config.js",
-    "bundle:min": "webpack --config webpack_release_min.config.js"
+    "bundle:regular": "esbuild ./lib/src/api.js --bundle --sourcemap --outfile=lib/chevrotain.js",
+    "bundle:min": "esbuild ./lib/src/api.js --bundle --minify --sourcemap --outfile=lib/chevrotain.min.js",
+    "bundle:esm:regular": "esbuild ./lib/src/api.js --bundle --sourcemap --format=esm --outfile=lib/chevrotain.mjs",
+    "bundle:esm:min": "esbuild ./lib/src/api.js --bundle --minify --format=esm --sourcemap --outfile=lib/chevrotain.min.mjs"
   },
   "dependencies": {
     "regexp-to-ast": "0.5.0"
@@ -89,7 +90,6 @@
     "@types/sinon-chai": "^3.2.0",
     "chai": "^4.1.2",
     "error-stack-parser": "^2.0.6",
-    "esm": "^3.2.25",
     "gitty": "^3.6.0",
     "jsdom": "16.4.0",
     "jsonfile": "^6.0.1",
@@ -101,9 +101,9 @@
     "typedoc": "^0.20.28",
     "typescript": "4.2.2",
     "vuepress": "^1.4.1",
-    "webpack": "5.24.2",
-    "webpack-cli": "^4.1.0",
-    "xregexp": "^5.0.1"
+    "xregexp": "^5.0.1",
+    "gen-esm-wrapper": "^1.0.4",
+    "esbuild": "^0.8.53"
   },
   "typings": "lib/chevrotain.d.ts",
   "mocha": {

packages/chevrotain/src/api.ts

@@ -1,19 +1,34 @@
 /* istanbul ignore file - tricky to import some things from this module during testing */
+import * as apiDefs from "../api"
 
 // semantic version
+import { VERSION } from "./version"
 export { VERSION } from "./version"
 
+import { Lexer, LexerDefinitionErrorType } from "./scan/lexer_public"
+export { Lexer, LexerDefinitionErrorType } from "./scan/lexer_public"
+
+import {
+  CstParser,
+  EmbeddedActionsParser,
+  ParserDefinitionErrorType,
+  EMPTY_ALT
+} from "./parse/parser/parser"
 export {
   CstParser,
   EmbeddedActionsParser,
   ParserDefinitionErrorType,
   EMPTY_ALT
 } from "./parse/parser/parser"
 
-export { Lexer, LexerDefinitionErrorType } from "./scan/lexer_public"
-
-// Tokens utilities
-
+import {
+  createToken,
+  createTokenInstance,
+  EOF,
+  tokenLabel,
+  tokenMatcher,
+  tokenName
+} from "./scan/tokens_public"
 export {
   createToken,
   createTokenInstance,
@@ -23,14 +38,24 @@ export {
   tokenName
 } from "./scan/tokens_public"
 
-// Other Utilities
-
+import {
+  defaultGrammarResolverErrorProvider,
+  defaultGrammarValidatorErrorProvider,
+  defaultParserErrorProvider
+} from "./parse/errors_public"
 export {
   defaultGrammarResolverErrorProvider,
   defaultGrammarValidatorErrorProvider,
   defaultParserErrorProvider
 } from "./parse/errors_public"
 
+import {
+  EarlyExitException,
+  isRecognitionException,
+  MismatchedTokenException,
+  NotAllInputParsedException,
+  NoViableAltException
+} from "./parse/exceptions_public"
 export {
   EarlyExitException,
   isRecognitionException,
@@ -39,10 +64,18 @@ export {
   NoViableAltException
 } from "./parse/exceptions_public"
 
-export { defaultLexerErrorProvider } from "./scan/lexer_errors_public"
-
-// grammar reflection API
-
+import {
+  Alternation,
+  Alternative,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Rule,
+  Terminal
+} from "./parse/grammar/gast/gast_public"
 export {
   Alternation,
   Alternative,
@@ -56,21 +89,88 @@ export {
   Terminal
 } from "./parse/grammar/gast/gast_public"
 
-// GAST Utilities
-
+import {
+  serializeGrammar,
+  serializeProduction
+} from "./parse/grammar/gast/gast_public"
 export {
   serializeGrammar,
   serializeProduction
 } from "./parse/grammar/gast/gast_public"
 
+import { GAstVisitor } from "./parse/grammar/gast/gast_visitor_public"
 export { GAstVisitor } from "./parse/grammar/gast/gast_visitor_public"
 
+import {
+  assignOccurrenceIndices,
+  resolveGrammar,
+  validateGrammar
+} from "./parse/grammar/gast/gast_resolver_public"
 export {
   assignOccurrenceIndices,
   resolveGrammar,
   validateGrammar
 } from "./parse/grammar/gast/gast_resolver_public"
 
+import { createSyntaxDiagramsCode } from "./diagrams/render_public"
+export { createSyntaxDiagramsCode } from "./diagrams/render_public"
+
+import {
+  generateParserFactory,
+  generateParserModule
+} from "./generate/generate_public"
+export {
+  generateParserFactory,
+  generateParserModule
+} from "./generate/generate_public"
+
+const api: typeof apiDefs = {
+  VERSION,
+  Lexer,
+  LexerDefinitionErrorType,
+  // @ts-ignore
+  CstParser,
+  // @ts-ignore
+  EmbeddedActionsParser,
+  ParserDefinitionErrorType,
+  EMPTY_ALT,
+  createToken,
+  createTokenInstance,
+  EOF,
+  tokenLabel,
+  tokenMatcher,
+  tokenName,
+  defaultGrammarResolverErrorProvider,
+  defaultGrammarValidatorErrorProvider,
+  defaultParserErrorProvider,
+  EarlyExitException,
+  isRecognitionException,
+  MismatchedTokenException,
+  NotAllInputParsedException,
+  NoViableAltException,
+  Alternation,
+  Alternative,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Rule,
+  Terminal,
+  serializeGrammar,
+  serializeProduction,
+  GAstVisitor,
+  assignOccurrenceIndices,
+  resolveGrammar,
+  validateGrammar,
+  createSyntaxDiagramsCode,
+  generateParserFactory,
+  generateParserModule
+}
+
+export default api
+
 /* istanbul ignore next */
 export function clearCache() {
   console.warn(
@@ -80,13 +180,6 @@ export function clearCache() {
   )
 }
 
-export { createSyntaxDiagramsCode } from "./diagrams/render_public"
-
-export {
-  generateParserFactory,
-  generateParserModule
-} from "./generate/generate_public"
-
 export class Parser {
   constructor() {
     throw new Error(

packages/chevrotain/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "module": "commonjs",
+    "module": "CommonJS",
     "removeComments": false,
     "target": "ES5",
     "outDir": "lib",