feat(@formatjs/cli): revamp underlying extraction

We replaced the underlying AST extractor from babel to typescript. This
makes the extractor a lot more future-proof due to babel having to
constantly include new plugin/syntax while TS just works out of the box.
Future new syntax would just be TS upgrade.

BREAKING CHANGE: Remove `--messages-dir` option. This was primarily used
to eagerly write out output in the babel plugin since we don't know when
the execution will be done. This is not the case with the CLI.
`--out-file` should be used instead.

BREAKING CHANGE: Remove `--module-source-name` option. This is not used.

package.json

@@ -26,11 +26,7 @@
   },
   "devDependencies": {
     "@babel/core": "^7.10.4",
-    "@babel/plugin-proposal-class-properties": "^7.8.3",
-    "@babel/plugin-proposal-optional-chaining": "7",
-    "@babel/preset-env": "^7.9.5",
-    "@babel/preset-react": "^7.10.4",
-    "@babel/preset-typescript": "^7.9.0",
+    "@babel/plugin-syntax-jsx": "^7.10.4",
     "@bazel/bazelisk": "^1.5.0",
     "@bazel/buildifier": "^3.3.0",
     "@bazel/ibazel": "^0.13.1",
@@ -59,7 +55,6 @@
     "@types/tar": "^4.0.3",
     "@typescript-eslint/eslint-plugin": "^3.5.0",
     "@typescript-eslint/parser": "^3.5.0",
-    "babel-plugin-const-enum": "^1.0.1",
     "benchmark": "^2.1.4",
     "chalk": "^4.0.0",
     "cldr-core": "^36.0.0",

packages/cli/BUILD

@@ -53,13 +53,12 @@ ts_compile(
     deps = SRC_DEPS,
 )
 
-# TODO: Add integration tests
 jest_test(
     name = "unit",
     srcs = [
         "package.json",
         "tests/extract/unit.test.ts",
-    ] + SRCS,
+    ] + SRCS + glob(["tests/extract/__snapshots__/*.snap"]),
     deps = [
         "@npm//eslint",
         "//packages/babel-plugin-react-intl:types",

packages/cli/package.json

@@ -31,28 +31,20 @@
     "url": "git+ssh://git@github.com/formatjs/formatjs.git"
   },
   "dependencies": {
-    "@babel/plugin-proposal-class-properties": "^7.8.3",
-    "@babel/plugin-proposal-optional-chaining": "7",
-    "@babel/preset-env": "^7.9.5",
-    "@babel/preset-typescript": "^7.9.0",
     "@formatjs/ts-transformer": "^2.4.2",
-    "@types/babel__core": "^7.1.7",
     "@types/lodash": "^4.14.150",
     "@types/loud-rejection": "^2.0.0",
     "@types/node": "14",
-    "babel-plugin-const-enum": "^1.0.1",
     "babel-plugin-react-intl": "^7.9.0",
     "chalk": "^4.0.0",
     "commander": "5.1.0",
     "fs-extra": "^9.0.0",
     "glob": "^7.1.6",
     "lodash": "^4.17.15",
-    "loud-rejection": "^2.2.0"
+    "loud-rejection": "^2.2.0",
+    "typescript": "^3.8"
   },
   "bugs": {
     "url": "https://github.com/formatjs/formatjs/issues"
-  },
-  "peerDependencies": {
-    "@babel/core": "^7.0.0"
   }
 }

packages/cli/src/cli.ts

@@ -34,14 +34,6 @@ async function main(argv: string[]) {
         'The input language is expected to be TypeScript or ES2017 with JSX.',
       ].join('\n')
     )
-    .option(
-      '--messages-dir <dir>',
-      [
-        'The target location where the plugin will output a `.json` file corresponding to each ',
-        'component from which React Intl messages were extracted. If not provided, the extracted ',
-        'message descriptors will be printed to standard output.',
-      ].join('')
-    )
     .option(
       '--out-file <path>',
       [
@@ -68,13 +60,6 @@ async function main(argv: string[]) {
       ].join(''),
       false
     )
-    .option(
-      '--module-source-name <name>',
-      [
-        'The ES6 module source name of the React Intl package. Defaults to: `"react-intl"`, ',
-        'but can be changed to another name/path to React Intl.',
-      ].join('')
-    )
     .option(
       '--remove-default-message',
       'Remove `defaultMessage` field in generated js after extraction',
@@ -130,16 +115,15 @@ async function main(argv: string[]) {
       for (const f of files) {
         processedFiles.push(
           ...globSync(f, {
-            cwd: process.cwd(),
             ignore: cmdObj.ignore,
           })
         );
       }
+
       await extract(processedFiles, {
         outFile: cmdObj.outFile,
         idInterpolationPattern:
           cmdObj.idInterpolationPattern || '[sha1:contenthash:base64:6]',
-        messagesDir: cmdObj.messagesDir,
         extractSourceLocation: cmdObj.extractSourceLocation,
         moduleSourceName: cmdObj.moduleSourceName,
         removeDefaultMessage: cmdObj.removeDefaultMessage,

packages/cli/src/extract.ts

@@ -1,13 +1,19 @@
-import {
-  ExtractionResult,
-  OptionsSchema,
-  ExtractedMessageDescriptor,
-} from 'babel-plugin-react-intl';
-import * as babel from '@babel/core';
+import {OptionsSchema} from 'babel-plugin-react-intl';
 import {warn, getStdinAsString} from './console_utils';
-import {outputJSONSync} from 'fs-extra';
-import {interpolateName} from '@formatjs/ts-transformer';
+import {outputJSONSync, readFile} from 'fs-extra';
+import {
+  interpolateName,
+  transform,
+  Opts,
+  MessageDescriptor,
+} from '@formatjs/ts-transformer';
 import {IOptions as GlobOptions} from 'glob';
+import * as ts from 'typescript';
+
+export interface ExtractionResult<M = Record<string, string>> {
+  messages: MessageDescriptor[];
+  meta: M;
+}
 
 export type ExtractCLIOptions = Omit<ExtractOptions, 'overrideIdFn'> & {
   outFile?: string;
@@ -18,148 +24,118 @@ export type ExtractOptions = OptionsSchema & {
   throws?: boolean;
   idInterpolationPattern?: string;
   readFromStdin?: boolean;
-};
+} & Pick<Opts, 'onMsgExtracted' | 'onMetaExtracted'>;
 
-function getBabelConfig(
-  reactIntlOptions: ExtractCLIOptions,
-  extraBabelOptions: Partial<babel.TransformOptions> = {}
-): babel.TransformOptions {
-  return {
-    babelrc: false,
-    configFile: false,
-    parserOpts: {
-      plugins: [
-        'asyncGenerators',
-        'bigInt',
-        'classPrivateMethods',
-        'classPrivateProperties',
-        'classProperties',
-        'decorators-legacy',
-        'doExpressions',
-        'dynamicImport',
-        'exportDefaultFrom',
-        'functionBind',
-        'functionSent',
-        'importMeta',
-        'jsx',
-        'logicalAssignment',
-        'nullishCoalescingOperator',
-        'numericSeparator',
-        'objectRestSpread',
-        'optionalCatchBinding',
-        'optionalChaining',
-        'partialApplication',
-        'placeholders',
-        'throwExpressions',
-        'topLevelAwait',
-        'typescript',
-      ],
+function processFile(
+  source: string,
+  fn: string,
+  {idInterpolationPattern, ...opts}: Opts & {idInterpolationPattern?: string}
+) {
+  let messages: MessageDescriptor[] = [];
+  let meta: Record<string, string> = {};
+  if (!opts.overrideIdFn && idInterpolationPattern) {
+    opts = {
+      ...opts,
+      overrideIdFn: (id, defaultMessage, description, fileName) =>
+        id ||
+        interpolateName(
+          {
+            resourcePath: fileName,
+          } as any,
+          idInterpolationPattern,
+          {
+            content: description
+              ? `${defaultMessage}#${description}`
+              : defaultMessage,
+          }
+        ),
+      onMsgExtracted(_, msgs) {
+        messages = messages.concat(msgs);
+      },
+      onMetaExtracted(_, m) {
+        meta = m;
+      },
+    };
+  }
+
+  ts.transpileModule(source, {
+    compilerOptions: {
+      allowJs: true,
+      target: ts.ScriptTarget.ESNext,
+      noEmit: true,
+    },
+    fileName: fn,
+    transformers: {
+      before: [transform(opts)],
     },
-    // We need to use require.resolve here, or otherwise the lookup is based on the current working
-    // directory of the CLI.
-    plugins: [[require.resolve('babel-plugin-react-intl'), reactIntlOptions]],
-    highlightCode: true,
-    // Extraction of string messages does not output the transformed JavaScript.
-    sourceMaps: false,
-    ...extraBabelOptions,
-  };
+  });
+  return {messages, meta};
 }
 
 export async function extract(
   files: readonly string[],
-  {idInterpolationPattern, throws, readFromStdin, ...babelOpts}: ExtractOptions
+  {throws, readFromStdin, ...opts}: ExtractOptions
 ): Promise<ExtractionResult[]> {
   if (readFromStdin) {
     // Read from stdin
     if (process.stdin.isTTY) {
       warn('Reading source file from TTY.');
     }
-    if (!babelOpts.overrideIdFn && idInterpolationPattern) {
-      babelOpts = {
-        ...babelOpts,
-        overrideIdFn: (id, defaultMessage, description) =>
-          id ||
-          interpolateName(
-            {
-              resourcePath: 'dummy',
-            } as any,
-            idInterpolationPattern,
-            {content: defaultMessage + (description ? '#' + description : '')}
-          ),
-      };
-    }
     const stdinSource = await getStdinAsString();
-    const babelResult = babel.transformSync(
-      stdinSource,
-      getBabelConfig(babelOpts)
-    );
-
-    return [
-      ((babelResult as babel.BabelFileResult).metadata as any)[
-        'react-intl'
-      ] as ExtractionResult,
-    ];
+    return [processFile(stdinSource, 'dummy', opts)];
   }
 
   const results = await Promise.all(
-    files.map(filename => {
-      if (!babelOpts.overrideIdFn && idInterpolationPattern) {
-        babelOpts = {
-          ...babelOpts,
-          overrideIdFn: (id, defaultMessage, description) =>
-            id ||
-            interpolateName(
-              {
-                resourcePath: filename,
-              } as any,
-              idInterpolationPattern,
-              {
-                content: description
-                  ? `${defaultMessage}#${description}`
-                  : defaultMessage,
-              }
-            ),
-        };
+    files.map(async fn => {
+      try {
+        const source = await readFile(fn, 'utf8');
+        return processFile(source, fn, opts);
+      } catch (e) {
+        if (throws) {
+          throw e;
+        } else {
+          warn(e);
+        }
       }
-      const promise = babel.transformFileAsync(
-        filename,
-        getBabelConfig(babelOpts, {filename: filename})
-      );
-      return throws ? promise : promise.catch(e => warn(e));
     })
   );
-  return results
-    .filter(r => r && r.metadata)
-    .map(
-      r =>
-        ((r as babel.BabelFileResult).metadata as any)[
-          'react-intl'
-        ] as ExtractionResult
-    );
+
+  return results.filter((r): r is ExtractionResult => !!r);
 }
 
 export default async function extractAndWrite(
   files: readonly string[],
   opts: ExtractCLIOptions
 ) {
   const {outFile, throws, ...extractOpts} = opts;
-  if (outFile) {
-    extractOpts.messagesDir = undefined;
-  }
   const extractionResults = await extract(files, extractOpts);
-  const printMessagesToStdout = extractOpts.messagesDir == null && !outFile;
-  const extractedMessages = new Map<string, ExtractedMessageDescriptor>();
+  const printMessagesToStdout = !outFile;
+  const extractedMessages = new Map<string, MessageDescriptor>();
+
   for (const {messages} of extractionResults) {
-    for (const message of messages ?? []) {
+    for (const message of messages) {
       const {id, description, defaultMessage} = message;
+      if (!id) {
+        const error = new Error(
+          `[FormatJS CLI] Missing message id for message: 
+${JSON.stringify(message, undefined, 2)}`
+        );
+        if (throws) {
+          throw error;
+        } else {
+          warn(error.message);
+        }
+        continue;
+      }
+
       if (extractedMessages.has(id)) {
         const existing = extractedMessages.get(id)!;
         if (
           description !== existing.description ||
           defaultMessage !== existing.defaultMessage
         ) {
           const error = new Error(
-            `[React Intl] Duplicate message id: "${id}", ` +
+            `[FormatJS CLI] Duplicate message id: "${id}", ` +
               'but the `description` and/or `defaultMessage` are different.'
           );
           if (throws) {
@@ -169,7 +145,7 @@ export default async function extractAndWrite(
           }
         }
       }
-      extractedMessages.set(message.id, message);
+      extractedMessages.set(id, message);
     }
   }
   const results = Array.from(extractedMessages.values());