feat(@formatjs/cli): add extract & compile to public API, fix #1939

packages/cli/index.ts

@@ -2,7 +2,14 @@ export {
   default as extractAndWrite,
   extract,
   ExtractCLIOptions,
+  ExtractOpts,
 } from './src/extract';
 export {MessageDescriptor} from '@formatjs/ts-transformer';
 export {FormatFn, CompileFn} from './src/formatters/default';
 export {Element, Comparator} from 'json-stable-stringify';
+export {
+  default as compileAndWrite,
+  compile,
+  CompileCLIOpts,
+  Opts as CompileOpts,
+} from './src/compile';

packages/cli/src/cli.ts

@@ -174,12 +174,12 @@ If this is not provided, result will be printed to stdout`
       `Whether to compile to AST. See https://formatjs.io/docs/guides/advanced-usage#pre-parsing-messages
 for more information`
     )
-    .action(async (filePattern: string, {outFile, ...opts}: CompileCLIOpts) => {
+    .action(async (filePattern: string, opts: CompileCLIOpts) => {
       const files = globSync(filePattern);
       if (!files.length) {
         throw new Error(`No input file found with pattern ${filePattern}`);
       }
-      await compile(files, outFile, opts);
+      await compile(files, opts);
     });
 
   if (argv.length < 3) {

packages/cli/src/compile.ts

@@ -1,23 +1,40 @@
 import {parse, MessageFormatElement} from 'intl-messageformat-parser';
-import {outputFileSync, readJSON} from 'fs-extra';
+import {outputFile, readJSON} from 'fs-extra';
 import * as stringify from 'json-stable-stringify';
 import {resolveBuiltinFormatter} from './formatters';
 
 export type CompileFn = (msgs: any) => Record<string, string>;
 
 export interface CompileCLIOpts extends Opts {
+  /**
+   * The target file that contains compiled messages.
+   */
   outFile?: string;
 }
 export interface Opts {
+  /**
+   * Whether to compile message into AST instead of just string
+   */
   ast?: boolean;
+  /**
+   * Path to a formatter file that converts <translation_files> to
+   * `Record<string, string>` so we can compile.
+   */
   format?: string;
 }
-export default async function compile(
-  inputFiles: string[],
-  outFile?: string,
-  {ast, format}: Opts = {}
-) {
-  const formatter = resolveBuiltinFormatter(format);
+
+/**
+ * Aggregate `inputFiles` into a single JSON blob and compile.
+ * Also checks for conflicting IDs.
+ * Then returns the serialized result as a `string` since key order
+ * makes a difference in some vendor.
+ * @param inputFiles Input files
+ * @param opts Options
+ * @returns serialized result in string format
+ */
+export async function compile(inputFiles: string[], opts: Opts = {}) {
+  const {ast, format} = opts;
+  const formatter = await resolveBuiltinFormatter(format);
 
   const messages: Record<string, string> = {};
   const idsWithFileName: Record<string, string> = {};
@@ -46,14 +63,28 @@ Message from ${compiled[id]}: ${inputFile}
     const msgAst = parse(message);
     results[id] = ast ? msgAst : message;
   }
-  const serializedResult = stringify(results, {
+  return stringify(results, {
     space: 2,
     cmp: formatter.compareMessages || undefined,
   });
-  if (!outFile) {
-    process.stdout.write(serializedResult);
-    process.stdout.write('\n');
-  } else {
-    outputFileSync(outFile, serializedResult);
+}
+
+/**
+ * Aggregate `inputFiles` into a single JSON blob and compile.
+ * Also checks for conflicting IDs and write output to `outFile`.
+ * @param inputFiles Input files
+ * @param compileOpts options
+ * @returns A `Promise` that resolves if file was written successfully
+ */
+export default async function compileAndWrite(
+  inputFiles: string[],
+  compileOpts: CompileCLIOpts = {}
+) {
+  const {outFile, ...opts} = compileOpts;
+  const serializedResult = await compile(inputFiles, opts);
+  if (outFile) {
+    return outputFile(outFile, serializedResult);
   }
+  process.stdout.write(serializedResult);
+  process.stdout.write('\n');
 }

packages/cli/src/extract.ts

@@ -1,5 +1,5 @@
 import {warn, getStdinAsString} from './console_utils';
-import {readFile, outputFileSync} from 'fs-extra';
+import {readFile, outputFile} from 'fs-extra';
 import {
   interpolateName,
   transform,
@@ -11,28 +11,59 @@ import * as ts from 'typescript';
 import {resolveBuiltinFormatter} from './formatters';
 import * as stringify from 'json-stable-stringify';
 export interface ExtractionResult<M = Record<string, string>> {
+  /**
+   * List of extracted messages
+   */
   messages: MessageDescriptor[];
+  /**
+   * Metadata extracted w/ `pragma`
+   */
   meta: M;
 }
 
 export interface ExtractedMessageDescriptor extends MessageDescriptor {
+  /**
+   * Line number
+   */
   line?: number;
+  /**
+   * Column number
+   */
   col?: number;
 }
 
 export type ExtractCLIOptions = Omit<
-  ExtractOptions,
+  ExtractOpts,
   'overrideIdFn' | 'onMsgExtracted' | 'onMetaExtracted'
 > & {
+  /**
+   * Output File
+   */
   outFile?: string;
+  /**
+   * Ignore file glob pattern
+   */
   ignore?: GlobOptions['ignore'];
-  format?: string;
 };
 
-export type ExtractOptions = Opts & {
+export type ExtractOpts = Opts & {
+  /**
+   * Whether to throw an error if we had any issues with
+   * 1 of the source files
+   */
   throws?: boolean;
+  /**
+   * Message ID interpolation pattern
+   */
   idInterpolationPattern?: string;
+  /**
+   * Whether we read from stdin instead of a file
+   */
   readFromStdin?: boolean;
+  /**
+   * Path to a formatter file that controls the shape of JSON file from `outFile`.
+   */
+  format?: string;
 } & Pick<Opts, 'onMsgExtracted' | 'onMetaExtracted'>;
 
 function calculateLineColFromOffset(
@@ -100,46 +131,48 @@ function processFile(
   return {messages, meta};
 }
 
+/**
+ * Extract strings from source files
+ * @param files list of files
+ * @param extractOpts extract options
+ * @returns messages serialized as JSON string since key order
+ * matters for some `format`
+ */
 export async function extract(
   files: readonly string[],
-  {throws, readFromStdin, ...opts}: ExtractOptions
-): Promise<ExtractionResult[]> {
+  extractOpts: ExtractOpts
+) {
+  const {throws, readFromStdin, ...opts} = extractOpts;
+  let rawResults: Array<ExtractionResult | undefined>;
   if (readFromStdin) {
     // Read from stdin
     if (process.stdin.isTTY) {
       warn('Reading source file from TTY.');
     }
     const stdinSource = await getStdinAsString();
-    return [processFile(stdinSource, 'dummy', opts)];
+    rawResults = [processFile(stdinSource, 'dummy', opts)];
+  } else {
+    rawResults = await Promise.all(
+      files.map(async fn => {
+        try {
+          const source = await readFile(fn, 'utf8');
+          return processFile(source, fn, opts);
+        } catch (e) {
+          if (throws) {
+            throw e;
+          } else {
+            warn(e);
+          }
+        }
+      })
+    );
   }
 
-  const results = await Promise.all(
-    files.map(async fn => {
-      try {
-        const source = await readFile(fn, 'utf8');
-        return processFile(source, fn, opts);
-      } catch (e) {
-        if (throws) {
-          throw e;
-        } else {
-          warn(e);
-        }
-      }
-    })
+  const formatter = await resolveBuiltinFormatter(opts.format);
+  const extractionResults = rawResults.filter(
+    (r): r is ExtractionResult => !!r
   );
 
-  return results.filter((r): r is ExtractionResult => !!r);
-}
-
-export default async function extractAndWrite(
-  files: readonly string[],
-  opts: ExtractCLIOptions
-) {
-  const {outFile, throws, format, ...extractOpts} = opts;
-  const formatter = resolveBuiltinFormatter(format);
-
-  const extractionResults = await extract(files, extractOpts);
-
   const extractedMessages = new Map<string, MessageDescriptor>();
 
   for (const {messages} of extractionResults) {
@@ -183,14 +216,27 @@ ${JSON.stringify(message, undefined, 2)}`
   for (const {id, ...msg} of messages) {
     results[id] = msg;
   }
-  const serializedResult = stringify(formatter.format(results), {
+  return stringify(formatter.format(results), {
     space: 2,
     cmp: formatter.compareMessages || undefined,
   });
+}
+
+/**
+ * Extract strings from source files, also writes to a file.
+ * @param files list of files
+ * @param extractOpts extract options
+ * @returns A Promise that resolves if output file was written successfully
+ */
+export default async function extractAndWrite(
+  files: readonly string[],
+  extractOpts: ExtractCLIOptions
+) {
+  const {outFile, ...opts} = extractOpts;
+  const serializedResult = await extract(files, opts);
   if (outFile) {
-    outputFileSync(outFile, serializedResult);
-  } else {
-    process.stdout.write(serializedResult);
-    process.stdout.write('\n');
+    return outputFile(outFile, serializedResult);
   }
+  process.stdout.write(serializedResult);
+  process.stdout.write('\n');
 }

packages/cli/src/formatters/index.ts

@@ -5,7 +5,7 @@ import * as simple from './simple';
 import * as lokalise from './lokalise';
 import * as crowdin from './crowdin';
 
-export function resolveBuiltinFormatter(format?: string) {
+export async function resolveBuiltinFormatter(format?: string) {
   if (!format) {
     return defaultFormatter;
   }
@@ -22,7 +22,7 @@ export function resolveBuiltinFormatter(format?: string) {
       return crowdin;
   }
   try {
-    return require(format);
+    return import(format);
   } catch (e) {
     console.error(`Cannot resolve formatter ${format}`);
     throw e;

website/docs/tooling/cli.md

@@ -186,3 +186,48 @@ export const compareMessages: Comparator = () => {};
 ```
 
 Take a look at our [builtin formatter code](https://github.com/formatjs/formatjs/tree/main/packages/cli/src/formatters) for some examples.
+
+## Node API
+
+`@formatjs/cli` can also be consumed programmatically like below:
+
+### Extraction
+
+```tsx
+import {extract} from '@formatjs/cli';
+
+const resultAsString: Promise<string> = extract(files, {
+  idInterpolationPattern: '[sha512:contenthash:base64:6]',
+});
+```
+
+### Compilation
+
+```tsx
+import {compile} from '@formatjs/cli';
+
+const resultAsString: Promise<string> = compile(files, {
+  ast: true,
+});
+```
+
+### Custom Formatter
+
+```tsx
+import {FormatFn, CompileFn, Comparator} from '@formatjs/cli';
+
+export const format: FormatFn = msgs => msgs;
+
+// Sort key reverse alphabetically
+export const compareMessages = (el1, el2) => {
+  return el1.key < el2.key ? 1 : -1;
+};
+
+export const compile: CompileFn = msgs => {
+  const results: Record<string, string> = {};
+  for (const k in msgs) {
+    results[k] = msgs[k].defaultMessage!;
+  }
+  return results;
+};
+```