Skip to content

Commit

Permalink
WIP: Use actual modules for the ember extractors -- however, formatjs is
Browse files Browse the repository at this point in the history
not setup to handle modules.
  • Loading branch information
NullVoxPopuli committed Apr 5, 2024
1 parent fbb0e83 commit 57d6473
Show file tree
Hide file tree
Showing 11 changed files with 1,632 additions and 641 deletions.
18 changes: 12 additions & 6 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@
"@lerna-lite/list": "2",
"@lerna-lite/version": "2",
"@napi-rs/cli": "^2.11.0",
"@swc-node/register": "^1.5.4",
"@swc/core": "^1.3.56",
"@swc/jest": "^0.2.23",
"@swc-node/register": "^1.9.0",
"@swc/core": "^1.4.12",
"@swc/jest": "^0.2.36",
"@taplo/cli": "^0.5.0",
"@testing-library/jest-dom": "^6.0.0",
"@testing-library/react": "^14.0.0",
Expand Down Expand Up @@ -101,7 +101,7 @@
"http-server": "^14.0.0",
"husky": "^8.0.0",
"jasmine-expect": "^5.0.0",
"jest": "29",
"jest": "^29.7.0",
"jest-environment-jsdom": "29",
"json-stable-stringify": "^1.0.1",
"karma": "^6.0.4",
Expand All @@ -123,8 +123,8 @@
"rimraf": "^3.0.2",
"serialize-javascript": "^6.0.0",
"test262-harness": "10",
"ts-jest": "29",
"ts-loader": "^9.1.2",
"ts-jest": "^29.1.2",
"ts-loader": "^9.5.1",
"tsd": "^0.30.0",
"tslib": "^2.4.0",
"typescript": "5.2.2",
Expand All @@ -145,6 +145,12 @@
"source-map-js": "1.0.2"
},
"pnpm": {
"overrides": {
"ember-template-recast": "^6.1.4",
"@glimmer/reference": "^0.91.1",
"@glimmer/syntax": "^0.91.1",
"@glimmer/validator": "^0.91.1"
},
"packageExtensions": {
"ts-jest": {
"dependencies": {
Expand Down
55 changes: 30 additions & 25 deletions packages/cli-lib/src/extract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ import {
Opts,
interpolateName,
} from '@formatjs/ts-transformer'
import {outputFile, readFile} from 'fs-extra'
import {debug, getStdinAsString, warn, writeStdout} from './console_utils'
import { outputFile, readFile } from 'fs-extra'
import { debug, getStdinAsString, warn, writeStdout } from './console_utils'

import {parse} from '@formatjs/icu-messageformat-parser'
import {hoistSelectors} from '@formatjs/icu-messageformat-parser/manipulator'
import {printAST} from '@formatjs/icu-messageformat-parser/printer'
import { parse } from '@formatjs/icu-messageformat-parser'
import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator'
import { printAST } from '@formatjs/icu-messageformat-parser/printer'
import stringify from 'json-stable-stringify'
import {Formatter, resolveBuiltinFormatter} from './formatters'
import {parseScript} from './parse_script'
import { Formatter, resolveBuiltinFormatter } from './formatters'
import { parseScript } from './parse_script'
export interface ExtractionResult<M = Record<string, string>> {
/**
* List of extracted messages
Expand Down Expand Up @@ -81,18 +81,18 @@ function calculateLineColFromOffset(
start?: number
): Pick<ExtractedMessageDescriptor, 'line' | 'col'> {
if (!start) {
return {line: 1, col: 1}
return { line: 1, col: 1 }
}
const chunk = text.slice(0, start)
const lines = chunk.split('\n')
const lastLine = lines[lines.length - 1]
return {line: lines.length, col: lastLine.length}
return { line: lines.length, col: lastLine.length }
}

async function processFile(
source: string,
fn: string,
{idInterpolationPattern, ...opts}: Opts & {idInterpolationPattern?: string}
{ idInterpolationPattern, ...opts }: Opts & { idInterpolationPattern?: string }
) {
let messages: ExtractedMessageDescriptor[] = []
let meta: Record<string, string> | undefined
Expand Down Expand Up @@ -140,11 +140,10 @@ async function processFile(
idInterpolationPattern,
{
content: description
? `${defaultMessage}#${
typeof description === 'string'
? description
: stringify(description)
}`
? `${defaultMessage}#${typeof description === 'string'
? description
: stringify(description)
}`
: defaultMessage,
}
),
Expand All @@ -156,15 +155,21 @@ async function processFile(
const scriptParseFn = parseScript(opts, fn)
if (fn.endsWith('.vue')) {
debug('Processing %s using vue extractor', fn)
const {parseFile} = await import('./vue_extractor.js')
const { parseFile } = await import('./vue_extractor.js')
parseFile(source, fn, scriptParseFn)
} else if (fn.endsWith('.hbs')) {
debug('Processing %s using hbs extractor', fn)
const {parseFile} = await import('./hbs_extractor')
// SAFETY: The TS config does not understand that this is a
// safe way to import a module in a cjs project
// @ts-ignore
const { parseFile } = await import('./hbs_extractor.mjs')
parseFile(source, fn, opts)
} else if (fn.endsWith('.gts') || fn.endsWith('.gjs')) {
debug('Processing %s as gts/gjs file', fn)
const {parseFile} = await import('./gts_extractor')
// SAFETY: The TS config does not understand that this is a
// safe way to import a module in a cjs project
// @ts-ignore
const { parseFile } = await import('./gts_extractor.mjs')
parseFile(source, fn, opts)
} else {
debug('Processing %s using typescript extractor', fn)
Expand All @@ -175,7 +180,7 @@ async function processFile(
debug('Extracted meta:', meta)
messages.forEach(m => (m.meta = meta))
}
return {messages, meta}
return { messages, meta }
}

/**
Expand All @@ -189,7 +194,7 @@ export async function extract(
files: readonly string[],
extractOpts: ExtractOpts
) {
const {throws, readFromStdin, flatten, ...opts} = extractOpts
const { throws, readFromStdin, flatten, ...opts } = extractOpts
let rawResults: Array<ExtractionResult | undefined>
if (readFromStdin) {
debug(`Reading input from stdin`)
Expand Down Expand Up @@ -222,9 +227,9 @@ export async function extract(

const extractedMessages = new Map<string, MessageDescriptor>()

for (const {messages} of extractionResults) {
for (const { messages } of extractionResults) {
for (const message of messages) {
const {id, description, defaultMessage} = message
const { id, description, defaultMessage } = message
if (!id) {
const error = new Error(
`[FormatJS CLI] Missing message id for message:
Expand All @@ -246,7 +251,7 @@ ${JSON.stringify(message, undefined, 2)}`
) {
const error = new Error(
`[FormatJS CLI] Duplicate message id: "${id}", ` +
'but the `description` and/or `defaultMessage` are different.'
'but the `description` and/or `defaultMessage` are different.'
)
if (throws) {
throw error
Expand All @@ -260,7 +265,7 @@ ${JSON.stringify(message, undefined, 2)}`
}
const results: Record<string, Omit<MessageDescriptor, 'id'>> = {}
const messages = Array.from(extractedMessages.values())
for (const {id, ...msg} of messages) {
for (const { id, ...msg } of messages) {
if (flatten && msg.defaultMessage) {
msg.defaultMessage = printAST(hoistSelectors(parse(msg.defaultMessage)))
}
Expand All @@ -285,7 +290,7 @@ export default async function extractAndWrite(
files: readonly string[],
extractOpts: ExtractCLIOptions
) {
const {outFile, ...opts} = extractOpts
const { outFile, ...opts } = extractOpts
const serializedResult = (await extract(files, opts)) + '\n'
if (outFile) {
debug('Writing output file:', outFile)
Expand Down
18 changes: 18 additions & 0 deletions packages/cli-lib/src/gts_extractor.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { parseFile as parseHbsFile } from './hbs_extractor.mjs'
import { parseScript } from './parse_script.js'
import { Preprocessor } from 'content-tag'
let p = new Preprocessor()

export function parseFile(source: string, fileName: string, options: any) {
const scriptParseFn = parseScript(options, fileName)
const transformedSource = p.process(source, { filename: fileName })

scriptParseFn(transformedSource)

// extract template from transformed source to then run through hbs processor
const parseResult = p.parse(source, { filename: fileName })

for (let parsed of parseResult) {
parseHbsFile(parsed.contents, fileName, options)
}
}
25 changes: 0 additions & 25 deletions packages/cli-lib/src/gts_extractor.ts

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {transform} from 'ember-template-recast'
import type {AST} from '@glimmer/syntax'
import {Opts} from '@formatjs/ts-transformer'
import { transform } from 'ember-template-recast'
import type { AST } from '@glimmer/syntax'
import { Opts } from '@formatjs/ts-transformer'

function extractText(
node: AST.MustacheStatement | AST.SubExpression,
Expand Down Expand Up @@ -42,7 +42,7 @@ function extractText(
}

export function parseFile(source: string, fileName: string, options: any) {
let visitor = function () {
let visitor = function() {
return {
MustacheStatement(node: AST.MustacheStatement) {
extractText(node, fileName, options)
Expand Down
33 changes: 0 additions & 33 deletions packages/cli-lib/tests/unit/gjs_extractor.test.ts

This file was deleted.

95 changes: 65 additions & 30 deletions packages/cli-lib/tests/unit/gts_extractor.test.ts
Original file line number Diff line number Diff line change
@@ -1,33 +1,68 @@
import {MessageDescriptor, interpolateName} from '@formatjs/ts-transformer'
import {parseFile} from '../../src/gts_extractor'
import {readFile} from 'fs-extra'
import {join} from 'path'
import { MessageDescriptor, interpolateName } from '@formatjs/ts-transformer'
import { readFile } from 'fs-extra'
import { join } from 'path'

test('gts_extractor', async function () {
let messages: MessageDescriptor[] = []
const fixturePath = join(__dirname, './fixtures/comp.gts')
parseFile(await readFile(fixturePath, 'utf8'), fixturePath, {
onMsgExtracted(_: any, msgs: any) {
messages = messages.concat(msgs)
},
overrideIdFn: (
id: any,
defaultMessage: any,
description: any,
fileName: any
) =>
id ||
interpolateName(
{
resourcePath: fileName,
} as any,
'[sha512:contenthash:base64:6]',
{
content: description
? `${defaultMessage}#${description}`
: defaultMessage,
}
),
describe('gts_extractor', () => {
test('gts files', async function() {
const { parseFile } = await import('../../src/gts_extractor.mjs');

let messages: MessageDescriptor[] = []
const fixturePath = join(__dirname, './fixtures/comp.gts')
parseFile(await readFile(fixturePath, 'utf8'), fixturePath, {
onMsgExtracted(_: any, msgs: any) {
messages = messages.concat(msgs)
},
overrideIdFn: (
id: any,
defaultMessage: any,
description: any,
fileName: any
) =>
id ||
interpolateName(
{
resourcePath: fileName,
} as any,
'[sha512:contenthash:base64:6]',
{
content: description
? `${defaultMessage}#${description}`
: defaultMessage,
}
),
})
expect(messages).toMatchSnapshot()
})
expect(messages).toMatchSnapshot()

test('gjs files', async function() {
const { parseFile } = await import('../../src/gts_extractor.mjs');

let messages: MessageDescriptor[] = []
const fixturePath = join(__dirname, './fixtures/comp.gjs')
parseFile(await readFile(fixturePath, 'utf8'), fixturePath, {
onMsgExtracted(_: any, msgs: any) {
messages = messages.concat(msgs)
},
overrideIdFn: (
id: any,
defaultMessage: any,
description: any,
fileName: any
) =>
id ||
interpolateName(
{
resourcePath: fileName,
} as any,
'[sha512:contenthash:base64:6]',
{
content: description
? `${defaultMessage}#${description}`
: defaultMessage,
}
),
})
expect(messages).toMatchSnapshot()
})

})
Loading

0 comments on commit 57d6473

Please sign in to comment.