Skip to content

Commit

Permalink
feat: add ESM tools in gax (#1459)
Browse files Browse the repository at this point in the history
This PR does the following:

1. Adds 3 babel plugins to help transform ESM code to CJS: one to transform `path.dirname(fileURLToPath(import.meta)` to `__dirname`, another to turn `isEsm` to false, and lastly a final one to turn proxyquire into esmock.
3. Adds an ESM-path in compileProtos (searches for source code one level deeper, and generates `protos.js` and `protos.cjs` in es6 and amd, respectively)
  • Loading branch information
ddelgrosso1 committed Sep 19, 2023
1 parent 6052c9a commit 0fb1cf9
Show file tree
Hide file tree
Showing 15 changed files with 3,878 additions and 9 deletions.
6 changes: 6 additions & 0 deletions tools/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,17 @@
"google-proto-files": "^4.0.0",
"protobufjs-cli": "1.1.2",
"rimraf": "^5.0.1",
"@babel/core": "^7.22.5",
"@babel/traverse": "^7.22.5",
"uglify-js": "^3.17.0",
"walkdir": "^0.4.0"
},
"repository": "googleapis/gax-nodejs",
"devDependencies": {
"@babel/cli": "^7.22.5",
"@babel/types": "^7.22.5",
"@types/babel__core": "^7.20.1",
"@types/babel__traverse": "^7.20.1",
"@types/mocha": "^9.0.0",
"@types/ncp": "^2.0.1",
"@types/uglify-js": "^3.17.0",
Expand Down
68 changes: 60 additions & 8 deletions tools/src/compileProtos.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,13 @@ function updateDtsTypes(dts: string, enums: Set<string>): string {
}

function fixJsFile(js: string): string {
// 0. fix protobufjs import: we don't want the libraries to
// depend on protobufjs, so we re-export it from google-gax
js = js.replace(
'import * as $protobuf from "protobufjs/minimal"',
'import {protobufMinimal as $protobuf} from "google-gax/build/src/protobuf.js"'
);

// 1. fix protobufjs require: we don't want the libraries to
// depend on protobufjs, so we re-export it from google-gax
js = js.replace(
Expand Down Expand Up @@ -211,13 +218,19 @@ function fixDtsFile(dts: string): string {
* @param {string[]} protoJsonFiles List of JSON files to parse
* @return {Promise<string[]>} Resolves to an array of proto files.
*/
async function buildListOfProtos(protoJsonFiles: string[]): Promise<string[]> {
async function buildListOfProtos(
protoJsonFiles: string[],
esm?: boolean
): Promise<string[]> {
const result: string[] = [];
for (const file of protoJsonFiles) {
const directory = path.dirname(file);
const content = await readFile(file);
const list = JSON.parse(content.toString()).map((filePath: string) =>
path.join(directory, normalizePath(filePath))
// If we're in ESM, we're going to be in a directory level below normal
esm
? path.join(directory, '..', normalizePath(filePath))
: path.join(directory, normalizePath(filePath))
);
result.push(...list);
}
Expand All @@ -236,7 +249,8 @@ async function buildListOfProtos(protoJsonFiles: string[]): Promise<string[]> {
async function compileProtos(
rootName: string,
protos: string[],
skipJson = false
skipJson = false,
esm = false
): Promise<void> {
if (!skipJson) {
// generate protos.json file from proto list
Expand All @@ -261,7 +275,9 @@ async function compileProtos(
}

// generate protos/protos.js from protos.json
const jsOutput = path.join('protos', 'protos.js');
const jsOutput = esm
? path.join('protos', 'protos.cjs')
: path.join('protos', 'protos.js');
const pbjsArgs4js = [
'-r',
rootName,
Expand All @@ -281,9 +297,34 @@ async function compileProtos(
jsResult = fixJsFile(jsResult);
await writeFile(jsOutput, jsResult);

let jsOutputEsm;
if (esm) {
jsOutputEsm = path.join('protos', 'protos.js');
const pbjsArgs4jsEsm = [
'-r',
rootName,
'--target',
'static-module',
'-p',
'protos',
'-p',
gaxProtos,
'-o',
jsOutputEsm,
'-w',
'es6',
];
pbjsArgs4jsEsm.push(...protos);
await pbjsMain(pbjsArgs4jsEsm);

let jsResult = (await readFile(jsOutputEsm)).toString();
jsResult = fixJsFile(jsResult);
await writeFile(jsOutputEsm, jsResult);
}

// generate protos/protos.d.ts
const tsOutput = path.join('protos', 'protos.d.ts');
const pbjsArgs4ts = [jsOutput, '-o', tsOutput];
const pbjsArgs4ts = [esm ? jsOutputEsm! : jsOutput, '-o', tsOutput];
await pbtsMain(pbjsArgs4ts);

let tsResult = (await readFile(tsOutput)).toString();
Expand Down Expand Up @@ -330,27 +371,38 @@ export async function generateRootName(directories: string[]): Promise<string> {
export async function main(parameters: string[]): Promise<void> {
const protoJsonFiles: string[] = [];
let skipJson = false;
let esm = false;
const directories: string[] = [];
for (const parameter of parameters) {
if (parameter === '--skip-json') {
skipJson = true;
continue;
}
if (parameter === '--esm') {
esm = true;
continue;
}
// it's not an option so it's a directory
const directory = parameter;
directories.push(directory);
protoJsonFiles.push(...(await findProtoJsonFiles(directory)));
}
const rootName = await generateRootName(directories);
const protos = await buildListOfProtos(protoJsonFiles);
await compileProtos(rootName, protos, skipJson);
if (esm) {
const esmProtos = await buildListOfProtos(protoJsonFiles, esm);
await compileProtos(rootName, esmProtos, skipJson, esm);
}
const protos = await buildListOfProtos(protoJsonFiles, esm);
await compileProtos(rootName, protos, skipJson, esm);
}

/**
* Shows the usage information.
*/
function usage() {
console.log(`Usage: node ${process.argv[1]} [--skip-json] directory ...`);
console.log(
`Usage: node ${process.argv[1]} [--skip-json] [--esm] directory ...`
);
console.log(
`Finds all files matching ${PROTO_LIST_REGEX} in the given directories.`
);
Expand Down
64 changes: 64 additions & 0 deletions tools/src/replaceESMMockingLib.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import {Visitor, types} from '@babel/core';

export interface PluginOptions {
opts?: {
fromLibName?: string;
toLibName?: string;
};
}

export default function replaceESMMockingLib(): {
visitor: Visitor<PluginOptions>;
} {
return {
visitor: {
ImportDeclaration(path, state) {
const opts = state.opts || {};
const fromLib = opts.fromLibName || 'esmock';
const toLib = opts.toLibName || 'proxyquire';
const {node} = path;

node.specifiers.forEach(spec => {
if (spec.local.name !== fromLib) {
return;
}
spec.local.name = spec.local.name.replace(fromLib, toLib);
});

if (node.source.value !== fromLib) {
return;
}
node.source.value = node.source.value.replace(fromLib, toLib);
},
CallExpression(path, state) {
const opts = state.opts || {};
const fromLib = opts.fromLibName || 'esmock';
const toLib = opts.toLibName || 'proxyquire';
const {node} = path;

if (types.isIdentifier(node.callee)) {
if (node.callee.name !== fromLib) {
return;
}

node.callee.name = toLib;
path.parentPath.replaceWith(types.expressionStatement(node));
}
},
},
};
}
58 changes: 58 additions & 0 deletions tools/src/replaceImportMetaUrl.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// eslint-disable-next-line node/no-extraneous-import
import {smart} from '@babel/template';
import {Statement} from '@babel/types';
import {Visitor} from '@babel/core';

export interface PluginOptions {
opts?: {
replacementValue?: string;
};
}

export default function replaceImportMetaUrl(): {
visitor: Visitor<PluginOptions>;
} {
return {
visitor: {
CallExpression(path, state) {
const opts = state.opts || {};
const replacementValue = opts.replacementValue || '__dirname';
const {node} = path;
if (
node.callee.type === 'MemberExpression' &&
node.callee.property.type === 'Identifier' &&
node.callee.property.name === 'dirname' &&
node.arguments[0].type === 'CallExpression' &&
node.arguments[0].callee.type === 'Identifier' &&
node.arguments[0].callee.name === 'fileURLToPath' &&
node.arguments[0].arguments[0].type === 'MemberExpression' &&
node.arguments[0].arguments[0].object.type === 'MetaProperty' &&
node.arguments[0].arguments[0].object.meta.type === 'Identifier' &&
node.arguments[0].arguments[0].object.meta.name === 'import' &&
node.arguments[0].arguments[0].object.property.type ===
'Identifier' &&
node.arguments[0].arguments[0].object.property.name === 'meta' &&
node.arguments[0].arguments[0].property.type === 'Identifier' &&
node.arguments[0].arguments[0].property.name === 'url'
) {
const replacement = smart.ast`${replacementValue}` as Statement;
path.replaceWith(replacement);
}
},
},
};
}
44 changes: 44 additions & 0 deletions tools/src/toggleESMFlagVariable.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import {Visitor, types} from '@babel/core';

export interface PluginOptions {
opts?: {
variableIdentifier?: string;
replacementValue?: boolean;
};
}

export default function toggleESMFlagVariable(): {
visitor: Visitor<PluginOptions>;
} {
return {
visitor: {
VariableDeclarator(path, state) {
const opts = state.opts || {};
const variableIdentifier = opts.variableIdentifier || 'isEsm';
const replacementValue = opts.replacementValue || false;
const {node} = path;
const identifier = node.id as types.Identifier;
if (
identifier.name === variableIdentifier &&
node.init?.type === 'BooleanLiteral'
) {
node.init.value = replacementValue;
}
},
},
};
}

0 comments on commit 0fb1cf9

Please sign in to comment.