chore(javascript): build util packages first when building all (#371)

* chore(javascript): build util packages first when building all * chore: remove duplication WIP * chore: fix rollup config * chore: fix for `all` as client * chore: fix lint error * chore: fail-safe clean * chore: support `yarn build utils` * chore: update the script for build:utils * chore: build utils only if they aren't built yet * chore: update .cache_version * chore: fix bugs * chore: fix lint error * chore: fix lint error * chore: enable build commands via docker * chore: remove unnecessary utils build after generation
algolia · Apr 14, 2022 · fab6c18 · fab6c18
1 parent aecafb5
commit fab6c18
Show file tree

Hide file tree

Showing 7 changed files with 115 additions and 168 deletions.
diff --git a/clients/algoliasearch-client-javascript/package.json b/clients/algoliasearch-client-javascript/package.json
@@ -6,8 +6,8 @@
   ],
   "scripts": {
     "build": "CLIENT=${0:-all} yarn rollup -c rollup.config.js",
-    "build:utils": "yarn build client-common && yarn build requester-browser-xhr && yarn build requester-node-http",
-    "clean": "rm -rf packages/*/dist",
+    "build:utils": "yarn build utils",
+    "clean": "rm -rf packages/*/dist || true",
     "clean:utils": "yarn workspace @experimental-api-clients-automation/client-common clean && yarn workspace @experimental-api-clients-automation/requester-node-http clean && yarn workspace @experimental-api-clients-automation/requester-browser-xhr clean",
     "release": "shipjs prepare",
     "release:bump": "lerna version ${0:-patch} --no-changelog --no-git-tag-version --no-push --exact --yes",

diff --git a/clients/algoliasearch-client-javascript/rollup.config.js b/clients/algoliasearch-client-javascript/rollup.config.js
@@ -15,6 +15,9 @@ const client = process.env.CLIENT?.replace(
 );
 const UTILS = ['client-common', 'requester-browser-xhr', 'requester-node-http'];
 
+const CLIENT_ALL = 'all';
+const CLIENT_UTILS = 'utils';
+
 function createLicence(name, version) {
   return `/*! ${name}.umd.js | ${version} | © Algolia, inc. | https://github.com/algolia/algoliasearch-client-javascript */`;
 }
@@ -45,64 +48,70 @@ function createBundlers({ output, clientPath }) {
 }
 
 function getAvailableClients() {
-  const exception = new Set([
-    'client-common',
-    'requester-browser-xhr',
-    'requester-node-http',
-  ]);
-
   // ['algoliasearch', 'client-abtesting', ... ]
   const availableClients = fs
     .readdirSync('packages/')
-    .filter((_client) => !exception.has(_client));
+    .filter((_client) => !UTILS.includes(_client));
 
-  return client === 'all'
+  return client === CLIENT_ALL
     ? availableClients
     : availableClients.filter((availableClient) => availableClient === client);
 }
 
-function initPackagesConfig() {
-  if (UTILS.includes(client)) {
-    const commonOptions = {
-      input: 'index.ts',
-      formats: ['cjs-node', 'esm-node'],
-      external: [],
-      dependencies: [],
-    };
+function getUtilConfigs() {
+  const commonOptions = {
+    input: 'index.ts',
+    formats: ['cjs-node', 'esm-node'],
+    external: [],
+    dependencies: [],
+  };
 
-    const availableUtils = [
-      // Common
-      {
-        ...commonOptions,
-        output: 'client-common',
-        package: 'client-common',
-        name: '@experimental-api-clients-automation/client-common',
-      },
-      // Browser requester
-      {
-        ...commonOptions,
-        output: 'requester-browser-xhr',
-        package: 'requester-browser-xhr',
-        name: '@experimental-api-clients-automation/requester-browser-xhr',
-        external: ['dom'],
-        dependencies: ['@experimental-api-clients-automation/client-common'],
-      },
-      // Node requester
-      {
-        ...commonOptions,
-        output: 'requester-node-http',
-        package: 'requester-node-http',
-        name: '@experimental-api-clients-automation/requester-node-http',
-        external: ['https', 'http', 'url'],
-        dependencies: ['@experimental-api-clients-automation/client-common'],
-      },
-    ];
+  return [
+    // Common
+    {
+      ...commonOptions,
+      output: 'client-common',
+      package: 'client-common',
+      name: '@experimental-api-clients-automation/client-common',
+    },
+    // Browser requester
+    {
+      ...commonOptions,
+      output: 'requester-browser-xhr',
+      package: 'requester-browser-xhr',
+      name: '@experimental-api-clients-automation/requester-browser-xhr',
+      external: ['dom'],
+      dependencies: ['@experimental-api-clients-automation/client-common'],
+    },
+    // Node requester
+    {
+      ...commonOptions,
+      output: 'requester-node-http',
+      package: 'requester-node-http',
+      name: '@experimental-api-clients-automation/requester-node-http',
+      external: ['https', 'http', 'url'],
+      dependencies: ['@experimental-api-clients-automation/client-common'],
+    },
+  ];
+}
 
-    return client === 'all'
-      ? availableUtils
-      : availableUtils.filter(
-          (availableUtil) => availableUtil.package === client
-        );
+function isClientBuilt(client) {
+  // Checking existence of `dist` folder doesn't really guarantee the built files are up-to-date.
+  // However, on the CI, it's very likely.
+  // For the local environment, we simply return `false`, which will trigger unnecessary builds.
+  // We can come back here and improve this part (checking if `dist` folder exists and if it's up-to-date).
+  return process.env.CI
+    ? fs.existsSync(path.resolve('packages', client, 'dist'))
+    : false;
+}
+
+function getPackageConfigs() {
+  if (client === CLIENT_UTILS) {
+    return getUtilConfigs();
+  }
+
+  if (UTILS.includes(client)) {
+    return getUtilConfigs().filter((config) => config.package === client);
   }
 
   const availableClients = getAvailableClients();
@@ -111,7 +120,7 @@ function initPackagesConfig() {
     throw new Error(`No clients matches '${client}'.`);
   }
 
-  return availableClients.flatMap((packageName) => {
+  const configs = availableClients.flatMap((packageName) => {
     const isAlgoliasearchClient = packageName === 'algoliasearch';
     const commonConfig = {
       package: packageName,
@@ -157,12 +166,17 @@ function initPackagesConfig() {
       },
     ];
   });
+
+  return [
+    ...getUtilConfigs().filter((config) => !isClientBuilt(config.package)),
+    ...configs,
+  ];
 }
 
-const packagesConfig = initPackagesConfig();
+const packageConfigs = getPackageConfigs();
 const rollupConfig = [];
 
-packagesConfig.forEach((packageConfig) => {
+packageConfigs.forEach((packageConfig) => {
   const clientPath = path.resolve('packages', packageConfig.package);
   const clientPackage = JSON.parse(
     fs.readFileSync(path.resolve(clientPath, 'package.json'))

diff --git a/package.json b/package.json
@@ -13,7 +13,7 @@
   ],
   "scripts": {
     "build:eslint": "yarn workspace eslint-plugin-automation-custom build && yarn install",
-    "clean": "rm -rf **/dist **/build **/node_modules **/.gradle",
+    "clean": "rm -rf **/dist **/build **/node_modules **/.gradle || true",
     "cli": "yarn workspace scripts ts-node --transpile-only ./index.ts",
     "docker": "docker exec -it dev yarn cli $*",
     "docker:build": "./scripts/docker/build.sh",

diff --git a/scripts/buildClients.ts b/scripts/buildClients.ts
@@ -1,76 +1,47 @@
-import { run } from './common';
+import {
+  createGeneratorKey,
+  GENERATORS,
+  LANGUAGES,
+  run,
+  toAbsolutePath,
+} from './common';
 import { getLanguageFolder } from './config';
 import { createSpinner } from './oraLog';
-import type { Generator } from './types';
-
-const multiBuildLanguage = new Set(['javascript']);
 
 /**
- * Build JavaScript utils packages used in generated clients.
+ * Build all client for a language at the same time, for those who live in the same folder.
  */
-export async function buildJSClientUtils(
-  verbose: boolean,
-  client?: string
-): Promise<void> {
-  if (!client || client === 'all') {
-    const spinner = createSpinner('building JavaScript utils', verbose).start();
-    await run('yarn workspace algoliasearch-client-javascript clean:utils', {
-      verbose,
-    });
-    await run('yarn workspace algoliasearch-client-javascript build:utils', {
-      verbose,
-    });
-
-    spinner.succeed();
-    return;
-  }
-
-  const spinner = createSpinner(
-    `building JavaScript ${client} utils`,
-    verbose
-  ).start();
-
-  await run(
-    `yarn workspace @experimental-api-clients-automation/${client} clean`,
-    { verbose }
-  );
-  await run(`yarn workspace algoliasearch-client-javascript build ${client}`, {
-    verbose,
-  });
-
-  spinner.succeed();
-}
+async function buildPerLanguage({
+  language,
+  client,
+  verbose,
+}: {
+  language: string;
+  client: string;
+  verbose: boolean;
+}): Promise<void> {
+  const spinner = createSpinner(`building '${language}'`, verbose).start();
+  const cwd = toAbsolutePath(getLanguageFolder(language));
+  const generator =
+    client === 'all'
+      ? null
+      : GENERATORS[createGeneratorKey({ language, client })];
 
-/**
- * Build only a specific client for one language, used by javascript for example.
- */
-async function buildPerClient(
-  { language, key, additionalProperties: { packageName } }: Generator,
-  verbose: boolean
-): Promise<void> {
-  const spinner = createSpinner(`building ${key}`, verbose).start();
   switch (language) {
     case 'javascript':
-      await run(`yarn workspace ${packageName} clean`, { verbose });
+      await run(`yarn clean`, { cwd, verbose });
       await run(
-        `yarn workspace algoliasearch-client-javascript build ${packageName}`,
-        { verbose }
+        `yarn build ${
+          client === 'all'
+            ? ''
+            : generator?.additionalProperties.buildFile ?? client
+        }`,
+        {
+          cwd,
+          verbose,
+        }
       );
       break;
-    default:
-  }
-  spinner.succeed();
-}
-
-/**
- * Build all client for a language at the same time, for those who live in the same folder.
- */
-async function buildAllClients(
-  language: string,
-  verbose: boolean
-): Promise<void> {
-  const spinner = createSpinner(`building '${language}'`, verbose).start();
-  switch (language) {
     case 'java':
       await run(
         `./gradle/gradlew --no-daemon -p ${getLanguageFolder(
@@ -89,35 +60,19 @@ async function buildAllClients(
 }
 
 export async function buildClients(
-  allGenerators: Generator[],
+  language: string,
+  client: string,
   verbose: boolean
 ): Promise<void> {
-  const langs = [...new Set(allGenerators.map((gen) => gen.language))];
+  const languages = language === 'all' ? LANGUAGES : [language];
 
-  // We exclude `javascript-algoliasearch` from the build batch because it
-  // is made of built generated clients and can cause race issue when executed
-  // together.
-  const jsAlgoliasearch = allGenerators.find(
-    (gen) => gen.key === 'javascript-algoliasearch'
+  await Promise.all(
+    languages.map((lang) =>
+      buildPerLanguage({
+        language: lang,
+        client,
+        verbose,
+      })
+    )
   );
-  const generators = allGenerators.filter(
-    (gen) => gen.key !== 'javascript-algoliasearch'
-  );
-
-  await Promise.all([
-    Promise.all(
-      generators
-        .filter(({ language }) => multiBuildLanguage.has(language))
-        .map((gen) => buildPerClient(gen, verbose))
-    ),
-    Promise.all(
-      langs
-        .filter((lang) => !multiBuildLanguage.has(lang))
-        .map((lang) => buildAllClients(lang, verbose))
-    ),
-  ]);
-
-  if (jsAlgoliasearch) {
-    await buildPerClient(jsAlgoliasearch, verbose);
-  }
 }
diff --git a/scripts/common.ts b/scripts/common.ts
@@ -37,6 +37,7 @@ export const GENERATORS: Record<string, Generator> = {
     client: 'algoliasearch',
     key: 'javascript-algoliasearch',
     additionalProperties: {
+      buildFile: 'algoliasearch',
       packageName: '@experimental-api-clients-automation/algoliasearch',
       packageVersion:
         openapitools['generator-cli'].generators[

diff --git a/scripts/generate.ts b/scripts/generate.ts
@@ -1,6 +1,5 @@
 import path from 'path';
 
-import { buildJSClientUtils } from './buildClients';
 import { buildSpecs } from './buildSpecs';
 import {
   buildCustomGenerators,
@@ -144,11 +143,5 @@ export async function generate(
     if (!(CI && lang === 'javascript')) {
       await formatter(lang, getLanguageFolder(lang), verbose);
     }
-
-    // JavaScript utils are tested independently, we only build them
-    // during dev to ease the process
-    if (!CI && lang === 'javascript') {
-      await buildJSClientUtils(verbose, 'all');
-    }
   }
 }
diff --git a/scripts/index.ts b/scripts/index.ts
@@ -2,7 +2,7 @@
 import { Argument, program } from 'commander';
 import inquirer from 'inquirer';
 
-import { buildClients, buildJSClientUtils } from './buildClients';
+import { buildClients } from './buildClients';
 import { buildSpecs } from './buildSpecs';
 import {
   CI,
@@ -157,23 +157,7 @@ buildCommand
         : CLIENTS;
       client = await promptClient(client, interactive, clientList);
 
-      // We build the JavaScript utils before generated clients as they
-      // rely on them
-      if (
-        shouldBuildJs &&
-        (!client || client === 'all' || CLIENTS_JS_UTILS.includes(client))
-      ) {
-        await buildJSClientUtils(Boolean(verbose), client);
-      }
-
-      await buildClients(
-        generatorList({
-          language,
-          client,
-          clientList: CLIENTS_JS,
-        }),
-        Boolean(verbose)
-      );
+      await buildClients(language, client, Boolean(verbose));
     }
   );