From 3fb6913aed21a0afb5ff677d09b32188e86599e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Fri, 25 Feb 2022 16:06:10 +0100 Subject: [PATCH 1/4] fix(ci): build algoliasearch with clients --- .github/actions/setup/action.yml | 12 ++++++++++++ .github/workflows/check.yml | 5 ++--- .github/workflows/process-release.yml | 2 +- scripts/generate.ts | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 6a8a4e5b161..8a71dae9960 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -38,6 +38,7 @@ runs: echo "::set-output name=SCRIPTS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- scripts | wc -l)" echo "::set-output name=JS_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript | wc -l)" + echo "::set-output name=JS_ALGOLIASEARCH_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript/packages/algoliasearch clients/algoliasearch-client-javascript/packages/client-search clients/algoliasearch-client-javascript/packages/client-analytics clients/algoliasearch-client-javascript/packages/client-personalization | wc -l)" echo "::set-output name=JS_COMMON_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript/packages/client-common clients/algoliasearch-client-javascript/packages/requester-browser-xhr clients/algoliasearch-client-javascript/packages/requester-node-http | wc -l)" echo "::set-output name=JS_TEMPLATE_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- templates/javascript | wc -l)" @@ -69,9 +70,20 @@ runs: shell: bash run: | base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} + algoliasearch_changed=${{ steps.diff.outputs.JS_ALGOLIASEARCH_CHANGED > 0 }} matrix=$(./scripts/ci/create-client-matrix.sh javascript $base_changed ${{ github.base_ref }}) + if [[ $algoliasearch_changed == 'true' || $base_changed == 'true' ]]; then + algoliasearch_client='{"name": "algoliasearch","folder": "clients/algoliasearch-client-javascript/packages/algoliasearch"}' + + if [[ $matrix == '{"client":["no-run"]}' ]]; then + matrix="{'client': [$algoliasearch_client]}" + else + matrix="{'client': $(echo $matrix | jq ".client + [$algoliasearch_client]" -c)}" + fi + fi + if [[ $matrix == '{"client":["no-run"]}' ]]; then run="false" else diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c1fb46f3b73..9b47ecd11e2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -8,7 +8,7 @@ on: - main env: - CACHE_VERSION: '5' + CACHE_VERSION: '6' concurrency: group: ${{ github.ref }} @@ -76,7 +76,6 @@ jobs: - client-common - requester-browser-xhr - requester-node-http - - algoliasearch steps: - uses: actions/checkout@v2 @@ -122,7 +121,7 @@ jobs: key: ${{ runner.os }}-${{ env.CACHE_VERSION }}-js-client-${{ matrix.client.name }}-${{ hashFiles(format('{0}/**', matrix.client.folder)) }}-${{ hashFiles(format('specs/bundled/{0}.yml', matrix.client.name)) }} - name: Generate ${{ matrix.client.name }} client - if: steps.cache.outputs.cache-hit != 'true' + if: steps.cache.outputs.cache-hit != 'true' && matrix.client.name != 'algoliasearch' run: yarn cli generate javascript ${{ matrix.client.name }} - name: Check diff with pushed client diff --git a/.github/workflows/process-release.yml b/.github/workflows/process-release.yml index 4189d0682dd..037d1503555 100644 --- a/.github/workflows/process-release.yml +++ b/.github/workflows/process-release.yml @@ -6,7 +6,7 @@ on: - closed env: - CACHE_VERSION: '5' + CACHE_VERSION: '6' jobs: build: diff --git a/scripts/generate.ts b/scripts/generate.ts index 5a6d5a594c1..94adb46adfa 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -69,7 +69,7 @@ export async function generate( if (!CI || lang !== 'javascript') { await formatter(lang, getLanguageFolder(lang), verbose); } - if (lang === 'javascript') { + if (!CI && lang === 'javascript') { const spinner = createSpinner( 'cleaning JavaScript client utils', verbose From 97ba91438021d6a57fbe9f60a594f9b83ac950df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Mon, 28 Feb 2022 09:25:29 +0100 Subject: [PATCH 2/4] allow utils build --- .github/workflows/check.yml | 2 +- clients.config.json | 32 ++++++------- .../package.json | 2 +- .../rollup.config.js | 8 ++-- openapitools.json | 2 +- scripts/buildClients.ts | 33 +++++++++++++ scripts/common.ts | 7 ++- scripts/generate.ts | 1 + scripts/index.ts | 46 ++++++++++++++++--- 9 files changed, 102 insertions(+), 31 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 9b47ecd11e2..f437d558734 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -93,7 +93,7 @@ jobs: - name: Build ${{ matrix.client }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn workspace algoliasearch-client-javascript build:utils ${{ matrix.client }} + run: yarn workspace algoliasearch-client-javascript build ${{ matrix.client }} client_javascript: timeout-minutes: 10 diff --git a/clients.config.json b/clients.config.json index 817a6a11af3..06e736304c1 100644 --- a/clients.config.json +++ b/clients.config.json @@ -1,19 +1,19 @@ { - "java": { - "folder": "clients/algoliasearch-client-java-2", - "tests": { - "extension": ".test.java", - "outputFolder": "src/test/java/com/algolia" - } - }, - "javascript": { - "folder": "clients/algoliasearch-client-javascript", - "tests": { - "extension": ".test.ts", - "outputFolder": "src" - } - }, - "php": { - "folder": "clients/algoliasearch-client-php" + "java": { + "folder": "clients/algoliasearch-client-java-2", + "tests": { + "extension": ".test.java", + "outputFolder": "src/test/java/com/algolia" } + }, + "javascript": { + "folder": "clients/algoliasearch-client-javascript", + "tests": { + "extension": ".test.ts", + "outputFolder": "src" + } + }, + "php": { + "folder": "clients/algoliasearch-client-php" + } } diff --git a/clients/algoliasearch-client-javascript/package.json b/clients/algoliasearch-client-javascript/package.json index 69cd7909f47..a937006c9b3 100644 --- a/clients/algoliasearch-client-javascript/package.json +++ b/clients/algoliasearch-client-javascript/package.json @@ -6,7 +6,7 @@ ], "private": true, "scripts": { - "build:utils": "UTILS=${0:-all} yarn rollup -c rollup.config.js", + "build:utils": "yarn build client-common && yarn build requester-browser-xhr && yarn build requester-node-http", "build": "CLIENT=${0:-all} yarn rollup -c rollup.config.js", "clean:utils": "yarn workspace @algolia/client-common clean && yarn workspace @algolia/requester-node-http clean && yarn workspace @algolia/requester-browser-xhr clean", "clean": "rm -rf packages/*/dist", diff --git a/clients/algoliasearch-client-javascript/rollup.config.js b/clients/algoliasearch-client-javascript/rollup.config.js index c76ceccb03b..c00d3528bb6 100644 --- a/clients/algoliasearch-client-javascript/rollup.config.js +++ b/clients/algoliasearch-client-javascript/rollup.config.js @@ -13,7 +13,7 @@ import { version } from './version'; // Retrieve package to build const client = process.env.CLIENT?.replace('@algolia/', ''); -const utils = process.env.UTILS; +const UTILS = ['client-common', 'requester-browser-xhr', 'requester-node-http']; function createLicence(name) { return `/*! ${name}.umd.js | ${version} | © Algolia, inc. | https://github.com/algolia/algoliasearch-client-javascript */`; @@ -64,7 +64,7 @@ function getAvailableClients() { } function initPackagesConfig() { - if (utils) { + if (UTILS.includes(client)) { const commonOptions = { input: 'index.ts', formats: ['cjs-node', 'esm-node'], @@ -100,10 +100,10 @@ function initPackagesConfig() { }, ]; - return utils === 'all' + return client === 'all' ? availableUtils : availableUtils.filter( - (availableUtil) => availableUtil.package === utils + (availableUtil) => availableUtil.package === client ); } diff --git a/openapitools.json b/openapitools.json index c7f40ae154b..9ed8a7f65f0 100644 --- a/openapitools.json +++ b/openapitools.json @@ -403,4 +403,4 @@ } } } -} \ No newline at end of file +} diff --git a/scripts/buildClients.ts b/scripts/buildClients.ts index 082505e8410..48836322a73 100644 --- a/scripts/buildClients.ts +++ b/scripts/buildClients.ts @@ -5,6 +5,39 @@ import type { Generator } from './types'; const multiBuildLanguage = new Set(['javascript']); +/** + * Build JavaScript utils packages used in generated clients. + */ +export async function buildJSClientUtils( + verbose: boolean, + client?: string +): Promise { + if (!client || client === 'all') { + const spinner = createSpinner('building JavaScript utils', verbose).start(); + await run('yarn workspace algoliasearch-client-javascript clean:utils', { + verbose, + }); + await run('yarn workspace algoliasearch-client-javascript build:utils', { + verbose, + }); + + spinner.succeed(); + return; + } + + const spinner = createSpinner( + `building JavaScript ${client} utils`, + verbose + ).start(); + + await run(`yarn workspace @algolia/${client} clean`, { verbose }); + await run(`yarn workspace algoliasearch-client-javascript build ${client}`, { + verbose, + }); + + spinner.succeed(); +} + /** * Build only a specific client for one language, used by javascript for example. */ diff --git a/scripts/common.ts b/scripts/common.ts index 41dcf7643d9..fdacf2c7852 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -27,7 +27,12 @@ export const CLIENTS = [ ...new Set(Object.values(GENERATORS).map((gen) => gen.client)), ]; -export const CLIENTS_JS = CLIENTS.concat([]); +export const CLIENTS_JS_UTILS = [ + 'client-common', + 'requester-browser-xhr', + 'requester-node-http', +]; +export const CLIENTS_JS = ['algoliasearch', ...CLIENTS]; /** * Takes a generator key in the form 'language-client' and returns the Generator object. diff --git a/scripts/generate.ts b/scripts/generate.ts index 94adb46adfa..2e627a34aab 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -77,6 +77,7 @@ export async function generate( await run('yarn workspace algoliasearch-client-javascript clean:utils', { verbose, }); + spinner.text = 'building JavaScript client utils'; await run('yarn workspace algoliasearch-client-javascript build:utils', { verbose, diff --git a/scripts/index.ts b/scripts/index.ts index 8ccae24a1a9..d2897e6788a 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -2,12 +2,13 @@ import { Argument, program } from 'commander'; import inquirer from 'inquirer'; -import { buildClients } from './buildClients'; +import { buildClients, buildJSClientUtils } from './buildClients'; import { buildSpecs } from './buildSpecs'; import { CI, CLIENTS, CLIENTS_JS, + CLIENTS_JS_UTILS, createGeneratorKey, DOCKER, GENERATORS, @@ -91,9 +92,22 @@ function generatorList({ return langsTodo .flatMap((lang) => - clientsTodo.map( - (cli) => GENERATORS[createGeneratorKey({ language: lang, client: cli })] - ) + clientsTodo.map((cli) => { + // Edge case for `algoliasearch`, which is not a generated client but + // use the same build system + if (cli === 'algoliasearch') { + return { + language: 'javascript', + client: cli, + key: `javascript-${cli}`, + additionalProperties: { + packageName: cli, + }, + }; + } + + return GENERATORS[createGeneratorKey({ language: lang, client: cli })]; + }) ) .filter(Boolean); } @@ -135,7 +149,9 @@ buildCommand ) ) .addArgument( - new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS_JS)) + new Argument('[client]', 'The client').choices( + ['all'].concat([...CLIENTS_JS_UTILS, ...CLIENTS_JS]) + ) ) .option('-v, --verbose', 'make the compilation verbose') .option('-i, --interactive', 'open prompt to query parameters') @@ -146,10 +162,26 @@ buildCommand { verbose, interactive } ) => { language = await promptLanguage(language, interactive); - client = await promptClient(client, interactive, CLIENTS_JS); + client = await promptClient(client, interactive, [ + ...CLIENTS_JS_UTILS, + ...CLIENTS_JS, + ]); + + // We build the JavaScript utils before generated clients as they + // rely on them + if ( + language === 'javascript' && + (!client || client === 'all' || CLIENTS_JS_UTILS.includes(client)) + ) { + await buildJSClientUtils(Boolean(verbose), client); + } await buildClients( - generatorList({ language, client, clientList: CLIENTS_JS }), + generatorList({ + language, + client, + clientList: CLIENTS_JS, + }), Boolean(verbose) ); } From 0585295ab2b5440644105c52088045a545bac6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Mon, 28 Feb 2022 15:46:52 +0100 Subject: [PATCH 3/4] merge issue --- .github/workflows/process-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/process-release.yml b/.github/workflows/process-release.yml index cc6ff13cdba..58506f8ad7b 100644 --- a/.github/workflows/process-release.yml +++ b/.github/workflows/process-release.yml @@ -21,7 +21,7 @@ jobs: token: ${{ secrets.TOKEN_RELEASE_BOT }} - run: git checkout chore/release - + - run: git submodule update - name: Setup From 183253398c683ac2c9417677fa8cdf8954b7d60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Mon, 28 Feb 2022 16:10:04 +0100 Subject: [PATCH 4/4] apply changes from suggestion --- openapitools.json | 2 +- scripts/common.ts | 47 ++++++++++++++++--------- scripts/generate.ts | 19 +++------- scripts/index.ts | 21 +++-------- scripts/release/create-release-issue.ts | 2 +- 5 files changed, 42 insertions(+), 49 deletions(-) diff --git a/openapitools.json b/openapitools.json index 9ed8a7f65f0..c7f40ae154b 100644 --- a/openapitools.json +++ b/openapitools.json @@ -403,4 +403,4 @@ } } } -} +} \ No newline at end of file diff --git a/scripts/common.ts b/scripts/common.ts index 288cbbb8c97..56ee68556ff 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -13,33 +13,48 @@ export const DOCKER = Boolean(process.env.DOCKER); // This script is run by `yarn workspace ...`, which means the current working directory is `./script` export const ROOT_DIR = path.resolve(process.cwd(), '..'); -export const GENERATORS = Object.fromEntries( - Object.entries(openapitools['generator-cli'].generators).map(([key, gen]) => { - return [ - key, - { - ...gen, - output: gen.output.replace('#{cwd}/', ''), - ...splitGeneratorKey(key), - }, - ]; - }) +export const GENERATORS: Record = { + // Default `algoliasearch` package as it's built similarly to generated clients + 'javascript-algoliasearch': { + language: 'javascript', + client: 'algoliasearch', + key: 'javascript-algoliasearch', + additionalProperties: { + packageName: 'algoliasearch', + packageVersion: '0', + }, + }, +}; + +// Build `GENERATORS` from the openapitools file +Object.entries(openapitools['generator-cli'].generators).forEach( + ([key, gen]) => { + GENERATORS[key] = { + ...gen, + output: gen.output.replace('#{cwd}/', ''), + ...splitGeneratorKey(key), + }; + } ); export const LANGUAGES = [ ...new Set(Object.values(GENERATORS).map((gen) => gen.language)), ]; -export const CLIENTS = [ - ...new Set(Object.values(GENERATORS).map((gen) => gen.client)), -]; - export const CLIENTS_JS_UTILS = [ 'client-common', 'requester-browser-xhr', 'requester-node-http', ]; -export const CLIENTS_JS = ['algoliasearch', ...CLIENTS]; + +export const CLIENTS_JS = [ + 'algoliasearch', + ...new Set(Object.values(GENERATORS).map((gen) => gen.client)), +]; + +export const CLIENTS = CLIENTS_JS.filter( + (client) => client !== 'algoliasearch' +); /** * Takes a generator key in the form 'language-client' and returns the Generator object. diff --git a/scripts/generate.ts b/scripts/generate.ts index 2340f22fc26..39cfb119b0e 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -1,3 +1,4 @@ +import { buildJSClientUtils } from './buildClients'; import { buildSpecs } from './buildSpecs'; import { CI, run, runIfExists } from './common'; import { getLanguageFolder } from './config'; @@ -67,21 +68,11 @@ export async function generate( if (!(CI && lang === 'javascript')) { await formatter(lang, getLanguageFolder(lang), verbose); } - if (!CI && lang === 'javascript') { - const spinner = createSpinner( - 'cleaning JavaScript client utils', - verbose - ).start(); - await run('yarn workspace algoliasearch-client-javascript clean:utils', { - verbose, - }); - - spinner.text = 'building JavaScript client utils'; - await run('yarn workspace algoliasearch-client-javascript build:utils', { - verbose, - }); - spinner.succeed(); + // JavaScript utils are tested independently, we only build them + // during dev to ease the process + if (!CI && lang === 'javascript') { + await buildJSClientUtils(verbose, 'all'); } } diff --git a/scripts/index.ts b/scripts/index.ts index d2897e6788a..930c182a35e 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -92,22 +92,9 @@ function generatorList({ return langsTodo .flatMap((lang) => - clientsTodo.map((cli) => { - // Edge case for `algoliasearch`, which is not a generated client but - // use the same build system - if (cli === 'algoliasearch') { - return { - language: 'javascript', - client: cli, - key: `javascript-${cli}`, - additionalProperties: { - packageName: cli, - }, - }; - } - - return GENERATORS[createGeneratorKey({ language: lang, client: cli })]; - }) + clientsTodo.map( + (cli) => GENERATORS[createGeneratorKey({ language: lang, client: cli })] + ) ) .filter(Boolean); } @@ -170,7 +157,7 @@ buildCommand // We build the JavaScript utils before generated clients as they // rely on them if ( - language === 'javascript' && + (language === 'javascript' || language === 'all') && (!client || client === 'all' || CLIENTS_JS_UTILS.includes(client)) ) { await buildJSClientUtils(Boolean(verbose), client); diff --git a/scripts/release/create-release-issue.ts b/scripts/release/create-release-issue.ts index d425a2597f5..41faf38f927 100755 --- a/scripts/release/create-release-issue.ts +++ b/scripts/release/create-release-issue.ts @@ -28,7 +28,7 @@ function readVersions(): Versions { Object.values(GENERATORS).forEach((gen) => { if (!versions[gen.language]) { versions[gen.language] = { - current: gen.additionalProperties.packageVersion, + current: gen.additionalProperties?.packageVersion, langName: gen.language, next: undefined, };