From 587c13f1ce63b7126f44ec1bbeb05d1e2a16bd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Wed, 30 Mar 2022 11:34:06 +0200 Subject: [PATCH 1/4] chore: cache custom generators --- .gitignore | 1 + scripts/buildSpecs.ts | 37 ++++++++------------- scripts/common.ts | 76 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index a41aa410dd9..662cf6f91d3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ build pom.xml dist +.cache .openapi-generator diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts index 022449cd746..0429ddd4cae 100644 --- a/scripts/buildSpecs.ts +++ b/scripts/buildSpecs.ts @@ -1,9 +1,8 @@ import fsp from 'fs/promises'; -import { hashElement } from 'folder-hash'; import yaml from 'js-yaml'; -import { exists, run, toAbsolutePath } from './common'; +import { checkForCache, exists, run, toAbsolutePath } from './common'; import { createSpinner } from './oraLog'; import type { Spec } from './pre-gen/setHostsOptions'; @@ -53,30 +52,22 @@ async function buildSpec( let hash = ''; if (useCache) { - const spinner = createSpinner( - `checking cache for '${client}'`, + const { cacheExists, hash: newCache } = await checkForCache( + { + job: `'${client}' specs`, + folder: toAbsolutePath('specs/'), + buildFiles: [`bundled/${client}.yml`], + filesToCache: [client, 'common'], + cacheFile, + }, verbose - ).start(); - // check if file and cache exist - if (await exists(toAbsolutePath(`specs/bundled/${client}.yml`))) { - // compare with stored cache - const specHash = (await hashElement(toAbsolutePath(`specs/${client}`))) - .hash; - const commonHash = (await hashElement(toAbsolutePath(`specs/common`))) - .hash; - hash = `${specHash}-${commonHash}`; - if (await exists(cacheFile)) { - const storedHash = (await fsp.readFile(cacheFile)).toString(); - if (storedHash === hash) { - spinner.succeed( - `skipped building ${client} spec because the files did not change` - ); - return; - } - } + ); + + if (cacheExists) { + return; } - spinner.info(`cache not found for ${client}' spec`); + hash = newCache; } const spinner = createSpinner(`building ${client} spec`, verbose).start(); diff --git a/scripts/common.ts b/scripts/common.ts index ba8ec31f33e..f710eac2e39 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -2,6 +2,7 @@ import fsp from 'fs/promises'; import path from 'path'; import execa from 'execa'; // https://github.com/sindresorhus/execa/tree/v5.1.1 +import { hashElement } from 'folder-hash'; import openapitools from '../openapitools.json'; @@ -194,11 +195,86 @@ export async function gitCommit({ ); } +export async function checkForCache( + { + job, + folder, + buildFiles, + filesToCache, + cacheFile, + }: { + job: string; + folder: string; + buildFiles: string[]; + filesToCache: string[]; + cacheFile: string; + }, + verbose: boolean +): Promise<{ cacheExists: boolean; hash: string }> { + const spinner = createSpinner(`checking cache for ${job}`, verbose).start(); + const buildFilesExists = buildFiles.every((buildFile) => + exists(`${folder}/${buildFile}`).then((res) => res) + ); + const cache = { + cacheExists: false, + hash: '', + }; + + if (!buildFilesExists) { + spinner.info(`cache not found for ${job}`); + return cache; + } + + for (const fileToCache of filesToCache) { + const fileHash = (await hashElement(`${folder}/${fileToCache}`)).hash; + + cache.hash = `${cache.hash}-${fileHash}`; + } + + if (await exists(cacheFile)) { + const storedHash = (await fsp.readFile(cacheFile)).toString(); + if (storedHash === cache.hash) { + spinner.succeed(`job skipped, cache found for ${job}`); + return { + cacheExists: true, + hash: cache.hash, + }; + } + } + + spinner.info(`cache not found for ${job}`); + + return cache; +} + export async function buildCustomGenerators(verbose: boolean): Promise { + const cacheFile = toAbsolutePath('generators/.cache'); + const { cacheExists, hash } = await checkForCache( + { + job: 'custom generators', + folder: toAbsolutePath('generators/'), + buildFiles: ['build', '.gradle'], + filesToCache: ['src', 'build.gradle', 'settings.gradle'], + cacheFile, + }, + verbose + ); + + if (cacheExists) { + return; + } + const spinner = createSpinner('building custom generators', verbose).start(); + await run('./gradle/gradlew --no-daemon -p generators assemble', { verbose, }); + + if (hash) { + spinner.text = 'storing custom generators cache'; + await fsp.writeFile(cacheFile, hash); + } + spinner.succeed(); } From 63917b6bff5d4c2bffed6155f9182976ed6bc28a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Wed, 30 Mar 2022 12:19:46 +0200 Subject: [PATCH 2/4] apply changes from suggestion --- scripts/buildSpecs.ts | 2 +- scripts/common.ts | 34 ++++++++++++++++------------------ scripts/types.ts | 13 +++++++++++++ 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts index 0429ddd4cae..1f211c9f46c 100644 --- a/scripts/buildSpecs.ts +++ b/scripts/buildSpecs.ts @@ -56,7 +56,7 @@ async function buildSpec( { job: `'${client}' specs`, folder: toAbsolutePath('specs/'), - buildFiles: [`bundled/${client}.yml`], + generatedFiles: [`bundled/${client}.yml`], filesToCache: [client, 'common'], cacheFile, }, diff --git a/scripts/common.ts b/scripts/common.ts index f710eac2e39..fac7ced408d 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -7,7 +7,12 @@ import { hashElement } from 'folder-hash'; import openapitools from '../openapitools.json'; import { createSpinner } from './oraLog'; -import type { Generator, RunOptions } from './types'; +import type { + CheckForCache, + CheckForCacheOptions, + Generator, + RunOptions, +} from './types'; export const CI = Boolean(process.env.CI); export const DOCKER = Boolean(process.env.DOCKER); @@ -199,30 +204,23 @@ export async function checkForCache( { job, folder, - buildFiles, + generatedFiles, filesToCache, cacheFile, - }: { - job: string; - folder: string; - buildFiles: string[]; - filesToCache: string[]; - cacheFile: string; - }, + }: CheckForCacheOptions, verbose: boolean -): Promise<{ cacheExists: boolean; hash: string }> { +): Promise { const spinner = createSpinner(`checking cache for ${job}`, verbose).start(); - const buildFilesExists = buildFiles.every((buildFile) => - exists(`${folder}/${buildFile}`).then((res) => res) - ); - const cache = { + const cache: CheckForCache = { cacheExists: false, hash: '', }; - if (!buildFilesExists) { - spinner.info(`cache not found for ${job}`); - return cache; + for (const buildFile of generatedFiles) { + if ((await exists(`${folder}/${buildFile}`)) === false) { + spinner.info(`cache not found for ${job}`); + return cache; + } } for (const fileToCache of filesToCache) { @@ -253,7 +251,7 @@ export async function buildCustomGenerators(verbose: boolean): Promise { { job: 'custom generators', folder: toAbsolutePath('generators/'), - buildFiles: ['build', '.gradle'], + generatedFiles: ['build'], filesToCache: ['src', 'build.gradle', 'settings.gradle'], cacheFile, }, diff --git a/scripts/types.ts b/scripts/types.ts index 3b2382a58ea..31aff2867a2 100644 --- a/scripts/types.ts +++ b/scripts/types.ts @@ -1,5 +1,18 @@ import type config from '../config/clients.config.json'; +export type CheckForCacheOptions = { + job: string; + folder: string; + generatedFiles: string[]; + filesToCache: string[]; + cacheFile: string; +}; + +export type CheckForCache = { + cacheExists: boolean; + hash: string; +}; + export type Generator = Record & { language: string; client: string; From 4baa527b433f03464c748976449fffd1f1b7b832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Wed, 30 Mar 2022 12:34:30 +0200 Subject: [PATCH 3/4] avoid losing hash when missing output files --- scripts/common.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/common.ts b/scripts/common.ts index fac7ced408d..f2d2af4d70e 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -215,13 +215,13 @@ export async function checkForCache( cacheExists: false, hash: '', }; - - for (const buildFile of generatedFiles) { - if ((await exists(`${folder}/${buildFile}`)) === false) { - spinner.info(`cache not found for ${job}`); - return cache; - } - } + const generatedFilesExists = ( + await Promise.all( + generatedFiles.map((generatedFile) => + exists(`${folder}/${generatedFile}`).then((res) => res) + ) + ) + ).every((exist) => exist); for (const fileToCache of filesToCache) { const fileHash = (await hashElement(`${folder}/${fileToCache}`)).hash; @@ -229,7 +229,8 @@ export async function checkForCache( cache.hash = `${cache.hash}-${fileHash}`; } - if (await exists(cacheFile)) { + // We only skip if both the cache and the generated file exists + if (generatedFilesExists && (await exists(cacheFile))) { const storedHash = (await fsp.readFile(cacheFile)).toString(); if (storedHash === cache.hash) { spinner.succeed(`job skipped, cache found for ${job}`); From 7162df13b382ef6145f15db77e800ecfcc332c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Wed, 30 Mar 2022 12:37:30 +0200 Subject: [PATCH 4/4] oops --- scripts/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/common.ts b/scripts/common.ts index f2d2af4d70e..a01c6165a0b 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -218,7 +218,7 @@ export async function checkForCache( const generatedFilesExists = ( await Promise.all( generatedFiles.map((generatedFile) => - exists(`${folder}/${generatedFile}`).then((res) => res) + exists(`${folder}/${generatedFile}`) ) ) ).every((exist) => exist);