diff --git a/libs/native-federation-core/src/lib/core/build-for-federation.ts b/libs/native-federation-core/src/lib/core/build-for-federation.ts index 5e367413..5e1d2e4c 100644 --- a/libs/native-federation-core/src/lib/core/build-for-federation.ts +++ b/libs/native-federation-core/src/lib/core/build-for-federation.ts @@ -14,6 +14,8 @@ import { FederationOptions } from './federation-options'; import { writeFederationInfo } from './write-federation-info'; import { writeImportMap } from './write-import-map'; import { logger } from '../utils/logger'; +import { getCachePath } from './bundle-caching'; +import { normalizePackageName } from '../utils/normalize'; import { AbortedError } from '../utils/errors'; export interface BuildParams { @@ -27,9 +29,7 @@ export const defaultBuildParams: BuildParams = { skipShared: false, }; -// Externals cache const sharedPackageInfoCache: SharedInfo[] = []; -const cachedSharedPackages = new Set(); export async function buildForFederation( config: NormalizedFederationConfig, @@ -64,7 +64,23 @@ export async function buildForFederation( ? describeExposed(config, fedOptions) : artefactInfo.exposes; - if (!buildParams.skipShared) { + const cacheProjectFolder = normalizePackageName(config.name); + if (cacheProjectFolder.length < 1) { + logger.warn( + "Project name in 'federation.config.js' is empty, defaulting to root cache folder (could collide with other projects in the workspace).", + ); + } + + const pathToCache = getCachePath( + fedOptions.workspaceRoot, + cacheProjectFolder, + ); + + if (!buildParams.skipShared && sharedPackageInfoCache.length > 0) { + logger.info('Checksum matched, re-using cached externals.'); + } + + if (!buildParams.skipShared && sharedPackageInfoCache.length === 0) { const { sharedBrowser, sharedServer, separateBrowser, separateServer } = splitShared(config.shared); @@ -76,6 +92,7 @@ export async function buildForFederation( fedOptions, externals, 'browser', + { pathToCache, bundleName: 'browser-shared' }, ); logger.measure( @@ -84,9 +101,7 @@ export async function buildForFederation( ); sharedPackageInfoCache.push(...sharedPackageInfoBrowser); - Object.keys(sharedBrowser).forEach((packageName) => - cachedSharedPackages.add(packageName), - ); + if (signal?.aborted) throw new AbortedError( '[buildForFederation] After shared-browser bundle', @@ -101,15 +116,14 @@ export async function buildForFederation( fedOptions, externals, 'node', + { pathToCache, bundleName: 'node-shared' }, ); logger.measure( start, '[build artifacts] - To bundle all shared node externals', ); sharedPackageInfoCache.push(...sharedPackageInfoServer); - Object.keys(sharedServer).forEach((packageName) => - cachedSharedPackages.add(packageName), - ); + if (signal?.aborted) throw new AbortedError('[buildForFederation] After shared-node bundle'); } @@ -122,15 +136,14 @@ export async function buildForFederation( config, fedOptions, 'browser', + pathToCache, ); logger.measure( start, '[build artifacts] - To bundle all separate browser externals', ); sharedPackageInfoCache.push(...separatePackageInfoBrowser); - Object.keys(separateBrowser).forEach((packageName) => - cachedSharedPackages.add(packageName), - ); + if (signal?.aborted) throw new AbortedError( '[buildForFederation] After separate-browser bundle', @@ -145,15 +158,13 @@ export async function buildForFederation( config, fedOptions, 'node', + pathToCache, ); logger.measure( start, '[build artifacts] - To bundle all separate node externals', ); sharedPackageInfoCache.push(...separatePackageInfoServer); - Object.keys(separateServer).forEach((packageName) => - cachedSharedPackages.add(packageName), - ); } if (signal?.aborted) @@ -203,6 +214,7 @@ async function bundleSeparate( config: NormalizedFederationConfig, fedOptions: FederationOptions, platform: 'node' | 'browser', + pathToCache: string, ) { const bundlePromises = Object.entries(separateBrowser).map( async ([key, shared]) => { @@ -216,6 +228,10 @@ async function bundleSeparate( fedOptions, filteredExternals, platform, + { + pathToCache, + bundleName: `${platform}-${normalizePackageName(key)}`, + }, ); }, ); @@ -233,7 +249,6 @@ function splitShared( const separateServer: Record = {}; for (const key in shared) { - if (cachedSharedPackages.has(key)) continue; const obj = shared[key]; if (obj.platform === 'node' && obj.build === 'default') { sharedServer[key] = obj; diff --git a/libs/native-federation-core/src/lib/core/bundle-caching.ts b/libs/native-federation-core/src/lib/core/bundle-caching.ts new file mode 100644 index 00000000..f063567f --- /dev/null +++ b/libs/native-federation-core/src/lib/core/bundle-caching.ts @@ -0,0 +1,115 @@ +import path from 'path'; +import fs from 'fs'; +import crypto from 'crypto'; +import { NormalizedSharedConfig } from '../config/federation-config'; +import { SharedInfo } from '@softarc/native-federation-runtime'; +import { logger } from '../utils/logger'; + +export const getCachePath = (workspaceRoot: string, project: string) => + path.join(workspaceRoot, 'node_modules/.cache/native-federation', project); + +export const getFilename = (title: string) => { + return `${title}.meta.json`; +}; + +export const getChecksum = ( + shared: Record, +): string => { + const denseExternals = Object.keys(shared) + .sort() + .reduce((clean, external) => { + return ( + clean + + ':' + + external + + (shared[external].version ? `@${shared[external].version}` : '') + ); + }, 'deps'); + + return crypto.createHash('sha256').update(denseExternals).digest('hex'); +}; + +export const cacheEntry = (pathToCache: string, fileName: string) => ({ + getMetadata: ( + checksum: string, + ): + | { + checksum: string; + externals: SharedInfo[]; + files: string[]; + } + | undefined => { + const metadataFile = path.join(pathToCache, fileName); + if (!fs.existsSync(pathToCache) || !fs.existsSync(metadataFile)) + return undefined; + + const cachedResult: { + checksum: string; + externals: SharedInfo[]; + files: string[]; + } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8')); + if (cachedResult.checksum !== checksum) return undefined; + return cachedResult; + }, + persist: (payload: { + checksum: string; + externals: SharedInfo[]; + files: string[]; + }) => { + fs.writeFileSync( + path.join(pathToCache, fileName), + JSON.stringify(payload), + 'utf-8', + ); + }, + copyFiles: (fullOutputPath: string) => { + const metadataFile = path.join(pathToCache, fileName); + if (!fs.existsSync(metadataFile)) + throw new Error( + 'Error copying artifacts to dist, metadata file could not be found.', + ); + + const cachedResult: { + externals: SharedInfo[]; + files: string[]; + } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8')); + + fs.mkdirSync(path.dirname(fullOutputPath), { recursive: true }); + + cachedResult.files.forEach((file) => { + const cachedFile = path.join(pathToCache, file); + const distFileName = path.join(fullOutputPath, file); + + if (fs.existsSync(cachedFile)) { + fs.copyFileSync(cachedFile, distFileName); + } + }); + }, + clear: () => { + const metadataFile = path.join(pathToCache, fileName); + if (!fs.existsSync(pathToCache)) { + fs.mkdirSync(pathToCache, { recursive: true }); + logger.debug(`Creating cache folder '${pathToCache}' for '${fileName}'.`); + return; + } + if (!fs.existsSync(metadataFile)) { + logger.debug( + `Could not purge cached bundle, metadata file '${metadataFile}' does not exist.`, + ); + return; + } + + const cachedResult: { + checksum: string; + externals: SharedInfo[]; + files: string[]; + } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8')); + + cachedResult.files.forEach((file) => { + const cachedFile = path.join(pathToCache, file); + if (fs.existsSync(cachedFile)) fs.unlinkSync(cachedFile); + }); + + fs.unlinkSync(metadataFile); + }, +}); diff --git a/libs/native-federation-core/src/lib/core/bundle-shared.ts b/libs/native-federation-core/src/lib/core/bundle-shared.ts index 725993cc..37fed5ef 100644 --- a/libs/native-federation-core/src/lib/core/bundle-shared.ts +++ b/libs/native-federation-core/src/lib/core/bundle-shared.ts @@ -8,7 +8,6 @@ import { bundle } from '../utils/build-utils'; import { getPackageInfo, PackageInfo } from '../utils/package-info'; import { SharedInfo } from '@softarc/native-federation-runtime'; import { FederationOptions } from './federation-options'; -import { copySrcMapIfExists } from '../utils/copy-src-map-if-exists'; import { logger } from '../utils/logger'; import crypto from 'crypto'; import { DEFAULT_EXTERNAL_LIST } from './default-external-list'; @@ -18,6 +17,7 @@ import { isSourceFile, rewriteChunkImports, } from '../utils/rewrite-chunk-imports'; +import { cacheEntry, getChecksum, getFilename } from './bundle-caching'; export async function bundleShared( sharedBundles: Record, @@ -25,17 +25,31 @@ export async function bundleShared( fedOptions: FederationOptions, externals: string[], platform: 'browser' | 'node' = 'browser', + cacheOptions: { pathToCache: string; bundleName: string }, ): Promise> { + const checksum = getChecksum(sharedBundles); const folder = fedOptions.packageJson ? path.dirname(fedOptions.packageJson) : fedOptions.workspaceRoot; - const cachePath = path.join( - fedOptions.workspaceRoot, - 'node_modules/.cache/native-federation', + const bundleCache = cacheEntry( + cacheOptions.pathToCache, + getFilename(cacheOptions.bundleName), ); - fs.mkdirSync(cachePath, { recursive: true }); + const cacheMetadata = bundleCache.getMetadata(checksum); + if (cacheMetadata) { + logger.debug( + `Checksum of ${cacheOptions.bundleName} matched, Skipped artifact bundling`, + ); + bundleCache.copyFiles( + path.join(fedOptions.workspaceRoot, fedOptions.outputPath), + ); + return cacheMetadata.externals; + } + + // Delete older packages if checksum didnt match + bundleCache.clear(); const inferredPackageInfos = Object.keys(sharedBundles) .filter((packageName) => !sharedBundles[packageName].packageInfo) @@ -67,11 +81,11 @@ export async function bundleShared( fedOptions.outputPath, ); - const exptedResults = allEntryPoints.map((ep) => + const expectedResults = allEntryPoints.map((ep) => path.join(fullOutputPath, ep.outName), ); const entryPoints = allEntryPoints.filter( - (ep) => !fs.existsSync(path.join(cachePath, ep.outName)), + (ep) => !fs.existsSync(path.join(cacheOptions.pathToCache, ep.outName)), ); if (entryPoints.length > 0) { @@ -98,7 +112,7 @@ export async function bundleShared( entryPoints, tsConfigPath: fedOptions.tsConfig, external: [...additionalExternals, ...externals], - outdir: cachePath, + outdir: cacheOptions.pathToCache, mappedPaths: config.sharedMappings, dev: fedOptions.dev, kind: 'shared-package', @@ -108,9 +122,7 @@ export async function bundleShared( }); const cachedFiles = bundleResult.map((br) => path.basename(br.fileName)); - rewriteImports(cachedFiles, cachePath); - - copyCacheToOutput(cachedFiles, cachePath, fullOutputPath); + rewriteImports(cachedFiles, cacheOptions.pathToCache); } catch (e) { logger.error('Error bundling shared npm package '); if (e instanceof Error) { @@ -143,35 +155,9 @@ export async function bundleShared( throw e; } - const resultCacheFile = createCacheFileName( - configState, - sharedBundles, - fedOptions, - cachePath, - platform, - ); - - if (fs.existsSync(resultCacheFile)) { - const cachedResult: SharedInfo[] = JSON.parse( - fs.readFileSync(resultCacheFile, 'utf-8'), - ); - const cachedFiles = cachedResult.map((cr) => cr.outFileName); - - // Chunks are overwritten by the bundler, so we need to reprocess them - rewriteImports(cachedFiles, cachePath); - - copyCacheToOutput(cachedFiles, cachePath, fullOutputPath); - return cachedResult; - } - - const outFileNames = [...exptedResults]; + const outFileNames = [...expectedResults]; - const result = buildResult( - packageInfos, - sharedBundles, - outFileNames, - fedOptions, - ); + const result = buildResult(packageInfos, sharedBundles, outFileNames); // TODO: Decide whether/when to add .map files const chunks = bundleResult.filter( @@ -182,10 +168,14 @@ export async function bundleShared( addChunksToResult(chunks, result, fedOptions.dev); - fs.writeFileSync( - resultCacheFile, - JSON.stringify(result, undefined, 2), - 'utf-8', + bundleCache.persist({ + checksum, + externals: result, + files: bundleResult.map((r) => r.fileName.split('/').pop() ?? r.fileName), + }); + + bundleCache.copyFiles( + path.join(fedOptions.workspaceRoot, fedOptions.outputPath), ); return result; @@ -200,19 +190,6 @@ function rewriteImports(cachedFiles: string[], cachePath: string) { } } -function copyCacheToOutput( - cachedFiles: string[], - cachePath: string, - fullOutputPath: string, -) { - for (const fileName of cachedFiles) { - const cachedFile = path.join(cachePath, fileName); - const distFileName = path.join(fullOutputPath, fileName); - copyFileIfExists(cachedFile, distFileName); - copySrcMapIfExists(cachedFile, distFileName); - } -} - function createOutName( pi: PackageInfo, configState: string, @@ -228,28 +205,10 @@ function createOutName( return outName; } -function createCacheFileName( - configState: string, - sharedBundles: Record, - fedOptions: FederationOptions, - cachePath: string, - platform: string, -) { - const resultCacheState = configState + JSON.stringify(sharedBundles); - const resultHash = calcHash(resultCacheState); - const dev = fedOptions.dev ? '-dev' : ''; - const resultCacheFile = path.join( - cachePath, - 'result-' + resultHash + '-' + platform + dev + '.json', - ); - return resultCacheFile; -} - function buildResult( packageInfos: PackageInfo[], sharedBundles: Record, outFileNames: string[], - fedOptions: FederationOptions, ) { return packageInfos.map((pi) => { const shared = sharedBundles[pi.packageName]; @@ -312,11 +271,3 @@ function calcHash(hashBase: string) { .substring(0, 10); return hash; } - -function copyFileIfExists(cachedFile: string, fullOutputPath: string) { - fs.mkdirSync(path.dirname(fullOutputPath), { recursive: true }); - - if (fs.existsSync(cachedFile)) { - fs.copyFileSync(cachedFile, fullOutputPath); - } -} diff --git a/libs/native-federation-core/src/lib/core/remove-unused-deps.ts b/libs/native-federation-core/src/lib/core/remove-unused-deps.ts index 6770d58a..ec819acb 100644 --- a/libs/native-federation-core/src/lib/core/remove-unused-deps.ts +++ b/libs/native-federation-core/src/lib/core/remove-unused-deps.ts @@ -6,6 +6,7 @@ import { NormalizedFederationConfig } from '../config/federation-config'; import { getPackageInfo, PackageInfo } from '../utils/package-info'; import { getExternalImports as extractExternalImports } from '../utils/get-external-imports'; import { MappedPath } from '../utils/mapped-paths'; +import { normalizePackageName } from '../utils/normalize'; export function removeUnusedDeps( config: NormalizedFederationConfig, @@ -116,11 +117,11 @@ function addTransientDeps(packages: Set, workspaceRoot: string) { } function getExternalImports(pInfo: PackageInfo, workspaceRoot: string) { - const encodedPackageName = pInfo.packageName.replace(/[^A-Za-z0-9]/g, '_'); + const encodedPackageName = normalizePackageName(pInfo.packageName); const cacheFileName = `${encodedPackageName}-${pInfo.version}.deps.json`; const cachePath = path.join( workspaceRoot, - 'node_modules/.cache/native-federation', + 'node_modules/.cache/native-federation/_externals-metadata', ); const cacheFilePath = path.join(cachePath, cacheFileName); diff --git a/libs/native-federation-core/src/lib/utils/copy-src-map-if-exists.ts b/libs/native-federation-core/src/lib/utils/copy-src-map-if-exists.ts deleted file mode 100644 index 9905e033..00000000 --- a/libs/native-federation-core/src/lib/utils/copy-src-map-if-exists.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as fs from 'fs'; - -export function copySrcMapIfExists(cachedFile: string, fullOutputPath: string) { - const mapSrc = cachedFile + '.map'; - const mapDest = fullOutputPath + '.map'; - - if (fs.existsSync(mapSrc)) { - fs.copyFileSync(mapSrc, mapDest); - } -} diff --git a/libs/native-federation-core/src/lib/utils/normalize.ts b/libs/native-federation-core/src/lib/utils/normalize.ts index 4b0814b9..4c81b515 100644 --- a/libs/native-federation-core/src/lib/utils/normalize.ts +++ b/libs/native-federation-core/src/lib/utils/normalize.ts @@ -15,3 +15,8 @@ export function normalize(path: string, trailingSlash?: boolean): string { return cand; } + +export function normalizePackageName(fileName: string) { + const sanitized = fileName.replace(/[^A-Za-z0-9]/g, '_'); + return sanitized.startsWith('_') ? sanitized.slice(1) : sanitized; +}