diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index b3cd0b3762..a84868b91f 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -6,11 +6,12 @@ import { flags, FlagInput, ContentstackClient, - pathValidator, log, handleAndLogError, configHandler, getLogPath, + CLIProgressManager, + cliux, } from '@contentstack/cli-utilities'; import { Context, ImportConfig } from '../../../types'; @@ -181,6 +182,16 @@ export default class ImportCommand extends Command { } } + if (flags.branch) { + CLIProgressManager.initializeGlobalSummary( + `IMPORT-${flags.branch}`, + flags.branch, + `IMPORTING DATA INTO "${flags.branch}" BRANCH`, + ); + } else { + CLIProgressManager.initializeGlobalSummary(`IMPORT`, flags.branch, 'IMPORTING CONTENT'); + } + const moduleImporter = new ModuleImporter(managementAPIClient, importConfig); const result = await moduleImporter.start(); backupDir = importConfig.backupDir; @@ -192,16 +203,48 @@ export default class ImportCommand extends Command { log.success(successMessage, importConfig.context); } - log.success(`The log has been stored at '${getLogPath()}'`, importConfig.context); - log.info(`The backup content has been stored at '${backupDir}'`, importConfig.context); + CLIProgressManager.printGlobalSummary(); + this.logSuccessAndBackupMessages(backupDir, importConfig); } catch (error) { handleAndLogError(error); - log.info(`The log has been stored at '${getLogPath()}'`); - if (importConfig?.backupDir) { - log.info(`The backup content has been stored at '${importConfig?.backupDir}'`); - } else { - log.info('No backup directory was created due to early termination'); - } + this.logAndPrintErrorDetails(error, importConfig); + } + } + + private logAndPrintErrorDetails(error: unknown, importConfig: any) { + cliux.print('\n'); + const logPath = getLogPath(); + const logMsg = `The log has been stored at '${logPath}'`; + + const backupDir = importConfig?.backupDir; + const backupDirMsg = backupDir + ? `The backup content has been stored at '${backupDir}'` + : 'No backup directory was created due to early termination'; + + log.info(logMsg); + log.info(backupDirMsg); + + const showConsoleLogs = configHandler.get('log')?.showConsoleLogs; + if (!showConsoleLogs) { + cliux.print(`Error: ${error}`, { color: 'red' }); + cliux.print(logMsg, { color: 'blue' }); + cliux.print(backupDirMsg, { color: 'blue' }); + } + } + + private logSuccessAndBackupMessages(backupDir: string, importConfig: any) { + cliux.print('\n'); + const logPath = getLogPath(); + const logMsg = `The log has been stored at '${logPath}'`; + const backupDirMsg = `The backup content has been stored at '${backupDir}'`; + + log.success(logMsg, importConfig.context); + log.info(backupDirMsg, importConfig.context); + + const showConsoleLogs = configHandler.get('log')?.showConsoleLogs; + if (!showConsoleLogs) { + cliux.print(logMsg, { color: 'blue' }); + cliux.print(backupDirMsg, { color: 'blue' }); } } diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index f3637b2212..103d8b0101 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -1,12 +1,12 @@ import { resolve } from 'path'; import { AuditFix } from '@contentstack/cli-audit'; import messages, { $t } from '@contentstack/cli-audit/lib/messages'; -import { addLocale, cliux, ContentstackClient, Logger } from '@contentstack/cli-utilities'; +import { addLocale, cliux, ContentstackClient, Logger, log, handleAndLogError } from '@contentstack/cli-utilities'; import startModuleImport from './modules'; import startJSModuleImport from './modules-js'; import { ImportConfig, Modules } from '../types'; -import { backupHandler, log, validateBranch, masterLocalDetails, sanitizeStack, initLogger, trace } from '../utils'; +import { backupHandler, validateBranch, masterLocalDetails, sanitizeStack, initLogger, trace } from '../utils'; class ModuleImporter { private managementAPIClient: ContentstackClient; @@ -50,15 +50,9 @@ class ModuleImporter { if ( !this.importConfig.skipAudit && (!this.importConfig.moduleName || - [ - 'content-types', - 'global-fields', - 'entries', - 'extensions', - 'workflows', - 'custom-roles', - 'assets' - ].includes(this.importConfig.moduleName)) + ['content-types', 'global-fields', 'entries', 'extensions', 'workflows', 'custom-roles', 'assets'].includes( + this.importConfig.moduleName, + )) ) { if (!(await this.auditImportData(logger))) { return { noSuccessMsg: true }; @@ -77,7 +71,7 @@ class ModuleImporter { } async import() { - log(this.importConfig, `Starting to import content version ${this.importConfig.contentVersion}`, 'info'); + log.info(`Starting to import content version ${this.importConfig.contentVersion}`, this.importConfig.context); // checks for single module or all modules if (this.importConfig.singleModuleImport) { @@ -87,7 +81,7 @@ class ModuleImporter { } async importByModuleByName(moduleName: Modules) { - log(this.importConfig, `Starting import of ${moduleName} module`, 'info'); + log.info(`Starting import of ${moduleName} module`, this.importConfig.context); // import the modules by name // calls the module runner which inturn calls the module itself // NOTE: Implement a mechanism to determine whether module is new or old @@ -113,10 +107,9 @@ class ModuleImporter { // use the algorithm to determine the parallel and sequential execution of modules for (let moduleName of this.importConfig.modules.types) { if (this.importConfig.globalModules.includes(moduleName) && this.importConfig['exclude-global-modules']) { - log( - this.importConfig, + log.warn( `Skipping the import of the global module '${moduleName}', as it already exists in the stack.`, - 'warn', + this.importConfig.context, ); continue; } @@ -150,24 +143,18 @@ class ModuleImporter { } else if (this.importConfig.modules.types.length) { this.importConfig.modules.types .filter((val) => - [ - 'content-types', - 'global-fields', - 'entries', - 'extensions', - 'workflows', - 'custom-roles', - 'assets' - ].includes(val), + ['content-types', 'global-fields', 'entries', 'extensions', 'workflows', 'custom-roles', 'assets'].includes( + val, + ), ) .forEach((val) => { args.push('--modules', val); }); } args.push('--modules', 'field-rules'); - log(this.importConfig, 'Starting audit process', 'info'); + log.info('Starting audit process', this.importConfig.context); const result = await AuditFix.run(args); - log(this.importConfig, 'Audit process completed', 'info'); + log.info('Audit process completed', this.importConfig.context); if (result) { const { hasFix, config } = result; @@ -193,7 +180,7 @@ class ModuleImporter { return true; } catch (error) { - log(this.importConfig, `Audit failed with following error. ${error}`, 'error'); + handleAndLogError(error, { ...this.importConfig.context }); } } } diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index ffddbbb626..401215364f 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -34,6 +34,7 @@ export default class ImportAssets extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'assets'; + this.currentModuleName = 'Assets'; this.assetsPath = join(this.importConfig.backupDir, 'assets'); this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'assets'); @@ -54,33 +55,46 @@ export default class ImportAssets extends BaseClass { */ async start(): Promise { try { - // NOTE Step 1: Import folders and create uid mapping file - log.debug('Starting folder import process', this.importConfig.context); - await this.importFolders(); - - // NOTE Step 2: Import versioned assets and create it mapping files (uid, url) - if (this.assetConfig.includeVersionedAssets) { - const versionsPath = `${this.assetsPath}/versions`; - if (existsSync(versionsPath)) { - log.debug('Starting versioned assets import', this.importConfig.context); - await this.importAssets(true); - } else { - log.info('No Versioned assets found to import', this.importConfig.context); - } + log.debug('Starting assets import process...', this.importConfig.context); + + // Step 1: Analyze import data + const [foldersCount, assetsCount, versionedAssetsCount, publishableAssetsCount] = await this.withLoadingSpinner( + 'ASSETS: Analyzing import data...', + () => this.analyzeImportData(), + ); + + // Step 2: Initialize progress tracking + const progress = this.createNestedProgress(this.currentModuleName); + this.initializeProgress(progress, { + foldersCount, + assetsCount, + versionedAssetsCount, + publishableAssetsCount, + }); + + // Step 3: Perform import steps based on data + if (foldersCount > 0) { + await this.executeStep(progress, 'Asset Folders', 'Importing folder structure...', () => this.importFolders()); } - // NOTE Step 3: Import Assets and create it mapping files (uid, url) - log.debug('Starting assets import', this.importConfig.context); - await this.importAssets(); + if (this.assetConfig.includeVersionedAssets && versionedAssetsCount > 0) { + await this.executeStep(progress, 'Versioned Assets', 'Importing versioned assets...', () => + this.importAssets(true), + ); + } + + if (assetsCount > 0) { + await this.executeStep(progress, 'Asset Upload', 'Uploading asset files...', () => this.importAssets()); + } - // NOTE Step 4: Publish assets - if (!this.importConfig.skipAssetsPublish) { - log.debug('Starting assets publishing', this.importConfig.context); - await this.publish(); + if (!this.importConfig.skipAssetsPublish && publishableAssetsCount > 0) { + await this.executeStep(progress, 'Asset Publishing', 'Publishing assets...', () => this.publish()); } + this.completeProgress(true); log.success('Assets imported successfully!', this.importConfig.context); } catch (error) { + this.completeProgress(false, error?.message || 'Asset import failed'); handleAndLogError(error, { ...this.importConfig.context }); } } @@ -105,11 +119,18 @@ export default class ImportAssets extends BaseClass { const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }: any) => { this.assetsFolderMap[uid] = response.uid; + this.progressManager?.tick(true, `folder: ${name || uid}`, null, 'Asset Folders'); log.debug(`Created folder: ${name} (Mapped ${uid} → ${response.uid})`, this.importConfig.context); log.success(`Created folder: '${name}'`, this.importConfig.context); }; - const onReject = ({ error, apiData: { name } = { name: '' } }: any) => { + const onReject = ({ error, apiData: { name, uid } = { name: '', uid: '' } }: any) => { + this.progressManager?.tick( + false, + `folder: ${name || uid}`, + error?.message || 'Failed to create folder', + 'Asset Folders', + ); log.error(`${name} folder creation failed.!`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, name }); }; @@ -171,6 +192,8 @@ export default class ImportAssets extends BaseClass { const processName = isVersion ? 'import versioned assets' : 'import assets'; const indexFileName = isVersion ? 'versioned-assets.json' : 'assets.json'; const basePath = isVersion ? join(this.assetsPath, 'versions') : this.assetsPath; + const progressProcessName = isVersion ? 'Versioned Assets' : 'Asset Upload'; + log.debug(`Importing ${processName} from ${basePath}`, this.importConfig.context); const fs = new FsUtility({ basePath, indexFileName }); @@ -182,11 +205,18 @@ export default class ImportAssets extends BaseClass { const onSuccess = ({ response = {}, apiData: { uid, url, title } = undefined }: any) => { this.assetsUidMap[uid] = response.uid; this.assetsUrlMap[url] = response.url; + this.progressManager?.tick(true, `asset: ${title || uid}`, null, progressProcessName); log.debug(`Created asset: ${title} (Mapped ${uid} → ${response.uid})`, this.importConfig.context); log.success(`Created asset: '${title}'`, this.importConfig.context); }; - const onReject = ({ error, apiData: { title } = undefined }: any) => { + const onReject = ({ error, apiData: { title, uid } = undefined }: any) => { + this.progressManager?.tick( + false, + `asset: ${title || uid}`, + error?.message || 'Failed to upload asset', + progressProcessName, + ); log.error(`${title} asset upload failed.!`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, title }); }; @@ -318,10 +348,17 @@ export default class ImportAssets extends BaseClass { log.debug(`Found ${indexerCount} asset chunks to publish`, this.importConfig.context); const onSuccess = ({ apiData: { uid, title } = undefined }: any) => { + this.progressManager?.tick(true, `published: ${title || uid}`, null, 'Asset Publishing'); log.success(`Asset '${uid}: ${title}' published successfully`, this.importConfig.context); }; const onReject = ({ error, apiData: { uid, title } = undefined }: any) => { + this.progressManager?.tick( + false, + `publish failed: ${title || uid}`, + error?.message || 'Failed to publish asset', + 'Asset Publishing', + ); log.error(`Asset '${uid}: ${title}' not published`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, uid, title }); }; @@ -441,4 +478,92 @@ export default class ImportAssets extends BaseClass { } return importOrder; } + + private async analyzeImportData(): Promise<[number, number, number, number]> { + const foldersCount = this.countFolders(); + const assetsCount = await this.countAssets(this.assetsPath, 'assets.json'); + + let versionedAssetsCount = 0; + if (this.assetConfig.includeVersionedAssets && existsSync(`${this.assetsPath}/versions`)) { + versionedAssetsCount = await this.countAssets(`${this.assetsPath}/versions`, 'versioned-assets.json'); + } + + let publishableAssetsCount = 0; + if (!this.importConfig.skipAssetsPublish) { + publishableAssetsCount = await this.countPublishableAssets(); + } + + log.debug( + `Analysis complete: ${foldersCount} folders, ${assetsCount} assets, ${versionedAssetsCount} versioned, ${publishableAssetsCount} publishable`, + this.importConfig.context, + ); + + return [foldersCount, assetsCount, versionedAssetsCount, publishableAssetsCount]; + } + + private initializeProgress( + progress: any, + counts: { + foldersCount: number; + assetsCount: number; + versionedAssetsCount: number; + publishableAssetsCount: number; + }, + ) { + const { foldersCount, assetsCount, versionedAssetsCount, publishableAssetsCount } = counts; + + if (foldersCount > 0) { + progress.addProcess('Asset Folders', foldersCount); + } + if (versionedAssetsCount > 0) { + progress.addProcess('Versioned Assets', versionedAssetsCount); + } + if (assetsCount > 0) { + progress.addProcess('Asset Upload', assetsCount); + } + if (publishableAssetsCount > 0) { + progress.addProcess('Asset Publishing', publishableAssetsCount); + } + } + + private countFolders(): number { + const foldersPath = pResolve(this.assetsRootPath, 'folders.json'); + const folders = this.fs.readFile(foldersPath) || []; + return Array.isArray(folders) ? folders.length : 0; + } + + private async countAssets(basePath: string, indexFileName: string): Promise { + const fsUtil = new FsUtility({ basePath, indexFileName }); + let count = 0; + + for (const _ of values(fsUtil.indexFileContent)) { + const chunkData = await fsUtil.readChunkFiles.next().catch(() => ({})); + count += values(chunkData as Record[]).length; + } + + return count; + } + + private async countPublishableAssets(): Promise { + const fsUtil = new FsUtility({ basePath: this.assetsPath, indexFileName: 'assets.json' }); + let count = 0; + + for (const _ of values(fsUtil.indexFileContent)) { + const chunkData = await fsUtil.readChunkFiles.next().catch(() => ({})); + const publishableAssets = filter( + values(chunkData as Record[]), + ({ publish_details }) => !isEmpty(publish_details), + ); + count += publishableAssets.length; + } + + return count; + } + + private async executeStep(progress: any, name: string, status: string, action: () => Promise): Promise { + progress.startProcess(name).updateStatus(status, name); + log.debug(`Starting ${name.toLowerCase()}`, this.importConfig.context); + await action(); + progress.completeProcess(name, true); + } } diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 587eb6ae13..eb6dc57cd0 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -16,8 +16,8 @@ import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; import { RoleData } from '@contentstack/management/types/stack/role'; +import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; -import { log } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; import cloneDeep from 'lodash/cloneDeep'; @@ -90,6 +90,8 @@ export default abstract class BaseClass { public importConfig: ImportConfig; public modulesConfig: any; + protected progressManager: CLIProgressManager | null = null; + protected currentModuleName: string = ''; constructor({ importConfig, stackAPIClient }: Omit) { this.client = stackAPIClient; @@ -101,6 +103,51 @@ export default abstract class BaseClass { return this.client; } + static printFinalSummary(): void { + CLIProgressManager.printGlobalSummary(); + } + + /** + * Create simple progress manager + */ + protected createSimpleProgress(moduleName: string, total?: number): CLIProgressManager { + this.currentModuleName = moduleName; + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; // Default to true for better UX + this.progressManager = CLIProgressManager.createSimple(moduleName, total, showConsoleLogs); + return this.progressManager; + } + + /** + * Create nested progress manager + */ + protected createNestedProgress(moduleName: string): CLIProgressManager { + this.currentModuleName = moduleName; + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; // Default to true for better UX + this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs); + return this.progressManager; + } + + /** + * Complete progress manager + */ + protected completeProgress(success: boolean = true, error?: string): void { + this.progressManager?.complete(success, error); + this.progressManager = null; + } + + protected async withLoadingSpinner(message: string, action: () => Promise): Promise { + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + + if (showConsoleLogs) { + // If console logs are enabled, don't show spinner, just execute the action + return await action(); + } + return await CLIProgressManager.withLoadingSpinner(message, action); + } + /** * @method delay * @param {number} ms number @@ -209,7 +256,7 @@ export default abstract class BaseClass { // info: Batch No. 20 of import assets is complete if (currentIndexer) batchMsg += `Current chunk processing is (${currentIndexer}/${indexerCount})`; - log(this.importConfig, `Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete`, 'success'); + log.info(`Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete`); } if (this.importConfig.modules.assets.displayExecutionTime) { @@ -325,20 +372,20 @@ export default abstract class BaseClass { return this.stack.globalField({ api_version: '3.2' }).create(apiData).then(onSuccess).catch(onReject); case 'update-gfs': let globalFieldUid = apiData.uid ?? apiData.global_field?.uid; - return this.stack - .globalField(globalFieldUid, { api_version: '3.2' }) - .fetch() - .then(async (gf) => { - const { uid, ...updatePayload } = cloneDeep(apiData); - Object.assign(gf, updatePayload); - try { - const response = await gf.update(); - return onSuccess(response); - } catch (error) { - return onReject(error); - } - }) - .catch(onReject); + return this.stack + .globalField(globalFieldUid, { api_version: '3.2' }) + .fetch() + .then(async (gf) => { + const { uid, ...updatePayload } = cloneDeep(apiData); + Object.assign(gf, updatePayload); + try { + const response = await gf.update(); + return onSuccess(response); + } catch (error) { + return onReject(error); + } + }) + .catch(onReject); case 'create-environments': return this.stack .environment() diff --git a/packages/contentstack-import/src/import/modules/environments.ts b/packages/contentstack-import/src/import/modules/environments.ts index 6ff7cf1908..bd4f92b7f9 100644 --- a/packages/contentstack-import/src/import/modules/environments.ts +++ b/packages/contentstack-import/src/import/modules/environments.ts @@ -22,6 +22,7 @@ export default class ImportEnvironments extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'environments'; + this.currentModuleName = 'Environments'; this.environmentsConfig = importConfig.modules.environments; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'environments'); this.environmentsFolderPath = join(this.importConfig.backupDir, this.environmentsConfig.dirName); @@ -38,51 +39,29 @@ export default class ImportEnvironments extends BaseClass { * @returns {Promise} Promise */ async start(): Promise { - log.debug('Checking for environments folder existence', this.importConfig.context); + try { + log.debug('Starting environments import process...', this.importConfig.context); - //Step1 check folder exists or not - if (fileHelper.fileExistsSync(this.environmentsFolderPath)) { - log.debug(`Found environments folder: ${this.environmentsFolderPath}`, this.importConfig.context); - this.environments = fsUtil.readFile(join(this.environmentsFolderPath, 'environments.json'), true) as Record< - string, - unknown - >; - const envCount = Object.keys(this.environments || {}).length; - log.debug(`Loaded ${envCount} environment items from file`, this.importConfig.context); - } else { - log.info(`No Environments Found - '${this.environmentsFolderPath}'`, this.importConfig.context); - return; - } + const [environmentsCount] = await this.analyzeEnvironments(); + if (environmentsCount === 0) { + log.info('No Environments Found', this.importConfig.context); + return; + } - log.debug('Creating environments mapper directory', this.importConfig.context); - await fsUtil.makeDirectory(this.mapperDirPath); - log.debug('Loading existing environment UID mappings', this.importConfig.context); - this.envUidMapper = fileHelper.fileExistsSync(this.envUidMapperPath) - ? (fsUtil.readFile(join(this.envUidMapperPath), true) as Record) - : {}; + const progress = this.createSimpleProgress(this.currentModuleName, environmentsCount); + await this.prepareEnvironmentMapper(); - if (Object.keys(this.envUidMapper)?.length > 0) { - const envUidCount = Object.keys(this.envUidMapper || {}).length; - log.debug(`Loaded existing environment UID data: ${envUidCount} items`, this.importConfig.context); - } else { - log.debug('No existing environment UID mappings found', this.importConfig.context); - } + progress.updateStatus('Importing environments...'); + await this.importEnvironments(); - log.debug('Starting environment import process', this.importConfig.context); - await this.importEnvironments(); + await this.processImportResults(); - log.debug('Processing environment import results', this.importConfig.context); - if (this.envSuccess?.length) { - fsUtil.writeFile(this.envSuccessPath, this.envSuccess); - log.debug(`Written ${this.envSuccess.length} successful environments to file`, this.importConfig.context); - } - - if (this.envFailed?.length) { - fsUtil.writeFile(this.envFailsPath, this.envFailed); - log.debug(`Written ${this.envFailed.length} failed environments to file`, this.importConfig.context); + this.completeProgress(true); + log.success('Environments have been imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Environments import failed'); + handleAndLogError(error, { ...this.importConfig.context }); } - - log.success('Environments have been imported successfully!', this.importConfig.context); } async importEnvironments() { @@ -99,6 +78,7 @@ export default class ImportEnvironments extends BaseClass { const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }: any) => { this.envSuccess.push(response); this.envUidMapper[uid] = response.uid; + this.progressManager?.tick(true, `environment: ${name || uid}`); log.success(`Environment '${name}' imported successfully`, this.importConfig.context); log.debug(`Environment UID mapping: ${uid} → ${response.uid}`, this.importConfig.context); fsUtil.writeFile(this.envUidMapperPath, this.envUidMapper); @@ -108,15 +88,22 @@ export default class ImportEnvironments extends BaseClass { const err = error?.message ? JSON.parse(error.message) : error; const { name, uid } = apiData; log.debug(`Environment '${name}' (${uid}) failed to import`, this.importConfig.context); + if (err?.errors?.name) { log.debug(`Environment '${name}' already exists, fetching details`, this.importConfig.context); const res = await this.getEnvDetails(name); this.envUidMapper[uid] = res?.uid || ' '; fsUtil.writeFile(this.envUidMapperPath, this.envUidMapper); + this.progressManager?.tick(true, `environment: ${name || uid} (already exists)`); log.info(`Environment '${name}' already exists`, this.importConfig.context); log.debug(`Added existing environment UID mapping: ${uid} → ${res?.uid}`, this.importConfig.context); } else { this.envFailed.push(apiData); + this.progressManager?.tick( + false, + `environment: ${name || uid}`, + error?.message || 'Failed to import environment', + ); handleAndLogError(error, { ...this.importConfig.context, name }, `Environment '${name}' failed to be import`); } }; @@ -157,6 +144,8 @@ export default class ImportEnvironments extends BaseClass { this.importConfig.context, ); log.debug(`Skipping environment serialization for: ${environment.uid}`, this.importConfig.context); + // Still tick progress for skipped environments + this.progressManager?.tick(true, `environment: ${environment.name} (skipped - already exists)`); apiOptions.entity = undefined; } else { log.debug(`Processing environment: ${environment.name}`, this.importConfig.context); @@ -179,4 +168,57 @@ export default class ImportEnvironments extends BaseClass { handleAndLogError(error, { ...this.importConfig.context, envName }); }); } + + private async analyzeEnvironments(): Promise<[number]> { + return this.withLoadingSpinner('ENVIRONMENTS: Analyzing import data...', async () => { + log.debug('Checking for environments folder existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.environmentsFolderPath)) { + log.info(`No Environments Found - '${this.environmentsFolderPath}'`, this.importConfig.context); + return [0]; + } + + log.debug(`Found environments folder: ${this.environmentsFolderPath}`, this.importConfig.context); + + this.environments = fsUtil.readFile(join(this.environmentsFolderPath, 'environments.json'), true) as Record< + string, + unknown + >; + + const count = Object.keys(this.environments || {}).length; + log.debug(`Loaded ${count} environment items from file`, this.importConfig.context); + return [count]; + }); + } + + private async prepareEnvironmentMapper(): Promise { + log.debug('Creating environments mapper directory', this.importConfig.context); + await fsUtil.makeDirectory(this.mapperDirPath); + + log.debug('Loading existing environment UID mappings', this.importConfig.context); + this.envUidMapper = fileHelper.fileExistsSync(this.envUidMapperPath) + ? (fsUtil.readFile(this.envUidMapperPath, true) as Record) + : {}; + + const count = Object.keys(this.envUidMapper || {}).length; + if (count > 0) { + log.debug(`Loaded existing environment UID data: ${count} items`, this.importConfig.context); + } else { + log.debug('No existing environment UID mappings found', this.importConfig.context); + } + } + + private async processImportResults(): Promise { + log.debug('Processing environment import results', this.importConfig.context); + + if (this.envSuccess?.length) { + fsUtil.writeFile(this.envSuccessPath, this.envSuccess); + log.debug(`Written ${this.envSuccess.length} successful environments to file`, this.importConfig.context); + } + + if (this.envFailed?.length) { + fsUtil.writeFile(this.envFailsPath, this.envFailed); + log.debug(`Written ${this.envFailed.length} failed environments to file`, this.importConfig.context); + } + } } diff --git a/packages/contentstack-import/src/import/modules/extensions.ts b/packages/contentstack-import/src/import/modules/extensions.ts index 37bc1eb619..0c1c09da52 100644 --- a/packages/contentstack-import/src/import/modules/extensions.ts +++ b/packages/contentstack-import/src/import/modules/extensions.ts @@ -26,6 +26,7 @@ export default class ImportExtensions extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'extensions'; + this.currentModuleName = 'Extensions'; this.extensionsConfig = importConfig.modules.extensions; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'extensions'); this.extensionsFolderPath = join(this.importConfig.backupDir, this.extensionsConfig.dirName); @@ -45,82 +46,49 @@ export default class ImportExtensions extends BaseClass { * @returns {Promise} Promise */ async start(): Promise { - log.debug('Checking for extensions folder existence', this.importConfig.context); - - //Step1 check folder exists or not - if (fileHelper.fileExistsSync(this.extensionsFolderPath)) { - log.debug(`Found extensions folder: ${this.extensionsFolderPath}`, this.importConfig.context); - - this.extensions = fsUtil.readFile(join(this.extensionsFolderPath, 'extensions.json'), true) as Record< - string, - Record - >; - - // Check if extensions file was read successfully - if (!this.extensions) { - log.info(`No extensions found in file - '${join(this.extensionsFolderPath, 'extensions.json')}'`, this.importConfig.context); + try { + log.debug('Starting Create process...', this.importConfig.context); + const [extensionsCount] = await this.analyzeExtensions(); + if (extensionsCount === 0) { + log.info('No extensions found to import', this.importConfig.context); return; } - - const extensionsCount = Object.keys(this.extensions || {}).length; - log.debug(`Loaded ${extensionsCount} extension items from file`, this.importConfig.context); - } else { - log.info(`No Extensions Found - '${this.extensionsFolderPath}'`, this.importConfig.context); - return; - } - log.debug('Creating extensions mapper directory', this.importConfig.context); - await fsUtil.makeDirectory(this.mapperDirPath); - - log.debug('Loading existing extensions UID data', this.importConfig.context); - this.extUidMapper = fileHelper.fileExistsSync(this.extUidMapperPath) - ? (fsUtil.readFile(join(this.extUidMapperPath), true) as Record) || {} - : {}; + const progress = this.createNestedProgress(this.currentModuleName); + progress.addProcess('Create', extensionsCount); - if (this.extUidMapper && Object.keys(this.extUidMapper || {}).length > 0) { - const extUidCount = Object.keys(this.extUidMapper || {}).length; - log.debug(`Loaded existing extensions UID data: ${extUidCount} items`, this.importConfig.context); - } else { - log.debug('No existing extensions UID data found', this.importConfig.context); - } + await this.prepareExtensionMapper(); + log.debug('Checking content types in extension scope', this.importConfig.context); - // Check whether the scope of an extension contains content-types in scope - // Remove the scope and store the scope with uid in pending extensions - log.debug('Checking content types in extension scope', this.importConfig.context); - this.getContentTypesInScope(); - - log.debug('Starting extensions import', this.importConfig.context); - await this.importExtensions(); - - // Update the uid of the extension - log.debug('Updating extension UIDs', this.importConfig.context); - this.updateUidExtension(); - - // Note: if any extensions present, then update it - if (this.importConfig.replaceExisting && this.existingExtensions.length > 0) { - log.debug(`Replacing ${this.existingExtensions.length} existing extensions`, this.importConfig.context); - await this.replaceExtensions().catch((error: Error) => { - log.debug('Error replacing extensions', this.importConfig.context); - handleAndLogError(error, { ...this.importConfig.context}); - }); - } + this.getContentTypesInScope(); - log.debug('Processing extensions import results', this.importConfig.context); - if (this.extSuccess?.length) { - fsUtil.writeFile(this.extSuccessPath, this.extSuccess); - log.debug(`Written ${this.extSuccess.length} successful extensions to file`, this.importConfig.context); - } + progress.startProcess('Create').updateStatus('Importing extensions...', 'Create'); + log.debug('Starting Create', this.importConfig.context); + await this.importExtensions(); + progress.completeProcess('Create', true); - if (this.extFailed?.length) { - fsUtil.writeFile(this.extFailsPath, this.extFailed); - log.debug(`Written ${this.extFailed.length} failed extensions to file`, this.importConfig.context); - } + log.debug('Updating extension UIDs', this.importConfig.context); + this.updateUidExtension(); + + if (this.importConfig.replaceExisting && this.existingExtensions.length > 0) { + progress.addProcess('Update', this.existingExtensions.length); + progress.startProcess('Update').updateStatus('Updating existing extensions...', 'Update'); + await this.replaceExtensions(); + progress.completeProcess('Update', true); + } + await this.processExtensionResults(); + + this.completeProgress(true); log.success('Extensions have been imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Create failed'); + handleAndLogError(error, { ...this.importConfig.context }); + } } async importExtensions(): Promise { - log.debug('Starting extensions import process', this.importConfig.context); + log.debug('Starting Create process', this.importConfig.context); if (this.extensions === undefined || isEmpty(this.extensions)) { log.info('No Extensions Found', this.importConfig.context); return; @@ -132,30 +100,48 @@ export default class ImportExtensions extends BaseClass { const onSuccess = ({ response, apiData: { uid, title } = { uid: null, title: '' } }: any) => { this.extSuccess.push(response); this.extUidMapper[uid] = response.uid; + this.progressManager?.tick(true, `extension: ${title || uid}`, null, 'Create'); log.success(`Extension '${title}' imported successfully`, this.importConfig.context); log.debug(`Extension import completed: ${title} (${uid})`, this.importConfig.context); fsUtil.writeFile(this.extUidMapperPath, this.extUidMapper); }; const onReject = ({ error, apiData }: any) => { - const { title } = apiData; + const { title, uid } = apiData; log.debug(`Extension '${title}' import failed`, this.importConfig.context); - + if (error?.errors?.title) { if (this.importConfig.replaceExisting) { this.existingExtensions.push(apiData); + this.progressManager?.tick( + true, + `extension: ${title || uid} (marked for replacement)`, + null, + 'Create', + ); log.debug(`Extension '${title}' marked for replacement`, this.importConfig.context); + } else { + this.progressManager?.tick(true, `extension: ${title || uid} (already exists)`, null, 'Create'); } if (!this.importConfig.skipExisting) { log.info(`Extension '${title}' already exists`, this.importConfig.context); } } else { this.extFailed.push(apiData); + this.progressManager?.tick( + false, + `extension: ${title || uid}`, + error?.message || 'Failed to import extension', + 'Create', + ); handleAndLogError(error, { ...this.importConfig.context, title }, `Extension '${title}' failed to be import`); } }; - log.debug(`Using concurrency limit: ${this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1}`, this.importConfig.context); + log.debug( + `Using concurrency limit: ${this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1}`, + this.importConfig.context, + ); await this.makeConcurrentCall( { apiContent, @@ -171,28 +157,35 @@ export default class ImportExtensions extends BaseClass { undefined, false, ); - - log.debug('Extensions import process completed', this.importConfig.context); + + log.debug('Create process completed', this.importConfig.context); } async replaceExtensions(): Promise { log.debug(`Replacing ${this.existingExtensions.length} existing extensions`, this.importConfig.context); - + const onSuccess = ({ response, apiData: { uid, title } = { uid: null, title: '' } }: any) => { this.extSuccess.push(response); this.extUidMapper[uid] = response.uid; - log.success(`Extension '${title}' replaced successfully`, this.importConfig.context); - log.debug(`Extension replacement completed: ${title} (${uid})`, this.importConfig.context); + this.progressManager?.tick(true, `extension: ${title || uid} (updated)`, null, 'Update'); + log.success(`Extension '${title}' updated successfully`, this.importConfig.context); + log.debug(`Extension update completed: ${title} (${uid})`, this.importConfig.context); fsUtil.writeFile(this.extUidMapperPath, this.extUidMapper); }; const onReject = ({ error, apiData }: any) => { - log.debug(`Extension '${apiData.title}' replacement failed`, this.importConfig.context); + const { title, uid } = apiData; this.extFailed.push(apiData); - handleAndLogError(error, { ...this.importConfig.context, title: apiData.title }, `Extension '${apiData.title}' failed to replace`); + this.progressManager?.tick( + false, + `extension: ${title || uid}`, + error?.message || 'Failed to update extension', + 'Update', + ); + log.debug(`Extension '${title}' update failed`, this.importConfig.context); + handleAndLogError(error, { ...this.importConfig.context, title }, `Extension '${title}' failed to be updated`); }; - log.debug(`Using concurrency limit for replacement: ${this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1}`, this.importConfig.context); await this.makeConcurrentCall( { apiContent: this.existingExtensions, @@ -207,7 +200,7 @@ export default class ImportExtensions extends BaseClass { }, this.replaceExtensionHandler.bind(this), ); - + log.debug('Extensions replacement process completed', this.importConfig.context); } @@ -221,10 +214,10 @@ export default class ImportExtensions extends BaseClass { isLastRequest: boolean; }) { log.debug(`Processing extension replacement: ${extension.title}`, this.importConfig.context); - + return new Promise(async (resolve, reject) => { log.debug(`Searching for existing extension: ${extension.title}`, this.importConfig.context); - + const { items: [extensionsInStack] = [] }: any = await this.stack .extension() .query({ query: { title: extension.title } }) @@ -237,10 +230,13 @@ export default class ImportExtensions extends BaseClass { }); reject(true); }); - + if (extensionsInStack) { - log.debug(`Found existing extension in stack: ${extension.title} (${extensionsInStack.uid})`, this.importConfig.context); - + log.debug( + `Found existing extension in stack: ${extension.title} (${extensionsInStack.uid})`, + this.importConfig.context, + ); + const extensionPayload = this.stack.extension(extension.uid); Object.assign(extensionPayload, extensionsInStack, cloneDeep(extension), { uid: extensionsInStack.uid, @@ -248,7 +244,7 @@ export default class ImportExtensions extends BaseClass { _version: extensionsInStack._version, stackHeaders: extensionsInStack.stackHeaders, }); - + log.debug(`Updating extension: ${extension.title}`, this.importConfig.context); return extensionPayload .update() @@ -281,16 +277,19 @@ export default class ImportExtensions extends BaseClass { getContentTypesInScope() { log.debug('Processing content types in extension scope', this.importConfig.context); - + const extension = values(this.extensions); let processedExtensions = 0; - + extension.forEach((ext: ExtensionType) => { let ct: any = ext?.scope?.content_types || []; if ((ct.length === 1 && ct[0] !== '$all') || ct?.length > 1) { - log.info(`Removing the content-types ${ct.join(',')} from the extension ${ext.title} ...`, this.importConfig.context); + log.info( + `Removing the content-types ${ct.join(',')} from the extension ${ext.title} ...`, + this.importConfig.context, + ); log.debug(`Extension '${ext.title}' has ${ct.length} content types in scope`, this.importConfig.context); - + const { uid, scope } = ext; this.extensionObject.push({ uid, scope }); delete ext.scope; @@ -298,29 +297,93 @@ export default class ImportExtensions extends BaseClass { processedExtensions++; } }); - + log.debug(`Processed ${processedExtensions} extensions with content type scope`, this.importConfig.context); log.debug(`Total extensions with pending scope: ${this.extensionObject.length}`, this.importConfig.context); } updateUidExtension() { log.debug('Updating extension UIDs in pending extensions', this.importConfig.context); - + let updatedCount = 0; for (let i in this.extensionObject) { const originalUid = this.extensionObject[i].uid as string; this.extensionObject[i].uid = this.extUidMapper[originalUid]; if (this.extUidMapper[originalUid]) { updatedCount++; - log.debug(`Updated extension UID: ${originalUid} → ${this.extUidMapper[originalUid]}`, this.importConfig.context); + log.debug( + `Updated extension UID: ${originalUid} → ${this.extUidMapper[originalUid]}`, + this.importConfig.context, + ); } } - + log.debug(`Updated ${updatedCount} extension UIDs in pending extensions`, this.importConfig.context); - + if (this.extensionObject.length > 0) { fsUtil.writeFile(this.extPendingPath, this.extensionObject); log.debug(`Written ${this.extensionObject.length} pending extensions to file`, this.importConfig.context); } } + + private async analyzeExtensions(): Promise<[number]> { + return this.withLoadingSpinner('EXTENSIONS: Analyzing import data...', async () => { + log.debug('Checking for extensions folder existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.extensionsFolderPath)) { + log.info(`No Extensions Found - '${this.extensionsFolderPath}'`, this.importConfig.context); + return [0]; + } + + log.debug(`Found extensions folder: ${this.extensionsFolderPath}`, this.importConfig.context); + + this.extensions = fsUtil.readFile(join(this.extensionsFolderPath, 'extensions.json'), true) as Record< + string, + Record + >; + + if (!this.extensions) { + log.info( + `No extensions found in file - '${join(this.extensionsFolderPath, 'extensions.json')}'`, + this.importConfig.context, + ); + return [0]; + } + + const count = Object.keys(this.extensions).length; + log.debug(`Loaded ${count} extension items from file`, this.importConfig.context); + return [count]; + }); + } + + private async prepareExtensionMapper(): Promise { + log.debug('Creating extensions mapper directory', this.importConfig.context); + await fsUtil.makeDirectory(this.mapperDirPath); + + log.debug('Loading existing extensions UID data', this.importConfig.context); + this.extUidMapper = fileHelper.fileExistsSync(this.extUidMapperPath) + ? (fsUtil.readFile(this.extUidMapperPath, true) as Record) || {} + : {}; + + const count = Object.keys(this.extUidMapper || {}).length; + if (count > 0) { + log.debug(`Loaded existing extensions UID data: ${count} items`, this.importConfig.context); + } else { + log.debug('No existing extensions UID data found', this.importConfig.context); + } + } + + private async processExtensionResults(): Promise { + log.debug('Processing Create results', this.importConfig.context); + + if (this.extSuccess?.length) { + fsUtil.writeFile(this.extSuccessPath, this.extSuccess); + log.debug(`Written ${this.extSuccess.length} successful extensions to file`, this.importConfig.context); + } + + if (this.extFailed?.length) { + fsUtil.writeFile(this.extFailsPath, this.extFailed); + log.debug(`Written ${this.extFailed.length} failed extensions to file`, this.importConfig.context); + } + } } diff --git a/packages/contentstack-import/src/import/modules/global-fields.ts b/packages/contentstack-import/src/import/modules/global-fields.ts index e7af748bf4..caacfdad2a 100644 --- a/packages/contentstack-import/src/import/modules/global-fields.ts +++ b/packages/contentstack-import/src/import/modules/global-fields.ts @@ -9,12 +9,11 @@ import * as path from 'path'; import { isEmpty, cloneDeep } from 'lodash'; import { cliux, sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; import { GlobalFieldData, GlobalField } from '@contentstack/management/types/stack/globalField'; -import { fsUtil,fileHelper, lookupExtension, removeReferenceFields } from '../../utils'; +import { fsUtil, fileHelper, lookupExtension, removeReferenceFields } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; import { gfSchemaTemplate } from '../../utils/global-field-helper'; - export default class ImportGlobalFields extends BaseClass { private gFsMapperPath: string; private gFsFolderPath: string; @@ -44,6 +43,7 @@ export default class ImportGlobalFields extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'global-fields'; + this.currentModuleName = 'Global Fields'; this.config = importConfig; this.gFsConfig = importConfig.modules['global-fields']; this.gFs = []; @@ -58,111 +58,120 @@ export default class ImportGlobalFields extends BaseClass { this.gFsFailsPath = path.resolve(sanitizePath(this.config.data), 'mapper', 'global_fields', 'fails.json'); this.gFsSuccessPath = path.resolve(sanitizePath(this.config.data), 'mapper', 'global_fields', 'success.json'); this.gFsUidMapperPath = path.resolve(sanitizePath(this.config.data), 'mapper', 'global_fields', 'uid-mapping.json'); - this.gFsPendingPath = path.resolve(sanitizePath(this.config.data), 'mapper', 'global_fields', 'pending_global_fields.js'); - this.marketplaceAppMapperPath = path.join(sanitizePath(this.config.data), 'mapper', 'marketplace_apps', 'uid-mapping.json'); + this.gFsPendingPath = path.resolve( + sanitizePath(this.config.data), + 'mapper', + 'global_fields', + 'pending_global_fields.js', + ); + this.marketplaceAppMapperPath = path.join( + sanitizePath(this.config.data), + 'mapper', + 'marketplace_apps', + 'uid-mapping.json', + ); } - async start(): Promise { - log.debug('Reading global fields from file', this.importConfig.context); - - this.gFs = fsUtil.readFile(path.join(this.gFsFolderPath, this.gFsConfig.fileName)) as Record[]; - - if (!this.gFs || isEmpty(this.gFs)) { - log.info('No global fields found to import', this.importConfig.context); - return; - } - - const gfsCount = Array.isArray(this.gFs) ? this.gFs.length : Object.keys(this.gFs).length; - log.debug(`Loaded ${gfsCount} global field items from file`, this.importConfig.context); - - log.debug('Creating global fields mapper directory', this.importConfig.context); - await fsUtil.makeDirectory(this.gFsMapperPath); - - log.debug('Loading existing global fields UID data', this.importConfig.context); - if (fileHelper.fileExistsSync(this.gFsUidMapperPath)) { - this.gFsUidMapper = (fsUtil.readFile(this.gFsUidMapperPath) || {}) as Record; - const gfsUidCount = Object.keys(this.gFsUidMapper || {}).length; - log.debug(`Loaded existing global fields UID data: ${gfsUidCount} items`, this.importConfig.context); - } else { - log.debug('No existing global fields UID data found', this.importConfig.context); - } - - log.debug('Loading installed extensions data', this.importConfig.context); - this.installedExtensions = ( - ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } - ).extension_uid; - - const installedExtCount = Object.keys(this.installedExtensions || {}).length; - log.debug(`Loaded ${installedExtCount} installed extension references`, this.importConfig.context); + /** + * @method start + * @returns {Promise} Promise + */ + async start(): Promise { + try { + log.debug('Starting global fields import process...', this.importConfig.context); + const [globalFieldsCount] = await this.analyzeGlobalFields(); + if (globalFieldsCount === 0) { + log.info('No global fields found to import', this.importConfig.context); + return; + } - log.debug('Starting global fields seeding process', this.importConfig.context); - await this.seedGFs(); - - if (this.pendingGFs?.length) { - fsUtil.writeFile(this.gFsPendingPath, this.pendingGFs); - log.debug(`Written ${this.pendingGFs.length} pending global fields to file`, this.importConfig.context); - } - - log.success('Created Global Fields', this.importConfig.context); - - log.debug('Starting global fields update process', this.importConfig.context); - await this.updateGFs(); - if (this.pendingGFs?.length) fsUtil.writeFile(this.gFsPendingPath, this.pendingGFs); - log.success('Updated Global Fields', this.importConfig.context); - - if (this.importConfig.replaceExisting && this.existingGFs.length > 0) { - log.debug(`Replacing ${this.existingGFs.length} existing global fields`, this.importConfig.context); - await this.replaceGFs().catch((error: Error) => { - log.debug('Error replacing global fields', this.importConfig.context); - handleAndLogError(error, { ...this.importConfig.context}); - }); - } + const progress = this.createNestedProgress(this.currentModuleName); + progress.addProcess('Create', globalFieldsCount); + progress.addProcess('Update', globalFieldsCount); - log.debug('Processing global fields import results', this.importConfig.context); - if (this.createdGFs?.length) { - fsUtil.writeFile(this.gFsSuccessPath, this.createdGFs); - log.debug(`Written ${this.createdGFs.length} successful global fields to file`, this.importConfig.context); - } + await this.prepareGlobalFieldMapper(); - if (this.failedGFs?.length) { - fsUtil.writeFile(this.gFsFailsPath, this.failedGFs); - log.debug(`Written ${this.failedGFs.length} failed global fields to file`, this.importConfig.context); - } + // Step 1: Create global fields + progress + .startProcess('Create') + .updateStatus('Creating global fields...', 'Create'); + log.info('Starting Create process', this.importConfig.context); + await this.seedGFs(); + progress.completeProcess('Create', true); + + // Step 2: Update global fields with references + progress.startProcess('Update').updateStatus('Updating global fields', 'Update'); + log.info('Starting Update process', this.importConfig.context); + await this.updateGFs(); + progress.completeProcess('Update', true); + + // Step 3: Replace existing global fields if needed + if (this.importConfig.replaceExisting && this.existingGFs.length > 0) { + progress.addProcess('Global Fields Replacement', this.existingGFs.length); + progress + .startProcess('Global Fields Replacement') + .updateStatus('Replacing existing global fields...', 'Global Fields Replacement'); + log.info('Starting global fields replacement process', this.importConfig.context); + await this.replaceGFs(); + progress.completeProcess('Global Fields Replacement', true); + } - log.success('Global fields import has been completed!', this.importConfig.context); + await this.processGlobalFieldResults(); + + this.completeProgress(true); + log.success('Global fields import has been completed!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Global fields import failed'); + handleAndLogError(error, { ...this.importConfig.context }); + } } async seedGFs(): Promise { log.debug('Starting global fields seeding process', this.importConfig.context); - + const gfsToSeed = Array.isArray(this.gFs) ? this.gFs.length : Object.keys(this.gFs).length; log.debug(`Seeding ${gfsToSeed} global fields`, this.importConfig.context); - + const onSuccess = ({ response: globalField, apiData: { uid } = undefined }: any) => { this.createdGFs.push(globalField); this.gFsUidMapper[uid] = globalField; + this.progressManager?.tick(true, `global field: ${globalField.uid}`, null, 'Create'); log.success(`Global field ${globalField.uid} created successfully`, this.importConfig.context); - log.debug(`Global field creation completed: ${globalField.uid}`, this.importConfig.context); + log.debug(`Global field Create completed: ${globalField.uid}`, this.importConfig.context); }; - + const onReject = ({ error, apiData: globalField = undefined }: any) => { const uid = globalField?.global_field?.uid; - log.debug(`Global field '${uid}' creation failed`, this.importConfig.context); - + log.debug(`Global field '${uid}' Create failed`, this.importConfig.context); + if (error?.errors?.title) { if (this.importConfig.replaceExisting) { this.existingGFs.push(globalField); + this.progressManager?.tick( + true, + `global field: ${uid} (marked for replacement)`, + null, + 'Create', + ); log.debug(`Global field '${uid}' marked for replacement`, this.importConfig.context); + } else { + this.progressManager?.tick(true, `global field: ${uid} (already exists)`, null, 'Create'); } if (!this.importConfig.skipExisting) { log.info(`Global fields '${uid}' already exist`, this.importConfig.context); } } else { + this.progressManager?.tick( + false, + `global field: ${uid}`, + error?.message || 'Failed to create global field', + 'Create', + ); handleAndLogError(error, { ...this.importConfig.context, uid }, `Global fields '${uid}' failed to import`); this.failedGFs.push({ uid }); } }; - + log.debug(`Using concurrency limit for seeding: ${this.reqConcurrency}`, this.importConfig.context); const result = await this.makeConcurrentCall({ processName: 'Import global fields', @@ -176,7 +185,7 @@ export default class ImportGlobalFields extends BaseClass { }, concurrencyLimit: this.reqConcurrency, }); - + log.debug('Global fields seeding process completed', this.importConfig.context); return result; } @@ -189,48 +198,56 @@ export default class ImportGlobalFields extends BaseClass { serializeGFs(apiOptions: ApiOptions): ApiOptions { const { apiData: globalField } = apiOptions; log.debug(`Serializing global field: ${globalField.uid}`, this.importConfig.context); - + const updatedGF = cloneDeep(gfSchemaTemplate); updatedGF.global_field.uid = globalField.uid; updatedGF.global_field.title = globalField.title; - + log.debug(`Global field serialization completed: ${globalField.uid}`, this.importConfig.context); apiOptions.apiData = updatedGF; return apiOptions; } async updateGFs(): Promise { - log.debug('Starting global fields update process', this.importConfig.context); - + log.debug('Starting Update process', this.importConfig.context); + const gfsToUpdate = Array.isArray(this.gFs) ? this.gFs.length : Object.keys(this.gFs).length; log.debug(`Updating ${gfsToUpdate} global fields`, this.importConfig.context); - + const onSuccess = ({ response: globalField, apiData: { uid } = undefined }: any) => { - log.info(`Updated the global field ${uid}`, this.importConfig.context); + this.progressManager?.tick(true, `global field: ${uid}`, null, 'Update'); + log.success(`Updated the global field ${uid}`, this.importConfig.context); log.debug(`Global field update completed: ${uid}`, this.importConfig.context); }; - + const onReject = ({ error, apiData: { uid } = undefined }: any) => { + this.progressManager?.tick( + false, + `global field: ${uid}`, + error?.message || 'Failed to update global field', + 'Update', + ); log.debug(`Global field '${uid}' update failed`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, uid }, `Failed to update the global field '${uid}'`); }; - + log.debug(`Using concurrency limit for updates: ${this.reqConcurrency}`, this.importConfig.context); - const result = await this.makeConcurrentCall({ - processName: 'Update Global Fields', - apiContent: this.gFs, - apiParams: { - reject: onReject.bind(this), - resolve: onSuccess.bind(this), - entity: 'update-gfs', - includeParamOnCompletion: true, + const result = await this.makeConcurrentCall( + { + processName: 'Update Global Fields', + apiContent: this.gFs, + apiParams: { + reject: onReject.bind(this), + resolve: onSuccess.bind(this), + entity: 'update-gfs', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.reqConcurrency, }, - concurrencyLimit: this.reqConcurrency, - }, - this.updateSerializedGFs.bind(this), + this.updateSerializedGFs.bind(this), ); - - log.debug('Global fields update process completed', this.importConfig.context); + + log.debug('Update process completed', this.importConfig.context); return result; } @@ -244,57 +261,61 @@ export default class ImportGlobalFields extends BaseClass { isLastRequest: boolean; }) { log.debug(`Processing global field update: ${globalField.uid}`, this.importConfig.context); - + return new Promise(async (resolve, reject) => { log.debug(`Looking up extensions for global field: ${globalField.uid}`, this.importConfig.context); lookupExtension(this.config, globalField.schema, this.config.preserveStackVersion, this.installedExtensions); - + let flag = { supressed: false }; log.debug(`Removing reference fields for global field: ${globalField.uid}`, this.importConfig.context); await removeReferenceFields(globalField.schema, flag, this.stack); - + if (flag.supressed) { - log.debug(`Global field '${globalField.uid}' has suppressed references, adding to pending`, this.importConfig.context); + log.debug( + `Global field '${globalField.uid}' has suppressed references, adding to pending`, + this.importConfig.context, + ); this.pendingGFs.push(globalField.uid); log.info(`Global field '${globalField.uid}' will be updated later`, this.importConfig.context); return resolve(true); } - + log.debug(`Fetching existing global field: ${globalField.uid}`, this.importConfig.context); return this.stack - .globalField(globalField.uid, { api_version: '3.2' }) - .fetch() - .then((response: GlobalField) => { - log.debug(`Updating global field: ${globalField.uid}`, this.importConfig.context); - Object.assign(response, globalField); - return response.update(); - }) - .then((response: GlobalField) => { - log.debug(`Global field update successful: ${globalField.uid}`, this.importConfig.context); - apiParams.resolve({ - response, - apiData: globalField, + .globalField(globalField.uid, { api_version: '3.2' }) + .fetch() + .then((response: GlobalField) => { + log.debug(`Updating global field: ${globalField.uid}`, this.importConfig.context); + Object.assign(response, globalField); + return response.update(); + }) + .then((response: GlobalField) => { + log.debug(`Global field update successful: ${globalField.uid}`, this.importConfig.context); + apiParams.resolve({ + response, + apiData: globalField, + }); + resolve(true); + }) + .catch((error: unknown) => { + log.debug(`Global field update failed: ${globalField.uid}`, this.importConfig.context); + apiParams.reject({ + error, + apiData: globalField, + }); + reject(true); }); - resolve(true); - }) - .catch((error: unknown) => { - log.debug(`Global field update failed: ${globalField.uid}`, this.importConfig.context); - apiParams.reject({ - error, - apiData: globalField, - }); - reject(true); - }); }); } async replaceGFs(): Promise { log.debug(`Replacing ${this.existingGFs.length} existing global fields`, this.importConfig.context); - + const onSuccess = ({ response: globalField, apiData }: any) => { const uid = apiData?.uid ?? apiData?.global_field?.uid ?? 'unknown'; this.createdGFs.push(globalField); this.gFsUidMapper[uid] = globalField; + this.progressManager?.tick(true, `global field: ${uid} (replaced)`, null, 'Global Fields Replacement'); fsUtil.writeFile(this.gFsUidMapperPath, this.gFsUidMapper); log.success(`Global field '${uid}' replaced successfully`, this.importConfig.context); log.debug(`Global field replacement completed: ${uid}`, this.importConfig.context); @@ -302,12 +323,23 @@ export default class ImportGlobalFields extends BaseClass { const onReject = ({ error, apiData }: any) => { const uid = apiData?.uid ?? apiData?.global_field?.uid ?? 'unknown'; + this.progressManager?.tick( + false, + `global field: ${uid}`, + error?.message || 'Failed to replace global field', + 'Global Fields Replacement', + ); log.debug(`Global field '${uid}' replacement failed`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, uid }, `Global fields '${uid}' failed to replace`); this.failedGFs.push({ uid }); }; - log.debug(`Using concurrency limit for replacement: ${this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1}`, this.importConfig.context); + log.debug( + `Using concurrency limit for replacement: ${ + this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1 + }`, + this.importConfig.context, + ); await this.makeConcurrentCall( { apiContent: this.existingGFs, @@ -324,7 +356,7 @@ export default class ImportGlobalFields extends BaseClass { undefined, false, ); - + log.debug('Global fields replacement process completed', this.importConfig.context); } @@ -337,14 +369,80 @@ export default class ImportGlobalFields extends BaseClass { const { apiData: globalField } = apiOptions; const uid = globalField?.uid ?? globalField?.global_field?.uid ?? 'unknown'; log.debug(`Serializing global field replacement: ${uid}`, this.importConfig.context); - + const globalFieldPayload = this.stack.globalField(globalField.uid, { api_version: '3.2' }); Object.assign(globalFieldPayload, cloneDeep(globalField), { stackHeaders: globalFieldPayload.stackHeaders, }); - + log.debug(`Global field replacement serialization completed: ${uid}`, this.importConfig.context); apiOptions.apiData = globalFieldPayload; return apiOptions; } + + private async analyzeGlobalFields(): Promise<[number]> { + return this.withLoadingSpinner('GLOBAL FIELDS: Analyzing import data...', async () => { + log.debug('Checking for global fields folder existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.gFsFolderPath)) { + log.info(`No global fields found - '${this.gFsFolderPath}'`, this.importConfig.context); + return [0]; + } + + log.debug(`Found global fields folder: ${this.gFsFolderPath}`, this.importConfig.context); + this.gFs = fsUtil.readFile(path.join(this.gFsFolderPath, this.gFsConfig.fileName)) as Record[]; + if (!this.gFs || isEmpty(this.gFs)) { + log.info( + `No global fields found in file - '${path.join(this.gFsFolderPath, this.gFsConfig.fileName)}'`, + this.importConfig.context, + ); + return [0]; + } + + const count = Array.isArray(this.gFs) ? this.gFs?.length : Object.keys(this.gFs)?.length; + log.debug(`Loaded ${count} global field items from file`, this.importConfig.context); + return [count]; + }); + } + + private async prepareGlobalFieldMapper(): Promise { + log.debug('Creating global fields mapper directory', this.importConfig.context); + await fsUtil.makeDirectory(this.gFsMapperPath); + + log.debug('Loading existing global fields UID data', this.importConfig.context); + if (fileHelper.fileExistsSync(this.gFsUidMapperPath)) { + this.gFsUidMapper = (fsUtil.readFile(this.gFsUidMapperPath) || {}) as Record; + const gfsUidCount = Object.keys(this.gFsUidMapper || {}).length; + log.debug(`Loaded existing global fields UID data: ${gfsUidCount} items`, this.importConfig.context); + } else { + log.debug('No existing global fields UID data found', this.importConfig.context); + } + + log.debug('Loading installed extensions data', this.importConfig.context); + this.installedExtensions = ( + (fsUtil.readFile(this.marketplaceAppMapperPath) as any) || { extension_uid: {} } + ).extension_uid; + + const installedExtCount = Object.keys(this.installedExtensions || {}).length; + log.debug(`Loaded ${installedExtCount} installed extension references`, this.importConfig.context); + } + + private async processGlobalFieldResults(): Promise { + log.debug('Processing global fields import results', this.importConfig.context); + + if (this.pendingGFs?.length) { + fsUtil.writeFile(this.gFsPendingPath, this.pendingGFs); + log.debug(`Written ${this.pendingGFs.length} pending global fields to file`, this.importConfig.context); + } + + if (this.createdGFs?.length) { + fsUtil.writeFile(this.gFsSuccessPath, this.createdGFs); + log.debug(`Written ${this.createdGFs.length} successful global fields to file`, this.importConfig.context); + } + + if (this.failedGFs?.length) { + fsUtil.writeFile(this.gFsFailsPath, this.failedGFs); + log.debug(`Written ${this.failedGFs.length} failed global fields to file`, this.importConfig.context); + } + } } diff --git a/packages/contentstack-import/src/import/modules/locales.ts b/packages/contentstack-import/src/import/modules/locales.ts index 752adc6416..ee0e73a4bc 100644 --- a/packages/contentstack-import/src/import/modules/locales.ts +++ b/packages/contentstack-import/src/import/modules/locales.ts @@ -6,7 +6,7 @@ */ import * as path from 'path'; -import { values, isEmpty, filter, pick } from 'lodash'; +import { values, isEmpty, filter, pick, keys } from 'lodash'; import { cliux, sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; import { fsUtil, formatError, fileHelper } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; @@ -41,6 +41,7 @@ export default class ImportLocales extends BaseClass { super({ importConfig, stackAPIClient }); this.config = importConfig; this.config.context.module = 'locales'; + this.currentModuleName = 'Locales'; this.localeConfig = importConfig.modules.locales; this.masterLanguage = importConfig.masterLocale; this.masterLanguageConfig = importConfig.modules.masterLocale; @@ -57,126 +58,58 @@ export default class ImportLocales extends BaseClass { this.langUidMapperPath = path.resolve(sanitizePath(this.config.data), 'mapper', 'languages', 'uid-mapper.json'); } - async start(): Promise { - log.debug('Loading locales from file', this.config.context); - - this.languages = fsUtil.readFile(path.join(this.langFolderPath, this.localeConfig.fileName)) as Record< - string, - unknown - >[]; - if (!this.languages || isEmpty(this.languages)) { - log.info('No languages found to import', this.config.context); - return; - } - log.debug(`Found ${values(this.languages).length} languages to import`, this.config.context); - - log.debug('Loading source master language configuration', this.config.context); - this.sourceMasterLanguage = fsUtil.readFile( - path.join(this.langFolderPath, this.masterLanguageConfig.fileName), - ) as Record; - log.debug('Loaded source master language configuration', this.config.context); - - log.debug('Creating languages mapper directory', this.config.context); - await fileHelper.makeDirectory(this.langMapperPath); - log.debug('Created languages mapper directory', this.config.context); - - log.debug('Loading existing language UID mappings', this.config.context); - if (fileHelper.fileExistsSync(this.langUidMapperPath)) { - this.langUidMapper = fsUtil.readFile(this.langUidMapperPath) || {}; - const langUidCount = Object.keys(this.langUidMapper || {}).length; - log.debug(`Loaded existing language UID data: ${langUidCount} items`, this.config.context); - } else { - log.debug('No existing language UID mappings found', this.config.context); - } + async start(): Promise { + try { + log.debug('Starting locales import process...', this.config.context); + const [localesCount] = await this.analyzeLocales(); + if (localesCount === 0) { + log.info('No languages found to import', this.config.context); + return; + } - log.debug('Checking and updating master locale', this.config.context); - await this.checkAndUpdateMasterLocale().catch((error) => { - handleAndLogError(error, { ...this.config.context }); - }); + const progress = this.setupLocalesProgress(localesCount); + this.prepareLocalesMapper(); - log.debug('Creating locales', this.config.context); - await this.createLocales().catch((error) => { - handleAndLogError(error, { ...this.config.context }); - Promise.reject('Failed to import locales'); - }); + await this.processMasterLocale(progress); + await this.processLocaleCreation(progress); + await this.processLocaleUpdate(progress); - log.debug('Writing failed locales to file', this.config.context); - fsUtil.writeFile(this.langFailsPath, this.failedLocales); - log.debug(`Written ${this.failedLocales.length} failed locales to file`, this.config.context); + log.debug('Writing failed locales to file', this.config.context); + fsUtil.writeFile(this.langFailsPath, this.failedLocales); + log.debug(`Written ${this.failedLocales.length} failed locales to file`, this.config.context); - log.debug('Updating locales', this.config.context); - await this.updateLocales().catch((error) => { + this.completeProgress(true); + log.success('Languages have been imported successfully!', this.config.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Locales import failed'); handleAndLogError(error, { ...this.config.context }); - Promise.reject('Failed to update locales'); - }); - - log.success('Languages have been imported successfully!', this.config.context); + } } - async checkAndUpdateMasterLocale(): Promise { + async checkAndUpdateMasterLocale(): Promise { log.debug('Checking and updating master locale', this.config.context); - let sourceMasterLangDetails = (this.sourceMasterLanguage && Object.values(this.sourceMasterLanguage)) || []; - log.debug(`Source master language details count: ${sourceMasterLangDetails.length}`, this.config.context); + const sourceMasterLangDetails = this.getSourceMasterLangDetails(); + if (!sourceMasterLangDetails) return; - if (sourceMasterLangDetails?.[0]?.code === this.masterLanguage?.code) { - log.debug(`Master locale code matches: ${this.masterLanguage?.code}`, this.config.context); + if (this.masterLanguage?.code !== sourceMasterLangDetails?.code) { + this.logCodeMismatch(sourceMasterLangDetails.code); + return; + } - log.debug('Fetching current master language details from stack', this.config.context); - let masterLangDetails = await this.stackAPIClient - .locale(this.masterLanguage['code']) - .fetch() - .catch((error: Error) => { - log.debug('Error fetching master language details', this.config.context); - handleAndLogError(error, { ...this.config.context }); - }); - - if ( - masterLangDetails?.name?.toString().toUpperCase() !== - sourceMasterLangDetails[0]['name']?.toString().toUpperCase() - ) { - log.debug('Master language name differs between source and destination', this.config.context); - log.debug(`Current master language name: ${masterLangDetails['name']}`, this.config.context); - log.debug(`Source master language name: ${sourceMasterLangDetails[0]['name']}`, this.config.context); - - cliux.print('WARNING!!! The master language name for the source and destination is different.', { - color: 'yellow', - }); - cliux.print(`Old Master language name: ${masterLangDetails['name']}`, { color: 'red' }); - cliux.print(`New Master language name: ${sourceMasterLangDetails[0]['name']}`, { color: 'green' }); - - const langUpdateConfirmation: boolean = await cliux.inquire({ - type: 'confirm', - message: 'Are you sure you want to update name of master language?', - name: 'confirmation', - }); - - if (langUpdateConfirmation) { - log.debug('User confirmed master language name update', this.config.context); - let langUid = sourceMasterLangDetails[0] && sourceMasterLangDetails[0]['uid']; - let sourceMasterLanguage = this.sourceMasterLanguage[langUid]; - if (!sourceMasterLanguage) { - log.info(`Master language details not found with id ${langUid} to update`, this.config.context); - } - - log.debug(`Updating master language name: ${sourceMasterLanguage.name}`, this.config.context); - - const langUpdateRequest = this.stackAPIClient.locale(sourceMasterLanguage.code); - langUpdateRequest.name = sourceMasterLanguage.name; - await langUpdateRequest.update().catch(function (error: Error) { - log.debug('Error updating master language name', this.config.context); - handleAndLogError(error, { ...this.config.context }); - }); - log.success('Master Languages name have been updated successfully!', this.config.context); - } else { - log.debug('User declined master language name update', this.config.context); - } - } else { - log.debug('Master language names match, no update needed', this.config.context); - } - } else { - log.debug('Master language codes do not match', this.config.context); + log.debug(`Master locale code matches: ${this.masterLanguage.code}`, this.config.context); + const masterLangDetails = await this.fetchTargetMasterLocale(); + if (!masterLangDetails) return; + + if ( + masterLangDetails?.name?.toString().toUpperCase() === sourceMasterLangDetails['name']?.toString().toUpperCase() + ) { + this.tickProgress(true, `${masterLangDetails.name} (no update needed)`); + log.debug('Master language names match, no update required', this.config.context); + return; } + + await this.handleNameMismatch(sourceMasterLangDetails, masterLangDetails); } async createLocales(): Promise { @@ -188,14 +121,21 @@ export default class ImportLocales extends BaseClass { log.debug(`Creating ${languagesToCreate.length} locales (excluding master locale)`, this.config.context); const onSuccess = ({ response = {}, apiData: { uid, code } = undefined }: any) => { + this.createdLocales.push(response.uid); this.langUidMapper[uid] = response.uid; - this.createdLocales.push(pick(response, [...this.localeConfig.requiredKeys])); + this.progressManager?.tick(true, `locale: ${code}`, null, 'Locale Create'); log.info(`Created locale: '${code}'`, this.config.context); log.debug(`Locale UID mapping: ${uid} → ${response.uid}`, this.config.context); fsUtil.writeFile(this.langUidMapperPath, this.langUidMapper); }; const onReject = ({ error, apiData: { uid, code } = undefined }: any) => { + this.progressManager?.tick( + false, + `locale: ${code}`, + error?.message || 'Failed to create locale', + 'Locale Create', + ); if (error?.errorCode === 247) { log.info(formatError(error), this.config.context); } else { @@ -218,7 +158,7 @@ export default class ImportLocales extends BaseClass { }); } - async updateLocales(): Promise { + async updateLocales(): Promise { log.debug(`Updating ${values(this.languages).length} locales`, this.config.context); const onSuccess = ({ response = {}, apiData: { uid, code } = undefined }: any) => { @@ -234,7 +174,7 @@ export default class ImportLocales extends BaseClass { }; return await this.makeConcurrentCall({ - processName: 'Update locales', + processName: 'Locale Update locales', apiContent: values(this.languages), apiParams: { reject: onReject.bind(this), @@ -245,4 +185,171 @@ export default class ImportLocales extends BaseClass { concurrencyLimit: this.reqConcurrency, }); } -} \ No newline at end of file + + private async analyzeLocales(): Promise<[number]> { + return this.withLoadingSpinner('LOCALES: Analyzing import data...', async () => { + log.debug('Loading locales from file', this.config.context); + + this.languages = fsUtil.readFile(path.join(this.langFolderPath, this.localeConfig.fileName)) as Record< + string, + unknown + >[]; + + if (!this.languages || isEmpty(this.languages)) { + log.info('No languages found to import', this.config.context); + return [0]; + } + + this.sourceMasterLanguage = fsUtil.readFile( + path.join(this.langFolderPath, this.masterLanguageConfig.fileName), + ) as Record; + + log.debug('Loaded source master language configuration', this.config.context); + + const localesCount = keys(this.languages || {})?.length; + log.debug(`Found ${localesCount} languages to import`, this.config.context); + return [localesCount]; + }); + } + + private setupLocalesProgress(localesCount: number) { + const progress = this.createNestedProgress(this.currentModuleName); + progress.addProcess('Master Locale ', 1); + if (localesCount > 0) { + progress.addProcess('Locale Create', localesCount); + progress.addProcess('Locale Update', localesCount); + } + return progress; + } + + private async prepareLocalesMapper(): Promise { + log.debug('Creating languages mapper directory', this.config.context); + fileHelper.makeDirectory(this.langMapperPath); + log.debug('Created languages mapper directory', this.config.context); + + if (fileHelper.fileExistsSync(this.langUidMapperPath)) { + this.langUidMapper = fsUtil.readFile(this.langUidMapperPath) || {}; + const langUidCount = Object.keys(this.langUidMapper).length; + log.debug(`Loaded existing language UID data: ${langUidCount} items`, this.config.context); + } else { + log.debug('No existing language UID mappings found', this.config.context); + } + } + + private async processMasterLocale(progress: any): Promise { + progress.startProcess('Master Locale ').updateStatus('Checking master locale...', 'Master Locale '); + log.debug('Checking and updating master locale', this.config.context); + + try { + await this.checkAndUpdateMasterLocale(); + progress.completeProcess('Master Locale ', true); + } catch (error) { + progress.completeProcess('Master Locale ', false); + //NOTE:- Continue locale creation in case of master locale error + handleAndLogError(error, { ...this.config.context }); + } + } + + private async processLocaleCreation(progress: any): Promise { + progress.startProcess('Locale Create').updateStatus('Creating locales...', 'Locale Create'); + log.debug('Creating locales', this.config.context); + + try { + await this.createLocales(); + progress.completeProcess('Locale Create', true); + } catch (error) { + progress.completeProcess('Locale Create', false); + throw error; + } + } + + private async processLocaleUpdate(progress: any): Promise { + progress.startProcess('Locale Update').updateStatus('Updating locales...', 'Locale Update'); + log.debug('Updating locales', this.config.context); + + try { + await this.updateLocales(); + progress.completeProcess('Locale Update', true); + } catch (error) { + progress.completeProcess('Locale Update', false); + throw error; + } + } + + private getSourceMasterLangDetails(): Record | null { + const details = this.sourceMasterLanguage && Object.values(this.sourceMasterLanguage); + const lang = details?.[0]; + + if (!lang) { + log.info('No source master language details found', this.config.context); + return null; + } + + return lang as Record; + } + + private async fetchTargetMasterLocale(): Promise | null> { + try { + log.debug('Fetching current master language details from stack', this.config.context); + return await this.stackAPIClient.locale(this.masterLanguage.code).fetch(); + } catch (error) { + log.debug('Error fetching master language details', this.config.context); + handleAndLogError(error, { ...this.config.context }); + return null; + } + } + + private logCodeMismatch(sourceCode: string): void { + const targetCode = this.masterLanguage?.code; + const message = `master locale: codes differ (${sourceCode} vs ${targetCode})`; + + this.tickProgress(true, message); + log.debug(`Master language codes do not match. Source: ${sourceCode}, Target: ${targetCode}`, this.config.context); + } + + private async handleNameMismatch(source: Record, target: Record): Promise { + log.debug('Master language name differs between source and destination', this.config.context); + log.debug(`Current: ${target.name}, Source: ${source.name}`, this.config.context); + + cliux.print('WARNING!!! The master language name for the source and destination is different.', { + color: 'yellow', + }); + cliux.print('WARNING!!! The master language name for the source and destination is different.', { + color: 'yellow', + }); + cliux.print(`Old Master language name: ${target.name}`, { color: 'red' }); + cliux.print(`New Master language name: ${source.name}`, { color: 'green' }); + + const langUpdateConfirmation: boolean = await cliux.inquire({ + type: 'confirm', + message: 'Are you sure you want to update name of master language?', + name: 'confirmation', + }); + + if (!langUpdateConfirmation) { + this.tickProgress(true, `${target.name} (skipped update)`); + log.info('Master language update cancelled by user', this.config.context); + return; + } + + log.debug('User confirmed master language update', this.config.context); + try { + const updatePayload = { ...source, uid: target.uid }; + const langUpdateRequest = this.stackAPIClient.locale(source.code); + langUpdateRequest.name = source.name; + await langUpdateRequest.update(updatePayload); + this.tickProgress(true, `${source.name} (updated)`); + log.success( + `Successfully updated master language name from '${target.name}' to '${source.name}'`, + this.config.context, + ); + } catch (error) { + this.tickProgress(false, source.name, error?.message || 'Failed to update master locale'); + throw error; + } + } + + private tickProgress(success: boolean, message: string, error?: string): void { + this.progressManager?.tick(success, `master locale: ${message}`, error || null, 'Master Locale '); + } +} diff --git a/packages/contentstack-import/src/import/modules/stack.ts b/packages/contentstack-import/src/import/modules/stack.ts index eabfc6c3c8..3e04b683c2 100644 --- a/packages/contentstack-import/src/import/modules/stack.ts +++ b/packages/contentstack-import/src/import/modules/stack.ts @@ -2,10 +2,9 @@ import { join } from 'node:path'; import { fileHelper, fsUtil } from '../../utils'; import BaseClass from './base-class'; import { ModuleClassParams } from '../../types'; -import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { log, handleAndLogError, messageHandler } from '@contentstack/cli-utilities'; export default class ImportStack extends BaseClass { - // classname private stackSettingsPath: string; private envUidMapperPath: string; private stackSettings: Record | null = null; @@ -13,47 +12,105 @@ export default class ImportStack extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); + this.importConfig.context.module = 'stack'; + this.currentModuleName = 'Stack Settings'; this.stackSettingsPath = join(this.importConfig.backupDir, 'stack', 'settings.json'); this.envUidMapperPath = join(this.importConfig.backupDir, 'mapper', 'environments', 'uid-mapping.json'); } + /** + * @method start + * @returns {Promise} Promise + */ async start(): Promise { - if (this.importConfig.management_token) { - log.info( - 'Skipping stack settings import: Operation is not supported when using a management token.', - this.importConfig.context, - ); - return; - } + try { + log.debug('Starting stack settings import process...', this.importConfig.context); - if (fileHelper.fileExistsSync(this.stackSettingsPath)) { - this.stackSettings = fsUtil.readFile(this.stackSettingsPath, true) as Record; - } else { - log.info('No stack setting found!', this.importConfig.context); - return; - } + if (this.importConfig.management_token) { + log.info( + 'Skipping stack settings import: Operation is not supported when using a management token.', + this.importConfig.context, + ); + return; + } + + const [canImport] = await this.analyzeStackSettings(); + + if (!canImport) { + log.info('Stack settings import skipped', this.importConfig.context); + return; + } + + const progress = this.createSimpleProgress(this.currentModuleName, 1); - if (fileHelper.fileExistsSync(this.envUidMapperPath)) { - this.envUidMapper = fsUtil.readFile(this.envUidMapperPath, true) as Record; - } else { - log.warn( - 'Skipping stack settings import. Please run the environments migration first.', - this.importConfig.context, - ); - return; + progress.updateStatus('Importing stack settings...'); + log.info('Starting stack settings import process', this.importConfig.context); + await this.importStackSettings(); + + this.completeProgress(true); + log.success('Stack settings imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, 'Stack settings import failed'); + handleAndLogError(error, { ...this.importConfig.context }); } + } + private async importStackSettings(): Promise { + log.debug('Processing stack settings for import', this.importConfig.context); + + // Update environment UID mapping if live preview is configured if (this.stackSettings?.live_preview && this.stackSettings?.live_preview['default-env']) { const oldEnvUid = this.stackSettings.live_preview['default-env']; const mappedEnvUid = this.envUidMapper[oldEnvUid]; - this.stackSettings.live_preview['default-env'] = mappedEnvUid; - } - try { - await this.stack.addSettings(this.stackSettings); - log.success('Successfully imported stack', this.importConfig.context); - } catch (error) { - handleAndLogError(error, { ...this.importConfig.context }); + if (mappedEnvUid) { + this.stackSettings.live_preview['default-env'] = mappedEnvUid; + log.debug(`Updated live preview environment: ${oldEnvUid} → ${mappedEnvUid}`, this.importConfig.context); + } else { + log.debug(`No mapping found for live preview environment: ${oldEnvUid}`, this.importConfig.context); + } } + + log.debug('Applying stack settings to target stack', this.importConfig.context); + await this.stack.addSettings(this.stackSettings); + + this.progressManager?.tick(true, 'stack settings applied'); + log.debug('Stack settings applied successfully', this.importConfig.context); + } + + private async analyzeStackSettings(): Promise<[boolean]> { + return this.withLoadingSpinner('STACK SETTINGS: Analyzing import data...', async () => { + log.debug('Checking for stack settings file existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.stackSettingsPath)) { + log.info('No stack setting found!', this.importConfig.context); + return [false]; + } + + log.debug(`Found stack settings file: ${this.stackSettingsPath}`, this.importConfig.context); + + this.stackSettings = fsUtil.readFile(this.stackSettingsPath, true) as Record; + + if (!this.stackSettings) { + log.info('Stack settings file is empty or invalid', this.importConfig.context); + return [false]; + } + + log.debug('Loading environment UID mappings', this.importConfig.context); + if (fileHelper.fileExistsSync(this.envUidMapperPath)) { + this.envUidMapper = fsUtil.readFile(this.envUidMapperPath, true) as Record; + const envMappingCount = Object.keys(this.envUidMapper || {}).length; + log.debug(`Loaded ${envMappingCount} environment UID mappings`, this.importConfig.context); + } else { + log.warn( + 'Skipping stack settings import. Please run the environments migration first.', + this.importConfig.context, + ); + return [false]; + } + + log.debug('Stack settings analysis completed successfully', this.importConfig.context); + return [true]; + }); } } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 2ff64e60a2..b8e53cb898 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -25,6 +25,7 @@ export default class ImportTaxonomies extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'taxonomies'; + this.currentModuleName = 'Taxonomies'; this.taxonomiesConfig = importConfig.modules.taxonomies; this.taxonomiesMapperDirPath = join(importConfig.backupDir, 'mapper', 'taxonomies'); this.termsMapperDirPath = join(this.taxonomiesMapperDirPath, 'terms'); @@ -40,37 +41,28 @@ export default class ImportTaxonomies extends BaseClass { * @returns {Promise} Promise */ async start(): Promise { - log.debug('Checking for taxonomies folder existence', this.importConfig.context); - - //Step1 check folder exists or not - if (fileHelper.fileExistsSync(this.taxonomiesFolderPath)) { - log.debug(`Found taxonomies folder: ${this.taxonomiesFolderPath}`, this.importConfig.context); - this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record< - string, - unknown - >; - const taxonomyCount = Object.keys(this.taxonomies || {}).length; - log.debug(`Loaded ${taxonomyCount} taxonomy items from file`, this.importConfig.context); - } else { - log.info(`No Taxonomies Found! - '${this.taxonomiesFolderPath}'`, this.importConfig.context); - return; - } + try { + log.debug('Starting taxonomies import process...', this.importConfig.context); - //Step 2 create taxonomies & terms mapper directory - log.debug('Creating mapper directories', this.importConfig.context); - await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); - await fsUtil.makeDirectory(this.termsMapperDirPath); - log.debug('Created taxonomies and terms mapper directories', this.importConfig.context); - - // Step 3 import taxonomies - log.debug('Starting taxonomies import', this.importConfig.context); - await this.importTaxonomies(); + const [taxonomiesCount] = await this.analyzeTaxonomies(); + if (taxonomiesCount === 0) { + log.info('No taxonomies found to import', this.importConfig.context); + return; + } - //Step 4 create taxonomy & related terms success & failure file - log.debug('Creating success and failure files', this.importConfig.context); - this.createSuccessAndFailedFile(); + const progress = this.createSimpleProgress(this.currentModuleName, taxonomiesCount); + await this.prepareMapperDirectories(); + progress.updateStatus('Importing taxonomies...'); + log.debug('Starting taxonomies import', this.importConfig.context); + await this.importTaxonomies(); + this.createSuccessAndFailedFile(); + this.completeProgress(true); log.success('Taxonomies imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Taxonomies import failed'); + handleAndLogError(error, { ...this.importConfig.context }); + } } /** @@ -97,10 +89,10 @@ export default class ImportTaxonomies extends BaseClass { this.createdTaxonomies[taxonomyUID] = apiData?.taxonomy; this.createdTerms[taxonomyUID] = apiData?.terms; + this.progressManager?.tick(true, `taxonomy: ${taxonomyName || taxonomyUID} (${termsCount} terms)`); log.success(`Taxonomy '${taxonomyUID}' imported successfully!`, this.importConfig.context); - log.debug(`Created taxonomy '${taxonomyName}' with ${termsCount} terms`, this.importConfig.context); log.debug( - `Taxonomy details: ${JSON.stringify({ uid: taxonomyUID, name: taxonomyName, termsCount })}`, + `Taxonomy '${taxonomyName}' imported with ${termsCount} terms successfully!`, this.importConfig.context, ); }; @@ -108,41 +100,36 @@ export default class ImportTaxonomies extends BaseClass { const onReject = ({ error, apiData }: any) => { const taxonomyUID = apiData?.taxonomy?.uid; const taxonomyName = apiData?.taxonomy?.name; - - log.debug(`Taxonomy '${taxonomyUID}' failed to import`, this.importConfig.context); - if (error?.status === 409 && error?.statusText === 'Conflict') { log.info(`Taxonomy '${taxonomyUID}' already exists!`, this.importConfig.context); log.debug(`Adding existing taxonomy '${taxonomyUID}' to created list`, this.importConfig.context); this.createdTaxonomies[taxonomyUID] = apiData?.taxonomy; this.createdTerms[taxonomyUID] = apiData?.terms; + this.progressManager?.tick(true, `taxonomy: ${taxonomyName || taxonomyUID}`); } else { - log.debug(`Adding taxonomy '${taxonomyUID}' to failed list`, this.importConfig.context); - if (error?.errorMessage || error?.message) { - const errorMsg = error?.errorMessage || error?.errors?.taxonomy || error?.errors?.term || error?.message; - log.error(`Taxonomy '${taxonomyUID}' failed to be import! ${errorMsg}`, this.importConfig.context); - } else { - handleAndLogError( - error, - { ...this.importConfig.context, taxonomyUID }, - `Taxonomy '${taxonomyUID}' failed to import`, - ); - } this.failedTaxonomies[taxonomyUID] = apiData?.taxonomy; this.failedTerms[taxonomyUID] = apiData?.terms; + + this.progressManager?.tick( + false, + `taxonomy: ${taxonomyName || taxonomyUID}`, + error?.message || 'Failed to import taxonomy', + ); + handleAndLogError( + error, + { ...this.importConfig.context, taxonomyUID }, + `Taxonomy '${taxonomyUID}' failed to be imported`, + ); } }; - log.debug( - `Using concurrency limit: ${this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1}`, - this.importConfig.context, - ); + log.debug(`Using concurrency limit: ${this.importConfig.fetchConcurrency || 2}`, this.importConfig.context); await this.makeConcurrentCall( { apiContent, processName: 'import taxonomies', apiParams: { - serializeData: this.serializeTaxonomy.bind(this), + serializeData: this.serializeTaxonomiesData.bind(this), reject: onReject, resolve: onSuccess, entity: 'import-taxonomy', @@ -158,23 +145,27 @@ export default class ImportTaxonomies extends BaseClass { } /** - * @method serializeTaxonomy + * @method serializeTaxonomiesData * @param {ApiOptions} apiOptions ApiOptions * @returns {ApiOptions} ApiOptions */ - serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { - const { apiData } = apiOptions; - const taxonomyUID = apiData?.uid; + serializeTaxonomiesData(apiOptions: ApiOptions): ApiOptions { + const { apiData: taxonomyData } = apiOptions; + log.debug( + `Serializing taxonomy: ${taxonomyData.taxonomy?.name} (${taxonomyData.taxonomy?.uid})`, + this.importConfig.context, + ); + + const taxonomyUID = taxonomyData?.uid; const filePath = join(this.taxonomiesFolderPath, `${taxonomyUID}.json`); - log.debug(`Serializing taxonomy: ${taxonomyUID}`, this.importConfig.context); log.debug(`Looking for taxonomy file: ${filePath}`, this.importConfig.context); if (fileHelper.fileExistsSync(filePath)) { const taxonomyDetails = fsUtil.readFile(filePath, true) as Record; log.debug(`Successfully loaded taxonomy details from ${filePath}`, this.importConfig.context); - const termCount = Object.keys(taxonomyDetails?.terms || {}).length; - log.debug(`Taxonomy has ${termCount} term entries`, this.importConfig.context); + const termCount = Object.keys(taxonomyDetails?.terms || {}).length; + log.debug(`Taxonomy has ${termCount} term entries`, this.importConfig.context); apiOptions.apiData = { filePath, taxonomy: taxonomyDetails?.taxonomy, terms: taxonomyDetails?.terms }; } else { log.debug(`File does not exist for taxonomy: ${taxonomyUID}`, this.importConfig.context); @@ -234,4 +225,33 @@ export default class ImportTaxonomies extends BaseClass { ); } } + + private async analyzeTaxonomies(): Promise<[number]> { + return this.withLoadingSpinner('TAXONOMIES: Analyzing import data...', async () => { + log.debug('Checking for taxonomies folder existence', this.importConfig.context); + + if (fileHelper.fileExistsSync(this.taxonomiesFolderPath)) { + log.debug(`Found taxonomies folder: ${this.taxonomiesFolderPath}`, this.importConfig.context); + + this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record< + string, + unknown + >; + + const taxonomyCount = Object.keys(this.taxonomies || {}).length; + log.debug(`Loaded ${taxonomyCount} taxonomy items from file`, this.importConfig.context); + return [taxonomyCount]; + } else { + log.info(`No Taxonomies Found! - '${this.taxonomiesFolderPath}'`, this.importConfig.context); + return [0]; + } + }); + } + + private async prepareMapperDirectories(): Promise { + log.debug('Creating mapper directories', this.importConfig.context); + await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); + await fsUtil.makeDirectory(this.termsMapperDirPath); + log.debug('Created taxonomies and terms mapper directories', this.importConfig.context); + } } diff --git a/packages/contentstack-import/src/utils/import-config-handler.ts b/packages/contentstack-import/src/utils/import-config-handler.ts index bb737fb915..9a351102b8 100644 --- a/packages/contentstack-import/src/utils/import-config-handler.ts +++ b/packages/contentstack-import/src/utils/import-config-handler.ts @@ -131,6 +131,9 @@ const setupConfig = async (importCmdFlags: any): Promise => { config['exclude-global-modules'] = importCmdFlags['exclude-global-modules']; } + // Set progress supported module to check and display console logs + configHandler.set('log.progressSupportedModule', 'import'); + // Add authentication details to config for context tracking config.authenticationMethod = authenticationMethod; log.debug('Import configuration setup completed', { ...config });