diff --git a/.talismanrc b/.talismanrc index 1e0ed04a36..0fd8d87921 100644 --- a/.talismanrc +++ b/.talismanrc @@ -37,4 +37,14 @@ fileignoreconfig: checksum: 449a5e3383631a6f78d1291aa3c28c91681879289398f0a933158fba5c5d5acf - filename: packages/contentstack-import/src/commands/cm/stacks/import.ts checksum: 0dbf0a6bc447206260b8acd41b85781d60ca50c948bb3ca62f444f97d64d1fb2 +- filename: packages/contentstack-utilities/src/interfaces/index.ts + checksum: d0b0042e643ce0c0489b86f15f3b64f60a837c2ae928b6275028e5e0184b0a7a +- filename: packages/contentstack-variants/src/import/attribute.ts + checksum: 03e764ee2032c44d9493f2be194f91a2337026b7fd8037df90240327e6bcaabb +- filename: packages/contentstack-variants/src/import/audiences.ts + checksum: f24697ef86e928bb4d16f93c021b647639cc344a7f02463d79d69f9434ebed56 +- filename: packages/contentstack-variants/src/import/events.ts + checksum: 6cb014b5518ffe204a9f894ad801c05e2ef91a1692049168f74dd12a224363c4 +- filename: packages/contentstack-import/src/import/modules/personalize.ts + checksum: 1311a613177160637e21b3983b281b384c2cb15837d001a398b67afef30a393a version: "1.0" diff --git a/packages/contentstack-export/src/export/module-exporter.ts b/packages/contentstack-export/src/export/module-exporter.ts index e45e422afb..5564a8b15e 100644 --- a/packages/contentstack-export/src/export/module-exporter.ts +++ b/packages/contentstack-export/src/export/module-exporter.ts @@ -50,8 +50,11 @@ class ModuleExporter { // Reset progress manager for each branch (except the first one which was initialized in export command) if (index >= 0) { CLIProgressManager.clearGlobalSummary(); - CLIProgressManager.initializeGlobalSummary(`EXPORT-${branch.uid}`, branch.uid, `EXPORTING "${branch.uid}" BRANCH CONTENT`,); - + CLIProgressManager.initializeGlobalSummary( + `EXPORT-${branch.uid}`, + branch.uid, + `EXPORTING "${branch.uid}" BRANCH CONTENT`, + ); } log.info(`Exporting content from branch ${branch.uid}`, this.exportConfig.context); @@ -97,7 +100,7 @@ class ModuleExporter { } else { //NOTE - new modules support only ts if (this.exportConfig.onlyTSModules.indexOf(moduleName) === -1) { - await startJSModuleExport({ + await startJSModuleExport({ stackAPIClient: this.stackAPIClient, exportConfig: this.exportConfig, moduleName, diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index b5f532ae57..33e7753839 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -64,41 +64,41 @@ export default class ExportAssets extends BaseClass { // Add sub-processes if (typeof assetsFolderCount === 'number' && assetsFolderCount > 0) { - progress.addProcess('Asset Folders', assetsFolderCount); + progress.addProcess('Folders', assetsFolderCount); } if (typeof assetsCount === 'number' && assetsCount > 0) { - progress.addProcess('Asset Metadata', assetsCount); - progress.addProcess('Asset Downloads', assetsCount); + progress.addProcess('Metadata', assetsCount); + progress.addProcess('Downloads', assetsCount); } try { // Process asset folders if (typeof assetsFolderCount === 'number' && assetsFolderCount > 0) { - progress.startProcess('Asset Folders').updateStatus('Fetching folder structure...', 'Asset Folders'); + progress.startProcess('Folders').updateStatus('Fetching folder structure...', 'Folders'); await this.getAssetsFolders(assetsFolderCount); - progress.completeProcess('Asset Folders', true); + progress.completeProcess('Folders', true); } // Process asset metadata if (typeof assetsCount === 'number' && assetsCount > 0) { - progress.startProcess('Asset Metadata').updateStatus('Fetching asset information...', 'Asset Metadata'); + progress.startProcess('Metadata').updateStatus('Fetching asset information...', 'Metadata'); await this.getAssets(assetsCount); - progress.completeProcess('Asset Metadata', true); + progress.completeProcess('Metadata', true); } // Get versioned assets if (!isEmpty(this.versionedAssets) && this.assetConfig.includeVersionedAssets) { log.debug('Fetching versioned assets metadata...', this.exportConfig.context); - progress.updateStatus('Processing versioned assets...', 'Asset Metadata'); + progress.updateStatus('Processing versioned assets...', 'Metadata'); await this.getVersionedAssets(); } // Download all assets if (typeof assetsCount === 'number' && assetsCount > 0) { - progress.startProcess('Asset Downloads').updateStatus('Downloading asset files...', 'Asset Downloads'); + progress.startProcess('Downloads').updateStatus('Downloading asset files...', 'Downloads'); log.debug('Starting download of all assets...', this.exportConfig.context); await this.downloadAssets(); - progress.completeProcess('Asset Downloads', true); + progress.completeProcess('Downloads', true); } this.completeProgress(true); @@ -128,13 +128,13 @@ export default class ExportAssets extends BaseClass { if (!isEmpty(items)) { this.assetsFolder.push(...items); items.forEach((folder: any) => { - this.progressManager?.tick(true, `folder: ${folder.name || folder.uid}`, null, 'Asset Folders'); + this.progressManager?.tick(true, `folder: ${folder.name || folder.uid}`, null, 'Folders'); }); } }; const onReject = ({ error }: any) => { - this.progressManager?.tick(false, 'asset folder', error?.message || 'Failed to fetch folder', 'Asset Folders'); + this.progressManager?.tick(false, 'asset folder', error?.message || 'Failed to fetch folder', 'Folders'); handleAndLogError(error, { ...this.exportConfig.context }); }; @@ -197,7 +197,7 @@ export default class ExportAssets extends BaseClass { } const onReject = ({ error }: any) => { - this.progressManager?.tick(false, 'asset', error?.message || 'Failed to fetch asset', 'Asset Metadata'); + this.progressManager?.tick(false, 'asset', error?.message || 'Failed to fetch asset', 'Metadata'); handleAndLogError(error, { ...this.exportConfig.context }, messageHandler.parse('ASSET_QUERY_FAILED')); }; @@ -219,7 +219,7 @@ export default class ExportAssets extends BaseClass { fs?.writeIntoFile(items, { mapKeyVal: true }); // Track progress for each asset with process name items.forEach((asset: any) => { - this.progressManager?.tick(true, `asset: ${asset.filename || asset.uid}`, null, 'Asset Metadata'); + this.progressManager?.tick(true, `asset: ${asset.filename || asset.uid}`, null, 'Metadata'); }); } }; @@ -418,7 +418,7 @@ export default class ExportAssets extends BaseClass { } else { data.pipe(assetWriterStream); } - this.progressManager?.tick(true, `Downloaded asset: ${asset.filename || asset.uid}`, null, 'Asset Downloads'); + this.progressManager?.tick(true, `Downloaded asset: ${asset.filename || asset.uid}`, null, 'Downloads'); log.success(messageHandler.parse('ASSET_DOWNLOAD_SUCCESS', asset.filename, asset.uid), this.exportConfig.context); }; @@ -428,7 +428,7 @@ export default class ExportAssets extends BaseClass { false, `Failed to download asset: ${asset.filename || asset.uid}`, null, - 'Asset Downloads', + 'Downloads', ); handleAndLogError( error, diff --git a/packages/contentstack-export/src/export/modules/entries.ts b/packages/contentstack-export/src/export/modules/entries.ts index 22fd4a0012..51554499b7 100644 --- a/packages/contentstack-export/src/export/modules/entries.ts +++ b/packages/contentstack-export/src/export/modules/entries.ts @@ -109,7 +109,7 @@ export default class EntriesExport extends BaseClass { } if (this.exportVariantEntry) { - progress.addProcess('Variant Entries', totalEntriesCount); + progress.addProcess('Variant Entries', 0); } } diff --git a/packages/contentstack-export/src/export/modules/index.ts b/packages/contentstack-export/src/export/modules/index.ts index f13bc4fa3d..4ac791eb62 100644 --- a/packages/contentstack-export/src/export/modules/index.ts +++ b/packages/contentstack-export/src/export/modules/index.ts @@ -1,5 +1,6 @@ import { handleAndLogError } from '@contentstack/cli-utilities'; import { ModuleClassParams } from '../../types'; +import '../../utils/strategy-registrations'; export default async function startModuleExport(modulePayload: ModuleClassParams) { try { diff --git a/packages/contentstack-export/src/export/modules/marketplace-apps.ts b/packages/contentstack-export/src/export/modules/marketplace-apps.ts index b9a43ec8d3..78701e3c90 100644 --- a/packages/contentstack-export/src/export/modules/marketplace-apps.ts +++ b/packages/contentstack-export/src/export/modules/marketplace-apps.ts @@ -71,19 +71,19 @@ export default class ExportMarketplaceApps extends BaseClass { const progress = this.createNestedProgress(this.currentModuleName); // Add processes based on what we found - progress.addProcess('Apps Fetch', appsCount); - progress.addProcess('App Processing', appsCount); // Manifests and configurations + progress.addProcess('Fetch', appsCount); + progress.addProcess('Fetch config & manifest', appsCount); // Manifests and configurations // Fetch stack specific apps - progress.startProcess('Apps Fetch').updateStatus('Fetching marketplace apps...', 'Apps Fetch'); + progress.startProcess('Fetch').updateStatus('Fetching marketplace apps...', 'Fetch'); await this.exportApps(); - progress.completeProcess('Apps Fetch', true); + progress.completeProcess('Fetch', true); // Process apps (manifests and configurations) if (this.installedApps.length > 0) { - progress.startProcess('App Processing').updateStatus('Processing app manifests and configurations...', 'App Processing'); + progress.startProcess('Fetch config & manifest').updateStatus('Processing app manifests and configurations...', 'Fetch config & manifest'); await this.getAppManifestAndAppConfig(); - progress.completeProcess('App Processing', true); + progress.completeProcess('Fetch config & manifest', true); } this.completeProgress(true); @@ -196,7 +196,7 @@ export default class ExportMarketplaceApps extends BaseClass { await this.getAppConfigurations(+index, app); // Track progress for each app processed - this.progressManager?.tick(true, `app: ${app.manifest?.name || app.uid}`, null, 'App Processing'); + this.progressManager?.tick(true, `app: ${app.manifest?.name || app.uid}`, null, 'Fetch config & manifest'); } const marketplaceAppsFilePath = pResolve(this.marketplaceAppPath, this.marketplaceAppConfig.fileName); @@ -346,7 +346,7 @@ export default class ExportMarketplaceApps extends BaseClass { // Track progress for each app fetched installation.forEach((app) => { - this.progressManager?.tick(true, `app: ${app.manifest?.name || app.uid}`, null, 'Apps Fetch'); + this.progressManager?.tick(true, `app: ${app.manifest?.name || app.uid}`, null, 'Fetch'); }); this.installedApps = this.installedApps.concat(installation); diff --git a/packages/contentstack-export/src/export/modules/stack.ts b/packages/contentstack-export/src/export/modules/stack.ts index bc9e6220ed..2bb9e19149 100644 --- a/packages/contentstack-export/src/export/modules/stack.ts +++ b/packages/contentstack-export/src/export/modules/stack.ts @@ -58,7 +58,7 @@ export default class ExportStack extends BaseClass { } if (!this.exportConfig.management_token) { - progress.addProcess('Stack Settings', 1); + progress.addProcess('Settings', 1); processCount++; } @@ -67,18 +67,18 @@ export default class ExportStack extends BaseClass { !this.exportConfig.hasOwnProperty('master_locale') && localesCount > 0 ) { - progress.addProcess('Locales', localesCount); + progress.addProcess('Locale', localesCount); processCount++; } else if (this.exportConfig.preserveStackVersion) { - progress.addProcess('Stack Export', 1); + progress.addProcess('Details', 1); processCount++; } // Execute processes if (!this.exportConfig.management_token) { - progress.startProcess('Stack Settings').updateStatus('Exporting stack settings...', 'Stack Settings'); + progress.startProcess('Settings').updateStatus('Exporting stack settings...', 'Settings'); await this.exportStackSettings(); - progress.completeProcess('Stack Settings', true); + progress.completeProcess('Settings', true); } else { log.info( 'Skipping stack settings export: Operation is not supported when using a management token.', @@ -91,9 +91,9 @@ export default class ExportStack extends BaseClass { !this.exportConfig.hasOwnProperty('master_locale') && localesCount > 0 ) { - progress.startProcess('Locales').updateStatus('Fetching master locale...', 'Locales'); + progress.startProcess('Locale').updateStatus('Fetching master locale...', 'Locale'); const masterLocale = await this.getLocales(); - progress.completeProcess('Locales', true); + progress.completeProcess('Locale', true); if (masterLocale?.code) { this.exportConfig.master_locale = { code: masterLocale.code }; @@ -103,14 +103,14 @@ export default class ExportStack extends BaseClass { this.completeProgress(true); return masterLocale; } else if (this.exportConfig.preserveStackVersion) { - progress.startProcess('Stack Export').updateStatus('Exporting stack data...', 'Stack Export'); + progress.startProcess('Details').updateStatus('Exporting stack data...', 'Details'); const stackResult = await this.exportStack(); - progress.completeProcess('Stack Export', true); + progress.completeProcess('Details', true); this.completeProgress(true); return stackResult; } else { - log.debug('Master locale already set, skipping locale fetch', this.exportConfig.context); + log.debug('Locale locale already set, skipping locale fetch', this.exportConfig.context); } this.completeProgress(true); @@ -185,7 +185,7 @@ export default class ExportStack extends BaseClass { // Track progress for each locale processed items.forEach((locale: any) => { - this.progressManager?.tick(true, `locale: ${locale.name || locale.code}`, null, 'Locales'); + this.progressManager?.tick(true, `locale: ${locale.name || locale.code}`, null, 'Locale'); }); skip += this.stackConfig.limit || 100; @@ -200,14 +200,14 @@ export default class ExportStack extends BaseClass { return masterLocalObj; } else if (skip >= count) { log.error( - `Master locale not found in the stack ${this.exportConfig.source_stack}. Please ensure that the stack has a master locale.`, + `Locale locale not found in the stack ${this.exportConfig.source_stack}. Please ensure that the stack has a master locale.`, this.exportConfig.context, ); log.debug('Completed searching all locales without finding master locale', this.exportConfig.context); return; } else { log.debug( - `Master locale not found in current batch, continuing with skip: ${skip}`, + `Locale locale not found in current batch, continuing with skip: ${skip}`, this.exportConfig.context, ); return await this.getLocales(skip); @@ -221,7 +221,7 @@ export default class ExportStack extends BaseClass { `Error occurred while fetching locales for stack: ${this.exportConfig.source_stack}`, this.exportConfig.context, ); - this.progressManager?.tick(false, 'locale fetch', error?.message || 'Failed to fetch locales', 'Locales'); + this.progressManager?.tick(false, 'locale fetch', error?.message || 'Failed to fetch locales', 'Locale'); handleAndLogError( error, { ...this.exportConfig.context }, @@ -245,7 +245,7 @@ export default class ExportStack extends BaseClass { fsUtil.writeFile(stackFilePath, resp); // Track progress for stack export completion - this.progressManager?.tick(true, `stack: ${this.exportConfig.source_stack}`, null, 'Stack Export'); + this.progressManager?.tick(true, `stack: ${this.exportConfig.source_stack}`, null, 'Details'); log.success( `Stack details exported successfully for stack ${this.exportConfig.source_stack}`, @@ -256,7 +256,7 @@ export default class ExportStack extends BaseClass { }) .catch((error: any) => { log.debug(`Error occurred while exporting stack: ${this.exportConfig.source_stack}`, this.exportConfig.context); - this.progressManager?.tick(false, 'stack export', error?.message || 'Failed to export stack', 'Stack Export'); + this.progressManager?.tick(false, 'stack export', error?.message || 'Failed to export stack', 'Details'); handleAndLogError(error, { ...this.exportConfig.context }); }); } @@ -270,7 +270,7 @@ export default class ExportStack extends BaseClass { fsUtil.writeFile(pResolve(this.stackFolderPath, 'settings.json'), resp); // Track progress for stack settings completion - this.progressManager?.tick(true, 'stack settings', null, 'Stack Settings'); + this.progressManager?.tick(true, 'stack settings', null, 'Settings'); log.success('Exported stack settings successfully!', this.exportConfig.context); return resp; @@ -280,7 +280,7 @@ export default class ExportStack extends BaseClass { false, 'stack settings', error?.message || 'Failed to export stack settings', - 'Stack Settings', + 'Settings', ); handleAndLogError(error, { ...this.exportConfig.context }); }); diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 75bc960098..901dfd3433 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -60,13 +60,13 @@ export default class ExportTaxonomies extends BaseClass { const progress = this.createNestedProgress(this.currentModuleName); // Add sub-processes - progress.addProcess('Fetch Taxonomies', totalCount); - progress.addProcess('Export Taxonomies & Terms', totalCount); + progress.addProcess('Fetch', totalCount); + progress.addProcess('Taxonomies & Terms', totalCount); // Fetch taxonomies - progress.startProcess('Fetch Taxonomies').updateStatus('Fetching taxonomy metadata...', 'Fetch Taxonomies'); + progress.startProcess('Fetch').updateStatus('Fetching taxonomy metadata...', 'Fetch'); await this.getAllTaxonomies(); - progress.completeProcess('Fetch Taxonomies', true); + progress.completeProcess('Fetch', true); const actualTaxonomyCount = Object.keys(this.taxonomies)?.length; log.debug(`Found ${actualTaxonomyCount} taxonomies to export (API reported ${totalCount})`, this.exportConfig.context); @@ -74,16 +74,16 @@ export default class ExportTaxonomies extends BaseClass { // Update progress for export step if counts differ if (actualTaxonomyCount !== totalCount && actualTaxonomyCount > 0) { // Remove the old process and add with correct count - progress.addProcess('Export Taxonomies & Terms', actualTaxonomyCount); + progress.addProcess('Taxonomies & Terms', actualTaxonomyCount); } // Export detailed taxonomies if (actualTaxonomyCount > 0) { progress - .startProcess('Export Taxonomies & Terms') - .updateStatus('Exporting taxonomy details...', 'Export Taxonomies & Terms'); + .startProcess('Taxonomies & Terms') + .updateStatus('Exporting taxonomy details...', 'Taxonomies & Terms'); await this.exportTaxonomies(); - progress.completeProcess('Export Taxonomies & Terms', true); + progress.completeProcess('Taxonomies & Terms', true); } else { log.info('No taxonomies found to export detailed information', this.exportConfig.context); } @@ -98,7 +98,7 @@ export default class ExportTaxonomies extends BaseClass { } /** - * fetch all taxonomies in the provided stack + * Fetch in the provided stack * @param {number} skip * @returns {Promise} */ @@ -152,7 +152,7 @@ export default class ExportTaxonomies extends BaseClass { } // Track progress for each taxonomy - this.progressManager?.tick(true, `taxonomy: ${taxonomyName || taxonomyUid}`, null, 'Fetch Taxonomies'); + this.progressManager?.tick(true, `taxonomy: ${taxonomyName || taxonomyUid}`, null, 'Fetch'); } log.debug( @@ -184,7 +184,7 @@ export default class ExportTaxonomies extends BaseClass { fsUtil.writeFile(filePath, response); // Track progress for each exported taxonomy - this.progressManager?.tick(true, `taxonomy: ${taxonomyName || uid}`, null, 'Export Taxonomies & Terms'); + this.progressManager?.tick(true, `taxonomy: ${taxonomyName || uid}`, null, 'Taxonomies & Terms'); log.success(messageHandler.parse('TAXONOMY_EXPORT_SUCCESS', taxonomyName || uid), this.exportConfig.context); }; @@ -197,7 +197,7 @@ export default class ExportTaxonomies extends BaseClass { false, `taxonomy: ${taxonomyName || uid}`, error?.message || 'Export failed', - 'Export Taxonomies & Terms', + 'Taxonomies & Terms', ); handleAndLogError( diff --git a/packages/contentstack-export/src/export/modules/workflows.ts b/packages/contentstack-export/src/export/modules/workflows.ts index fc49886758..b09d546a4b 100644 --- a/packages/contentstack-export/src/export/modules/workflows.ts +++ b/packages/contentstack-export/src/export/modules/workflows.ts @@ -52,17 +52,11 @@ export default class ExportWorkFlows extends BaseClass { } // Create nested progress manager for complex workflow processing - const progress = this.createNestedProgress(this.currentModuleName) - .addProcess('Fetch Workflows', totalCount) - .addProcess('Process Roles', totalCount * 2); // Estimate roles per workflow + const progress = this.createSimpleProgress(this.currentModuleName, totalCount) // Fetch workflows - progress.startProcess('Fetch Workflows').updateStatus('Fetching workflow definitions...', 'Fetch Workflows'); + progress.updateStatus('Fetching workflow definitions...'); await this.getWorkflows(); - progress.completeProcess('Fetch Workflows', true); - - // The role processing is handled within sanitizeAttribs, so marking it complete - progress.completeProcess('Process Roles', true); log.debug(`Retrieved ${Object.keys(this.workflows).length} workflows`, this.exportConfig.context); @@ -135,14 +129,14 @@ export default class ExportWorkFlows extends BaseClass { log.success(messageHandler.parse('WORKFLOW_EXPORT_SUCCESS', workflowName), this.exportConfig.context); // Track progress for each workflow - this.progressManager?.tick(true, `workflow: ${workflowName}`, null, 'Fetch Workflows'); + this.progressManager?.tick(true, `workflow: ${workflowName}`, null, 'Fetch'); } catch (error) { log.error(`Failed to process workflow: ${workflowName}`, this.exportConfig.context); this.progressManager?.tick( false, `workflow: ${workflowName}`, error?.message || 'Processing failed', - 'Fetch Workflows', + 'Fetch', ); } } @@ -167,11 +161,8 @@ export default class ExportWorkFlows extends BaseClass { const roleData = await this.getRoles(roleUid); stage.SYS_ACL.roles.uids[i] = roleData; - // Track progress for each role processed - this.progressManager?.tick(true, `role: ${roleUid}`, null, 'Process Roles'); } catch (error) { log.error(`Failed to fetch role ${roleUid}`, this.exportConfig.context); - this.progressManager?.tick(false, `role: ${roleUid}`, error?.message || 'Role fetch failed', 'Process Roles'); } } } diff --git a/packages/contentstack-export/src/utils/strategy-registrations.ts b/packages/contentstack-export/src/utils/strategy-registrations.ts new file mode 100644 index 0000000000..c4ce26d0c7 --- /dev/null +++ b/packages/contentstack-export/src/utils/strategy-registrations.ts @@ -0,0 +1,142 @@ +/** + * Progress Strategy Registrations for Export Modules + * This file registers progress calculation strategies for all export modules + * to ensure correct item counts in the final summary. + */ + +import { + ProgressStrategyRegistry, + PrimaryProcessStrategy, + CustomProgressStrategy, + DefaultProgressStrategy +} from '@contentstack/cli-utilities'; + +// Register strategy for Content Types - simple module +ProgressStrategyRegistry.register( + 'CONTENT TYPES', + new DefaultProgressStrategy() +); + +// Register strategy for Assets - use Asset Metadata as primary process +ProgressStrategyRegistry.register( + 'ASSETS', + new PrimaryProcessStrategy('Metadata') +); + +// Register strategy for Global Fields - simple module +ProgressStrategyRegistry.register( + 'GLOBAL FIELDS', + new DefaultProgressStrategy() +); + +// Register strategy for Extensions - simple module +ProgressStrategyRegistry.register( + 'EXTENSIONS', + new DefaultProgressStrategy() +); + +// Register strategy for Environments - simple module +ProgressStrategyRegistry.register( + 'ENVIRONMENTS', + new DefaultProgressStrategy() +); + +// Register strategy for Locales - simple module +ProgressStrategyRegistry.register( + 'LOCALES', + new DefaultProgressStrategy() +); + +// Register strategy for Labels - simple module +ProgressStrategyRegistry.register( + 'LABELS', + new DefaultProgressStrategy() +); + +// Register strategy for Webhooks - simple module +ProgressStrategyRegistry.register( + 'WEBHOOKS', + new DefaultProgressStrategy() +); + +// Register strategy for Workflows - simple module +ProgressStrategyRegistry.register( + 'WORKFLOWS', + new DefaultProgressStrategy() +); + +// Register strategy for Custom Roles - simple module +ProgressStrategyRegistry.register( + 'CUSTOM ROLES', + new DefaultProgressStrategy() +); + +// Register strategy for Taxonomies - use Taxonomies & Terms as primary process +ProgressStrategyRegistry.register( + 'TAXONOMIES', + new PrimaryProcessStrategy('Taxonomies & Terms') +); + +// Register strategy for Marketplace Apps - complex module with app installations +ProgressStrategyRegistry.register( + 'MARKETPLACE APPS', + new CustomProgressStrategy((processes) => { + // For marketplace apps, count the actual apps exported + const appsExport = processes.get('Fetch'); + if (appsExport) { + return { + total: appsExport.total, + success: appsExport.successCount, + failures: appsExport.failureCount + }; + } + + // Fallback to setup process if no export process + const setup = processes.get('Setup'); + if (setup) { + return { + total: setup.total, + success: setup.successCount, + failures: setup.failureCount + }; + } + + return null; + }) +); + +// Register strategy for Stack Settings - use Settings as primary process +ProgressStrategyRegistry.register( + 'STACK', + new PrimaryProcessStrategy('Settings') +); + +// Register strategy for Personalize - complex module with projects/experiences +ProgressStrategyRegistry.register( + 'PERSONALIZE', + new CustomProgressStrategy((processes) => { + // For personalize, we want to count projects as the main metric + const projectExport = processes.get('Project Export'); + if (projectExport) { + return { + total: projectExport.total, + success: projectExport.successCount, + failures: projectExport.failureCount + }; + } + + // Fallback to any other main process + const mainProcess = Array.from(processes.values())[0]; + if (mainProcess) { + return { + total: mainProcess.total, + success: mainProcess.successCount, + failures: mainProcess.failureCount + }; + } + + return null; + }) +); + +export default ProgressStrategyRegistry; \ No newline at end of file diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 401215364f..e149612b37 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -74,21 +74,21 @@ export default class ImportAssets extends BaseClass { // Step 3: Perform import steps based on data if (foldersCount > 0) { - await this.executeStep(progress, 'Asset Folders', 'Importing folder structure...', () => this.importFolders()); + await this.executeStep(progress, 'Folders', 'Importing folder structure...', () => this.importFolders()); } if (this.assetConfig.includeVersionedAssets && versionedAssetsCount > 0) { - await this.executeStep(progress, 'Versioned Assets', 'Importing versioned assets...', () => + await this.executeStep(progress, 'Versions', 'Importing versioned assets...', () => this.importAssets(true), ); } if (assetsCount > 0) { - await this.executeStep(progress, 'Asset Upload', 'Uploading asset files...', () => this.importAssets()); + await this.executeStep(progress, 'Upload', 'Uploading asset files...', () => this.importAssets()); } if (!this.importConfig.skipAssetsPublish && publishableAssetsCount > 0) { - await this.executeStep(progress, 'Asset Publishing', 'Publishing assets...', () => this.publish()); + await this.executeStep(progress, 'Publish', 'Publishing assets...', () => this.publish()); } this.completeProgress(true); @@ -119,7 +119,7 @@ export default class ImportAssets extends BaseClass { const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }: any) => { this.assetsFolderMap[uid] = response.uid; - this.progressManager?.tick(true, `folder: ${name || uid}`, null, 'Asset Folders'); + this.progressManager?.tick(true, `folder: ${name || uid}`, null, 'Folders'); log.debug(`Created folder: ${name} (Mapped ${uid} → ${response.uid})`, this.importConfig.context); log.success(`Created folder: '${name}'`, this.importConfig.context); }; @@ -129,7 +129,7 @@ export default class ImportAssets extends BaseClass { false, `folder: ${name || uid}`, error?.message || 'Failed to create folder', - 'Asset Folders', + 'Folders', ); log.error(`${name} folder creation failed.!`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, name }); @@ -192,7 +192,7 @@ export default class ImportAssets extends BaseClass { const processName = isVersion ? 'import versioned assets' : 'import assets'; const indexFileName = isVersion ? 'versioned-assets.json' : 'assets.json'; const basePath = isVersion ? join(this.assetsPath, 'versions') : this.assetsPath; - const progressProcessName = isVersion ? 'Versioned Assets' : 'Asset Upload'; + const progressProcessName = isVersion ? 'Versions' : 'Upload'; log.debug(`Importing ${processName} from ${basePath}`, this.importConfig.context); @@ -348,7 +348,7 @@ export default class ImportAssets extends BaseClass { log.debug(`Found ${indexerCount} asset chunks to publish`, this.importConfig.context); const onSuccess = ({ apiData: { uid, title } = undefined }: any) => { - this.progressManager?.tick(true, `published: ${title || uid}`, null, 'Asset Publishing'); + this.progressManager?.tick(true, `published: ${title || uid}`, null, 'Publish'); log.success(`Asset '${uid}: ${title}' published successfully`, this.importConfig.context); }; @@ -357,7 +357,7 @@ export default class ImportAssets extends BaseClass { false, `publish failed: ${title || uid}`, error?.message || 'Failed to publish asset', - 'Asset Publishing', + 'Publish', ); log.error(`Asset '${uid}: ${title}' not published`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, uid, title }); @@ -513,16 +513,16 @@ export default class ImportAssets extends BaseClass { const { foldersCount, assetsCount, versionedAssetsCount, publishableAssetsCount } = counts; if (foldersCount > 0) { - progress.addProcess('Asset Folders', foldersCount); + progress.addProcess('Folders', foldersCount); } if (versionedAssetsCount > 0) { - progress.addProcess('Versioned Assets', versionedAssetsCount); + progress.addProcess('Versions', versionedAssetsCount); } if (assetsCount > 0) { - progress.addProcess('Asset Upload', assetsCount); + progress.addProcess('Upload', assetsCount); } if (publishableAssetsCount > 0) { - progress.addProcess('Asset Publishing', publishableAssetsCount); + progress.addProcess('Publish', publishableAssetsCount); } } diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index 9e3f05a63a..659087a94e 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -1,22 +1,15 @@ -/* eslint-disable no-prototype-builtins */ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - import * as path from 'path'; -import { isEmpty, find, cloneDeep, map } from 'lodash'; +import { find, cloneDeep, map } from 'lodash'; import { sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; -import { fsUtil, schemaTemplate, lookupExtension, lookUpTaxonomy } from '../../utils'; -import { ImportConfig, ModuleClassParams } from '../../types'; + +import { ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; import { updateFieldRules } from '../../utils/content-type-helper'; +import { fsUtil, schemaTemplate, lookupExtension, lookUpTaxonomy } from '../../utils'; export default class ContentTypesImport extends BaseClass { private cTsMapperPath: string; private cTsFolderPath: string; - private cTsFailsPath: string; private cTsSuccessPath: string; private gFsPendingPath: string; private pendingGFs: string[]; @@ -24,12 +17,8 @@ export default class ContentTypesImport extends BaseClass { private gFsFolderPath: string; private gFsMapperFolderPath: string; private gFs: Record[]; - private failedCTs: Record[]; private createdCTs: string[]; private cTs: Record[]; - private cTsUidMapper: Record; - private config: ImportConfig; - private stackAPIClient: any; private marketplaceAppMapperPath: string; private reqConcurrency: number; private ignoredFilesInContentTypesFolder: Map; @@ -54,16 +43,18 @@ export default class ContentTypesImport extends BaseClass { public taxonomies: Record; private extPendingPath: string; private isExtensionsUpdate = false; + private pendingExts: string[]; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'content-types'; + this.currentModuleName = 'Content Types'; this.cTsConfig = importConfig.modules['content-types']; this.gFsConfig = importConfig.modules['global-fields']; this.reqConcurrency = this.cTsConfig.writeConcurrency || this.importConfig.writeConcurrency; this.cTsFolderPath = path.join(sanitizePath(this.importConfig.data), sanitizePath(this.cTsConfig.dirName)); this.cTsMapperPath = path.join(sanitizePath(this.importConfig.data), 'mapper', 'content_types'); - this.cTsSuccessPath = path.join(sanitizePath(this.cTsMapperPath), 'success.json'); + this.cTsSuccessPath = path.join(sanitizePath(this.importConfig.data), 'mapper', 'content_types', 'success.json'); this.gFsFolderPath = path.resolve(sanitizePath(this.importConfig.data), sanitizePath(this.gFsConfig.dirName)); this.gFsMapperFolderPath = path.join(sanitizePath(importConfig.data), 'mapper', 'global_fields', 'success.json'); this.gFsPendingPath = path.join( @@ -91,83 +82,65 @@ export default class ContentTypesImport extends BaseClass { this.gFs = []; this.createdGFs = []; this.pendingGFs = []; - this.taxonomiesPath = path.join(sanitizePath(importConfig.data), 'mapper/taxonomies', 'success.json'); + this.pendingExts = []; + this.taxonomiesPath = path.join(sanitizePath(importConfig.data), 'mapper', 'taxonomies', 'success.json'); this.extPendingPath = path.join(sanitizePath(importConfig.data), 'mapper', 'extensions', 'pending_extensions.js'); } async start(): Promise { - /** - * read content type, check if it is necessary to read the entire dir - * Seed content types - * Update content types, lookup extension. - * Update pending global fields - * write field rules - */ - this.cTs = fsUtil.readFile(path.join(this.cTsFolderPath, 'schema.json')) as Record[]; - if (!this.cTs || isEmpty(this.cTs)) { - log.info('No content type found to import', this.importConfig.context); - return; - } - log.debug(`Found ${this.cTs.length} content types to import`, this.importConfig.context); + try { + log.debug('Starting content types import process...', this.importConfig.context); + await this.analyzeImportData(); - await fsUtil.makeDirectory(this.cTsMapperPath); - log.debug('Created content types mapper directory', this.importConfig.context); + if (!this.cTs?.length) { + log.info('No content type found to import', this.importConfig.context); + return; + } - this.installedExtensions = ( - ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } - ).extension_uid; - log.debug( - `Loaded ${Object.keys(this.installedExtensions)?.length} installed extensions`, - this.importConfig.context, - ); + await fsUtil.makeDirectory(this.cTsMapperPath); + log.debug('Created content types mapper directory', this.importConfig.context); + const progress = this.initializeProgress(); - this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record; - const taxonomyCount = Object.keys(this.taxonomies || {}).length; - log.debug(`Loaded ${taxonomyCount} taxonomy definitions`, this.importConfig.context); - - log.info('Starting content types seeding process', this.importConfig.context); - await this.seedCTs(); - if (this.createdCTs?.length) { - fsUtil.writeFile(this.cTsSuccessPath, this.createdCTs); - log.debug(`Written ${this.createdCTs.length} successful content types to file`, this.importConfig.context); - } - log.success('Created content types', this.importConfig.context); + if (this.cTs.length > 0) { + await this.handleContentTypesCreation(progress); + await this.handleContentTypesUpdate(progress); + } - log.info('Starting content types update process', this.importConfig.context); - await this.updateCTs(); - log.success('Updated content types with references', this.importConfig.context); + if (this.fieldRules.length > 0) { + fsUtil.writeFile(path.join(this.cTsFolderPath, 'field_rules_uid.json'), this.fieldRules); + log.debug(`Written ${this.fieldRules.length} field rules to file`, this.importConfig.context); + } - if (this.fieldRules.length > 0) { - fsUtil.writeFile(path.join(this.cTsFolderPath, 'field_rules_uid.json'), this.fieldRules); - log.debug(`Written ${this.fieldRules.length} field rules to file`, this.importConfig.context); - } + if (this.pendingExts.length > 0) { + await this.handlePendingExtensions(progress); + } - log.info('Updating the extensions...', this.importConfig.context); - await this.updatePendingExtensions(); - if (this.isExtensionsUpdate) { - log.success('Successfully updated the extensions.', this.importConfig.context); - } + if (this.pendingGFs.length > 0) { + await this.handlePendingGlobalFields(progress); + } - log.info('Starting pending global fields update', this.importConfig.context); - this.pendingGFs = fsUtil.readFile(this.gFsPendingPath) as any; - if (!this.pendingGFs || isEmpty(this.pendingGFs)) { - log.info('No pending global fields found to update', this.importConfig.context); - return; - } - await this.updatePendingGFs().catch((error) => { + this.completeProgress(true); + log.success('Content types have been imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Content types import failed'); handleAndLogError(error, { ...this.importConfig.context }); - }); - log.success('Updated pending global fields with content type with references', this.importConfig.context); - log.success('Content types have been imported successfully!', this.importConfig.context); + } } async seedCTs(): Promise { const onSuccess = ({ response: globalField, apiData: { content_type: { uid = null } = {} } = {} }: any) => { this.createdCTs.push(uid); - log.info(`${uid} content type seeded`, this.importConfig.context); + this.progressManager?.tick(true, `content type: ${uid}`, null, 'Create'); + log.success(`Content type '${uid}' created successfully`, this.importConfig.context); log.debug(`Successfully seeded content type: ${uid}`, this.importConfig.context); }; const onReject = ({ error, apiData: { content_type: { uid = null } = {} } = {} }: any) => { + this.progressManager?.tick( + false, + `content type: ${uid}`, + error?.message || 'Failed to create content type', + 'Create', + ); if (error.errorCode === 115 && (error.errors.uid || error.errors.title)) { log.info(`${uid} content type already exist`, this.importConfig.context); log.debug(`Skipping existing content type: ${uid}`, this.importConfig.context); @@ -208,14 +181,23 @@ export default class ContentTypesImport extends BaseClass { async updateCTs(): Promise { const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { + this.progressManager?.tick(true, `content type: ${uid}`, null, 'Update'); log.success(`'${uid}' updated with references`, this.importConfig.context); log.debug(`Content type update completed for: ${uid}`, this.importConfig.context); }; + const onReject = ({ error, apiData: { uid } }: any) => { + this.progressManager?.tick( + false, + `content type: ${uid}`, + error?.message || 'Failed to update content type', + 'Update', + ); handleAndLogError(error, { ...this.importConfig.context, uid }, `Content type '${uid}' update failed`); - throw new Error(`Content type '${uid}' update error`); }; + log.debug(`Starting to update ${this.cTs.length} content types with references`, this.importConfig.context); + return await this.makeConcurrentCall({ processName: 'Update content types', apiContent: this.cTs, @@ -276,18 +258,26 @@ export default class ContentTypesImport extends BaseClass { } async updatePendingGFs(): Promise { - this.pendingGFs = fsUtil.readFile(this.gFsPendingPath) as any; - log.info(`Found ${this.pendingGFs.length} pending global fields to update`, this.importConfig.context); - this.gFs = fsUtil.readFile(path.resolve(this.gFsFolderPath, this.gFsConfig.fileName)) as Record[]; + if (!this.pendingGFs || this.pendingGFs.length === 0) { + log.info('No pending global fields found to update', this.importConfig.context); + return; + } log.debug(`Found ${this.pendingGFs?.length || 0} pending global fields to update`, this.importConfig.context); log.debug(`Loaded ${this.gFs?.length || 0} global fields from file`, this.importConfig.context); const onSuccess = ({ response: globalField, apiData: { uid } = undefined }: any) => { - log.info(`Updated the global field ${uid} with content type references`, this.importConfig.context); + this.progressManager?.tick(true, `global field: ${uid}`, null, 'GF Update'); + log.success(`Updated the global field ${uid} with content type references`, this.importConfig.context); log.debug(`Global field update completed for: ${uid}`, this.importConfig.context); }; const onReject = ({ error, apiData: { uid } = undefined }: any) => { + this.progressManager?.tick( + false, + `global field: ${uid}`, + error?.message || 'Failed to update global field', + 'GF Update', + ); handleAndLogError(error, { ...this.importConfig.context, uid }, `Failed to update the global field '${uid}'`); }; @@ -360,12 +350,19 @@ export default class ContentTypesImport extends BaseClass { this.isExtensionsUpdate = true; const onSuccess = ({ response, apiData: { uid, title } = { uid: null, title: '' } }: any) => { + this.progressManager?.tick(true, `extension: ${response.title || title || uid}`, null, 'Ext Update'); log.success(`Successfully updated the '${response.title}' extension.`, this.importConfig.context); log.debug(`Extension update completed for: ${uid}`, this.importConfig.context); }; const onReject = ({ error, apiData }: any) => { - const { uid } = apiData; + const { uid, title } = apiData; + this.progressManager?.tick( + false, + `extension: ${title || uid}`, + error?.message || 'Failed to update extension', + 'Ext Update', + ); if (error?.errors?.title) { if (!this.importConfig.skipExisting) { log.info(`Extension '${uid}' already exists.`, this.importConfig.context); @@ -393,4 +390,95 @@ export default class ContentTypesImport extends BaseClass { false, ); } + + async analyzeImportData(): Promise { + const [cts, gfs, pendingGfs, pendingExt] = await this.withLoadingSpinner( + 'CONTENT TYPES: Analyzing import data...', + async () => { + const cts = fsUtil.readFile(path.join(this.cTsFolderPath, 'schema.json')); + const gfs = fsUtil.readFile(path.resolve(this.gFsFolderPath, this.gFsConfig.fileName)); + const pendingGfs = fsUtil.readFile(this.gFsPendingPath); + const pendingExt = fsUtil.readFile(this.extPendingPath); + return [cts, gfs, pendingGfs, pendingExt]; + }, + ); + + this.cTs = (Array.isArray(cts) ? cts : []) as Record[]; + this.gFs = (Array.isArray(gfs) ? gfs : []) as Record[]; + this.pendingGFs = (Array.isArray(pendingGfs) ? pendingGfs : []) as unknown as string[]; + this.pendingExts = (Array.isArray(pendingExt) ? pendingExt : []) as unknown as string[]; + + const marketplaceAppData = fsUtil.readFile(this.marketplaceAppMapperPath) as any; + this.installedExtensions = marketplaceAppData?.extension_uid || {}; + this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record; + + log.debug( + `Analysis complete: ${this.cTs?.length} content types, ${this.gFs?.length} global fields, ${ + this.pendingGFs?.length + } pending GFs, ${Object.keys(this.installedExtensions)?.length} extensions, ${ + Object.keys(this.taxonomies)?.length + } taxonomies`, + this.importConfig.context, + ); + } + + initializeProgress() { + const progress = this.createNestedProgress(this.currentModuleName); + if (this.cTs.length) { + progress.addProcess('Create', this.cTs.length); + progress.addProcess('Update', this.cTs.length); + } + if (this.pendingGFs.length) { + progress.addProcess('GF Update', this.pendingGFs.length); + } + if (this.pendingExts.length) { + progress.addProcess('Ext Update', this.pendingExts.length); + } + return progress; + } + + async handlePendingGlobalFields(progress: any) { + progress + .startProcess('GF Update') + .updateStatus('Updating global fields with content type references...', 'GF Update'); + + log.info('Starting pending global fields update process', this.importConfig.context); + await this.updatePendingGFs(); + progress.completeProcess('GF Update', true); + } + + async handleContentTypesCreation(progress: any) { + progress.startProcess('Create').updateStatus('Creating content types...', 'Create'); + + log.info('Starting content types seeding process', this.importConfig.context); + await this.seedCTs(); + + if (this.createdCTs?.length) { + fsUtil.writeFile(this.cTsSuccessPath, this.createdCTs); + log.debug(`Written ${this.createdCTs.length} successful content types to file`, this.importConfig.context); + } + + progress.completeProcess('Create', true); + } + + async handleContentTypesUpdate(progress: any) { + progress.startProcess('Update').updateStatus('Updating content types with references...', 'Update'); + + log.info('Starting Update process', this.importConfig.context); + await this.updateCTs(); + + progress.completeProcess('Update', true); + } + + async handlePendingExtensions(progress: any) { + progress.startProcess('Ext Update').updateStatus('Updating extensions...', 'Ext Update'); + + log.info('Starting pending extensions update process', this.importConfig.context); + await this.updatePendingExtensions(); + progress.completeProcess('Ext Update', true); + + if (this.isExtensionsUpdate) { + log.success('Successfully updated the extensions.', this.importConfig.context); + } + } } diff --git a/packages/contentstack-import/src/import/modules/custom-roles.ts b/packages/contentstack-import/src/import/modules/custom-roles.ts index 20bc8a3eae..040197dfd5 100644 --- a/packages/contentstack-import/src/import/modules/custom-roles.ts +++ b/packages/contentstack-import/src/import/modules/custom-roles.ts @@ -31,6 +31,7 @@ export default class ImportCustomRoles extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'custom-roles'; + this.currentModuleName = 'Custom Roles'; this.customRolesConfig = importConfig.modules.customRoles; this.customRolesMapperPath = join(this.importConfig.backupDir, 'mapper', 'custom-roles'); this.customRolesFolderPath = join(this.importConfig.backupDir, this.customRolesConfig.dirName); @@ -54,78 +55,31 @@ export default class ImportCustomRoles extends BaseClass { * @returns {Promise} Promise */ async start(): Promise { - log.debug('Checking for custom roles folder existence', this.importConfig.context); + try { + log.debug('Starting custom roles import process...', this.importConfig.context); + const [customRolesCount] = await this.analyzeCustomRoles(); + if (customRolesCount === 0) { + log.info(`No custom-rules are found - '${this.customRolesFolderPath}'`, this.importConfig.context); + return; + } - //Step1 check folder exists or not - if (fileHelper.fileExistsSync(this.customRolesFolderPath)) { - log.debug(`Found custom roles folder: ${this.customRolesFolderPath}`, this.importConfig.context); - - this.customRoles = fsUtil.readFile(join(this.customRolesFolderPath, this.customRolesConfig.fileName),true) as Record; - this.customRolesLocales = fsUtil.readFile(join(this.customRolesFolderPath, this.customRolesConfig.customRolesLocalesFileName),true) as Record; - } else { - log.info(`No custom-rules are found - '${this.customRolesFolderPath}'`, this.importConfig.context); - return; - } + const progress = this.createSimpleProgress(this.currentModuleName, customRolesCount); + await this.prepareForImport(); - //create webhooks in mapper directory - log.debug('Creating custom roles mapper directory', this.importConfig.context); - await fsUtil.makeDirectory(this.customRolesMapperPath); - - log.debug('Loading existing custom roles UID data', this.importConfig.context); - this.customRolesUidMapper = fileHelper.fileExistsSync(this.customRolesUidMapperPath) - ? (fsUtil.readFile(join(this.customRolesUidMapperPath), true) as Record) || {} - : {}; - - log.debug('Loading environments UID data', this.importConfig.context); - this.environmentsUidMap = fileHelper.fileExistsSync(this.envUidMapperFolderPath) - ? (fsUtil.readFile(join(this.envUidMapperFolderPath, 'uid-mapping.json'), true) as Record) || {} - : {}; - - log.debug('Loading entries UID data', this.importConfig.context); - this.entriesUidMap = fileHelper.fileExistsSync(this.entriesUidMapperFolderPath) - ? (fsUtil.readFile(join(this.entriesUidMapperFolderPath, 'uid-mapping.json'), true) as Record) || {} - : {}; - - if (this.customRolesUidMapper && Object.keys(this.customRolesUidMapper || {}).length > 0) { - const customRolesUidCount = Object.keys(this.customRolesUidMapper || {}).length; - log.debug(`Loaded existing custom roles UID data: ${customRolesUidCount} items`, this.importConfig.context); - } else { - log.debug('No existing custom roles UID data found', this.importConfig.context); - } + progress.updateStatus('Building locale mappings...'); + await this.getLocalesUidMap(); - if (this.environmentsUidMap && Object.keys(this.environmentsUidMap || {})?.length > 0) { - const envUidCount = Object.keys(this.environmentsUidMap || {}).length; - log.debug(`Loaded environments UID data: ${envUidCount} items`, this.importConfig.context); - } else { - log.debug('No environments UID data found', this.importConfig.context); - } - - if (this.entriesUidMap && Object.keys(this.entriesUidMap || {}).length > 0) { - const entriesUidCount = Object.keys(this.entriesUidMap || {}).length; - log.debug(`Loaded entries UID data: ${entriesUidCount} items`, this.importConfig.context); - } else { - log.debug('No entries UID data found', this.importConfig.context); - } + progress.updateStatus('Importing custom roles...'); + await this.importCustomRoles(); - //source and target stack locale map - log.debug('Getting locales UID mapping', this.importConfig.context); - await this.getLocalesUidMap(); - - log.debug('Starting custom roles import', this.importConfig.context); - await this.importCustomRoles(); + this.handleImportResults(); - log.debug('Processing custom roles import results', this.importConfig.context); - if (this.createdCustomRoles?.length) { - fsUtil.writeFile(this.createdCustomRolesPath, this.createdCustomRoles); - log.debug(`Written ${this.createdCustomRoles.length} successful custom roles to file`, this.importConfig.context); + this.completeProgress(true); + log.success('Custom roles have been imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Custom roles import failed'); + handleAndLogError(error, { ...this.importConfig.context }); } - - if (this.failedCustomRoles?.length) { - fsUtil.writeFile(this.customRolesFailsPath, this.failedCustomRoles); - log.debug(`Written ${this.failedCustomRoles.length} failed custom roles to file`, this.importConfig.context); - } - - log.success('Custom roles have been imported successfully!', this.importConfig.context); } async getLocalesUidMap(): Promise { @@ -140,9 +94,9 @@ export default class ImportCustomRoles extends BaseClass { }) .catch((error) => { log.debug('Error fetching target stack locales', this.importConfig.context); - handleAndLogError(error, { ...this.importConfig.context}); + handleAndLogError(error, { ...this.importConfig.context }); }); - + this.targetLocalesMap = {}; this.sourceLocalesMap = {}; @@ -162,7 +116,7 @@ export default class ImportCustomRoles extends BaseClass { const sourceLocaleKey = this.sourceLocalesMap[key] as string; this.localesUidMap[sourceLocaleKey] = this.targetLocalesMap[key]; } - + const localesMappingCount = Object.keys(this.localesUidMap || {}).length; log.debug(`Created ${localesMappingCount} locale UID mappings`, this.importConfig.context); } @@ -180,6 +134,7 @@ export default class ImportCustomRoles extends BaseClass { const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }: any) => { this.createdCustomRoles.push(response); this.customRolesUidMapper[uid] = response.uid; + this.progressManager?.tick(true, `custom role: ${name || uid}`); log.success(`custom-role '${name}' imported successfully`, this.importConfig.context); log.debug(`Custom role import completed: ${name} (${uid})`, this.importConfig.context); fsUtil.writeFile(this.customRolesUidMapperPath, this.customRolesUidMapper); @@ -191,20 +146,22 @@ export default class ImportCustomRoles extends BaseClass { log.debug(`Custom role '${name}' import failed`, this.importConfig.context); if (err?.errors?.name) { + this.progressManager?.tick(true, `custom role: ${name} (already exists)`); log.info(`custom-role '${name}' already exists`, this.importConfig.context); } else { this.failedCustomRoles.push(apiData); + this.progressManager?.tick(false, `custom role: ${name}`, error?.message || 'Failed to import custom role'); handleAndLogError(error, { ...this.importConfig.context, name }, `custom-role '${name}' failed to be import`); } }; - log.debug(`Using concurrency limit: ${this.importConfig.fetchConcurrency || 1}`, this.importConfig.context); + log.debug(`Using concurrency limit: ${this.importConfig.fetchConcurrency || 2}`, this.importConfig.context); await this.makeConcurrentCall( { apiContent, processName: 'create custom role', apiParams: { - serializeData: this.serializeWebhooks.bind(this), + serializeData: this.serializeCustomRoles.bind(this), reject: onReject.bind(this), resolve: onSuccess.bind(this), entity: 'create-custom-role', @@ -215,16 +172,16 @@ export default class ImportCustomRoles extends BaseClass { undefined, false, ); - + log.debug('Custom roles import process completed', this.importConfig.context); } /** - * @method serializeWebhooks + * @method serializeCustomRoles * @param {ApiOptions} apiOptions ApiOptions * @returns {ApiOptions} ApiOptions */ - serializeWebhooks(apiOptions: ApiOptions): ApiOptions { + serializeCustomRoles(apiOptions: ApiOptions): ApiOptions { const { apiData: customRole } = apiOptions; log.debug(`Serializing custom role: ${customRole.name} (${customRole.uid})`, this.importConfig.context); @@ -234,11 +191,18 @@ export default class ImportCustomRoles extends BaseClass { this.importConfig.context, ); log.debug(`Skipping custom role serialization for: ${customRole.uid}`, this.importConfig.context); + // Still tick progress for skipped custom roles + this.progressManager?.tick(true, `custom role: ${customRole.name} (skipped - already exists)`); apiOptions.entity = undefined; } else { + log.debug(`Processing custom role: ${customRole.name}`, this.importConfig.context); + let branchRuleExists: boolean = false; - log.debug(`Processing ${customRole.rules?.length || 0} rules for custom role: ${customRole.name}`, this.importConfig.context); - + log.debug( + `Processing ${customRole.rules?.length || 0} rules for custom role: ${customRole.name}`, + this.importConfig.context, + ); + forEach(customRole.rules, (rule: Record) => { rule = this.getTransformUidsFactory(rule); // rules.branch is required to create custom roles. @@ -247,7 +211,7 @@ export default class ImportCustomRoles extends BaseClass { log.debug(`Found branch rule in custom role: ${customRole.name}`, this.importConfig.context); } }); - + if (!branchRuleExists) { log.debug(`Adding default branch rule to custom role: ${customRole.name}`, this.importConfig.context); customRole.rules.push({ @@ -256,7 +220,7 @@ export default class ImportCustomRoles extends BaseClass { acl: { read: true }, }); } - + log.debug(`Custom role serialization completed: ${customRole.name}`, this.importConfig.context); apiOptions.apiData = customRole; } @@ -265,9 +229,9 @@ export default class ImportCustomRoles extends BaseClass { getTransformUidsFactory = (rule: Record) => { log.debug(`Transforming UIDs for rule module: ${rule.module}`, this.importConfig.context); - + if (rule.module === 'environment') { - if(!isEmpty(this.environmentsUidMap)){ + if (!isEmpty(this.environmentsUidMap)) { const originalEnvs = rule.environments?.length || 0; rule.environments = map(rule.environments, (env: any) => this.environmentsUidMap[env]); log.debug(`Transformed ${originalEnvs} environment UIDs for rule`, this.importConfig.context); @@ -275,7 +239,7 @@ export default class ImportCustomRoles extends BaseClass { log.debug('No environment UID mappings available for transformation', this.importConfig.context); } } else if (rule.module === 'locale') { - if(!isEmpty(this.localesUidMap)){ + if (!isEmpty(this.localesUidMap)) { const originalLocales = rule.locales?.length || 0; rule.locales = map(rule.locales, (locale: any) => this.localesUidMap[locale]); log.debug(`Transformed ${originalLocales} locale UIDs for rule`, this.importConfig.context); @@ -283,7 +247,7 @@ export default class ImportCustomRoles extends BaseClass { log.debug('No locale UID mappings available for transformation', this.importConfig.context); } } else if (rule.module === 'entry') { - if(!isEmpty(this.entriesUidMap)){ + if (!isEmpty(this.entriesUidMap)) { const originalEntries = rule.entries?.length || 0; rule.entries = map(rule.entries, (entry: any) => this.entriesUidMap[entry]); log.debug(`Transformed ${originalEntries} entry UIDs for rule`, this.importConfig.context); @@ -293,4 +257,70 @@ export default class ImportCustomRoles extends BaseClass { } return rule; }; + + private async analyzeCustomRoles(): Promise<[number]> { + return this.withLoadingSpinner('CUSTOM ROLES: Analyzing import data...', async () => { + log.debug('Checking for custom roles folder existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.customRolesFolderPath)) { + log.info(`No custom-rules are found - '${this.customRolesFolderPath}'`, this.importConfig.context); + return [0]; + } + + log.debug(`Found custom roles folder: ${this.customRolesFolderPath}`, this.importConfig.context); + + this.customRoles = fsUtil.readFile( + join(this.customRolesFolderPath, this.customRolesConfig.fileName), + true, + ) as Record; + + this.customRolesLocales = fsUtil.readFile( + join(this.customRolesFolderPath, this.customRolesConfig.customRolesLocalesFileName), + true, + ) as Record; + + const count = Object.keys(this.customRoles || {}).length; + log.debug(`Loaded ${count} custom roles from file`, this.importConfig.context); + return [count]; + }); + } + + private async prepareForImport(): Promise { + log.debug('Creating custom roles mapper directory', this.importConfig.context); + await fsUtil.makeDirectory(this.customRolesMapperPath); + + this.customRolesUidMapper = this.loadJsonFileIfExists(this.customRolesUidMapperPath, 'custom roles'); + this.environmentsUidMap = this.loadJsonFileIfExists( + join(this.envUidMapperFolderPath, 'uid-mapping.json'), + 'environments', + ); + this.entriesUidMap = this.loadJsonFileIfExists( + join(this.entriesUidMapperFolderPath, 'uid-mapping.json'), + 'entries', + ); + } + + private loadJsonFileIfExists(path: string, label: string): Record { + if (fileHelper.fileExistsSync(path)) { + const data = fsUtil.readFile(path, true) as Record; + const count = Object.keys(data || {}).length; + log.debug(`Loaded ${label}: ${count} items`, this.importConfig.context); + return data || {}; + } else { + log.debug(`No ${label} UID data found`, this.importConfig.context); + return {}; + } + } + + private handleImportResults() { + if (this.createdCustomRoles?.length) { + fsUtil.writeFile(this.createdCustomRolesPath, this.createdCustomRoles); + log.debug(`Written ${this.createdCustomRoles.length} successful custom roles to file`, this.importConfig.context); + } + + if (this.failedCustomRoles?.length) { + fsUtil.writeFile(this.customRolesFailsPath, this.failedCustomRoles); + log.debug(`Written ${this.failedCustomRoles.length} failed custom roles to file`, this.importConfig.context); + } + } } diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 1b6af4a3c5..d897c0235f 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -23,6 +23,7 @@ import { } from '../../utils'; import { ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; + export default class EntriesImport extends BaseClass { private assetUidMapperPath: string; private assetUidMapper: Record; @@ -61,6 +62,7 @@ export default class EntriesImport extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'entries'; + this.currentModuleName = 'Entries'; this.assetUidMapperPath = path.resolve(sanitizePath(importConfig.data), 'mapper', 'assets', 'uid-mapping.json'); this.assetUrlMapperPath = path.resolve(sanitizePath(importConfig.data), 'mapper', 'assets', 'url-mapping.json'); this.entriesMapperPath = path.resolve(sanitizePath(importConfig.data), 'mapper', 'entries'); @@ -106,120 +108,244 @@ export default class EntriesImport extends BaseClass { } async start(): Promise { - try { + try { + log.debug('Starting entries import process...', this.importConfig.context); + + const [contentTypesCount, localesCount, totalEntryTasks] = await this.analyzeEntryData(); + if (contentTypesCount === 0) { + log.info('No content types found for entry import', this.importConfig.context); + return; + } + + const progress = this.createNestedProgress(this.currentModuleName); + this.initializeProgress(progress, { + contentTypesCount, + localesCount, + totalEntryTasks, + }); + + // Step 1: Prepare content types + progress + .startProcess('CT Preparation') + .updateStatus('Preparing content types for entry import...', 'CT Preparation'); + await this.disableMandatoryCTReferences(); + progress.completeProcess('CT Preparation', true); + + // Step 2: Create entries + progress.startProcess('Create').updateStatus('Creating entries...', 'Create'); + await this.processEntryCreation(); + progress.completeProcess('Create', true); + + // Step 3: Replace existing entries if needed + if (this.importConfig.replaceExisting) { + progress.startProcess('Replace Existing').updateStatus('Replacing existing entries...', 'Replace Existing'); + await this.processEntryReplacement(); + progress.completeProcess('Replace Existing', true); + } + + // Step 4: Update entries with references + progress.startProcess('Reference Updates').updateStatus('Updating entry references...', 'Reference Updates'); + await this.processEntryReferenceUpdates(); + progress.completeProcess('Reference Updates', true); + + // Step 5: Restore content types + progress.startProcess('CT Restoration').updateStatus('Restoring content type references...', 'CT Restoration'); + await this.enableMandatoryCTReferences(); + progress.completeProcess('CT Restoration', true); + + // Step 6: Update field rules + progress.startProcess('Field Rules Update').updateStatus('Updating field rules...', 'Field Rules Update'); + await this.updateFieldRules(); + progress.completeProcess('Field Rules Update', true); + + // Step 7: Publish entries if not skipped + if (!this.importConfig.skipEntriesPublish) { + progress.startProcess('Publish').updateStatus('Publishing entries...', 'Publish'); + await this.processEntryPublishing(); + progress.completeProcess('Publish', true); + } + + // Step 8: Cleanup and finalization + progress.startProcess('Cleanup').updateStatus('Cleaning up auto-created entries...', 'Cleanup'); + await this.processCleanup(); + progress.completeProcess('Cleanup', true); + + this.completeProgress(true); + log.success('Entries imported successfully', this.importConfig.context); + } catch (error) { + this.createEntryDataForVariantEntry(); + this.completeProgress(false, (error as any)?.message || 'Entries import failed'); + handleAndLogError(error, { ...this.importConfig.context }); + throw new Error('Error while importing entries'); + } + } + + private async analyzeEntryData(): Promise<[number, number, number]> { + return this.withLoadingSpinner('ENTRIES: Analyzing import data...', async () => { + log.debug('Loading content types for entry analysis', this.importConfig.context); + this.cTs = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record[]; if (!this.cTs || isEmpty(this.cTs)) { - log.info('No content type found', this.importConfig.context); - return; + return [0, 0, 0]; } - log.debug(`Found ${this.cTs.length} content types for entry import`, this.importConfig.context); - + + log.debug('Loading installed extensions for entry processing', this.importConfig.context); this.installedExtensions = ( - ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } + (fsUtil.readFile(this.marketplaceAppMapperPath) as any) || { extension_uid: {} } ).extension_uid; - log.debug('Loaded installed extensions for entry processing', this.importConfig.context); + log.debug('Loading asset mappings', this.importConfig.context); this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record) || {}; this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record) || {}; - log.debug(`Loaded asset mappings - UIDs: ${Object.keys(this.assetUidMapper).length}, URLs: ${Object.keys(this.assetUrlMapper).length}`, this.importConfig.context); + log.debug('Loading taxonomy data', this.importConfig.context); this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record; - log.debug('Loaded taxonomy data for entry processing', this.importConfig.context); + log.debug('Setting up mapper directories', this.importConfig.context); fsUtil.makeDirectory(this.entriesMapperPath); - log.debug('Created entries mapper directory', this.importConfig.context); - - log.info('Preparing content types for entry import', this.importConfig.context); - await this.disableMandatoryCTReferences(); - + + log.debug('Loading locales data', this.importConfig.context); this.locales = values(fsUtil.readFile(this.localesPath) as Record[]); - this.locales.unshift(this.importConfig.master_locale); // adds master locale to the list - log.debug(`Processing entries for ${values(this.locales).length} locales`, this.importConfig.context); - - //Create Entries - log.info('Starting entry creation process', this.importConfig.context); - const entryRequestOptions = this.populateEntryCreatePayload(); - log.debug(`Generated ${entryRequestOptions.length} entry creation tasks`, this.importConfig.context); - - for (let entryRequestOption of entryRequestOptions) { - await this.createEntries(entryRequestOption); - } - log.success('Entry creation process completed', this.importConfig.context); - - if (this.importConfig.replaceExisting) { - // Note: Instead of using entryRequestOptions, we can prepare request options for replace, to avoid unnecessary operations - log.info('Starting entry replacement process', this.importConfig.context); - for (let entryRequestOption of entryRequestOptions) { - await this.replaceEntries(entryRequestOption).catch((error) => { - handleAndLogError(error, { ...this.importConfig.context, cTUid: entryRequestOption.cTUid, locale: entryRequestOption.locale }, 'Error while replacing existing entries'); - }); - } - log.success('Entry replacement process completed', this.importConfig.context); - } + this.locales.unshift(this.importConfig.master_locale); - log.debug('Writing entry UID mappings to file', this.importConfig.context); - await fileHelper.writeLargeFile(path.join(this.entriesMapperPath, 'uid-mapping.json'), this.entriesUidMapper); // TBD: manages mapper in one file, should find an alternative - fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + const contentTypesCount = this.cTs.length; + const localesCount = this.locales.length; + const totalEntryTasks = contentTypesCount * localesCount; - if (this.autoCreatedEntries?.length > 0) { - log.info(`Removing ${this.autoCreatedEntries.length} entries from master language which got created by default`, this.importConfig.context); - await this.removeAutoCreatedEntries().catch((error) => { - handleAndLogError(error, { ...this.importConfig.context }, 'Error while removing auto created entries in master locale'); - }); - log.success('Auto-created entries cleanup completed', this.importConfig.context); - } + log.debug( + `Analysis complete: ${contentTypesCount} content types, ${localesCount} locales, ${totalEntryTasks} total tasks`, + this.importConfig.context, + ); - // Update entries with references - log.info('Starting entry references update process', this.importConfig.context); - const entryUpdateRequestOptions = this.populateEntryUpdatePayload(); - log.debug(`Generated ${entryUpdateRequestOptions.length} entry update tasks`, this.importConfig.context); - - for (let entryUpdateRequestOption of entryUpdateRequestOptions) { - await this.updateEntriesWithReferences(entryUpdateRequestOption).catch((error) => { - handleAndLogError(error, { ...this.importConfig.context, cTUid: entryUpdateRequestOption.cTUid, locale: entryUpdateRequestOption.locale }, `Error while updating entries references of ${entryUpdateRequestOption.cTUid} in locale ${entryUpdateRequestOption.locale}`); - }); - } - fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); - log.success('Entry references update process completed', this.importConfig.context); + return [contentTypesCount, localesCount, totalEntryTasks]; + }); + } + + private initializeProgress( + progress: any, + counts: { contentTypesCount: number; localesCount: number; totalEntryTasks: number }, + ) { + const { contentTypesCount, localesCount, totalEntryTasks } = counts; + + // Add main processes + progress.addProcess('CT Preparation', contentTypesCount); + progress.addProcess('Create', totalEntryTasks); + + if (this.importConfig.replaceExisting) { + progress.addProcess('Replace Existing', totalEntryTasks); + } + + progress.addProcess('Reference Updates', totalEntryTasks); + progress.addProcess('CT Restoration', contentTypesCount); + progress.addProcess('Field Rules Update', 1); + + if (!this.importConfig.skipEntriesPublish) { + progress.addProcess('Publish', totalEntryTasks); + } + + progress.addProcess('Cleanup', 1); - log.info('Restoring content type changes', this.importConfig.context); - await this.enableMandatoryCTReferences().catch((error) => { - handleAndLogError(error, { ...this.importConfig.context }, 'Error while updating content type references'); + log.debug( + `Initialized progress tracking for ${contentTypesCount} content types across ${localesCount} locales`, + this.importConfig.context, + ); + } + + private async processEntryCreation(): Promise { + log.info('Starting entry creation process', this.importConfig.context); + const entryRequestOptions = this.populateEntryCreatePayload(); + log.debug(`Generated ${entryRequestOptions.length} entry creation tasks`, this.importConfig.context); + + for (let entryRequestOption of entryRequestOptions) { + await this.createEntries(entryRequestOption); + } + + log.debug('Writing entry UID mappings to file', this.importConfig.context); + await fileHelper.writeLargeFile(path.join(this.entriesMapperPath, 'uid-mapping.json'), this.entriesUidMapper); + fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + } + + private async processEntryReplacement(): Promise { + log.info('Starting Replace Existing process', this.importConfig.context); + const entryRequestOptions = this.populateEntryCreatePayload(); + + for (let entryRequestOption of entryRequestOptions) { + await this.replaceEntries(entryRequestOption).catch((error) => { + handleAndLogError( + error, + { + ...this.importConfig.context, + cTUid: entryRequestOption.cTUid, + locale: entryRequestOption.locale, + }, + 'Error while replacing existing entries', + ); }); - log.success('Content type references restored successfully', this.importConfig.context); + } + } - // Update field rule of content types which are got removed earlier - log.info('Updating the field rules of content type', this.importConfig.context); - await this.updateFieldRules().catch((error) => { - handleAndLogError(error, { ...this.importConfig.context }, 'Error while updating field rules of content type'); + private async processEntryReferenceUpdates(): Promise { + log.info('Starting entry references update process', this.importConfig.context); + const entryUpdateRequestOptions = this.populateEntryUpdatePayload(); + log.debug(`Generated ${entryUpdateRequestOptions.length} entry update tasks`, this.importConfig.context); + + for (let entryUpdateRequestOption of entryUpdateRequestOptions) { + await this.updateEntriesWithReferences(entryUpdateRequestOption).catch((error) => { + handleAndLogError( + error, + { + ...this.importConfig.context, + cTUid: entryUpdateRequestOption.cTUid, + locale: entryUpdateRequestOption.locale, + }, + `Error while updating entries references of ${entryUpdateRequestOption.cTUid} in locale ${entryUpdateRequestOption.locale}`, + ); }); - log.success('Entries imported successfully', this.importConfig.context); + } - // Publishing entries - if (!this.importConfig.skipEntriesPublish) { - log.info('Starting entry publishing process', this.importConfig.context); - this.envs = fileHelper.readFileSync(this.envPath); - log.debug(`Loaded ${Object.keys(this.envs).length} environments for publishing`, this.importConfig.context); - - for (let entryRequestOption of entryRequestOptions) { - await this.publishEntries(entryRequestOption).catch((error) => { - handleAndLogError(error, { ...this.importConfig.context, cTUid: entryRequestOption.cTUid, locale: entryRequestOption.locale }, `Error in publishing entries of ${entryRequestOption.cTUid} in locale ${entryRequestOption.locale}`); - }); - } - log.success('All the entries have been published successfully', this.importConfig.context); - } else { - log.info('Skipping entry publishing as per configuration', this.importConfig.context); - } + fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + } - log.debug('Creating entry data for variant entries', this.importConfig.context); - this.createEntryDataForVariantEntry(); - } catch (error) { - this.createEntryDataForVariantEntry(); - handleAndLogError(error, { ...this.importConfig.context }); - throw new Error('Error while importing entries'); + private async processEntryPublishing(): Promise { + log.info('Starting entry publishing process', this.importConfig.context); + this.envs = fileHelper.readFileSync(this.envPath); + log.debug(`Loaded ${Object.keys(this.envs).length} environments for publishing`, this.importConfig.context); + + const entryRequestOptions = this.populateEntryCreatePayload(); + for (let entryRequestOption of entryRequestOptions) { + await this.publishEntries(entryRequestOption).catch((error) => { + handleAndLogError( + error, + { + ...this.importConfig.context, + cTUid: entryRequestOption.cTUid, + locale: entryRequestOption.locale, + }, + `Error in publishing entries of ${entryRequestOption.cTUid} in locale ${entryRequestOption.locale}`, + ); + }); } } + private async processCleanup(): Promise { + if (this.autoCreatedEntries?.length > 0) { + log.info( + `Removing ${this.autoCreatedEntries.length} entries from master language which got created by default`, + this.importConfig.context, + ); + await this.removeAutoCreatedEntries().catch((error) => { + handleAndLogError( + error, + { ...this.importConfig.context }, + 'Error while removing auto created entries in master locale', + ); + }); + } + + log.debug('Creating entry data for variant entries', this.importConfig.context); + this.createEntryDataForVariantEntry(); + } + /** * The function `createEntryDataForVariantEntry` writes the `entriesForVariant` data to a JSON file * named `data-for-variant-entry.json`. @@ -233,13 +359,22 @@ export default class EntriesImport extends BaseClass { } async disableMandatoryCTReferences() { - log.debug(`Starting to disable mandatory CT references for ${this.cTs.length} content types`, this.importConfig.context); - + log.debug( + `Starting to disable mandatory CT references for ${this.cTs.length} content types`, + this.importConfig.context, + ); + const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { + this.progressManager?.tick(true, `content type: ${uid}`, null, 'CT Preparation'); log.success(`${uid} content type references removed temporarily`, this.importConfig.context); - log.debug(`Successfully processed content type: ${uid}`, this.importConfig.context); }; const onReject = ({ error, apiData: { uid } }: any) => { + this.progressManager?.tick( + false, + `content type: ${uid}`, + error?.message || 'Failed to update content type', + 'CT Preparation', + ); handleAndLogError(error, { ...this.importConfig.context, uid }); throw new Error(`${uid} content type references removal failed`); }; @@ -347,15 +482,21 @@ export default class EntriesImport extends BaseClass { const fs = new FsUtility({ basePath, indexFileName }); const indexer = fs.indexFileContent; const indexerCount = values(indexer).length; + if (indexerCount === 0) { log.debug(`No entries found for content type ${cTUid} in locale ${locale}`, this.importConfig.context); + this.progressManager?.tick(true, `${cTUid} - ${locale} (no entries)`, null, 'Create'); return Promise.resolve(); } - log.debug(`Starting to create entries for ${cTUid} in locale ${locale} - ${indexerCount} chunks to process`, this.importConfig.context); - + + log.debug( + `Starting to create entries for ${cTUid} in locale ${locale} - ${indexerCount} chunks to process`, + this.importConfig.context, + ); + const isMasterLocale = locale === this.importConfig?.master_locale?.code; log.debug(`Processing ${isMasterLocale ? 'master' : 'non-master'} locale: ${locale}`, this.importConfig.context); - + // Write created entries const entriesCreateFileHelper = new FsUtility({ moduleName: 'entries', @@ -380,47 +521,51 @@ export default class EntriesImport extends BaseClass { log.debug(`Found content type schema for ${cTUid}`, this.importConfig.context); const onSuccess = ({ response, apiData: entry, additionalInfo }: any) => { + this.progressManager?.tick(true, `${entry?.title} - ${entry?.uid}`, null, 'Create'); if (additionalInfo[entry.uid]?.isLocalized) { let oldUid = additionalInfo[entry.uid].entryOldUid; this.entriesForVariant.push({ content_type: cTUid, entry_uid: oldUid, locale }); - log.info(`Localized entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, this.importConfig.context); + log.info( + `Localized entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, + this.importConfig.context, + ); log.debug(`Mapped localized entry UID: ${entry.uid} → ${oldUid}`, this.importConfig.context); entry.uid = oldUid; entry.entryOldUid = oldUid; - entry.sourceEntryFilePath = path.join(sanitizePath(basePath), sanitizePath(additionalInfo.entryFileName)); // stores source file path temporarily + entry.sourceEntryFilePath = path.join(sanitizePath(basePath), sanitizePath(additionalInfo.entryFileName)); entriesCreateFileHelper.writeIntoFile({ [oldUid]: entry } as any, { mapKeyVal: true }); } else { - log.info(`Created entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, this.importConfig.context); + log.info( + `Created entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, + this.importConfig.context, + ); log.debug(`Created entry UID mapping: ${entry.uid} → ${response.uid}`, this.importConfig.context); this.entriesForVariant.push({ content_type: cTUid, entry_uid: entry.uid, locale }); - // This is for creating localized entries that do not have a counterpart in master locale. - // For example : To create entry1 in fr-fr, where en-us is the master locale - // entry1 will get created in en-us first, then fr-fr version will be created - // thus entry1 has to be removed from en-us at the end. + if (!isMasterLocale && !additionalInfo[entry.uid]?.isLocalized) { this.autoCreatedEntries.push({ cTUid, locale, entryUid: response.uid }); log.debug(`Marked entry for auto-cleanup: ${response.uid} in master locale`, this.importConfig.context); } this.entriesUidMapper[entry.uid] = response.uid; - entry.sourceEntryFilePath = path.join(sanitizePath(basePath), sanitizePath(additionalInfo.entryFileName)); // stores source file path temporarily - entry.entryOldUid = entry.uid; // stores old uid temporarily + entry.sourceEntryFilePath = path.join(sanitizePath(basePath), sanitizePath(additionalInfo.entryFileName)); + entry.entryOldUid = entry.uid; entriesCreateFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true }); } }; + const onReject = ({ error, apiData: entry, additionalInfo }: any) => { const { title, uid } = entry; - // NOTE Remove from list if any entry import failed + this.progressManager?.tick(false, `${title} - ${uid}`, 'Error while creating entries', 'Create'); this.entriesForVariant = this.entriesForVariant.filter( (item) => !(item.locale === locale && item.entry_uid === uid), ); log.debug(`Removed failed entry from variant list: ${uid}`, this.importConfig.context); - - // NOTE: write existing entries into files to handler later + if (error.errorCode === 119) { if (error?.errors?.title || error?.errors?.uid) { if (this.importConfig.replaceExisting) { entry.entryOldUid = uid; - entry.sourceEntryFilePath = path.join(sanitizePath(basePath), sanitizePath(additionalInfo.entryFileName)); // stores source file path temporarily + entry.sourceEntryFilePath = path.join(sanitizePath(basePath), sanitizePath(additionalInfo.entryFileName)); existingEntriesFileHelper.writeIntoFile({ [uid]: entry } as any, { mapKeyVal: true }); log.debug(`Queued existing entry for replacement: ${title} (${uid})`, this.importConfig.context); } @@ -439,7 +584,7 @@ export default class EntriesImport extends BaseClass { for (const index in indexer) { log.debug(`Processing chunk ${index} of ${indexerCount} for ${cTUid} in ${locale}`, this.importConfig.context); - + const chunk = await fs.readChunkFiles.next().catch((error) => { handleAndLogError(error, { ...this.importConfig.context, cTUid: cTUid, locale }); }); @@ -447,7 +592,7 @@ export default class EntriesImport extends BaseClass { if (chunk) { let apiContent = values(chunk as Record[]); log.debug(`Processing ${apiContent.length} entries in chunk ${index}`, this.importConfig.context); - + await this.makeConcurrentCall({ apiContent, processName, @@ -462,13 +607,13 @@ export default class EntriesImport extends BaseClass { additionalInfo: { contentType, locale, cTUid, entryFileName: indexer[index], isMasterLocale }, }, concurrencyLimit: this.importConcurrency, - }).then(() => { - entriesCreateFileHelper?.completeFile(true); - existingEntriesFileHelper?.completeFile(true); - log.success(`Created entries for content type ${cTUid} in locale ${locale}`, this.importConfig.context); }); } } + + entriesCreateFileHelper?.completeFile(true); + existingEntriesFileHelper?.completeFile(true); + log.success(`Created entries for content type ${cTUid} in locale ${locale}`, this.importConfig.context); } /** @@ -483,8 +628,11 @@ export default class EntriesImport extends BaseClass { } = apiOptions; try { - log.debug(`Serializing entry: ${entry.title} (${entry.uid}) for ${cTUid} in ${locale}`, this.importConfig.context); - + log.debug( + `Serializing entry: ${entry.title} (${entry.uid}) for ${cTUid} in ${locale}`, + this.importConfig.context, + ); + if (this.jsonRteCTs.indexOf(cTUid) > -1) { entry = removeUidsFromJsonRteFields(entry, contentType.schema); log.debug(`Removed UIDs from JSON RTE fields for entry: ${entry.uid}`, this.importConfig.context); @@ -501,7 +649,7 @@ export default class EntriesImport extends BaseClass { //will remove term if term doesn't exists in taxonomy lookUpTerms(contentType?.schema, entry, this.taxonomies, this.importConfig); log.debug(`Processed taxonomy terms for entry: ${entry.uid}`, this.importConfig.context); - + // will replace all old asset uid/urls with new ones entry = lookupAssets( { @@ -514,7 +662,7 @@ export default class EntriesImport extends BaseClass { this.installedExtensions, ); log.debug(`Processed asset lookups for entry: ${entry.uid}`, this.importConfig.context); - + delete entry.publish_details; // checking the entry is a localized one or not if (!isMasterLocale && this.entriesUidMapper.hasOwnProperty(entry.uid)) { @@ -525,7 +673,10 @@ export default class EntriesImport extends BaseClass { isLocalized: true, entryOldUid: entry.uid, }; - log.debug(`Prepared localized entry: ${entry.uid} → ${this.entriesUidMapper[entry.uid]}`, this.importConfig.context); + log.debug( + `Prepared localized entry: ${entry.uid} → ${this.entriesUidMapper[entry.uid]}`, + this.importConfig.context, + ); return apiOptions; } apiOptions.apiData = entry; @@ -549,7 +700,10 @@ export default class EntriesImport extends BaseClass { log.debug(`No existing entries found for replacement in ${cTUid} - ${locale}`, this.importConfig.context); return Promise.resolve(); } - log.debug(`Starting to replace entries for ${cTUid} in locale ${locale} - ${indexerCount} chunks to process`, this.importConfig.context); + log.debug( + `Starting to replace entries for ${cTUid} in locale ${locale} - ${indexerCount} chunks to process`, + this.importConfig.context, + ); // Write updated entries const entriesReplaceFileHelper = new FsUtility({ @@ -566,7 +720,10 @@ export default class EntriesImport extends BaseClass { log.debug(`Found content type schema for replacement: ${cTUid}`, this.importConfig.context); const onSuccess = ({ response, apiData: entry, additionalInfo }: any) => { - log.info(`Replaced entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, this.importConfig.context); + log.info( + `Replaced entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, + this.importConfig.context, + ); log.debug(`Replaced entry UID mapping: ${entry.uid} → ${response.uid}`, this.importConfig.context); this.entriesUidMapper[entry.uid] = response.uid; entriesReplaceFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true }); @@ -588,16 +745,22 @@ export default class EntriesImport extends BaseClass { }; for (const index in indexer) { - log.debug(`Processing replacement chunk ${index} of ${indexerCount} for ${cTUid} in ${locale}`, this.importConfig.context); - + log.debug( + `Processing replacement chunk ${index} of ${indexerCount} for ${cTUid} in ${locale}`, + this.importConfig.context, + ); + const chunk = await fs.readChunkFiles.next().catch((error) => { handleAndLogError(error, { ...this.importConfig.context, cTUid, locale }); }); if (chunk) { let apiContent = values(chunk as Record[]); - log.debug(`Processing ${apiContent.length} entries for replacement in chunk ${index}`, this.importConfig.context); - + log.debug( + `Processing ${apiContent.length} entries for replacement in chunk ${index}`, + this.importConfig.context, + ); + await this.makeConcurrentCall( { apiContent, @@ -614,10 +777,7 @@ export default class EntriesImport extends BaseClass { concurrencyLimit: this.importConcurrency, }, this.replaceEntriesHandler.bind(this), - ).then(() => { - entriesReplaceFileHelper?.completeFile(true); - log.success(`Replaced entries for content type ${cTUid} in locale ${locale}`, this.importConfig.context); - }); + ); } } } @@ -702,7 +862,10 @@ export default class EntriesImport extends BaseClass { log.debug(`No entries found for reference updates in ${cTUid} - ${locale}`, this.importConfig.context); return Promise.resolve(); } - log.debug(`Starting to update entries with references for ${cTUid} in locale ${locale} - ${indexerCount} chunks to process`, this.importConfig.context); + log.debug( + `Starting to update entries with references for ${cTUid} in locale ${locale} - ${indexerCount} chunks to process`, + this.importConfig.context, + ); const contentType = find(this.cTs, { uid: cTUid }); log.debug(`Found content type schema for reference updates: ${cTUid}`, this.importConfig.context); @@ -728,16 +891,22 @@ export default class EntriesImport extends BaseClass { }; for (const index in indexer) { - log.debug(`Processing reference update chunk ${index} of ${indexerCount} for ${cTUid} in ${locale}`, this.importConfig.context); - + log.debug( + `Processing reference update chunk ${index} of ${indexerCount} for ${cTUid} in ${locale}`, + this.importConfig.context, + ); + const chunk = await fs.readChunkFiles.next().catch((error) => { handleAndLogError(error, { ...this.importConfig.context, cTUid, locale }, 'Error'); }); if (chunk) { let apiContent = values(chunk as Record[]); - log.debug(`Processing ${apiContent.length} entries for reference updates in chunk ${index}`, this.importConfig.context); - + log.debug( + `Processing ${apiContent.length} entries for reference updates in chunk ${index}`, + this.importConfig.context, + ); + await this.makeConcurrentCall({ apiContent, processName, @@ -752,8 +921,6 @@ export default class EntriesImport extends BaseClass { additionalInfo: { contentType, locale, cTUid }, }, concurrencyLimit: this.importConcurrency, - }).then(() => { - log.success(`Updated entries for content type ${cTUid} in locale ${locale}`, this.importConfig.context); }); } } @@ -770,18 +937,21 @@ export default class EntriesImport extends BaseClass { additionalInfo: { cTUid, locale, contentType }, } = apiOptions; try { - log.debug(`Serializing entry update: ${entry.title} (${entry.uid}) for ${cTUid} in ${locale}`, this.importConfig.context); - + log.debug( + `Serializing entry update: ${entry.title} (${entry.uid}) for ${cTUid} in ${locale}`, + this.importConfig.context, + ); + const sourceEntryFilePath = entry.sourceEntryFilePath; const sourceEntry = ((fsUtil.readFile(sourceEntryFilePath) || {}) as Record)[entry.entryOldUid]; const newUid = this.entriesUidMapper[entry.entryOldUid]; - + log.debug(`Updating entry references: ${entry.entryOldUid} → ${newUid}`, this.importConfig.context); - + // Removing temp values delete entry.sourceEntryFilePath; delete entry.entryOldUid; - + if (this.jsonRteCTs.indexOf(cTUid) > -1 || this.rteCTs.indexOf(cTUid) > -1) { // the entries stored in eSuccessFilePath, have the same uids as the entries from source data entry = restoreJsonRteEntryRefs(entry, sourceEntry, contentType.schema, { @@ -791,7 +961,7 @@ export default class EntriesImport extends BaseClass { }); log.debug(`Restored JSON RTE entry references for: ${newUid}`, this.importConfig.context); } - + entry = lookupAssets( { content_type: contentType, @@ -803,7 +973,7 @@ export default class EntriesImport extends BaseClass { this.installedExtensions, ); log.debug(`Processed asset lookups for entry update: ${newUid}`, this.importConfig.context); - + entry = lookupEntries( { content_type: contentType, @@ -828,12 +998,21 @@ export default class EntriesImport extends BaseClass { async enableMandatoryCTReferences(): Promise { const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { + this.progressManager?.tick(true, `content type: ${uid}`, null, 'CT Restoration'); log.success(`${uid} content type references updated`, this.importConfig.context); }; + const onReject = ({ error, apiData: { uid } }: any) => { + this.progressManager?.tick( + false, + `content type: ${uid}`, + error?.message || 'Failed to restore content type', + 'CT Restoration', + ); handleAndLogError(error, { ...this.importConfig.context, uid }, 'Error'); throw new Error(`Failed to update references of content type ${uid}`); }; + return await this.makeConcurrentCall({ processName: 'Update content type references', apiContent: this.modifiedCTs, @@ -886,7 +1065,11 @@ export default class EntriesImport extends BaseClass { (item) => !(item.entry_uid === entryUid && item.locale === this.importConfig?.master_locale?.code), ); - handleAndLogError(error, { ...this.importConfig.context }, `Failed to remove auto created entry in master locale - entry uid ${entryUid}`); + handleAndLogError( + error, + { ...this.importConfig.context }, + `Failed to remove auto created entry in master locale - entry uid ${entryUid}`, + ); }; return await this.makeConcurrentCall({ processName: 'Remove auto created entry in master locale', @@ -906,76 +1089,104 @@ export default class EntriesImport extends BaseClass { let cTsWithFieldRules = fsUtil.readFile(path.join(this.cTsPath + '/field_rules_uid.json')) as Record[]; if (!cTsWithFieldRules || cTsWithFieldRules?.length === 0) { log.debug('No content types with field rules found to update', this.importConfig.context); + this.progressManager?.tick(true, 'Field rules update completed (no rules found)', null, 'Field Rules Update'); return; } + log.debug(`Found ${cTsWithFieldRules.length} content types with field rules to update`, this.importConfig.context); - - for (let cTUid of cTsWithFieldRules) { - log.debug(`Processing field rules for content type: ${cTUid}`, this.importConfig.context); - - const cTs: Record[] = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record< - string, - unknown - >[]; - const contentType: any = find(cTs, { uid: cTUid }); - - if (contentType.field_rules) { - log.debug(`Found ${contentType.field_rules.length} field rules for content type: ${cTUid}`, this.importConfig.context); - - const fieldDatatypeMap: { [key: string]: string } = {}; - for (let i = 0; i < contentType.schema?.length; i++) { - const field = contentType.schema[i].uid; - fieldDatatypeMap[field] = contentType.schema[i].data_type; - } - log.debug(`Built field datatype map for ${Object.keys(fieldDatatypeMap).length} fields`, this.importConfig.context); - - let fieldRuleLength = contentType.field_rules?.length; - let updatedRulesCount = 0; - - for (let k = 0; k < fieldRuleLength; k++) { - let fieldRuleConditionLength = contentType.field_rules[k].conditions?.length; - for (let i = 0; i < fieldRuleConditionLength; i++) { - if (fieldDatatypeMap[contentType.field_rules[k].conditions[i].operand_field] === 'reference') { - let fieldRulesValue = contentType.field_rules[k].conditions[i].value; - let fieldRulesArray = fieldRulesValue.split('.'); - let updatedValue = []; - - for (const element of fieldRulesArray) { - let splittedFieldRulesValue = element; - if (this.entriesUidMapper.hasOwnProperty(splittedFieldRulesValue)) { - updatedValue.push(this.entriesUidMapper[splittedFieldRulesValue]); - log.debug(`Updated field rule reference: ${splittedFieldRulesValue} → ${this.entriesUidMapper[splittedFieldRulesValue]}`, this.importConfig.context); - } else { - updatedValue.push(element); + + try { + for (let cTUid of cTsWithFieldRules) { + log.debug(`Processing field rules for content type: ${cTUid}`, this.importConfig.context); + + const cTs: Record[] = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record< + string, + unknown + >[]; + const contentType: any = find(cTs, { uid: cTUid }); + + if (contentType.field_rules) { + log.debug( + `Found ${contentType.field_rules.length} field rules for content type: ${cTUid}`, + this.importConfig.context, + ); + + const fieldDatatypeMap: { [key: string]: string } = {}; + for (let i = 0; i < contentType.schema?.length; i++) { + const field = contentType.schema[i].uid; + fieldDatatypeMap[field] = contentType.schema[i].data_type; + } + + let fieldRuleLength = contentType.field_rules?.length; + let updatedRulesCount = 0; + + for (let k = 0; k < fieldRuleLength; k++) { + let fieldRuleConditionLength = contentType.field_rules[k].conditions?.length; + for (let i = 0; i < fieldRuleConditionLength; i++) { + if (fieldDatatypeMap[contentType.field_rules[k].conditions[i].operand_field] === 'reference') { + let fieldRulesValue = contentType.field_rules[k].conditions[i].value; + let fieldRulesArray = fieldRulesValue.split('.'); + let updatedValue = []; + + for (const element of fieldRulesArray) { + let splittedFieldRulesValue = element; + if (this.entriesUidMapper.hasOwnProperty(splittedFieldRulesValue)) { + updatedValue.push(this.entriesUidMapper[splittedFieldRulesValue]); + log.debug( + `Updated field rule reference: ${splittedFieldRulesValue} → ${this.entriesUidMapper[splittedFieldRulesValue]}`, + this.importConfig.context, + ); + } else { + updatedValue.push(element); + } } + contentType.field_rules[k].conditions[i].value = updatedValue.join('.'); + updatedRulesCount++; } - contentType.field_rules[k].conditions[i].value = updatedValue.join('.'); - updatedRulesCount++; } } - } - - log.debug(`Updated ${updatedRulesCount} field rule references for content type: ${cTUid}`, this.importConfig.context); - - const contentTypeResponse: any = await this.stack - .contentType(contentType.uid) - .fetch() - .catch((error) => { + + log.debug( + `Updated ${updatedRulesCount} field rule references for content type: ${cTUid}`, + this.importConfig.context, + ); + + const contentTypeResponse: any = await this.stack + .contentType(contentType.uid) + .fetch() + .catch((error) => { + handleAndLogError(error, { ...this.importConfig.context, cTUid }); + }); + + if (!contentTypeResponse) { + log.debug(`Skipping field rules update for ${cTUid} - content type not found`, this.importConfig.context); + continue; + } + + contentTypeResponse.field_rules = contentType.field_rules; + await contentTypeResponse.update().catch((error: Error) => { handleAndLogError(error, { ...this.importConfig.context, cTUid }); }); - if (!contentTypeResponse) { - log.debug(`Skipping field rules update for ${cTUid} - content type not found`, this.importConfig.context); - continue; + log.success(`Updated the field rules of ${cTUid}`, this.importConfig.context); + } else { + log.info(`No field rules found in content type ${cTUid} to update`, this.importConfig.context); } - - contentTypeResponse.field_rules = contentType.field_rules; - await contentTypeResponse.update().catch((error: Error) => { - handleAndLogError(error, { ...this.importConfig.context, cTUid }); - }); - log.success(`Updated the field rules of ${cTUid}`, this.importConfig.context); - } else { - log.info(`No field rules found in content type ${cTUid} to update`, this.importConfig.context); } + + this.progressManager?.tick( + true, + `Updated field rules for ${cTsWithFieldRules.length} content types`, + null, + 'Field Rules Update', + ); + } catch (error) { + this.progressManager?.tick( + false, + 'Field rules update', + (error as any)?.message || 'Field rules update failed', + 'Field Rules Update', + ); + throw error; } } @@ -992,23 +1203,53 @@ export default class EntriesImport extends BaseClass { log.debug(`No entries found for publishing in ${cTUid} - ${locale}`, this.importConfig.context); return Promise.resolve(); } - log.debug(`Starting to publish entries for ${cTUid} in locale ${locale} - ${indexerCount} chunks to process`, this.importConfig.context); + log.debug( + `Starting to publish entries for ${cTUid} in locale ${locale} - ${indexerCount} chunks to process`, + this.importConfig.context, + ); const onSuccess = ({ response, apiData: { environments, entryUid, locales }, additionalInfo }: any) => { - log.success(`Published the entry: '${entryUid}' of Content Type '${cTUid}' and Locale '${locale}' in Environments '${environments?.join( - ',')}' and Locales '${locales?.join(',')}'`, this.importConfig.context); - log.debug(`Published entry ${entryUid} to ${environments?.length || 0} environments and ${locales?.length || 0} locales`, this.importConfig.context); + log.success( + `Published the entry: '${entryUid}' of Content Type '${cTUid}' and Locale '${locale}' in Environments '${environments?.join( + ',', + )}' and Locales '${locales?.join(',')}'`, + this.importConfig.context, + ); + log.debug( + `Published entry ${entryUid} to ${environments?.length || 0} environments and ${locales?.length || 0} locales`, + this.importConfig.context, + ); + this.progressManager?.tick( + true, + `Published the entry: '${entryUid}' of Content Type '${cTUid}' and Locale '${locale}`, + null, + 'Publish', + ); }; const onReject = ({ error, apiData: { environments, entryUid, locales }, additionalInfo }: any) => { - handleAndLogError(error, { ...this.importConfig.context, cTUid, locale }, `Failed to publish: '${entryUid}' entry of Content Type '${cTUid}' and Locale '${locale}' in Environments '${environments?.join( - ',')}' and Locales '${locales?.join(',')}'`); + handleAndLogError( + error, + { ...this.importConfig.context, cTUid, locale }, + `Failed to publish: '${entryUid}' entry of Content Type '${cTUid}' and Locale '${locale}' in Environments '${environments?.join( + ',', + )}' and Locales '${locales?.join(',')}'`, + ); + this.progressManager?.tick( + false, + `Failed to publish: '${entryUid}' entry of Content Type '${cTUid}' and Locale '${locale}'`, + `Failed to publish: '${entryUid}' entry of Content Type '${cTUid}' and Locale '${locale}'`, + 'Publish', + ); }; for (const index in indexer) { - log.debug(`Processing publish chunk ${index} of ${indexerCount} for ${cTUid} in ${locale}`, this.importConfig.context); - + log.debug( + `Processing publish chunk ${index} of ${indexerCount} for ${cTUid} in ${locale}`, + this.importConfig.context, + ); + const chunk = await fs.readChunkFiles.next().catch((error) => { - handleAndLogError(error, { ...this.importConfig.context, cTUid, locale }, ); + handleAndLogError(error, { ...this.importConfig.context, cTUid, locale }); }); if (chunk) { @@ -1025,9 +1266,9 @@ export default class EntriesImport extends BaseClass { return []; // Return an empty array if publish_details is empty }); apiContent = apiContentDuplicate; - + log.debug(`Processing ${apiContent.length} publishable entries in chunk ${index}`, this.importConfig.context); - + if (apiContent?.length === 0) { log.debug(`No publishable entries found in chunk ${index}`, this.importConfig.context); continue; @@ -1046,8 +1287,6 @@ export default class EntriesImport extends BaseClass { additionalInfo: { contentType, locale, cTUid }, }, concurrencyLimit: this.importConcurrency, - }).then(() => { - log.success(`Published entries for content type ${cTUid} in locale ${locale}`, this.importConfig.context); }); } } @@ -1093,4 +1332,4 @@ export default class EntriesImport extends BaseClass { apiOptions.apiData = requestObject; return apiOptions; } -} \ No newline at end of file +} diff --git a/packages/contentstack-import/src/import/modules/extensions.ts b/packages/contentstack-import/src/import/modules/extensions.ts index 0c1c09da52..0cec3980e2 100644 --- a/packages/contentstack-import/src/import/modules/extensions.ts +++ b/packages/contentstack-import/src/import/modules/extensions.ts @@ -71,10 +71,10 @@ export default class ImportExtensions extends BaseClass { this.updateUidExtension(); if (this.importConfig.replaceExisting && this.existingExtensions.length > 0) { - progress.addProcess('Update', this.existingExtensions.length); - progress.startProcess('Update').updateStatus('Updating existing extensions...', 'Update'); + progress.addProcess('Replace existing', this.existingExtensions.length); + progress.startProcess('Replace existing').updateStatus('Updating existing extensions...', 'Replace existing'); await this.replaceExtensions(); - progress.completeProcess('Update', true); + progress.completeProcess('Replace existing', true); } await this.processExtensionResults(); @@ -167,7 +167,7 @@ export default class ImportExtensions extends BaseClass { const onSuccess = ({ response, apiData: { uid, title } = { uid: null, title: '' } }: any) => { this.extSuccess.push(response); this.extUidMapper[uid] = response.uid; - this.progressManager?.tick(true, `extension: ${title || uid} (updated)`, null, 'Update'); + this.progressManager?.tick(true, `extension: ${title || uid} (updated)`, null, 'Replace existing'); log.success(`Extension '${title}' updated successfully`, this.importConfig.context); log.debug(`Extension update completed: ${title} (${uid})`, this.importConfig.context); fsUtil.writeFile(this.extUidMapperPath, this.extUidMapper); @@ -180,7 +180,7 @@ export default class ImportExtensions extends BaseClass { false, `extension: ${title || uid}`, error?.message || 'Failed to update extension', - 'Update', + 'Replace existing', ); log.debug(`Extension '${title}' update failed`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, title }, `Extension '${title}' failed to be updated`); diff --git a/packages/contentstack-import/src/import/modules/global-fields.ts b/packages/contentstack-import/src/import/modules/global-fields.ts index caacfdad2a..ad72e22f05 100644 --- a/packages/contentstack-import/src/import/modules/global-fields.ts +++ b/packages/contentstack-import/src/import/modules/global-fields.ts @@ -107,13 +107,13 @@ export default class ImportGlobalFields extends BaseClass { // Step 3: Replace existing global fields if needed if (this.importConfig.replaceExisting && this.existingGFs.length > 0) { - progress.addProcess('Global Fields Replacement', this.existingGFs.length); + progress.addProcess('Replace Existing', this.existingGFs.length); progress - .startProcess('Global Fields Replacement') - .updateStatus('Replacing existing global fields...', 'Global Fields Replacement'); - log.info('Starting global fields replacement process', this.importConfig.context); + .startProcess('Replace Existing') + .updateStatus('Replacing existing global fields...', 'Replace Existing'); + log.info('Starting Replace Existing process', this.importConfig.context); await this.replaceGFs(); - progress.completeProcess('Global Fields Replacement', true); + progress.completeProcess('Replace Existing', true); } await this.processGlobalFieldResults(); @@ -315,7 +315,7 @@ export default class ImportGlobalFields extends BaseClass { const uid = apiData?.uid ?? apiData?.global_field?.uid ?? 'unknown'; this.createdGFs.push(globalField); this.gFsUidMapper[uid] = globalField; - this.progressManager?.tick(true, `global field: ${uid} (replaced)`, null, 'Global Fields Replacement'); + this.progressManager?.tick(true, `global field: ${uid} (replaced)`, null, 'Replace Existing'); fsUtil.writeFile(this.gFsUidMapperPath, this.gFsUidMapper); log.success(`Global field '${uid}' replaced successfully`, this.importConfig.context); log.debug(`Global field replacement completed: ${uid}`, this.importConfig.context); @@ -327,7 +327,7 @@ export default class ImportGlobalFields extends BaseClass { false, `global field: ${uid}`, error?.message || 'Failed to replace global field', - 'Global Fields Replacement', + 'Replace Existing', ); log.debug(`Global field '${uid}' replacement failed`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, uid }, `Global fields '${uid}' failed to replace`); @@ -357,7 +357,7 @@ export default class ImportGlobalFields extends BaseClass { false, ); - log.debug('Global fields replacement process completed', this.importConfig.context); + log.debug('Replace Existing process completed', this.importConfig.context); } /** diff --git a/packages/contentstack-import/src/import/modules/index.ts b/packages/contentstack-import/src/import/modules/index.ts index 388e5e9c68..ed48efaf92 100644 --- a/packages/contentstack-import/src/import/modules/index.ts +++ b/packages/contentstack-import/src/import/modules/index.ts @@ -1,4 +1,5 @@ import { ModuleClassParams } from '../../types'; +import '../../utils/strategy-registrations'; export default async function startModuleImport(modulePayload: ModuleClassParams) { const { default: ModuleRunner } = await import(`./${modulePayload.moduleName}`); diff --git a/packages/contentstack-import/src/import/modules/labels.ts b/packages/contentstack-import/src/import/modules/labels.ts index dc6ac47cf9..1cea3d8dda 100644 --- a/packages/contentstack-import/src/import/modules/labels.ts +++ b/packages/contentstack-import/src/import/modules/labels.ts @@ -1,10 +1,10 @@ -import isEmpty from 'lodash/isEmpty'; -import values from 'lodash/values'; import omit from 'lodash/omit'; import { join } from 'node:path'; +import isEmpty from 'lodash/isEmpty'; +import values from 'lodash/values'; +import { log, handleAndLogError } from '@contentstack/cli-utilities'; import { fsUtil, fileHelper } from '../../utils'; -import { log, handleAndLogError } from '@contentstack/cli-utilities'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, LabelConfig } from '../../types'; @@ -23,6 +23,7 @@ export default class ImportLabels extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'labels'; + this.currentModuleName = 'Labels'; this.labelsConfig = importConfig.modules.labels; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'labels'); this.labelsFolderPath = join(this.importConfig.backupDir, this.labelsConfig.dirName); @@ -40,89 +41,81 @@ export default class ImportLabels extends BaseClass { * @returns {Promise} Promise */ async start(): Promise { - log.debug('Checking for labels folder existence', this.importConfig.context); - - //Step1 check folder exists or not - if (fileHelper.fileExistsSync(this.labelsFolderPath)) { - log.debug(`Found labels folder: ${this.labelsFolderPath}`, this.importConfig.context); - this.labels = fsUtil.readFile(join(this.labelsFolderPath, 'labels.json'), true) as Record; - - // Check if labels file was read successfully - if (!this.labels) { - log.info( - `No labels found in file - '${join(this.labelsFolderPath, 'labels.json')}'`, - this.importConfig.context, - ); + try { + log.debug('Starting labels import process...', this.importConfig.context); + const [labelsCount] = await this.analyzeLabels(); + if (labelsCount === 0) { + log.info('No labels found to import', this.importConfig.context); return; } - const labelCount = Object.keys(this.labels || {}).length; - log.debug(`Loaded ${labelCount} label items from file`, this.importConfig.context); - } else { - log.info(`No labels found - '${this.labelsFolderPath}'`, this.importConfig.context); - return; - } + const progress = this.createNestedProgress(this.currentModuleName); + progress.addProcess('Create', labelsCount); + progress.addProcess('Update', labelsCount); - //create labels in mapper directory - log.debug('Creating labels mapper directory', this.importConfig.context); - await fsUtil.makeDirectory(this.mapperDirPath); - log.debug('Loading existing label UID mappings', this.importConfig.context); - this.labelUidMapper = fileHelper.fileExistsSync(this.labelUidMapperPath) - ? (fsUtil.readFile(join(this.labelUidMapperPath), true) as Record) || {} - : {}; + await this.prepareLabelMapper(); - if (Object.keys(this.labelUidMapper || {}).length > 0) { - const labelUidCount = Object.keys(this.labelUidMapper || {}).length; - log.debug(`Loaded existing label UID data: ${labelUidCount} items`, this.importConfig.context); - } else { - log.debug('No existing label UID mappings found', this.importConfig.context); - } + // Step 1: Import labels (without parent references) + progress.startProcess('Create').updateStatus('Creating labels...', 'Create'); + log.info('Starting labels creation process', this.importConfig.context); + await this.importLabels(); + progress.completeProcess('Create', true); - log.debug('Starting labels import', this.importConfig.context); - await this.importLabels(); - //update parent in created label - log.debug('Starting labels update process', this.importConfig.context); - await this.updateLabels(); + // Step 2: Update labels with parent references + progress.startProcess('Update').updateStatus('Updating labels with parent references...', 'Update'); + log.info('Starting labels update process', this.importConfig.context); + await this.updateLabels(); + progress.completeProcess('Update', true); - log.debug('Processing labels import results', this.importConfig.context); - if (this.createdLabel?.length) { - fsUtil.writeFile(this.createdLabelPath, this.createdLabel); - log.debug(`Written ${this.createdLabel.length} successful labels to file`, this.importConfig.context); - } + this.processLabelResults(); - if (this.failedLabel?.length) { - fsUtil.writeFile(this.labelFailsPath, this.failedLabel); - log.debug(`Written ${this.failedLabel.length} failed labels to file`, this.importConfig.context); + this.completeProgress(true); + log.success('Labels have been imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Labels import failed'); + handleAndLogError(error, { ...this.importConfig.context }); } - - log.success('Labels have been imported successfully!', this.importConfig.context); } async importLabels() { + log.debug('Validating labels data', this.importConfig.context); if (this.labels === undefined || isEmpty(this.labels)) { log.info('No Labels Found', this.importConfig.context); return; } const apiContent = values(this.labels); + log.debug(`Starting to import ${apiContent.length} labels`, this.importConfig.context); const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }: any) => { this.labelUidMapper[uid] = response; + this.progressManager?.tick(true, `label: ${name || uid}`, null, 'Create'); log.success(`Label '${name}' imported successfully`, this.importConfig.context); + log.debug(`Label UID mapping: ${uid} → ${response.uid}`, this.importConfig.context); fsUtil.writeFile(this.labelUidMapperPath, this.labelUidMapper); }; const onReject = ({ error, apiData }: any) => { const err = error?.message ? JSON.parse(error.message) : error; - const { name } = apiData; + const { name, uid } = apiData; + log.debug(`Label '${name}' (${uid}) failed to import`, this.importConfig.context); + if (err?.errors?.name) { + this.progressManager?.tick(true, `label: ${name || uid} (already exists)`, null, 'Create'); log.info(`Label '${name}' already exists`, this.importConfig.context); } else { this.failedLabel.push(apiData); + this.progressManager?.tick( + false, + `label: ${name || uid}`, + error?.message || 'Failed to import label', + 'Create', + ); handleAndLogError(error, { ...this.importConfig.context, name }, `Label '${name}' failed to be import`); } }; + log.debug(`Using concurrency limit: ${this.importConfig.fetchConcurrency || 1}`, this.importConfig.context); await this.makeConcurrentCall( { apiContent, @@ -139,6 +132,8 @@ export default class ImportLabels extends BaseClass { undefined, false, ); + + log.debug('Labels creation process completed', this.importConfig.context); } /** @@ -153,6 +148,7 @@ export default class ImportLabels extends BaseClass { if (this.labelUidMapper.hasOwnProperty(label.uid)) { log.info(`Label '${label.name}' already exists. Skipping it to avoid duplicates!`, this.importConfig.context); log.debug(`Skipping label serialization for: ${label.uid}`, this.importConfig.context); + this.progressManager?.tick(true, `label: ${label.name} (skipped - already exists)`, null, 'Create'); apiOptions.entity = undefined; } else { let labelReq = label; @@ -176,17 +172,21 @@ export default class ImportLabels extends BaseClass { const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }: any) => { this.createdLabel.push(response); + this.progressManager?.tick(true, `label: ${name || uid}`, null, 'Update'); log.success(`Label '${name}' updated successfully`, this.importConfig.context); log.debug(`Label update completed: ${name} (${uid})`, this.importConfig.context); }; const onReject = ({ error, apiData }: any) => { - log.debug(`Label '${apiData?.name}' update failed`, this.importConfig.context); - handleAndLogError( - error, - { ...this.importConfig.context, name: apiData?.name }, - `Failed to update label '${apiData?.name}'`, + const { name, uid } = apiData; + this.progressManager?.tick( + false, + `label: ${name || uid}`, + error?.message || 'Failed to update label', + 'Update', ); + log.debug(`Label '${name}' update failed`, this.importConfig.context); + handleAndLogError(error, { ...this.importConfig.context, name: name }, `Failed to update label '${name}'`); }; log.debug( @@ -247,13 +247,73 @@ export default class ImportLabels extends BaseClass { log.debug(`Updated label '${label.name}' with parent references`, this.importConfig.context); } else { log.debug(`Label '${label.name}' has no parent labels, adding to created list`, this.importConfig.context); + this.progressManager?.tick(true, `label: ${label.name} (no parent update needed)`, null, 'Update'); apiOptions.entity = undefined; this.createdLabel.push(newLabel); } } else { log.debug(`Label '${label.name}' not found in UID mapper, skipping update`, this.importConfig.context); + this.progressManager?.tick(true, `label: ${label.name} (skipped - not found)`, null, 'Update'); apiOptions.entity = undefined; } return apiOptions; } + + private async analyzeLabels(): Promise<[number]> { + return this.withLoadingSpinner('LABELS: Analyzing import data...', async () => { + log.debug('Checking for labels folder existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.labelsFolderPath)) { + log.info(`No labels found - '${this.labelsFolderPath}'`, this.importConfig.context); + return [0]; + } + + log.debug(`Found labels folder: ${this.labelsFolderPath}`, this.importConfig.context); + + this.labels = fsUtil.readFile(join(this.labelsFolderPath, 'labels.json'), true) as Record; + + if (!this.labels) { + log.info( + `No labels found in file - '${join(this.labelsFolderPath, 'labels.json')}'`, + this.importConfig.context, + ); + return [0]; + } + + const count = Object.keys(this.labels || {}).length; + log.debug(`Loaded ${count} label items from file`, this.importConfig.context); + return [count]; + }); + } + + private async prepareLabelMapper(): Promise { + log.debug('Creating labels mapper directory', this.importConfig.context); + await fsUtil.makeDirectory(this.mapperDirPath); + + log.debug('Loading existing label UID mappings', this.importConfig.context); + this.labelUidMapper = fileHelper.fileExistsSync(this.labelUidMapperPath) + ? (fsUtil.readFile(join(this.labelUidMapperPath), true) as Record) || {} + : {}; + + const count = Object.keys(this.labelUidMapper || {}).length; + if (count > 0) { + log.debug(`Loaded existing label UID data: ${count} items`, this.importConfig.context); + } else { + log.debug('No existing label UID mappings found', this.importConfig.context); + } + } + + private processLabelResults() { + log.debug('Processing labels import results', this.importConfig.context); + + if (this.createdLabel?.length) { + fsUtil.writeFile(this.createdLabelPath, this.createdLabel); + log.debug(`Written ${this.createdLabel.length} successful labels to file`, this.importConfig.context); + } + + if (this.failedLabel?.length) { + fsUtil.writeFile(this.labelFailsPath, this.failedLabel); + log.debug(`Written ${this.failedLabel.length} failed labels to file`, this.importConfig.context); + } + } } diff --git a/packages/contentstack-import/src/import/modules/locales.ts b/packages/contentstack-import/src/import/modules/locales.ts index ee0e73a4bc..c262d007c6 100644 --- a/packages/contentstack-import/src/import/modules/locales.ts +++ b/packages/contentstack-import/src/import/modules/locales.ts @@ -121,21 +121,16 @@ export default class ImportLocales extends BaseClass { log.debug(`Creating ${languagesToCreate.length} locales (excluding master locale)`, this.config.context); const onSuccess = ({ response = {}, apiData: { uid, code } = undefined }: any) => { - this.createdLocales.push(response.uid); this.langUidMapper[uid] = response.uid; - this.progressManager?.tick(true, `locale: ${code}`, null, 'Locale Create'); + this.createdLocales.push(pick(response, [...this.localeConfig.requiredKeys])); + this.progressManager?.tick(true, `locale: ${code}`, null, 'Create'); log.info(`Created locale: '${code}'`, this.config.context); log.debug(`Locale UID mapping: ${uid} → ${response.uid}`, this.config.context); fsUtil.writeFile(this.langUidMapperPath, this.langUidMapper); }; const onReject = ({ error, apiData: { uid, code } = undefined }: any) => { - this.progressManager?.tick( - false, - `locale: ${code}`, - error?.message || 'Failed to create locale', - 'Locale Create', - ); + this.progressManager?.tick(false, `locale: ${code}`, error?.message || 'Failed to create locale', 'Create'); if (error?.errorCode === 247) { log.info(formatError(error), this.config.context); } else { @@ -164,10 +159,12 @@ export default class ImportLocales extends BaseClass { const onSuccess = ({ response = {}, apiData: { uid, code } = undefined }: any) => { log.info(`Updated locale: '${code}'`, this.config.context); log.debug(`Locale update completed for: ${code}`, this.config.context); + this.progressManager?.tick(true, `locale: ${code}`, null, 'Update'); fsUtil.writeFile(this.langSuccessPath, this.createdLocales); }; const onReject = ({ error, apiData: { uid, code } = undefined }: any) => { + this.progressManager?.tick(false, `locale: ${code}`, 'Failed to update locale', 'Update'); log.error(`Language '${code}' failed to update`, this.config.context); handleAndLogError(error, { ...this.config.context, code }); fsUtil.writeFile(this.langFailsPath, this.failedLocales); @@ -216,8 +213,8 @@ export default class ImportLocales extends BaseClass { const progress = this.createNestedProgress(this.currentModuleName); progress.addProcess('Master Locale ', 1); if (localesCount > 0) { - progress.addProcess('Locale Create', localesCount); - progress.addProcess('Locale Update', localesCount); + progress.addProcess('Create', localesCount); + progress.addProcess('Update', localesCount); } return progress; } @@ -251,27 +248,27 @@ export default class ImportLocales extends BaseClass { } private async processLocaleCreation(progress: any): Promise { - progress.startProcess('Locale Create').updateStatus('Creating locales...', 'Locale Create'); + progress.startProcess('Create').updateStatus('Creating locales...', 'Create'); log.debug('Creating locales', this.config.context); try { await this.createLocales(); - progress.completeProcess('Locale Create', true); + progress.completeProcess('Create', true); } catch (error) { - progress.completeProcess('Locale Create', false); + progress.completeProcess('Create', false); throw error; } } private async processLocaleUpdate(progress: any): Promise { - progress.startProcess('Locale Update').updateStatus('Updating locales...', 'Locale Update'); + progress.startProcess('Update').updateStatus('Updating locales...', 'Update'); log.debug('Updating locales', this.config.context); try { await this.updateLocales(); - progress.completeProcess('Locale Update', true); + progress.completeProcess('Update', true); } catch (error) { - progress.completeProcess('Locale Update', false); + progress.completeProcess('Update', false); throw error; } } @@ -304,11 +301,14 @@ export default class ImportLocales extends BaseClass { const message = `master locale: codes differ (${sourceCode} vs ${targetCode})`; this.tickProgress(true, message); - log.debug(`Master language codes do not match. Source: ${sourceCode}, Target: ${targetCode}`, this.config.context); + log.debug( + `Master Locale language codes do not match. Source: ${sourceCode}, Target: ${targetCode}`, + this.config.context, + ); } private async handleNameMismatch(source: Record, target: Record): Promise { - log.debug('Master language name differs between source and destination', this.config.context); + log.debug('Master Locale language name differs between source and destination', this.config.context); log.debug(`Current: ${target.name}, Source: ${source.name}`, this.config.context); cliux.print('WARNING!!! The master language name for the source and destination is different.', { @@ -317,8 +317,8 @@ export default class ImportLocales extends BaseClass { cliux.print('WARNING!!! The master language name for the source and destination is different.', { color: 'yellow', }); - cliux.print(`Old Master language name: ${target.name}`, { color: 'red' }); - cliux.print(`New Master language name: ${source.name}`, { color: 'green' }); + cliux.print(`Old Master Locale language name: ${target.name}`, { color: 'red' }); + cliux.print(`New Master Locale language name: ${source.name}`, { color: 'green' }); const langUpdateConfirmation: boolean = await cliux.inquire({ type: 'confirm', @@ -328,7 +328,7 @@ export default class ImportLocales extends BaseClass { if (!langUpdateConfirmation) { this.tickProgress(true, `${target.name} (skipped update)`); - log.info('Master language update cancelled by user', this.config.context); + log.info('Master Locale language update cancelled by user', this.config.context); return; } diff --git a/packages/contentstack-import/src/import/modules/marketplace-apps.ts b/packages/contentstack-import/src/import/modules/marketplace-apps.ts index 7db4b024be..056e6acc0e 100644 --- a/packages/contentstack-import/src/import/modules/marketplace-apps.ts +++ b/packages/contentstack-import/src/import/modules/marketplace-apps.ts @@ -36,9 +36,9 @@ import { getConfirmationToCreateApps, getDeveloperHubUrl, } from '../../utils'; +import BaseClass from './base-class'; -export default class ImportMarketplaceApps { - public importConfig: ImportConfig; +export default class ImportMarketplaceApps extends BaseClass { private mapperDirPath: string; private marketPlaceFolderPath: string; private marketPlaceUidMapperPath: string; @@ -54,9 +54,10 @@ export default class ImportMarketplaceApps { public appSdk: ContentstackMarketplaceClient; public existingNames: Set; - constructor({ importConfig }: ModuleClassParams) { - this.importConfig = importConfig; + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'marketplace-apps'; + this.currentModuleName = 'Marketplace Apps'; this.marketPlaceAppConfig = importConfig.modules.marketplace_apps; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'marketplace_apps'); this.marketPlaceFolderPath = join(this.importConfig.backupDir, this.marketPlaceAppConfig.dirName); @@ -74,56 +75,63 @@ export default class ImportMarketplaceApps { * @returns The function `start()` returns a `Promise`. */ async start(): Promise { - log.debug('Checking for marketplace apps folder existence', this.importConfig.context); - - if (fileHelper.fileExistsSync(this.marketPlaceFolderPath)) { - log.debug(`Found marketplace apps folder: ${this.marketPlaceFolderPath}`, this.importConfig.context); - this.marketplaceApps = fsUtil.readFile( - join(this.marketPlaceFolderPath, this.marketPlaceAppConfig.fileName), - true, - ) as Installation[]; - log.debug(`Found ${this.marketplaceApps?.length || 0} marketplace apps to import`, this.importConfig.context); - } else { - log.info(`No Marketplace apps are found - '${this.marketPlaceFolderPath}'`, this.importConfig.context); - return; - } - - if (isEmpty(this.marketplaceApps)) { - log.debug('No marketplace apps found to import', this.importConfig.context); - return Promise.resolve(); - } else if (!isAuthenticated()) { - cliux.print( - '\nWARNING!!! To import Marketplace apps, you must be logged in. Please check csdx auth:login --help to log in\n', - { color: 'yellow' }, - ); - log.info('Skipping marketplace apps import - user not authenticated', this.importConfig.context); - return Promise.resolve(); - } + try { + log.debug('Starting marketplace apps import process...', this.importConfig.context); - log.debug('Creating marketplace apps mapper directory', this.importConfig.context); - await fsUtil.makeDirectory(this.mapperDirPath); - log.debug('Created marketplace apps mapper directory', this.importConfig.context); + const [marketplaceAppsCount] = await this.analyzeMarketplaceApps(); + if (marketplaceAppsCount === 0) { + log.info('No marketplace apps found to import', this.importConfig.context); + return; + } - log.debug('Getting developer hub base URL', this.importConfig.context); - this.developerHubBaseUrl = this.importConfig.developerHubBaseUrl || (await getDeveloperHubUrl(this.importConfig)); - this.importConfig.developerHubBaseUrl = this.developerHubBaseUrl; - log.debug(`Using developer hub base URL: ${this.developerHubBaseUrl}`, this.importConfig.context); + if (!isAuthenticated()) { + cliux.print( + '\nWARNING!!! To import Marketplace apps, you must be logged in. Please check csdx auth:login --help to log in\n', + { color: 'yellow' }, + ); + log.info('Skipping marketplace apps import - user not authenticated', this.importConfig.context); + return; + } - // NOTE init marketplace app sdk - log.debug('Initializing marketplace SDK client', this.importConfig.context); - const host = this.developerHubBaseUrl.split('://').pop(); - this.appSdk = await marketplaceSDKClient({ host }); - log.debug('Initialized marketplace SDK client', this.importConfig.context); + const progress = this.createNestedProgress(this.currentModuleName); + const privateAppsCount = filter(this.marketplaceApps, { manifest: { visibility: 'private' } }).length; - log.debug('Getting organization UID', this.importConfig.context); - this.importConfig.org_uid = await getOrgUid(this.importConfig); - log.debug(`Using organization UID: ${this.importConfig.org_uid}`, this.importConfig.context); + progress.addProcess('Setup Environment', 1); + if (privateAppsCount > 0) { + progress.addProcess('Create Apps', privateAppsCount); + } + progress.addProcess('Install', marketplaceAppsCount); + + await this.prepareMarketplaceAppMapper(); + + // Step 1: Setup Environment SDK and authentication + progress.startProcess('Setup Environment').updateStatus('Setting up marketplace SDK and authentication...', 'Setup Environment'); + log.info('Setting up marketplace SDK and authentication', this.importConfig.context); + await this.setupMarketplaceEnvironment(); + progress.completeProcess('Setup Environment', true); + + // Step 2: Handle private apps creation (if any) + if (privateAppsCount > 0) { + progress + .startProcess('Create Apps') + .updateStatus('Creating private apps...', 'Create Apps'); + log.info('Starting private apps creation process', this.importConfig.context); + await this.handleAllPrivateAppsCreationProcess(); + progress.completeProcess('Create Apps', true); + } - // NOTE start the marketplace import process - log.debug('Starting marketplace apps import process', this.importConfig.context); - await this.importMarketplaceApps(); + // Step 3: Install marketplace apps + progress.startProcess('Install').updateStatus('Installing marketplace apps...', 'Install'); + log.info('Starting marketplace apps installation process', this.importConfig.context); + await this.importMarketplaceApps(); + progress.completeProcess('Install', true); - log.success('Marketplace apps have been imported successfully!', this.importConfig.context); + this.completeProgress(true); + log.success('Marketplace apps have been imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Marketplace apps import failed'); + handleAndLogError(error, { ...this.importConfig.context }); + } } /** @@ -143,10 +151,6 @@ export default class ImportMarketplaceApps { await this.getAndValidateEncryptionKey(this.importConfig.marketplaceAppEncryptionKey); } - // NOTE install all private apps which is not available for stack. - log.debug('Handling private apps creation process', this.importConfig.context); - await this.handleAllPrivateAppsCreationProcess(); - // NOTE getting all apps to validate if it's already installed in the stack to manage conflict log.debug('Getting all stack-specific apps for validation', this.importConfig.context); this.installedApps = await getAllStackSpecificApps(this.importConfig); @@ -318,6 +322,12 @@ export default class ImportMarketplaceApps { if (await this.isPrivateAppExistInDeveloperHub(app)) { // NOTE Found app already exist in the same org this.appUidMapping[app.uid] = app.uid; + this.progressManager?.tick( + true, + `${app.manifest.name} (already exists)`, + null, + 'Create Apps', + ); cliux.print(`App '${app.manifest.name}' already exist. skipping app recreation.!`, { color: 'yellow' }); log.debug(`App '${app.manifest.name}' already exists, skipping recreation`, this.importConfig.context); continue; @@ -347,6 +357,15 @@ export default class ImportMarketplaceApps { log.success(`Completed processing ${privateApps.length} private apps`, this.importConfig.context); } else { log.info('Skipping private apps creation on Developer Hub...', this.importConfig.context); + // Mark all private apps as skipped in progress + for (let app of privateApps) { + this.progressManager?.tick( + true, + `${app.manifest.name} (creation skipped)`, + null, + 'Create Apps', + ); + } } this.appOriginalName = undefined; @@ -515,6 +534,7 @@ export default class ImportMarketplaceApps { log.debug(`Retrying app creation with updated name: ${updatedApp.name}`, this.importConfig.context); return this.createPrivateApp(updatedApp, appSuffix + 1, true); } else { + this.progressManager?.tick(false, `${app.name}`, message, 'Create Apps'); trace(response, 'error', true); log.error(formatError(message), this.importConfig.context); @@ -539,12 +559,19 @@ export default class ImportMarketplaceApps { } } else if (response.uid) { // NOTE new app installation + this.progressManager?.tick(true, `${response.name}`, null, 'Create Apps'); log.success(`${response.name} app created successfully.!`, this.importConfig.context); log.debug(`App UID mapping: ${app.uid} → ${response.uid}`, this.importConfig.context); this.appUidMapping[app.uid] = response.uid; this.appNameMapping[this.appOriginalName] = response.name; log.debug(`App name mapping: ${this.appOriginalName} → ${response.name}`, this.importConfig.context); } else { + this.progressManager?.tick( + false, + `${app.name}`, + 'Unexpected response format', + 'Create Apps', + ); log.debug(`Unexpected response format for app: ${app.name}`, this.importConfig.context); } } @@ -566,6 +593,12 @@ export default class ImportMarketplaceApps { log.debug(`App not found in current stack, installing new app: ${app.manifest?.name}`, this.importConfig.context); // NOTE install new app if (app.manifest.visibility === 'private' && !this.importConfig.canCreatePrivateApp) { + this.progressManager?.tick( + true, + `${app.manifest.name} (skipped - private app not allowed)`, + null, + 'Install', + ); log.info(`Skipping the installation of the private app ${app.manifest.name}...`, this.importConfig.context); return Promise.resolve(); } @@ -582,6 +615,7 @@ export default class ImportMarketplaceApps { if (installation.installation_uid) { const appName = this.appNameMapping[app.manifest.name] || app.manifest.name || app.manifest.uid; + this.progressManager?.tick(true, `${appName}`, null, 'Install'); log.success(`${appName} app installed successfully.!`, this.importConfig.context); log.debug(`Installation UID: ${installation.installation_uid}`, this.importConfig.context); @@ -592,16 +626,24 @@ export default class ImportMarketplaceApps { log.debug(`Installation UID mapping: ${app.uid} → ${installation.installation_uid}`, this.importConfig.context); updateParam = { manifest: app.manifest, ...installation, configuration, server_configuration }; } else if (installation.message) { + this.progressManager?.tick(false, `${app.manifest?.name}`, installation.message, 'Install'); log.info(formatError(installation.message), this.importConfig.context); log.debug(`Installation failed for app: ${app.manifest?.name}`, this.importConfig.context); await confirmToCloseProcess(installation, this.importConfig); } } else if (!isEmpty(configuration) || !isEmpty(server_configuration)) { const appName = app.manifest.name || app.manifest.uid; + this.progressManager?.tick( + true, + `${appName} (already installed, updating config)`, + null, + 'Install', + ); log.info(`${appName} is already installed`, this.importConfig.context); log.debug(`Handling existing app configuration for: ${appName}`, this.importConfig.context); updateParam = await ifAppAlreadyExist(app, currentStackApp, this.importConfig); } else { + this.progressManager?.tick(true, `${app.manifest?.name} (already installed)`, null, 'Install'); log.debug( `App ${app.manifest?.name} is already installed with no configuration to update`, this.importConfig.context, @@ -680,4 +722,58 @@ export default class ImportMarketplaceApps { }); } } + + private async analyzeMarketplaceApps(): Promise<[number]> { + return this.withLoadingSpinner('MARKETPLACE APPS: Analyzing import data...', async () => { + log.debug('Checking for marketplace apps folder existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.marketPlaceFolderPath)) { + log.info(`No Marketplace apps are found - '${this.marketPlaceFolderPath}'`, this.importConfig.context); + return [0]; + } + + log.debug(`Found marketplace apps folder: ${this.marketPlaceFolderPath}`, this.importConfig.context); + + this.marketplaceApps = fsUtil.readFile( + join(this.marketPlaceFolderPath, this.marketPlaceAppConfig.fileName), + true, + ) as Installation[]; + + if (isEmpty(this.marketplaceApps)) { + log.debug('No marketplace apps found to import', this.importConfig.context); + return [0]; + } + + const count = this.marketplaceApps?.length || 0; + log.debug(`Found ${count} marketplace apps to import`, this.importConfig.context); + return [count]; + }); + } + + private async prepareMarketplaceAppMapper(): Promise { + log.debug('Creating marketplace apps mapper directory', this.importConfig.context); + fsUtil.makeDirectory(this.mapperDirPath); + log.debug('Created marketplace apps mapper directory', this.importConfig.context); + } + + private async setupMarketplaceEnvironment(): Promise { + try { + log.debug('Getting developer hub base URL', this.importConfig.context); + this.developerHubBaseUrl = this.importConfig.developerHubBaseUrl || (await getDeveloperHubUrl(this.importConfig)); + this.importConfig.developerHubBaseUrl = this.developerHubBaseUrl; + log.debug(`Using developer hub base URL: ${this.developerHubBaseUrl}`, this.importConfig.context); + + // NOTE init marketplace app sdk + log.debug('Initializing marketplace SDK client', this.importConfig.context); + const host = this.developerHubBaseUrl.split('://').pop(); + this.appSdk = await marketplaceSDKClient({ host }); + log.debug('Initialized marketplace SDK client', this.importConfig.context); + + log.debug('Getting organization UID', this.importConfig.context); + this.importConfig.org_uid = await getOrgUid(this.importConfig); + log.debug(`Using organization UID: ${this.importConfig.org_uid}`, this.importConfig.context); + } catch (error) { + throw error; + } + } } diff --git a/packages/contentstack-import/src/import/modules/personalize.ts b/packages/contentstack-import/src/import/modules/personalize.ts index 01295376a9..3e4444b9fe 100644 --- a/packages/contentstack-import/src/import/modules/personalize.ts +++ b/packages/contentstack-import/src/import/modules/personalize.ts @@ -1,85 +1,61 @@ import { Import } from '@contentstack/cli-variants'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import BaseClass from './base-class'; import { ImportConfig, ModuleClassParams } from '../../types'; -export default class ImportPersonalize { +export default class ImportPersonalize extends BaseClass { private config: ImportConfig; public personalizeConfig: ImportConfig['modules']['personalize']; - constructor({ importConfig }: ModuleClassParams) { + private readonly moduleDisplayMapper = { + events: 'Events', + attributes: 'Attributes', + audiences: 'Audiences', + experiences: 'Experiences', + }; + + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); this.config = importConfig; this.config.context.module = 'personalize'; + this.currentModuleName = 'Personalize'; this.personalizeConfig = importConfig.modules.personalize; } /** - * The `start` function in TypeScript asynchronously imports data based on a specified order using a - * module mapper. + * @method start + * @returns {Promise} Promise */ async start(): Promise { try { - if (!this.personalizeConfig.baseURL[this.config.region.name]) { - log.debug(`No baseURL found for region: ${this.config.region.name}`, this.config.context); - log.info('Skipping Personalize project import, personalize url is not set', this.config.context); - this.personalizeConfig.importData = false; + log.debug('Starting personalize import process...', this.config.context); + const [canImport, modulesCount] = await this.analyzePersonalize(); + if (!canImport) { + log.info('Personalize import skipped', this.config.context); return; } - if (this.config.management_token) { - log.debug('Management token detected, skipping personalize import', this.config.context); - log.info('Skipping Personalize project import when using management token', this.config.context); - return; - } - - log.debug('Starting personalize project import', this.config.context); - log.debug(`Base URL: ${this.personalizeConfig.baseURL[this.config.region.name]}`, this.config.context); - await new Import.Project(this.config).import(); - log.debug('Personalize project import completed', this.config.context); - - if (this.personalizeConfig.importData) { - log.debug('Personalize data import is enabled', this.config.context); - - const moduleMapper = { - events: Import.Events, - audiences: Import.Audiences, - attributes: Import.Attribute, - experiences: Import.Experiences, - }; + const progress = this.createNestedProgress(this.currentModuleName); - const order: (keyof typeof moduleMapper)[] = this.personalizeConfig - .importOrder as (keyof typeof moduleMapper)[]; + this.addProjectProcess(progress); + this.addModuleProcesses(progress, modulesCount); - log.debug(`Processing ${order.length} personalize modules in order: ${order.join(', ')}`, this.config.context); - const moduleTypes = Object.keys(moduleMapper || {}).join(', '); - log.debug(`Available module types: ${moduleTypes}`, this.config.context); + // Step 1: Import personalize project + await this.importProjects(progress); - for (const module of order) { - log.debug(`Starting import for personalize module: ${module}`, this.config.context); - const Module = moduleMapper[module]; - - if (!Module) { - log.debug(`Module ${module} not found in moduleMapper`, this.config.context); - continue; - } - - log.debug(`Creating instance of ${module} module`, this.config.context); - const moduleInstance = new Module(this.config); - - log.debug(`Importing ${module} module`, this.config.context); - await moduleInstance.import(); - - log.success(`Successfully imported personalize module: ${module}`, this.config.context); - log.debug(`Completed import for personalize module: ${module}`, this.config.context); - } - - log.debug('All personalize modules imported successfully', this.config.context); + // Step 2: Import personalize data modules (if enabled) + if (this.personalizeConfig.importData && modulesCount > 0) { + log.debug('Processing personalize modules...', this.config.context); + await this.importModules(progress); } else { - log.debug('Personalize data import is disabled', this.config.context); + log.debug('No personalize modules configured for processing', this.config.context); } + this.completeProgress(true); log.success('Personalize import completed successfully', this.config.context); } catch (error) { this.personalizeConfig.importData = false; // Stop personalize import if project creation fails + this.completeProgress(false, (error as any)?.message || 'Personalize import failed'); handleAndLogError(error, { ...this.config.context }); if (!this.personalizeConfig.importData) { log.debug('Personalize import data flag set to false due to error', this.config.context); @@ -87,4 +63,105 @@ export default class ImportPersonalize { } } } + + private addProjectProcess(progress: any) { + progress.addProcess('Projects', 1); + log.debug('Added Projects process to personalize progress', this.config.context); + } + + private addModuleProcesses(progress: any, moduleCount: number) { + if (moduleCount > 0) { + const order: (keyof typeof this.moduleDisplayMapper)[] = this.personalizeConfig + .importOrder as (keyof typeof this.moduleDisplayMapper)[]; + + log.debug(`Adding ${order.length} personalize module processes: ${order.join(', ')}`, this.config.context); + + for (const module of order) { + const processName = this.moduleDisplayMapper[module]; + progress.addProcess(processName, 1); + log.debug(`Added ${processName} process to personalize progress`, this.config.context); + } + } else { + log.debug('No personalize modules to add to progress', this.config.context); + } + } + + private async importProjects(progress: any): Promise { + progress.startProcess('Projects').updateStatus('Importing personalization projects...', 'Projects'); + log.debug('Starting projects import for personalization...', this.config.context); + + const projectInstance = new Import.Project(this.config); + projectInstance.setParentProgressManager(progress); + await projectInstance.import(); + + progress.completeProcess('Projects', true); + } + + private async importModules(progress: any): Promise { + const moduleMapper = { + events: Import.Events, + audiences: Import.Audiences, + attributes: Import.Attribute, + experiences: Import.Experiences, + }; + + const order: (keyof typeof moduleMapper)[] = this.personalizeConfig.importOrder as (keyof typeof moduleMapper)[]; + + log.debug(`Personalize import order: ${order.join(', ')}`, this.config.context); + + for (const module of order) { + log.debug(`Processing personalize module: ${module}`, this.config.context); + const processName = this.moduleDisplayMapper[module]; + const ModuleClass = moduleMapper[module]; + + if (ModuleClass) { + progress.startProcess(processName).updateStatus(`Importing ${module}...`, processName); + log.debug(`Starting import for module: ${module}`, this.config.context); + + if (this.personalizeConfig.importData) { + const importer = new ModuleClass(this.config); + importer.setParentProgressManager(progress); + await importer.import(); + + progress.completeProcess(processName, true); + log.debug(`Completed import for module: ${module}`, this.config.context); + } else { + log.debug(`Skipping ${module} - personalization not enabled`, this.config.context); + this.progressManager?.tick(true, `${module} skipped (no project)`, null, processName); + progress.completeProcess(processName, true); + log.info(`Skipped ${module} import - no personalize project found`, this.config.context); + } + } else { + log.debug(`Module not implemented: ${module}`, this.config.context); + progress.startProcess(processName).updateStatus(`Module not implemented: ${module}`, processName); + this.progressManager?.tick(false, `module: ${module}`, 'Module not implemented', processName); + progress.completeProcess(processName, false); + log.info(`Module not implemented: ${module}`, this.config.context); + } + } + + log.debug('All personalize modules processed', this.config.context); + } + + private async analyzePersonalize(): Promise<[boolean, number]> { + return this.withLoadingSpinner('PERSONALIZE: Analyzing import configuration...', async () => { + if (!this.personalizeConfig.baseURL[this.config.region.name]) { + log.debug(`No baseURL found for region: ${this.config.region.name}`, this.config.context); + log.info('Skipping Personalize project import, personalize url is not set', this.config.context); + this.personalizeConfig.importData = false; + return [false, 0]; + } + + if (this.config.management_token) { + log.debug('Management token detected, skipping personalize import', this.config.context); + log.info('Skipping Personalize project import when using management token', this.config.context); + return [false, 0]; + } + + const modulesCount = this.personalizeConfig.importData ? this.personalizeConfig.importOrder?.length || 0 : 0; + + log.debug(`Personalize analysis complete: canImport=true, modulesCount=${modulesCount}`, this.config.context); + return [true, modulesCount]; + }); + } } diff --git a/packages/contentstack-import/src/import/modules/stack.ts b/packages/contentstack-import/src/import/modules/stack.ts index 3e04b683c2..26010dbba9 100644 --- a/packages/contentstack-import/src/import/modules/stack.ts +++ b/packages/contentstack-import/src/import/modules/stack.ts @@ -13,7 +13,7 @@ export default class ImportStack extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'stack'; - this.currentModuleName = 'Stack Settings'; + this.currentModuleName = 'Stack'; this.stackSettingsPath = join(this.importConfig.backupDir, 'stack', 'settings.json'); this.envUidMapperPath = join(this.importConfig.backupDir, 'mapper', 'environments', 'uid-mapping.json'); } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index b8e53cb898..ab9410015a 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -58,7 +58,7 @@ export default class ImportTaxonomies extends BaseClass { this.createSuccessAndFailedFile(); this.completeProgress(true); - log.success('Taxonomies imported successfully!', this.importConfig.context); + log.success('Taxonomies imported successfully!', this.importConfig.context); } catch (error) { this.completeProgress(false, error?.message || 'Taxonomies import failed'); handleAndLogError(error, { ...this.importConfig.context }); diff --git a/packages/contentstack-import/src/import/modules/variant-entries.ts b/packages/contentstack-import/src/import/modules/variant-entries.ts index c89ded7d01..03047f6537 100644 --- a/packages/contentstack-import/src/import/modules/variant-entries.ts +++ b/packages/contentstack-import/src/import/modules/variant-entries.ts @@ -11,15 +11,18 @@ import { fsUtil, fileHelper, } from '../../utils'; +import BaseClass from './base-class'; -export default class ImportVarientEntries { +export default class ImportVariantEntries extends BaseClass { private config: ImportConfig; public personalize: ImportConfig['modules']['personalize']; private projectMapperFilePath: string; - constructor({ importConfig }: ModuleClassParams) { + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); this.config = importConfig; this.config.context.module = 'variant-entries'; + this.currentModuleName = 'Variant Entries'; this.personalize = importConfig.modules.personalize; this.projectMapperFilePath = path.resolve( sanitizePath(this.config.data), @@ -31,52 +34,90 @@ export default class ImportVarientEntries { } /** - * The `start` function in TypeScript is an asynchronous method that conditionally imports data using - * helper methods and logs any errors encountered. + * @method start + * @returns {Promise} Promise */ async start(): Promise { try { - log.debug(`Reading project mapper from: ${this.projectMapperFilePath}`, this.config.context); + log.debug('Starting variant entries import process...', this.config.context); - if (!fileHelper.fileExistsSync(this.projectMapperFilePath)) { - log.debug('Project mapper file does not exist', this.config.context); - log.info('Skipping entry variants import because no personalize project mapper found.', this.config.context); + const [hasProject] = await this.analyzeVariantEntries(); + + if (!hasProject) { + log.info('No variant entries found to import', this.config.context); return; } - const project = fsUtil.readFile(this.projectMapperFilePath) as ProjectStruct; - log.debug(`Project data loaded: ${JSON.stringify(project)}`, this.config.context); + const progress = this.createSimpleProgress(this.currentModuleName); - if (project && project.uid) { - log.debug(`Found personalize project: ${project.uid}`, this.config.context); - this.config.modules.personalize.project_id = project.uid; + progress.updateStatus('Importing variant entries...'); + log.info('Starting variant entries import process', this.config.context); + await this.importVariantEntries(); - log.debug('Initializing helper methods for variant entries import', this.config.context); - const helpers: ImportHelperMethodsConfig = { - lookUpTerms, - lookupAssets, - lookupEntries, - lookupExtension, - restoreJsonRteEntryRefs, - }; + this.completeProgress(true); + log.success('Variant entries imported successfully', this.config.context); + } catch (error) { + this.completeProgress(false, (error as any)?.message || 'Variant entries import failed'); + handleAndLogError(error, { ...this.config.context }); + } + } + + private async importVariantEntries(): Promise { + const project = fsUtil.readFile(this.projectMapperFilePath) as ProjectStruct; + log.debug(`Project data loaded: ${JSON.stringify(project)}`, this.config.context); + + if (project && project.uid) { + log.debug(`Found personalize project: ${project.uid}`, this.config.context); + this.config.modules.personalize.project_id = project.uid; - log.debug('Helper methods initialized successfully', this.config.context); - const helperTypes = Object.keys(helpers || {}).join(', '); - log.debug(`Helper method types available: ${helperTypes}`, this.config.context); + log.debug('Initializing helper methods for variant entries import', this.config.context); + const helpers: ImportHelperMethodsConfig = { + lookUpTerms, + lookupAssets, + lookupEntries, + lookupExtension, + restoreJsonRteEntryRefs, + }; - log.debug('Creating VariantEntries instance', this.config.context); - const variantEntriesImporter = new Import.VariantEntries(Object.assign(this.config, { helpers })); + log.debug('Helper methods initialized successfully', this.config.context); + const helperTypes = Object.keys(helpers || {}).join(', '); + log.debug(`Helper method types available: ${helperTypes}`, this.config.context); - log.debug('Starting variant entries import', this.config.context); - await variantEntriesImporter.import(); + log.debug('Creating VariantEntries instance', this.config.context); + const variantEntriesImporter = new Import.VariantEntries(Object.assign(this.config, { helpers })); - log.success('Variant entries imported successfully', this.config.context); + log.debug('Starting variant entries import', this.config.context); + await variantEntriesImporter.import(); + + this.progressManager?.tick(true, 'variant entries import completed'); + log.debug('Variant entries import completed successfully', this.config.context); + } else { + log.debug('No valid project found in mapper file', this.config.context); + this.progressManager?.tick(false, 'variant entries import', 'No personalize project linked'); + log.info('Skipping entry variants import because no personalize project is linked.', this.config.context); + } + } + + private async analyzeVariantEntries(): Promise<[boolean]> { + return this.withLoadingSpinner('VARIANT ENTRIES: Analyzing import data...', async () => { + log.debug(`Reading project mapper from: ${this.projectMapperFilePath}`, this.config.context); + + if (!fileHelper.fileExistsSync(this.projectMapperFilePath)) { + log.debug('Project mapper file does not exist', this.config.context); + log.info('Skipping entry variants import because no personalize project mapper found.', this.config.context); + return [false] as [boolean]; + } + + const project = fsUtil.readFile(this.projectMapperFilePath) as ProjectStruct; + const hasValidProject = !!(project && project.uid); // Convert to boolean + + if (hasValidProject) { + log.debug(`Found valid personalize project: ${project.uid}`, this.config.context); } else { log.debug('No valid project found in mapper file', this.config.context); - log.info('Skipping entry variants import because no personalize project is linked.', this.config.context); } - } catch (error) { - handleAndLogError(error, { ...this.config.context }); - } + + return [hasValidProject] as [boolean]; + }); } } diff --git a/packages/contentstack-import/src/import/modules/webhooks.ts b/packages/contentstack-import/src/import/modules/webhooks.ts index b78aec5a85..6426fff4de 100644 --- a/packages/contentstack-import/src/import/modules/webhooks.ts +++ b/packages/contentstack-import/src/import/modules/webhooks.ts @@ -3,7 +3,7 @@ import values from 'lodash/values'; import { join } from 'node:path'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; -import { formatError, fsUtil, fileHelper } from '../../utils'; +import { fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, WebhookConfig } from '../../types'; @@ -22,6 +22,7 @@ export default class ImportWebhooks extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'webhooks'; + this.currentModuleName = 'Webhooks'; this.webhooksConfig = importConfig.modules.webhooks; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'webhooks'); this.webhooksFolderPath = join(this.importConfig.backupDir, this.webhooksConfig.dirName); @@ -39,51 +40,30 @@ export default class ImportWebhooks extends BaseClass { * @returns {Promise} Promise */ async start(): Promise { - log.debug('Checking for webhooks folder existence', this.importConfig.context); + try { + log.debug('Starting webhooks import process...', this.importConfig.context); - //Step1 check folder exists or not - if (fileHelper.fileExistsSync(this.webhooksFolderPath)) { - this.webhooks = fsUtil.readFile(join(this.webhooksFolderPath, 'webhooks.json'), true) as Record; - log.debug(`Found webhooks folder: ${this.webhooksFolderPath}`, this.importConfig.context); - const webhookCount = Object.keys(this.webhooks || {}).length; - log.debug(`Loaded ${webhookCount} webhook items from file`, this.importConfig.context); - } else { - log.info(`No Webhooks Found - '${this.webhooksFolderPath}'`, this.importConfig.context); - return; - } + const [webhooksCount] = await this.analyzeWebhooks(); - //create webhooks in mapper directory - log.debug('Creating webhooks mapper directory', this.importConfig.context); - await fsUtil.makeDirectory(this.mapperDirPath); - log.debug('Created webhooks mapper directory', this.importConfig.context); + if (webhooksCount === 0) { + log.info(`No Webhooks Found - '${this.webhooksFolderPath}'`, this.importConfig.context); + return; + } - log.debug('Loading existing webhook UID mappings', this.importConfig.context); - this.webhookUidMapper = fileHelper.fileExistsSync(this.webhookUidMapperPath) - ? (fsUtil.readFile(join(this.webhookUidMapperPath), true) as Record) - : {}; + const progress = this.createSimpleProgress(this.currentModuleName, webhooksCount); + await this.prepareWebhookMapper(); - if (Object.keys(this.webhookUidMapper)?.length > 0) { - const webhookUidCount = Object.keys(this.webhookUidMapper || {}).length; - log.debug(`Loaded existing webhook UID data: ${webhookUidCount} items`, this.importConfig.context); - } else { - log.debug('No existing webhook UID mappings found', this.importConfig.context); - } + progress.updateStatus('Importing webhooks...'); + await this.importWebhooks(); - log.debug('Starting webhook import process', this.importConfig.context); - await this.importWebhooks(); + this.processWebhookResults(); - log.debug('Processing webhook import results', this.importConfig.context); - if (this.createdWebhooks?.length) { - fsUtil.writeFile(this.createdWebhooksPath, this.createdWebhooks); - log.debug(`Written ${this.createdWebhooks.length} successful webhooks to file`, this.importConfig.context); - } - - if (this.failedWebhooks?.length) { - fsUtil.writeFile(this.failedWebhooksPath, this.failedWebhooks); - log.debug(`Written ${this.failedWebhooks.length} failed webhooks to file`, this.importConfig.context); + this.completeProgress(true); + log.success('Webhooks have been imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Webhooks import failed'); + handleAndLogError(error, { ...this.importConfig.context }); } - - log.success('Webhooks have been imported successfully!', this.importConfig.context); } async importWebhooks() { @@ -99,6 +79,7 @@ export default class ImportWebhooks extends BaseClass { const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }: any) => { this.createdWebhooks.push(response); this.webhookUidMapper[uid] = response.uid; + this.progressManager?.tick(true, `webhook: ${name || uid}`); log.success(`Webhook '${name}' imported successfully`, this.importConfig.context); log.debug(`Webhook UID mapping: ${uid} → ${response.uid}`, this.importConfig.context); fsUtil.writeFile(this.webhookUidMapperPath, this.webhookUidMapper); @@ -108,10 +89,13 @@ export default class ImportWebhooks extends BaseClass { const err = error?.message ? JSON.parse(error.message) : error; const { name, uid } = apiData; log.debug(`Webhook '${name}' (${uid}) failed to import`, this.importConfig.context); + if (err?.errors?.name) { + this.progressManager?.tick(true, `webhook: ${name || uid} (already exists)`); log.info(`Webhook '${name}' already exists`, this.importConfig.context); } else { this.failedWebhooks.push(apiData); + this.progressManager?.tick(false, `webhook: ${name || uid}`, error?.message || 'Failed to import webhook'); handleAndLogError( error, { ...this.importConfig.context, webhookName: name }, @@ -153,6 +137,7 @@ export default class ImportWebhooks extends BaseClass { if (this.webhookUidMapper.hasOwnProperty(webhook.uid)) { log.info(`Webhook '${webhook.name}' already exists. Skipping it to avoid duplicates!`, this.importConfig.context); log.debug(`Skipping webhook serialization for: ${webhook.uid}`, this.importConfig.context); + this.progressManager?.tick(true, `webhook: ${webhook.name} (skipped - already exists)`); apiOptions.entity = undefined; } else { log.debug(`Processing webhook status configuration`, this.importConfig.context); @@ -166,4 +151,62 @@ export default class ImportWebhooks extends BaseClass { } return apiOptions; } + + private async analyzeWebhooks(): Promise<[number]> { + return this.withLoadingSpinner('WEBHOOKS: Analyzing import data...', async () => { + log.debug('Checking for webhooks folder existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.webhooksFolderPath)) { + log.info(`No Webhooks Found - '${this.webhooksFolderPath}'`, this.importConfig.context); + return [0]; + } + + log.debug(`Found webhooks folder: ${this.webhooksFolderPath}`, this.importConfig.context); + + this.webhooks = fsUtil.readFile(join(this.webhooksFolderPath, 'webhooks.json'), true) as Record; + + if (!this.webhooks) { + log.info( + `No webhooks found in file - '${join(this.webhooksFolderPath, 'webhooks.json')}'`, + this.importConfig.context, + ); + return [0]; + } + + const count = Object.keys(this.webhooks || {}).length; + log.debug(`Loaded ${count} webhook items from file`, this.importConfig.context); + return [count]; + }); + } + + private async prepareWebhookMapper(): Promise { + log.debug('Creating webhooks mapper directory', this.importConfig.context); + await fsUtil.makeDirectory(this.mapperDirPath); + + log.debug('Loading existing webhook UID mappings', this.importConfig.context); + this.webhookUidMapper = fileHelper.fileExistsSync(this.webhookUidMapperPath) + ? (fsUtil.readFile(join(this.webhookUidMapperPath), true) as Record) || {} + : {}; + + const count = Object.keys(this.webhookUidMapper || {}).length; + if (count > 0) { + log.debug(`Loaded existing webhook UID data: ${count} items`, this.importConfig.context); + } else { + log.debug('No existing webhook UID mappings found', this.importConfig.context); + } + } + + private processWebhookResults() { + log.debug('Processing webhook import results', this.importConfig.context); + + if (this.createdWebhooks?.length) { + fsUtil.writeFile(this.createdWebhooksPath, this.createdWebhooks); + log.debug(`Written ${this.createdWebhooks.length} successful webhooks to file`, this.importConfig.context); + } + + if (this.failedWebhooks?.length) { + fsUtil.writeFile(this.failedWebhooksPath, this.failedWebhooks); + log.debug(`Written ${this.failedWebhooks.length} failed webhooks to file`, this.importConfig.context); + } + } } diff --git a/packages/contentstack-import/src/import/modules/workflows.ts b/packages/contentstack-import/src/import/modules/workflows.ts index e06a04b248..b0059026bf 100644 --- a/packages/contentstack-import/src/import/modules/workflows.ts +++ b/packages/contentstack-import/src/import/modules/workflows.ts @@ -29,6 +29,7 @@ export default class ImportWorkflows extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); this.importConfig.context.module = 'workflows'; + this.currentModuleName = 'Workflows'; this.workflowsConfig = importConfig.modules.workflows; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'workflows'); this.workflowsFolderPath = join(this.importConfig.backupDir, this.workflowsConfig.dirName); @@ -47,60 +48,42 @@ export default class ImportWorkflows extends BaseClass { * @returns {Promise} Promise */ async start(): Promise { - log.debug('Checking for workflows folder existence', this.importConfig.context); + try { + log.debug('Starting workflows import process...', this.importConfig.context); - //Step1 check folder exists or not - if (fileHelper.fileExistsSync(this.workflowsFolderPath)) { - log.debug(`Found workflows folder: ${this.workflowsFolderPath}`, this.importConfig.context); - this.workflows = fsUtil.readFile(join(this.workflowsFolderPath, this.workflowsConfig.fileName), true) as Record< - string, - unknown - >; - const workflowCount = Object.keys(this.workflows || {}).length; - log.debug(`Loaded ${workflowCount} workflow items from file`, this.importConfig.context); - } else { - log.info(`No Workflows Found - '${this.workflowsFolderPath}'`, this.importConfig.context); - return; - } + const [workflowsCount] = await this.analyzeWorkflows(); - //create workflows in mapper directory - log.debug('Creating workflows mapper directory', this.importConfig.context); - await fsUtil.makeDirectory(this.mapperDirPath); - log.debug('Loading existing workflow UID mappings', this.importConfig.context); - this.workflowUidMapper = fileHelper.fileExistsSync(this.workflowUidMapperPath) - ? (fsUtil.readFile(join(this.workflowUidMapperPath), true) as Record) - : {}; + if (workflowsCount === 0) { + log.info(`No Workflows Found - '${this.workflowsFolderPath}'`, this.importConfig.context); + return; + } - if (Object.keys(this.workflowUidMapper)?.length > 0) { - const workflowUidCount = Object.keys(this.workflowUidMapper || {}).length; - log.debug(`Loaded existing workflow UID data: ${workflowUidCount} items`, this.importConfig.context); - } else { - log.debug('No existing workflow UID mappings found', this.importConfig.context); - } + const progress = this.createNestedProgress(this.currentModuleName); + progress.addProcess('Get Roles', 1); + progress.addProcess('Create', workflowsCount); - if (this.workflows === undefined || isEmpty(this.workflows)) { - log.info('No Workflow Found', this.importConfig.context); - return; - } + await this.prepareWorkflowMapper(); - //fetch all roles - log.debug('Fetching all roles for workflow processing', this.importConfig.context); - await this.getRoles(); - log.debug('Starting workflow import process', this.importConfig.context); - await this.importWorkflows(); + // Step 1: Fetch and setup roles + progress.startProcess('Get Roles').updateStatus('Fetching roles for workflow processing...', 'Get Roles'); + log.info('Fetching all roles for workflow processing', this.importConfig.context); + await this.getRoles(); + progress.completeProcess('Get Roles', true); - log.debug('Processing workflow import results', this.importConfig.context); - if (this.createdWorkflows?.length) { - fsUtil.writeFile(this.createdWorkflowsPath, this.createdWorkflows); - log.debug(`Written ${this.createdWorkflows.length} successful workflows to file`, this.importConfig.context); - } + // Step 2: Import workflows + progress.startProcess('Create').updateStatus('Importing workflows...', 'Create'); + log.info('Starting workflows import process', this.importConfig.context); + await this.importWorkflows(); + progress.completeProcess('Create', true); - if (this.failedWebhooks?.length) { - fsUtil.writeFile(this.failedWorkflowsPath, this.failedWebhooks); - log.debug(`Written ${this.failedWebhooks.length} failed workflows to file`, this.importConfig.context); - } + this.processWorkflowResults(); - log.success('Workflows have been imported successfully!', this.importConfig.context); + this.completeProgress(true); + log.success('Workflows have been imported successfully!', this.importConfig.context); + } catch (error) { + this.completeProgress(false, error?.message || 'Workflows import failed'); + handleAndLogError(error, { ...this.importConfig.context }); + } } async getRoles(): Promise { @@ -157,6 +140,12 @@ export default class ImportWorkflows extends BaseClass { response.workflow_stages, oldWorkflowStages, ).catch((error) => { + this.progressManager?.tick( + false, + `workflow: ${name || uid}`, + error?.message || 'Failed to update next available stages', + 'Create', + ); handleAndLogError(error, { ...this.importConfig.context, name }, `Workflow '${name}' update failed`); }); @@ -168,6 +157,7 @@ export default class ImportWorkflows extends BaseClass { this.createdWorkflows.push(response); this.workflowUidMapper[uid] = response.uid; + this.progressManager?.tick(true, `workflow: ${name || uid}`, null, 'Create'); log.success(`Workflow '${name}' imported successfully`, this.importConfig.context); log.debug(`Workflow UID mapping: ${uid} → ${response.uid}`, this.importConfig.context); fsUtil.writeFile(this.workflowUidMapperPath, this.workflowUidMapper); @@ -178,10 +168,18 @@ export default class ImportWorkflows extends BaseClass { const { name, uid } = apiData; log.debug(`Workflow '${name}' (${uid}) failed to import`, this.importConfig.context); const workflowExists = err?.errors?.name || err?.errors?.['workflow.name']; + if (workflowExists) { + this.progressManager?.tick(true, `workflow: ${name || uid} (already exists)`, null, 'Create'); log.info(`Workflow '${name}' already exists`, this.importConfig.context); } else { this.failedWebhooks.push(apiData); + this.progressManager?.tick( + false, + `workflow: ${name || uid}`, + error?.message || 'Failed to import workflow', + 'Create', + ); if (error.errors['workflow_stages.0.users']) { log.error( "Failed to import Workflows as you've specified certain roles in the Stage transition and access rules section. We currently don't import roles to the stack.", @@ -250,12 +248,20 @@ export default class ImportWorkflows extends BaseClass { */ serializeWorkflows(apiOptions: ApiOptions): ApiOptions { let { apiData: workflow } = apiOptions; + log.debug(`Serializing workflow: ${workflow.name} (${workflow.uid})`, this.importConfig.context); if (this.workflowUidMapper.hasOwnProperty(workflow.uid)) { log.info( `Workflow '${workflow.name}' already exists. Skipping it to avoid duplicates!`, this.importConfig.context, ); + log.debug(`Skipping workflow serialization for: ${workflow.uid}`, this.importConfig.context); + this.progressManager?.tick( + true, + `workflow: ${workflow.name} (skipped - already exists)`, + null, + 'Create', + ); apiOptions.entity = undefined; } else { if (workflow.admin_users !== undefined) { @@ -277,6 +283,7 @@ export default class ImportWorkflows extends BaseClass { } } + log.debug(`Workflow serialization completed: ${workflow.name}`, this.importConfig.context); apiOptions.apiData = workflow; } return apiOptions; @@ -287,9 +294,11 @@ export default class ImportWorkflows extends BaseClass { const { name } = apiData; this.updateRoleData({ workflowUid, stageIndex, roleData: apiData }); this.roleNameMap[name] = response?.uid; + log.debug(`Custom role '${name}' created successfully for workflow`, this.importConfig.context); }; const onReject = ({ error, apiData: { name } = { name: '' } }: any) => { + log.debug(`Custom role '${name}' creation failed`, this.importConfig.context); handleAndLogError(error, { ...this.importConfig.context, name }, `Failed to create custom roles '${name}'`); }; @@ -301,6 +310,10 @@ export default class ImportWorkflows extends BaseClass { } if (stage?.SYS_ACL?.roles?.uids?.length) { const apiContent = stage.SYS_ACL.roles.uids; + log.debug( + `Creating ${apiContent.length} custom roles for workflow stage ${stageIndex}`, + this.importConfig.context, + ); await this.makeConcurrentCall( { apiContent, @@ -333,6 +346,9 @@ export default class ImportWorkflows extends BaseClass { apiData: roleData, additionalInfo: { workflowUid, stageIndex }, } = apiOptions; + + log.debug(`Serializing custom role: ${roleData.name}`, this.importConfig.context); + if (!this.roleNameMap[roleData.name]) { // rules.branch is required to create custom roles. const branchRuleExists = find(roleData.rules, (rule: any) => rule.module === 'branch'); @@ -342,9 +358,11 @@ export default class ImportWorkflows extends BaseClass { branches: ['main'], acl: { read: true }, }); + log.debug(`Added branch rule to custom role: ${roleData.name}`, this.importConfig.context); } apiOptions = roleData; } else { + log.debug(`Custom role '${roleData.name}' already exists, skipping creation`, this.importConfig.context); apiOptions.entity = undefined; this.updateRoleData({ workflowUid, stageIndex, roleData }); } @@ -357,5 +375,66 @@ export default class ImportWorkflows extends BaseClass { const roles = workflowStage[stageIndex].SYS_ACL.roles.uids; const index = findIndex(roles, ['uid', roleData.uid]); roles[index >= 0 ? index : roles.length] = this.roleNameMap[roleData.name]; + log.debug(`Updated role data for workflow ${workflowUid}, stage ${stageIndex}`, this.importConfig.context); + } + + private async analyzeWorkflows(): Promise<[number]> { + return this.withLoadingSpinner('WORKFLOWS: Analyzing import data...', async () => { + log.debug('Checking for workflows folder existence', this.importConfig.context); + + if (!fileHelper.fileExistsSync(this.workflowsFolderPath)) { + log.info(`No Workflows Found - '${this.workflowsFolderPath}'`, this.importConfig.context); + return [0]; + } + + log.debug(`Found workflows folder: ${this.workflowsFolderPath}`, this.importConfig.context); + this.workflows = fsUtil.readFile(join(this.workflowsFolderPath, this.workflowsConfig.fileName), true) as Record< + string, + unknown + >; + + if (!this.workflows || isEmpty(this.workflows)) { + log.info( + `No workflows found in file - '${join(this.workflowsFolderPath, this.workflowsConfig.fileName)}'`, + this.importConfig.context, + ); + return [0]; + } + + const count = Object.keys(this.workflows || {}).length; + log.debug(`Loaded ${count} workflow items from file`, this.importConfig.context); + return [count]; + }); + } + + private async prepareWorkflowMapper(): Promise { + log.debug('Creating workflows mapper directory', this.importConfig.context); + await fsUtil.makeDirectory(this.mapperDirPath); + + log.debug('Loading existing workflow UID mappings', this.importConfig.context); + this.workflowUidMapper = fileHelper.fileExistsSync(this.workflowUidMapperPath) + ? (fsUtil.readFile(join(this.workflowUidMapperPath), true) as Record) || {} + : {}; + + const count = Object.keys(this.workflowUidMapper || {}).length; + if (count > 0) { + log.debug(`Loaded existing workflow UID data: ${count} items`, this.importConfig.context); + } else { + log.debug('No existing workflow UID mappings found', this.importConfig.context); + } + } + + private processWorkflowResults() { + log.debug('Processing workflow import results', this.importConfig.context); + + if (this.createdWorkflows?.length) { + fsUtil.writeFile(this.createdWorkflowsPath, this.createdWorkflows); + log.debug(`Written ${this.createdWorkflows.length} successful workflows to file`, this.importConfig.context); + } + + if (this.failedWebhooks?.length) { + fsUtil.writeFile(this.failedWorkflowsPath, this.failedWebhooks); + log.debug(`Written ${this.failedWebhooks.length} failed workflows to file`, this.importConfig.context); + } } } diff --git a/packages/contentstack-import/src/utils/strategy-registrations.ts b/packages/contentstack-import/src/utils/strategy-registrations.ts new file mode 100644 index 0000000000..1811430673 --- /dev/null +++ b/packages/contentstack-import/src/utils/strategy-registrations.ts @@ -0,0 +1,132 @@ +/** + * Progress Strategy Registrations for Import Modules + * This file registers progress calculation strategies for all import modules + * to ensure correct item counts in the final summary. + */ + +import { + ProgressStrategyRegistry, + PrimaryProcessStrategy, + CustomProgressStrategy, + DefaultProgressStrategy +} from '@contentstack/cli-utilities'; + +// Register strategy for Content Types - use Create as primary process +ProgressStrategyRegistry.register( + 'CONTENT TYPES', + new PrimaryProcessStrategy('Create') +); + +// Register strategy for Assets - use Asset Upload as primary process +ProgressStrategyRegistry.register( + 'ASSETS', + new PrimaryProcessStrategy('Upload') +); + +// Register strategy for Entries - use Entry Creation as primary process +ProgressStrategyRegistry.register( + 'ENTRIES', + new PrimaryProcessStrategy('Create') +); + +// Register strategy for Global Fields - use Create as primary process +ProgressStrategyRegistry.register( + 'GLOBAL FIELDS', + new PrimaryProcessStrategy('Create') +); + +// Register strategy for Extensions - simple module +ProgressStrategyRegistry.register( + 'EXTENSIONS', + new PrimaryProcessStrategy('Create') +); + +// Register strategy for Environments - uses default (no nested progress yet) +ProgressStrategyRegistry.register( + 'ENVIRONMENTS', + new DefaultProgressStrategy() +); + +// Register strategy for Locales - uses default (no nested progress yet) +ProgressStrategyRegistry.register( + 'LOCALES', + new PrimaryProcessStrategy('Create') +); + +// Register strategy for Labels - uses default (no nested progress yet) +ProgressStrategyRegistry.register( + 'LABELS', + new PrimaryProcessStrategy('Create') +); + +// Register strategy for Webhooks - uses default (no nested progress yet) +ProgressStrategyRegistry.register( + 'WEBHOOKS', + new DefaultProgressStrategy() +); + +// Register strategy for Workflows - uses default (no nested progress yet) +ProgressStrategyRegistry.register( + 'WORKFLOWS', + new PrimaryProcessStrategy('Create') +); + +// Register strategy for Custom Roles - uses default (no nested progress yet) +ProgressStrategyRegistry.register( + 'CUSTOM ROLES', + new DefaultProgressStrategy() +); + +// Register strategy for Taxonomies - uses default (no nested progress yet) +ProgressStrategyRegistry.register( + 'TAXONOMIES', + new DefaultProgressStrategy() +); + +// Register strategy for Marketplace Apps - complex module with app installations +ProgressStrategyRegistry.register( + 'MARKETPLACE APPS', + new PrimaryProcessStrategy('Apps Installation') +); + +// Register strategy for Stack Settings - simple module +ProgressStrategyRegistry.register( + 'STACK', + new DefaultProgressStrategy() +); + +// Register strategy for Personalize - complex module with projects/experiences +ProgressStrategyRegistry.register( + 'PERSONALIZE', + new CustomProgressStrategy((processes) => { + // For personalize import, count project imports as primary metric + const projectImport = processes.get('Project'); + if (projectImport) { + return { + total: projectImport.total, + success: projectImport.successCount, + failures: projectImport.failureCount + }; + } + + // Fallback to any other main process + const mainProcess = Array.from(processes.values())[0]; + if (mainProcess) { + return { + total: mainProcess.total, + success: mainProcess.successCount, + failures: mainProcess.failureCount + }; + } + + return null; + }) +); + +// Register strategy for Variant Entries - sub-process of entries +ProgressStrategyRegistry.register( + 'VARIANT ENTRIES', + new DefaultProgressStrategy() +); + +export default ProgressStrategyRegistry; \ No newline at end of file diff --git a/packages/contentstack-utilities/src/index.ts b/packages/contentstack-utilities/src/index.ts index 4cff246d16..2481183d77 100644 --- a/packages/contentstack-utilities/src/index.ts +++ b/packages/contentstack-utilities/src/index.ts @@ -78,4 +78,11 @@ export { default as TablePrompt } from './inquirer-table-prompt'; export { Logger }; export { default as authenticationHandler } from './authentication-handler'; export { v2Logger as log, cliErrorHandler, handleAndLogError, getLogPath } from './logger/log'; -export { CLIProgressManager, SummaryManager } from './progress-summary'; +export { + CLIProgressManager, + SummaryManager, + PrimaryProcessStrategy, + ProgressStrategyRegistry, + CustomProgressStrategy, + DefaultProgressStrategy +} from './progress-summary'; diff --git a/packages/contentstack-utilities/src/interfaces/index.ts b/packages/contentstack-utilities/src/interfaces/index.ts index aa6c117193..229c1b842c 100644 --- a/packages/contentstack-utilities/src/interfaces/index.ts +++ b/packages/contentstack-utilities/src/interfaces/index.ts @@ -155,6 +155,7 @@ export interface ModuleResult { successCount: number; failureCount: number; failures: Array<{ item: string; error: string }>; + processes?: Array<{ processName: string; [key: string]: any }>; } export interface SummaryOptions { @@ -162,3 +163,9 @@ export interface SummaryOptions { context?: any; branchName?: string; // Optional branch name for operations } + +export interface ProgressResult { + total: number; + success: number; + failures: number; +} \ No newline at end of file diff --git a/packages/contentstack-utilities/src/progress-summary/cli-progress-manager.ts b/packages/contentstack-utilities/src/progress-summary/cli-progress-manager.ts index d1ba0f0ceb..2392787998 100644 --- a/packages/contentstack-utilities/src/progress-summary/cli-progress-manager.ts +++ b/packages/contentstack-utilities/src/progress-summary/cli-progress-manager.ts @@ -4,6 +4,7 @@ import ProgressBar from 'cli-progress'; import SummaryManager from './summary-manager'; import { ProcessProgress, ProgressManagerOptions, Failure } from '../interfaces'; import { configHandler } from '..'; +import { ProgressStrategyRegistry } from './progress-strategy'; interface ProgressCallback { onModuleStart?: (moduleName: string) => void; @@ -56,7 +57,7 @@ export default class CLIProgressManager { this.multiBar = null; this.currentProcess = null; this.callbacks = {}; - this.branchName = ''; // Default branch name + this.branchName = ''; this.initializeProgress(); this.setupGlobalSummaryIntegration(); @@ -93,10 +94,56 @@ export default class CLIProgressManager { } /** - * Print the final summary for all modules + * Print the final summary for all modules using strategies */ static printGlobalSummary(): void { - CLIProgressManager.globalSummary?.printFinalSummary(); + if (!CLIProgressManager.globalSummary) { + return; + } + + // Apply strategy-based corrections before printing + CLIProgressManager.applyStrategyCorrections(); + + // Print the final summary + CLIProgressManager.globalSummary.printFinalSummary(); + } + + /** + * Apply strategy-based corrections to module data + */ + private static applyStrategyCorrections(): void { + if (!CLIProgressManager.globalSummary) return; + + const modules = Array.from(CLIProgressManager.globalSummary.getModules().values()); + + modules.forEach((module) => { + // Check if this module has a registered strategy + if (ProgressStrategyRegistry.has(module.name)) { + const strategy = ProgressStrategyRegistry.get(module.name); + + // Create a processes map from module data if available + const processesMap = new Map(); + + // If module has process data, populate the map + if (module.processes && Array.isArray(module.processes)) { + module.processes.forEach((processData: any) => { + if (processData.processName) { + processesMap.set(processData.processName, processData); + } + }); + } + + // Calculate corrected progress using strategy + const correctedResult = strategy.calculate(processesMap); + + if (correctedResult) { + // Update module with corrected counts + module.totalItems = correctedResult.total; + module.successCount = correctedResult.success; + module.failureCount = correctedResult.failures; + } + } + }); } /** @@ -154,6 +201,8 @@ export default class CLIProgressManager { CLIProgressManager.globalSummary?.startModule(name); }, onModuleComplete: (name, success, error) => { + // Register process data with summary manager before completing + this.registerProcessDataWithSummary(name); CLIProgressManager.globalSummary?.completeModule(name, success, error); }, onProgress: (name, success, itemName, error) => { @@ -166,6 +215,26 @@ export default class CLIProgressManager { } } + /** + * Register process data with summary manager for strategy calculations + */ + private registerProcessDataWithSummary(moduleName: string): void { + if (!CLIProgressManager.globalSummary) return; + + // Register each process with the summary manager + this.processes.forEach((processData, processName) => { + CLIProgressManager.globalSummary?.registerProcessData(moduleName, processName, { + processName, + total: processData.total, + current: processData.current, + successCount: processData.successCount, + failureCount: processData.failureCount, + status: processData.status, + failures: processData.failures, + }); + }); + } + /** * Set callbacks for external integration */ @@ -180,6 +249,32 @@ export default class CLIProgressManager { return name.charAt(0).toUpperCase() + name.slice(1).toLowerCase(); } + /** + * Format process name with smart truncation (modules should use short names) + */ + private formatProcessName(processName: string): string { + const cleaned = processName.trim(); + + if (cleaned.length <= 20) { + return cleaned; + } + + return cleaned.length <= 20 ? cleaned : cleaned.substring(0, 20) + '...'; + } + + /** + * Format percentage for consistent alignment (always 3 characters) + */ + private formatPercentage(percentage: number): string { + if (percentage === 100) { + return '100'; + } else if (percentage >= 10) { + return ` ${percentage}`; + } else { + return ` ${percentage}`; + } + } + private initializeProgress(): void { if (this.showConsoleLogs) { return; @@ -217,7 +312,7 @@ export default class CLIProgressManager { this.progressBar.start(this.total, 0, { label: chalk.gray(` └─ ${displayName}`.padEnd(25)), status: chalk.gray('Starting...'), - percentage: '0', + percentage: ' 0', }); } else { this.spinner = ora(`${chalk.bold(this.moduleName)}: Processing...`).start(); @@ -241,12 +336,12 @@ export default class CLIProgressManager { }; if (!this.showConsoleLogs) { - const truncatedName = processName.length > 20 ? processName.substring(0, 17) + '...' : processName; - const indentedLabel = ` ├─ ${truncatedName}`.padEnd(25); + const displayName = this.formatProcessName(processName); + const indentedLabel = ` ├─ ${displayName}`.padEnd(25); process.progressBar = this.multiBar.create(total, 0, { label: chalk.gray(indentedLabel), status: chalk.gray('Pending'), - percentage: '0', + percentage: ' 0', }); } @@ -254,6 +349,23 @@ export default class CLIProgressManager { return this; } + /** + * Update the total for a specific process (for dynamic totals after API calls) + */ + updateProcessTotal(processName: string, newTotal: number): this { + if (!this.enableNestedProgress) return this; + + const process = this.processes.get(processName); + if (process) { + process.total = newTotal; + if (process.progressBar && !this.showConsoleLogs) { + // Update the progress bar with the new total + process.progressBar.setTotal(newTotal); + } + } + return this; + } + /** * Start a specific process */ @@ -264,12 +376,12 @@ export default class CLIProgressManager { if (process) { process.status = 'active'; if (!this.showConsoleLogs && process.progressBar) { - const truncatedName = processName.length > 20 ? processName.substring(0, 17) + '...' : processName; - const indentedLabel = ` ├─ ${truncatedName}`.padEnd(25); + const displayName = this.formatProcessName(processName); + const indentedLabel = ` ├─ ${displayName}`.padEnd(25); process.progressBar.update(0, { label: chalk.yellow(indentedLabel), status: chalk.yellow('Processing'), - percentage: '0', + percentage: ' 0', }); } this.currentProcess = processName; @@ -288,15 +400,16 @@ export default class CLIProgressManager { process.status = success ? 'completed' : 'failed'; if (!this.showConsoleLogs && process.progressBar) { const percentage = Math.round((process.current / process.total) * 100); + const formattedPercentage = this.formatPercentage(percentage); const statusText = success ? chalk.green(`✓ Complete (${process.successCount}/${process.current})`) : chalk.red(`✗ Failed (${process.successCount}/${process.current})`); - const truncatedName = processName.length > 20 ? processName.substring(0, 17) + '...' : processName; - const indentedLabel = ` ├─ ${truncatedName}`.padEnd(25); + const displayName = this.formatProcessName(processName); + const indentedLabel = ` ├─ ${displayName}`.padEnd(25); process.progressBar.update(process.total, { label: success ? chalk.green(indentedLabel) : chalk.red(indentedLabel), status: statusText, - percentage: percentage.toString(), + percentage: formattedPercentage, }); } } @@ -312,22 +425,24 @@ export default class CLIProgressManager { const process = this.processes.get(processName); if (process && process.progressBar) { const percentage = Math.round((process.current / process.total) * 100); - const truncatedName = processName.length > 20 ? processName.substring(0, 17) + '...' : processName; - const indentedLabel = ` ├─ ${truncatedName}`.padEnd(25); + const formattedPercentage = this.formatPercentage(percentage); + const displayName = this.formatProcessName(processName); + const indentedLabel = ` ├─ ${displayName}`.padEnd(25); process.progressBar.update(process.current, { label: chalk.yellow(indentedLabel), status: chalk.yellow(message), - percentage: percentage.toString(), + percentage: formattedPercentage, }); } } else if (this.progressBar) { const percentage = Math.round(this.progressBar.getProgress() * 100); + const formattedPercentage = this.formatPercentage(percentage); const formattedName = this.formatModuleName(this.moduleName); const displayName = formattedName.length > 20 ? formattedName.substring(0, 17) + '...' : formattedName; this.progressBar.update(this.progressBar.getProgress() * this.total, { label: chalk.yellow(` └─ ${displayName}`.padEnd(25)), status: chalk.yellow(message), - percentage: percentage.toString(), + percentage: formattedPercentage, }); } else if (this.spinner) { this.spinner.text = `${chalk.bold(this.moduleName)}: ${message}`; @@ -371,14 +486,15 @@ export default class CLIProgressManager { // Only update progress bar if console logs are disabled if (!this.showConsoleLogs && process.progressBar) { const percentage = Math.round((process.current / process.total) * 100); + const formattedPercentage = this.formatPercentage(percentage); const statusText = `${process.successCount}✓ ${process.failureCount}✗`; - const truncatedName = targetProcess.length > 20 ? targetProcess.substring(0, 17) + '...' : targetProcess; - const indentedLabel = ` ├─ ${truncatedName}`.padEnd(25); + const displayName = this.formatProcessName(targetProcess); + const indentedLabel = ` ├─ ${displayName}`.padEnd(25); process.progressBar.increment(1, { label: chalk.cyan(indentedLabel), status: chalk.cyan(statusText), - percentage: percentage.toString(), + percentage: formattedPercentage, }); } } @@ -387,6 +503,7 @@ export default class CLIProgressManager { if (!this.showConsoleLogs) { if (this.progressBar) { const percentage = Math.round(((this.successCount + this.failureCount) / this.total) * 100); + const formattedPercentage = this.formatPercentage(percentage); const totalProcessed = this.successCount + this.failureCount; // Show completion status when finished, otherwise show running count @@ -405,7 +522,7 @@ export default class CLIProgressManager { this.progressBar.increment(1, { label: labelColor(` └─ ${displayName}`.padEnd(25)), status: statusText, - percentage: percentage.toString(), + percentage: formattedPercentage, }); } else if (this.spinner) { const total = this.successCount + this.failureCount; diff --git a/packages/contentstack-utilities/src/progress-summary/index.ts b/packages/contentstack-utilities/src/progress-summary/index.ts index 296a0790b7..5acb383d1d 100644 --- a/packages/contentstack-utilities/src/progress-summary/index.ts +++ b/packages/contentstack-utilities/src/progress-summary/index.ts @@ -1,4 +1,17 @@ import SummaryManager from './summary-manager'; import CLIProgressManager from './cli-progress-manager'; +import { + PrimaryProcessStrategy, + CustomProgressStrategy, + ProgressStrategyRegistry, + DefaultProgressStrategy, +} from './progress-strategy'; -export { SummaryManager, CLIProgressManager }; \ No newline at end of file +export { + SummaryManager, + CLIProgressManager, + PrimaryProcessStrategy, + CustomProgressStrategy, + ProgressStrategyRegistry, + DefaultProgressStrategy, +}; diff --git a/packages/contentstack-utilities/src/progress-summary/progress-strategy.ts b/packages/contentstack-utilities/src/progress-summary/progress-strategy.ts new file mode 100644 index 0000000000..c33c7f2c81 --- /dev/null +++ b/packages/contentstack-utilities/src/progress-summary/progress-strategy.ts @@ -0,0 +1,59 @@ +import { ProcessProgress, ProgressResult } from '../interfaces'; + +export interface ProgressCalculationStrategy { + calculate(processes: Map): ProgressResult | null; +} + +export class DefaultProgressStrategy implements ProgressCalculationStrategy { + calculate(): ProgressResult | null { + return null; // Use default aggregated counting + } +} + +export class PrimaryProcessStrategy implements ProgressCalculationStrategy { + constructor(private primaryProcessName: string) {} + + calculate(processes: Map): ProgressResult | null { + const primaryProcess = processes.get(this.primaryProcessName); + if (!primaryProcess) return null; + + return { + total: primaryProcess.total, + success: primaryProcess.successCount, + failures: primaryProcess.failureCount + }; + } +} + +export class CustomProgressStrategy implements ProgressCalculationStrategy { + constructor(private calculator: (processes: Map) => ProgressResult | null) {} + + calculate(processes: Map): ProgressResult | null { + return this.calculator(processes); + } +} + +// Registry +export class ProgressStrategyRegistry { + private static strategies = new Map(); + + static register(moduleName: string, strategy: ProgressCalculationStrategy): void { + this.strategies.set(moduleName.toUpperCase(), strategy); + } + + static get(moduleName: string): ProgressCalculationStrategy { + return this.strategies.get(moduleName.toUpperCase()) || new DefaultProgressStrategy(); + } + + static clear(): void { + this.strategies.clear(); + } + + static has(moduleName: string): boolean { + return this.strategies.has(moduleName.toUpperCase()); + } + + static getAllRegistered(): string[] { + return Array.from(this.strategies.keys()); + } +} diff --git a/packages/contentstack-utilities/src/progress-summary/summary-manager.ts b/packages/contentstack-utilities/src/progress-summary/summary-manager.ts index dbeae04703..48a12241e9 100644 --- a/packages/contentstack-utilities/src/progress-summary/summary-manager.ts +++ b/packages/contentstack-utilities/src/progress-summary/summary-manager.ts @@ -15,6 +15,10 @@ export default class SummaryManager { this.branchName = context.branchName || ''; } + getModules() { + return this.modules; + } + registerModule(moduleName: string, totalItems: number = 0): void { this.modules.set(moduleName, { name: moduleName, @@ -23,6 +27,7 @@ export default class SummaryManager { successCount: 0, failureCount: 0, failures: [], + processes: [], }); } @@ -46,6 +51,25 @@ export default class SummaryManager { } } + /** + * Register process data for strategy calculations + */ + registerProcessData(moduleName: string, processName: string, processData: any): void { + const module = this.modules.get(moduleName); + if (module) { + if (!module.processes) { + module.processes = []; + } + + const existingIndex = module.processes.findIndex((p: any) => p.processName === processName); + if (existingIndex >= 0) { + module.processes[existingIndex] = { processName, ...processData }; + } else { + module.processes.push({ processName, ...processData }); + } + } + } + updateModuleProgress(moduleName: string, success: boolean, itemName: string, error?: string): void { const module = this.modules.get(moduleName); if (module) { @@ -98,11 +122,10 @@ export default class SummaryManager { console.log( `${status} ${module.name.padEnd(20)} | ` + `${String(module.successCount).padStart(4)}/${String(totalCount).padStart(4)} items | ` + - `${successRate.padStart(6)}% | ` + + `${this.formatSuccessRate(successRate).padStart(6)} | ` + `${duration.padStart(8)}`, ); - // Show failures if any - TEMPORARILY DISABLED - will be shown in separate section later // if (module.failures.length > 0) { // console.log(chalk.red(` Failures (${module.failures.length}):`)); @@ -125,8 +148,6 @@ export default class SummaryManager { console.log(chalk.bold.red(`❌ ${this.operationName} failed`)); } - //TODO:- Smart Failure Summary - only show if there are failures - console.log(chalk.bold('='.repeat(80))); } @@ -155,4 +176,14 @@ export default class SummaryManager { if (total === 0) return '0'; return ((success / total) * 100).toFixed(1); } + + private formatSuccessRate(rate: string): string { + if (rate === '100.0') { + return '100%'; + } else if (parseFloat(rate) >= 10) { + return `${rate}%`; + } else { + return ` ${rate}%`; + } + } } diff --git a/packages/contentstack-variants/src/export/attributes.ts b/packages/contentstack-variants/src/export/attributes.ts index c7fe5f41b9..50596d376c 100644 --- a/packages/contentstack-variants/src/export/attributes.ts +++ b/packages/contentstack-variants/src/export/attributes.ts @@ -54,21 +54,21 @@ export default class ExportAttributes extends PersonalizationAdapter { const sanitizedAudience = omit(audience, this.audiencesConfig.invalidKeys); - // Update progress for each processed audience + // Update progress for each processed audience if (this.progressManager) { const processName = this.parentProgressManager ? 'Audiences' : undefined; this.updateProgress( @@ -126,7 +117,7 @@ export default class ExportAudiences extends PersonalizationAdapter { return; } - // Create progress manager - use parent if available, otherwise create simple let progress: any; + const processName = 'Events'; + if (this.parentProgressManager) { - // Use parent progress manager - we're part of the personalize modules process progress = this.parentProgressManager; this.progressManager = this.parentProgressManager; + progress.updateProcessTotal(processName, this.events.length); } else { - // Create our own progress for standalone execution - progress = this.createSimpleProgress('Events', this.events.length + 1); + progress = this.createSimpleProgress('Events', this.events.length); } log.debug(`Processing ${this.events.length} events`, this.exportConfig.context); - - // Update progress with process name - const processName = 'Events'; progress.updateStatus('Sanitizing events data...', processName); - + this.sanitizeAttribs(); log.debug('Events sanitization completed', this.exportConfig.context); @@ -80,9 +77,9 @@ export default class ExportEvents extends PersonalizationAdapter { log.debug(`Writing events to: ${eventsFilePath}`, this.exportConfig.context); fsUtil.writeFile(eventsFilePath, this.events); - // Final progress update + // Final progress update if (this.progressManager) { - this.updateProgress(true, `${this.events.length} events exported`, undefined, processName); + //this.updateProgress(true, `${this.events.length} events exported`, undefined, processName); } // Complete progress only if we're managing our own progress @@ -109,7 +106,6 @@ export default class ExportEvents extends PersonalizationAdapter { this.events?.map((event, index) => { const sanitizedEvent = omit(event, this.eventsConfig.invalidKeys); - // Update progress for each processed event if (this.progressManager) { const processName = this.parentProgressManager ? 'Events' : undefined; this.updateProgress( @@ -118,7 +114,7 @@ export default class ExportEvents extends PersonalizationAdapter { (event as any).key || (event as any).name || (event as any).uid || 'unknown' }`, undefined, - processName + processName, ); } diff --git a/packages/contentstack-variants/src/export/experiences.ts b/packages/contentstack-variants/src/export/experiences.ts index bf03986388..82872a44fc 100644 --- a/packages/contentstack-variants/src/export/experiences.ts +++ b/packages/contentstack-variants/src/export/experiences.ts @@ -64,24 +64,21 @@ export default class ExportExperiences extends PersonalizationAdapter if (this.parentProgressManager) { progress = this.parentProgressManager; this.progressManager = this.parentProgressManager; + progress.updateProcessTotal('Projects', this.projectsData?.length); } else { progress = this.createNestedProgress('Projects'); - progress.addProcess('Projects', 1); + progress.addProcess('Projects', this.projectsData?.length); progress.startProcess('Projects').updateStatus('Processing and exporting project data...', 'Projects'); } diff --git a/packages/contentstack-variants/src/export/variant-entries.ts b/packages/contentstack-variants/src/export/variant-entries.ts index a83440f439..5cad4994e4 100644 --- a/packages/contentstack-variants/src/export/variant-entries.ts +++ b/packages/contentstack-variants/src/export/variant-entries.ts @@ -71,6 +71,9 @@ export default class VariantEntries extends VariantAdapter { private attributesUidMapper: Record; private personalizeConfig: ImportConfig['modules']['personalize']; private attributeConfig: ImportConfig['modules']['personalize']['attributes']; + private attributeData: AttributeStruct[]; - constructor(public readonly config: ImportConfig) { + constructor(public readonly config: ImportConfig) { const conf: APIConfig = { config, baseURL: config.modules.personalize.baseURL[config.region.name], headers: { 'X-Project-Uid': config.modules.personalize.project_id }, }; super(Object.assign(config, conf)); - + this.personalizeConfig = this.config.modules.personalize; this.attributeConfig = this.personalizeConfig.attributes; this.mapperDirPath = resolve( @@ -31,61 +32,114 @@ export default class Attribute extends PersonalizationAdapter { this.attributesUidMapperPath = resolve(sanitizePath(this.attrMapperDirPath), 'uid-mapping.json'); this.attributesUidMapper = {}; this.config.context.module = 'attributes'; + this.attributeData = []; } /** * The function asynchronously imports attributes from a JSON file and creates them in the system. */ - async import() { - await this.init(); - await fsUtil.makeDirectory(this.attrMapperDirPath); - log.debug(`Created mapper directory: ${this.attrMapperDirPath}`, this.config.context); - - const { dirName, fileName } = this.attributeConfig; - const attributesPath = resolve( - sanitizePath(this.config.data), - sanitizePath(this.personalizeConfig.dirName), - sanitizePath(dirName), - sanitizePath(fileName), - ); + async import() { + try { + log.debug('Starting attributes import...', this.config.context); - log.debug(`Checking for attributes file: ${attributesPath}`, this.config.context); - - if (existsSync(attributesPath)) { - try { - const attributes = fsUtil.readFile(attributesPath, true) as AttributeStruct[]; - log.info(`Found ${attributes.length} attributes to import`, this.config.context); - - for (const attribute of attributes) { - const { key, name, description, uid } = attribute; - log.debug(`Processing attribute: ${name} - ${attribute.__type}`, this.config.context); - - // skip creating preset attributes, as they are already present in the system - if (attribute.__type === 'PRESET') { - log.debug(`Skipping preset attribute: ${name}`, this.config.context); - continue; - } - - try { - log.debug(`Creating custom attribute: ${name}`, this.config.context); - const attributeRes = await this.createAttribute({ key, name, description }); - //map old attribute uid to new attribute uid - //mapper file is used to check whether attribute created or not before creating audience - this.attributesUidMapper[uid] = attributeRes?.uid ?? ''; - log.debug(`Created attribute: ${uid} -> ${attributeRes?.uid}`, this.config.context); - } catch (error) { - handleAndLogError(error, this.config.context, `Failed to create attribute: ${name}`); - } + const [canImport, attributesCount] = await this.analyzeAttributes(); + if (!canImport) { + log.info('No attributes found to import', this.config.context); + // Still need to mark as complete for parent progress + if (this.parentProgressManager) { + this.parentProgressManager.tick(true, 'attributes module (no data)', null, 'Attributes'); } + return; + } - fsUtil.writeFile(this.attributesUidMapperPath, this.attributesUidMapper); - log.debug(`Saved ${Object.keys(this.attributesUidMapper).length} attribute mappings to: ${this.attributesUidMapperPath}`, this.config.context); - log.success('Attributes imported successfully', this.config.context); - } catch (error) { - handleAndLogError(error, this.config.context); + // If we have a parent progress manager, use it as a sub-module + // Otherwise create our own simple progress manager + let progress; + if (this.parentProgressManager) { + progress = this.parentProgressManager; + log.debug('Using parent progress manager for attributes import', this.config.context); + } else { + progress = this.createSimpleProgress('Attributes', attributesCount); + log.debug('Created standalone progress manager for attributes import', this.config.context); } - } else { - log.warn(`Attributes file not found: ${attributesPath}`, this.config.context); + + await this.init(); + await fsUtil.makeDirectory(this.attrMapperDirPath); + log.debug(`Created mapper directory: ${this.attrMapperDirPath}`, this.config.context); + + const { dirName, fileName } = this.attributeConfig; + log.info(`Processing ${attributesCount} attributes`, this.config.context); + + for (const attribute of this.attributeData) { + const { key, name, description, uid } = attribute; + if (!this.parentProgressManager) { + progress.updateStatus(`Processing attribute: ${name}...`); + } + log.debug(`Processing attribute: ${name} - ${attribute.__type}`, this.config.context); + + // skip creating preset attributes, as they are already present in the system + if (attribute.__type === 'PRESET') { + log.debug(`Skipping preset attribute: ${name}`, this.config.context); + this.updateProgress(true, `attribute: ${name} (preset - skipped)`, undefined, 'Attributes'); + continue; + } + + try { + log.debug(`Creating custom attribute: ${name}`, this.config.context); + const attributeRes = await this.createAttribute({ key, name, description }); + //map old attribute uid to new attribute uid + //mapper file is used to check whether attribute created or not before creating audience + this.attributesUidMapper[uid] = attributeRes?.uid ?? ''; + + this.updateProgress(true, `attribute: ${name}`, undefined, 'Attributes'); + log.debug(`Created attribute: ${uid} -> ${attributeRes?.uid}`, this.config.context); + } catch (error) { + this.updateProgress(false, `attribute: ${name}`, (error as any)?.message, 'Attributes'); + handleAndLogError(error, this.config.context, `Failed to create attribute: ${name}`); + } + } + + fsUtil.writeFile(this.attributesUidMapperPath, this.attributesUidMapper); + log.debug(`Saved ${Object.keys(this.attributesUidMapper).length} attribute mappings`, this.config.context); + + if (!this.parentProgressManager) { + this.completeProgress(true); + } + log.success( + `Attributes imported successfully! Total attributes: ${attributesCount} - personalization enabled`, + this.config.context, + ); + } catch (error) { + if (!this.parentProgressManager) { + this.completeProgress(false, (error as any)?.message || 'Attributes import failed'); + } + handleAndLogError(error, this.config.context); + throw error; } } + + private async analyzeAttributes(): Promise<[boolean, number]> { + return this.withLoadingSpinner('ATTRIBUTES: Analyzing import data...', async () => { + const { dirName, fileName } = this.attributeConfig; + const attributesPath = resolve( + sanitizePath(this.config.data), + sanitizePath(this.personalizeConfig.dirName), + sanitizePath(dirName), + sanitizePath(fileName), + ); + + log.debug(`Checking for attributes file: ${attributesPath}`, this.config.context); + + if (!existsSync(attributesPath)) { + log.warn(`Attributes file not found: ${attributesPath}`, this.config.context); + return [false, 0]; + } + + this.attributeData = fsUtil.readFile(attributesPath, true) as AttributeStruct[]; + const attributesCount = this.attributeData?.length || 0; + + log.debug(`Found ${attributesCount} attributes to import`, this.config.context); + return [attributesCount > 0, attributesCount]; + }); + } } diff --git a/packages/contentstack-variants/src/import/audiences.ts b/packages/contentstack-variants/src/import/audiences.ts index aee1ebe69b..6496bc5811 100644 --- a/packages/contentstack-variants/src/import/audiences.ts +++ b/packages/contentstack-variants/src/import/audiences.ts @@ -13,15 +13,16 @@ export default class Audiences extends PersonalizationAdapter { private personalizeConfig: ImportConfig['modules']['personalize']; private audienceConfig: ImportConfig['modules']['personalize']['audiences']; public attributeConfig: ImportConfig['modules']['personalize']['attributes']; + private audiences: AudienceStruct[]; - constructor(public readonly config: ImportConfig ) { + constructor(public readonly config: ImportConfig) { const conf: APIConfig = { config, baseURL: config.modules.personalize.baseURL[config.region.name], headers: { 'X-Project-Uid': config.modules.personalize.project_id }, }; super(Object.assign(config, conf)); - + this.personalizeConfig = this.config.modules.personalize; this.audienceConfig = this.personalizeConfig.audiences; this.attributeConfig = this.personalizeConfig.attributes; @@ -39,67 +40,124 @@ export default class Audiences extends PersonalizationAdapter { ); this.audiencesUidMapper = {}; this.config.context.module = 'audiences'; + this.audiences = []; } /** * The function asynchronously imports audiences from a JSON file and creates them in the system. */ - async import() { - await this.init(); - await fsUtil.makeDirectory(this.audienceMapperDirPath); - log.debug(`Created mapper directory: ${this.audienceMapperDirPath}`, this.config.context); - - const { dirName, fileName } = this.audienceConfig; - const audiencesPath = resolve( - sanitizePath(this.config.data), - sanitizePath(this.personalizeConfig.dirName), - sanitizePath(dirName), - sanitizePath(fileName), - ); + async import() { + try { + log.debug('Starting audiences import...', this.config.context); - log.debug(`Checking for audiences file: ${audiencesPath}`, this.config.context); - - if (existsSync(audiencesPath)) { - try { - const audiences = fsUtil.readFile(audiencesPath, true) as AudienceStruct[]; - log.info(`Found ${audiences.length} audiences to import`, this.config.context); - - const attributesUid = (fsUtil.readFile(this.attributesMapperPath, true) as Record) || {}; - log.debug(`Loaded ${Object.keys(attributesUid).length} attribute mappings for audience processing`, this.config.context); - - for (const audience of audiences) { - let { name, definition, description, uid } = audience; - log.debug(`Processing audience: ${name} (${uid})`, this.config.context); - - try { - //check whether reference attributes exists or not - if (definition.rules?.length) { - log.debug(`Processing ${definition.rules.length} definition rules for audience: ${name}`, this.config.context); - definition.rules = lookUpAttributes(definition.rules, attributesUid); - log.debug(`Processed definition rules, remaining rules: ${definition.rules.length}`, this.config.context); - } else { - log.debug(`No definition rules found for audience: ${name}`, this.config.context); - } - - log.debug(`Creating audience: ${name}`, this.config.context); - const audienceRes = await this.createAudience({ definition, name, description }); - //map old audience uid to new audience uid - //mapper file is used to check whether audience created or not before creating experience - this.audiencesUidMapper[uid] = audienceRes?.uid ?? ''; - log.debug(`Created audience: ${uid} -> ${audienceRes?.uid}`, this.config.context); - } catch (error) { - handleAndLogError(error, this.config.context, `Failed to create audience: ${name} (${uid})`); + const [canImport, audiencesCount] = await this.analyzeAudiences(); + if (!canImport) { + log.info('No audiences found to import', this.config.context); + // Still need to mark as complete for parent progress + if (this.parentProgressManager) { + this.parentProgressManager.tick(true, 'audiences module (no data)', null, 'Audiences'); + } + return; + } + + // If we have a parent progress manager, use it as a sub-module + // Otherwise create our own simple progress manager + let progress; + if (this.parentProgressManager) { + progress = this.parentProgressManager; + log.debug('Using parent progress manager for audiences import', this.config.context); + } else { + progress = this.createSimpleProgress('Audiences', audiencesCount); + log.debug('Created standalone progress manager for audiences import', this.config.context); + } + + await this.init(); + await fsUtil.makeDirectory(this.audienceMapperDirPath); + log.debug(`Created mapper directory: ${this.audienceMapperDirPath}`, this.config.context); + + const attributesUid = (fsUtil.readFile(this.attributesMapperPath, true) as Record) || {}; + log.debug( + `Loaded ${Object.keys(attributesUid).length} attribute mappings for audience processing`, + this.config.context, + ); + + for (const audience of this.audiences) { + let { name, definition, description, uid } = audience; + if (!this.parentProgressManager) { + progress.updateStatus(`Processing audience: ${name}...`); + } + log.debug(`Processing audience: ${name} (${uid})`, this.config.context); + + try { + //check whether reference attributes exists or not + if (definition.rules?.length) { + log.debug( + `Processing ${definition.rules.length} definition rules for audience: ${name}`, + this.config.context, + ); + definition.rules = lookUpAttributes(definition.rules, attributesUid); + log.debug(`Processed definition rules, remaining rules: ${definition.rules.length}`, this.config.context); + } else { + log.debug(`No definition rules found for audience: ${name}`, this.config.context); } + + log.debug(`Creating audience: ${name}`, this.config.context); + const audienceRes = await this.createAudience({ definition, name, description }); + //map old audience uid to new audience uid + //mapper file is used to check whether audience created or not before creating experience + this.audiencesUidMapper[uid] = audienceRes?.uid ?? ''; + + this.updateProgress(true, `audience: ${name}`, undefined, 'Audiences'); + log.debug(`Created audience: ${uid} -> ${audienceRes?.uid}`, this.config.context); + } catch (error) { + this.updateProgress(false, `audience: ${name}`, (error as any)?.message, 'Audiences'); + handleAndLogError(error, this.config.context, `Failed to create audience: ${name} (${uid})`); } + } + + fsUtil.writeFile(this.audiencesUidMapperPath, this.audiencesUidMapper); + log.debug(`Saved ${Object.keys(this.audiencesUidMapper).length} audience mappings`, this.config.context); - fsUtil.writeFile(this.audiencesUidMapperPath, this.audiencesUidMapper); - log.debug(`Saved ${Object.keys(this.audiencesUidMapper).length} audience mappings to: ${this.audiencesUidMapperPath}`, this.config.context); - log.success('Audiences imported successfully', this.config.context); - } catch (error) { - handleAndLogError(error, this.config.context); + // Only complete progress if we own the progress manager (no parent) + if (!this.parentProgressManager) { + this.completeProgress(true); } - } else { - log.warn(`Audiences file not found: ${audiencesPath}`, this.config.context); + + log.success( + `Audiences imported successfully! Total audiences: ${audiencesCount}`, + this.config.context, + ); + } catch (error) { + if (!this.parentProgressManager) { + this.completeProgress(false, (error as any)?.message || 'Audiences import failed'); + } + handleAndLogError(error, this.config.context); + throw error; } } + + private async analyzeAudiences(): Promise<[boolean, number]> { + return this.withLoadingSpinner('AUDIENCES: Analyzing import data...', async () => { + const { dirName, fileName } = this.audienceConfig; + const audiencesPath = resolve( + sanitizePath(this.config.data), + sanitizePath(this.personalizeConfig.dirName), + sanitizePath(dirName), + sanitizePath(fileName), + ); + + log.debug(`Checking for audiences file: ${audiencesPath}`, this.config.context); + + if (!existsSync(audiencesPath)) { + log.warn(`Audiences file not found: ${audiencesPath}`, this.config.context); + return [false, 0]; + } + + this.audiences = fsUtil.readFile(audiencesPath, true) as AudienceStruct[]; + const audiencesCount = this.audiences?.length || 0; + + log.debug(`Found ${audiencesCount} audiences to import`, this.config.context); + return [audiencesCount > 0, audiencesCount]; + }); + } } diff --git a/packages/contentstack-variants/src/import/events.ts b/packages/contentstack-variants/src/import/events.ts index 795838f791..46647e8722 100644 --- a/packages/contentstack-variants/src/import/events.ts +++ b/packages/contentstack-variants/src/import/events.ts @@ -10,7 +10,8 @@ export default class Events extends PersonalizationAdapter { private eventsUidMapperPath: string; private eventsUidMapper: Record; private personalizeConfig: ImportConfig['modules']['personalize']; - private eventsConfig: ImportConfig['modules']['personalize']['events']; + private eventConfig: ImportConfig['modules']['personalize']['events']; + private events: EventStruct[]; constructor(public readonly config: ImportConfig) { const conf: APIConfig = { @@ -19,65 +20,124 @@ export default class Events extends PersonalizationAdapter { headers: { 'X-Project-Uid': config.modules.personalize.project_id }, }; super(Object.assign(config, conf)); - + this.personalizeConfig = this.config.modules.personalize; - this.eventsConfig = this.personalizeConfig.events; + this.eventConfig = this.personalizeConfig.events; this.mapperDirPath = resolve( sanitizePath(this.config.backupDir), 'mapper', sanitizePath(this.personalizeConfig.dirName), ); - this.eventMapperDirPath = resolve(sanitizePath(this.mapperDirPath), sanitizePath(this.eventsConfig.dirName)); + this.eventMapperDirPath = resolve(sanitizePath(this.mapperDirPath), sanitizePath(this.eventConfig.dirName)); this.eventsUidMapperPath = resolve(sanitizePath(this.eventMapperDirPath), 'uid-mapping.json'); this.eventsUidMapper = {}; - this.config.context.module = 'events'; + this.events = []; } /** * The function asynchronously imports events from a JSON file and creates them in the system. */ async import() { - await this.init(); - await fsUtil.makeDirectory(this.eventMapperDirPath); - log.debug(`Created mapper directory: ${this.eventMapperDirPath}`, this.config.context); - - const { dirName, fileName } = this.eventsConfig; - const eventsPath = resolve( - sanitizePath(this.config.data), - sanitizePath(this.personalizeConfig.dirName), - sanitizePath(dirName), - sanitizePath(fileName), - ); + try { + log.debug('Starting events import...', this.config.context); + + const [canImport, eventsCount] = await this.analyzeEvents(); + if (!canImport) { + log.info('No events found to import', this.config.context); + // Still need to mark as complete for parent progress + if (this.parentProgressManager) { + this.parentProgressManager.tick(true, 'events module (no data)', null, 'Events'); + } + return; + } + + // Don't create own progress manager if we have a parent + let progress; + if (this.parentProgressManager) { + progress = this.parentProgressManager; + log.debug('Using parent progress manager for events import', this.config.context); + } else { + progress = this.createSimpleProgress('Events', eventsCount); + log.debug('Created standalone progress manager for events import', this.config.context); + } + + await this.init(); + await fsUtil.makeDirectory(this.eventMapperDirPath); + log.debug(`Created mapper directory: ${this.eventMapperDirPath}`, this.config.context); + + log.info(`Processing ${eventsCount} events`, this.config.context); + + for (const event of this.events) { + const { key, description, uid } = event; + if (!this.parentProgressManager) { + progress.updateStatus(`Processing event: ${key}...`); + } + log.debug(`Processing event: ${key} (${uid})`, this.config.context); - log.debug(`Checking for events file: ${eventsPath}`, this.config.context); - - if (existsSync(eventsPath)) { - try { - const events = fsUtil.readFile(eventsPath, true) as EventStruct[]; - log.info(`Found ${events.length} events to import`, this.config.context); - - for (const event of events) { - const { key, description, uid } = event; - log.debug(`Processing event: ${key} (${uid})`, this.config.context); - - try { - log.debug(`Creating event: ${key}`, this.config.context); - const eventsResponse = await this.createEvents({ key, description }); - this.eventsUidMapper[uid] = eventsResponse?.uid ?? ''; - log.debug(`Created event: ${uid} -> ${eventsResponse?.uid}`, this.config.context); - } catch (error) { - handleAndLogError(error, this.config.context, `Failed to create event: ${key} (${uid})`); + try { + log.debug(`Creating event: ${key}`, this.config.context); + const eventRes = await this.createEvents({ key, description }); + this.eventsUidMapper[uid] = eventRes?.uid ?? ''; + + // For parent progress manager, we don't need to specify process name as it will be handled automatically + if (this.parentProgressManager) { + this.updateProgress(true, `event: ${key}`); + } else { + this.updateProgress(true, `event: ${key}`, undefined, 'Events'); + } + log.debug(`Created event: ${uid} -> ${eventRes?.uid}`, this.config.context); + } catch (error) { + if (this.parentProgressManager) { + this.updateProgress(false, `event: ${key}`, (error as any)?.message); + } else { + this.updateProgress(false, `event: ${key}`, (error as any)?.message, 'Events'); } + handleAndLogError(error, this.config.context, `Failed to create event: ${key} (${uid})`); } + } - fsUtil.writeFile(this.eventsUidMapperPath, this.eventsUidMapper); - log.debug(`Saved ${Object.keys(this.eventsUidMapper).length} event mappings to: ${this.eventsUidMapperPath}`, this.config.context); - log.success('Events imported successfully', this.config.context); - } catch (error) { - handleAndLogError(error, this.config.context); + fsUtil.writeFile(this.eventsUidMapperPath, this.eventsUidMapper); + log.debug(`Saved ${Object.keys(this.eventsUidMapper).length} event mappings`, this.config.context); + + // Only complete progress if we own the progress manager (no parent) + if (!this.parentProgressManager) { + this.completeProgress(true); + } + log.success( + `Events imported successfully! Total events: ${eventsCount} - personalization enabled`, + this.config.context, + ); + } catch (error) { + if (!this.parentProgressManager) { + this.completeProgress(false, (error as any)?.message || 'Events import failed'); } - } else { - log.warn(`Events file not found: ${eventsPath}`, this.config.context); + handleAndLogError(error, this.config.context); + throw error; } } + + private async analyzeEvents(): Promise<[boolean, number]> { + return this.withLoadingSpinner('EVENTS: Analyzing import data...', async () => { + const { dirName, fileName } = this.eventConfig; + const eventsPath = resolve( + sanitizePath(this.config.data), + sanitizePath(this.personalizeConfig.dirName), + sanitizePath(dirName), + sanitizePath(fileName), + ); + + log.debug(`Checking for events file: ${eventsPath}`, this.config.context); + + if (!existsSync(eventsPath)) { + log.warn(`Events file not found: ${eventsPath}`, this.config.context); + return [false, 0]; + } + + this.events = fsUtil.readFile(eventsPath, true) as EventStruct[]; + const eventsCount = this.events?.length || 0; + + log.debug(`Found ${eventsCount} events to import`, this.config.context); + return [eventsCount > 0, eventsCount]; + }); + } } diff --git a/packages/contentstack-variants/src/import/experiences.ts b/packages/contentstack-variants/src/import/experiences.ts index 29c4db03da..daf3475b20 100644 --- a/packages/contentstack-variants/src/import/experiences.ts +++ b/packages/contentstack-variants/src/import/experiences.ts @@ -40,6 +40,7 @@ export default class Experiences extends PersonalizationAdapter { private personalizeConfig: ImportConfig['modules']['personalize']; private audienceConfig: ImportConfig['modules']['personalize']['audiences']; private experienceConfig: ImportConfig['modules']['personalize']['experiences']; + private experiences: ExperienceStruct[]; constructor(public readonly config: ImportConfig) { const conf: APIConfig = { @@ -102,32 +103,53 @@ export default class Experiences extends PersonalizationAdapter { this.audiencesUid = (fsUtil.readFile(this.audiencesMapperPath, true) as Record) || {}; this.eventsUid = (fsUtil.readFile(this.eventsMapperPath, true) as Record) || {}; this.config.context.module = 'experiences'; + this.experiences = []; } /** * The function asynchronously imports experiences from a JSON file and creates them in the system. */ async import() { - await this.init(); - await fsUtil.makeDirectory(this.expMapperDirPath); - log.debug(`Created mapper directory: ${this.expMapperDirPath}`, this.config.context); + try { + log.debug('Starting experiences import...', this.config.context); - if (existsSync(this.experiencesPath)) { - log.debug(`Loading experiences from: ${this.experiencesPath}`, this.config.context); - - try { - const experiences = fsUtil.readFile(this.experiencesPath, true) as ExperienceStruct[]; - log.info(`Found ${experiences.length} experiences to import`, this.config.context); - - for (const experience of experiences) { - const { uid, ...restExperienceData } = experience; - log.debug(`Processing experience: ${uid}`, this.config.context); - - //check whether reference audience exists or not that referenced in variations having __type equal to AudienceBasedVariation & targeting - let experienceReqObj: CreateExperienceInput = lookUpAudiences(restExperienceData, this.audiencesUid); - //check whether events exists or not that referenced in metrics - experienceReqObj = lookUpEvents(experienceReqObj, this.eventsUid); + const [canImport, experiencesCount] = await this.analyzeExperiences(); + if (!canImport) { + log.info('No experiences found to import', this.config.context); + // Still need to mark as complete for parent progress + if (this.parentProgressManager) { + this.parentProgressManager.tick(true, 'experiences module (no data)', null, 'Experiences'); + } + return; + } + // If we have a parent progress manager, use it as a sub-module + // Otherwise create our own simple progress manager + let progress; + if (this.parentProgressManager) { + progress = this.parentProgressManager; + log.debug('Using parent progress manager for experiences import', this.config.context); + } else { + progress = this.createSimpleProgress('Experiences', experiencesCount); + log.debug('Created standalone progress manager for experiences import', this.config.context); + } + + await this.init(); + await fsUtil.makeDirectory(this.expMapperDirPath); + log.debug(`Created mapper directory: ${this.expMapperDirPath}`, this.config.context); + + log.info(`Processing ${experiencesCount} experiences for import`, this.config.context); + + for (const experience of this.experiences) { + const { uid, ...restExperienceData } = experience; + log.debug(`Processing experience: ${uid}`, this.config.context); + + //check whether reference audience exists or not that referenced in variations having __type equal to AudienceBasedVariation & targeting + let experienceReqObj: CreateExperienceInput = lookUpAudiences(restExperienceData, this.audiencesUid); + //check whether events exists or not that referenced in metrics + experienceReqObj = lookUpEvents(experienceReqObj, this.eventsUid); + + try { const expRes = (await this.createExperience(experienceReqObj)) as ExperienceStruct; //map old experience uid to new experience uid this.experiencesUidMapper[uid] = expRes?.uid ?? ''; @@ -139,39 +161,79 @@ export default class Experiences extends PersonalizationAdapter { } catch (error) { handleAndLogError(error, this.config.context, `Failed to import experience versions for ${expRes.uid}`); } - } - - fsUtil.writeFile(this.experiencesUidMapperPath, this.experiencesUidMapper); - log.success('Experiences created successfully', this.config.context); - - log.info('Validating variant and variant group creation',this.config.context); - this.pendingVariantAndVariantGrpForExperience = values(cloneDeep(this.experiencesUidMapper)); - const jobRes = await this.validateVariantGroupAndVariantsCreated(); - fsUtil.writeFile(this.cmsVariantPath, this.cmsVariants); - fsUtil.writeFile(this.cmsVariantGroupPath, this.cmsVariantGroups); - - if (jobRes) { - log.success('Variant and variant groups created successfully', this.config.context); - } else { - log.error('Failed to create variants and variant groups', this.config.context); - this.personalizeConfig.importData = false; - } - if (this.personalizeConfig.importData) { - log.info('Attaching content types to experiences', this.config.context); - await this.attachCTsInExperience(); - log.success('Content types attached to experiences successfully', this.config.context); + this.updateProgress(true, `experience: ${experience.name || uid}`, undefined, 'Experiences'); + log.debug(`Successfully processed experience: ${uid}`, this.config.context); + } catch (error) { + this.updateProgress(false, `experience: ${experience.name || uid}`, (error as any)?.message, 'Experiences'); + handleAndLogError(error, this.config.context, `Failed to create experience: ${uid}`); } + } + + fsUtil.writeFile(this.experiencesUidMapperPath, this.experiencesUidMapper); + log.success('Experiences created successfully', this.config.context); - await this.createVariantIdMapper(); - } catch (error) { - handleAndLogError(error, this.config.context); + log.info('Validating variant and variant group creation',this.config.context); + this.pendingVariantAndVariantGrpForExperience = values(cloneDeep(this.experiencesUidMapper)); + const jobRes = await this.validateVariantGroupAndVariantsCreated(); + fsUtil.writeFile(this.cmsVariantPath, this.cmsVariants); + fsUtil.writeFile(this.cmsVariantGroupPath, this.cmsVariantGroups); + + if (jobRes) { + log.success('Variant and variant groups created successfully', this.config.context); + } else { + log.error('Failed to create variants and variant groups', this.config.context); + this.personalizeConfig.importData = false; } - } else { - log.warn(`Experiences file not found: ${this.experiencesPath}`, this.config.context); + + if (this.personalizeConfig.importData) { + log.info('Attaching content types to experiences', this.config.context); + await this.attachCTsInExperience(); + log.success('Content types attached to experiences successfully', this.config.context); + } + + await this.createVariantIdMapper(); + + // Only complete progress if we own the progress manager (no parent) + if (!this.parentProgressManager) { + this.completeProgress(true); + } + + log.success( + `Experiences imported successfully! Total experiences: ${experiencesCount} - personalization enabled`, + this.config.context, + ); + } catch (error) { + if (!this.parentProgressManager) { + this.completeProgress(false, (error as any)?.message || 'Experiences import failed'); + } + handleAndLogError(error, this.config.context); + throw error; } } + private async analyzeExperiences(): Promise<[boolean, number]> { + return this.withLoadingSpinner('EXPERIENCES: Analyzing import data...', async () => { + log.debug(`Checking for experiences file: ${this.experiencesPath}`, this.config.context); + + if (!existsSync(this.experiencesPath)) { + log.warn(`Experiences file not found: ${this.experiencesPath}`, this.config.context); + return [false, 0]; + } + + this.experiences = fsUtil.readFile(this.experiencesPath, true) as ExperienceStruct[]; + const experiencesCount = this.experiences?.length || 0; + + if (experiencesCount < 1) { + log.warn('No experiences found in file', this.config.context); + return [false, 0]; + } + + log.debug(`Found ${experiencesCount} experiences to import`, this.config.context); + return [true, experiencesCount]; + }); + } + /** * function import experience versions from a JSON file and creates them in the project. */ diff --git a/packages/contentstack-variants/src/import/project.ts b/packages/contentstack-variants/src/import/project.ts index 5eb8811504..dcc0c6c9f9 100644 --- a/packages/contentstack-variants/src/import/project.ts +++ b/packages/contentstack-variants/src/import/project.ts @@ -6,7 +6,8 @@ import { APIConfig, CreateProjectInput, ImportConfig, ProjectStruct } from '../t export default class Project extends PersonalizationAdapter { private projectMapperFolderPath: string; - + private projectsData: CreateProjectInput[]; + constructor(public readonly config: ImportConfig) { const conf: APIConfig = { config, @@ -14,7 +15,7 @@ export default class Project extends PersonalizationAdapter { headers: { organization_uid: config.org_uid }, }; super(Object.assign(config, conf)); - + this.projectMapperFolderPath = pResolve( sanitizePath(this.config.backupDir), 'mapper', @@ -22,42 +23,45 @@ export default class Project extends PersonalizationAdapter { 'projects', ); this.config.context.module = 'project'; + this.projectsData = []; } /** * The function asynchronously imports projects data from a file and creates projects based on the * data. */ - async import() { - const personalize = this.config.modules.personalize; - const { dirName, fileName } = personalize.projects; - const projectPath = join( - sanitizePath(this.config.data), - sanitizePath(personalize.dirName), - sanitizePath(dirName), - sanitizePath(fileName), - ); - - log.debug(`Checking for project file: ${projectPath}`, this.config.context); - - if (existsSync(projectPath)) { - const projects = JSON.parse(readFileSync(projectPath, 'utf8')) as CreateProjectInput[]; - log.debug(`Loaded ${projects?.length || 0} projects from file`, this.config.context); + async import() { + try { + log.debug('Starting personalize project import...', this.config.context); - if (!projects || projects.length < 1) { - this.config.modules.personalize.importData = false; // Stop personalize import if stack not connected to any project - log.warn('No projects found in file', this.config.context); + const [canImport, projectsCount] = await this.analyzeProjects(); + if (!canImport) { + log.info('No projects found to import', this.config.context); return; } - + + // If we have a parent progress manager, use it as a sub-module + // Otherwise create our own simple progress manager + let progress; + if (this.parentProgressManager) { + progress = this.parentProgressManager; + log.debug('Using parent progress manager for projects import', this.config.context); + } else { + progress = this.createSimpleProgress('Projects', projectsCount); + log.debug('Created standalone progress manager for projects import', this.config.context); + } + await this.init(); - - for (const project of projects) { + + for (const project of this.projectsData) { + if (!this.parentProgressManager) { + progress.updateStatus(`Creating project: ${project.name}...`); + } log.debug(`Processing project: ${project.name}`, this.config.context); - + const createProject = async (newName: void | string): Promise => { log.debug(`Creating project with name: ${newName || project.name}`, this.config.context); - + return await this.createProject({ name: newName || project.name, description: project.description, @@ -75,19 +79,70 @@ export default class Project extends PersonalizationAdapter { }); }; - const projectRes = await createProject(this.config.personalizeProjectName); - this.config.modules.personalize.project_id = projectRes.uid; - this.config.modules.personalize.importData = true; + try { + const projectRes = await createProject(this.config.personalizeProjectName); + this.config.modules.personalize.project_id = projectRes.uid; + this.config.modules.personalize.importData = true; + + await fsUtil.makeDirectory(this.projectMapperFolderPath); + fsUtil.writeFile(pResolve(sanitizePath(this.projectMapperFolderPath), 'projects.json'), projectRes); + + this.updateProgress(true, `project: ${project.name}`, undefined, 'Projects'); + log.success(`Project created successfully: ${projectRes.uid}`, this.config.context); + } catch (error) { + this.updateProgress(false, `project: ${project.name}`, (error as any)?.message, 'Projects'); + throw error; + } + } + + // Only complete progress if we own the progress manager (no parent) + if (!this.parentProgressManager) { + this.completeProgress(true); + } - await fsUtil.makeDirectory(this.projectMapperFolderPath); - fsUtil.writeFile(pResolve(sanitizePath(this.projectMapperFolderPath), 'projects.json'), projectRes); - - log.success(`Project created successfully: ${projectRes.uid}`, this.config.context); - log.debug(`Project data saved to: ${this.projectMapperFolderPath}/projects.json`, this.config.context); + log.success( + `Projects imported successfully! Total projects: ${projectsCount} - personalization enabled`, + this.config.context, + ); + } catch (error) { + this.config.modules.personalize.importData = false; + if (!this.parentProgressManager) { + this.completeProgress(false, (error as any)?.message || 'Project import failed'); } - } else { - this.config.modules.personalize.importData = false; // Stop personalize import if stack not connected to any project - log.warn(`Project file not found: ${projectPath}`, this.config.context); + throw error; } } + + private async analyzeProjects(): Promise<[boolean, number]> { + return this.withLoadingSpinner('PROJECT: Analyzing import data...', async () => { + const personalize = this.config.modules.personalize; + const { dirName, fileName } = personalize.projects; + const projectPath = join( + sanitizePath(this.config.data), + sanitizePath(personalize.dirName), + sanitizePath(dirName), + sanitizePath(fileName), + ); + + log.debug(`Checking for project file: ${projectPath}`, this.config.context); + + if (!existsSync(projectPath)) { + this.config.modules.personalize.importData = false; + log.warn(`Project file not found: ${projectPath}`, this.config.context); + return [false, 0]; + } + + this.projectsData = JSON.parse(readFileSync(projectPath, 'utf8')) as CreateProjectInput[]; + const projectsCount = this.projectsData?.length || 0; + + if (projectsCount < 1) { + this.config.modules.personalize.importData = false; + log.warn('No projects found in file', this.config.context); + return [false, 0]; + } + + log.debug(`Found ${projectsCount} projects to import`, this.config.context); + return [true, projectsCount]; + }); + } } diff --git a/packages/contentstack-variants/src/import/variant-entries.ts b/packages/contentstack-variants/src/import/variant-entries.ts index 006435db6b..98dcf51ba4 100644 --- a/packages/contentstack-variants/src/import/variant-entries.ts +++ b/packages/contentstack-variants/src/import/variant-entries.ts @@ -74,34 +74,35 @@ export default class VariantEntries extends VariantAdapter extends AdapterHelper impl * Update progress for a specific item */ protected updateProgress(success: boolean, itemName: string, error?: string, processName?: string): void { - this.progressManager?.tick(success, itemName, error, processName); + if (this.parentProgressManager) { + this.parentProgressManager.tick(success, itemName, error, processName); + } else if (this.progressManager) { + this.progressManager.tick(success, itemName, error, processName); + } } static printFinalSummary(): void {