From 37fa5d6e2bb142601c0d0586d5eb743caad78b33 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 1 Oct 2025 15:23:52 +0530 Subject: [PATCH 1/2] chore: remove deprecated JS & ContentVersion support, default to TypeScript modules, update export command to main only --- .../src/commands/cm/stacks/export.ts | 6 +- .../contentstack-export/src/config/index.ts | 2 - .../src/export/module-exporter.ts | 103 +- .../src/export/modules-js/assets.js | 445 ----- .../src/export/modules-js/content-types.js | 89 - .../src/export/modules-js/custom-roles.js | 91 - .../src/export/modules-js/entries.js | 200 --- .../src/export/modules-js/environments.js | 69 - .../src/export/modules-js/extensions.js | 66 - .../src/export/modules-js/global-fields.js | 121 -- .../src/export/modules-js/index.js | 8 - .../src/export/modules-js/labels.js | 63 - .../src/export/modules-js/locales.js | 71 - .../src/export/modules-js/marketplace-apps.js | 172 -- .../src/export/modules-js/stack.js | 99 -- .../src/export/modules-js/webhooks.js | 73 - .../src/export/modules-js/workflows.js | 102 -- .../src/types/default-config.ts | 2 - .../src/utils/common-helper.ts | 10 - .../contentstack-import/src/config/index.ts | 1 - .../src/import/module-importer.ts | 38 +- .../src/import/modules-js/assets.js | 498 ------ .../src/import/modules-js/content-types.js | 231 --- .../src/import/modules-js/custom-roles.js | 168 -- .../src/import/modules-js/entries.js | 1517 ----------------- .../src/import/modules-js/environments.js | 102 -- .../src/import/modules-js/extensions.js | 100 -- .../src/import/modules-js/global-fields.js | 123 -- .../src/import/modules-js/index.js | 6 - .../src/import/modules-js/labels.js | 172 -- .../src/import/modules-js/locales.js | 213 --- .../src/import/modules-js/marketplace-apps.js | 557 ------ .../src/import/modules-js/webhooks.js | 106 -- .../src/import/modules-js/workflows.js | 200 --- .../src/import/modules/marketplace-apps.ts | 10 +- .../src/types/default-config.ts | 1 - .../src/types/import-config.ts | 1 - .../src/utils/backup-handler.ts | 3 +- .../src/utils/import-path-resolver.ts | 28 +- .../contentstack-import/src/utils/index.ts | 1 - packages/contentstack-import/src/utils/log.ts | 38 - .../src/utils/marketplace-app-helper.ts | 5 - .../src/types/export-config.ts | 2 - .../src/types/import-config.ts | 1 - 44 files changed, 76 insertions(+), 5838 deletions(-) delete mode 100644 packages/contentstack-export/src/export/modules-js/assets.js delete mode 100644 packages/contentstack-export/src/export/modules-js/content-types.js delete mode 100644 packages/contentstack-export/src/export/modules-js/custom-roles.js delete mode 100644 packages/contentstack-export/src/export/modules-js/entries.js delete mode 100644 packages/contentstack-export/src/export/modules-js/environments.js delete mode 100644 packages/contentstack-export/src/export/modules-js/extensions.js delete mode 100644 packages/contentstack-export/src/export/modules-js/global-fields.js delete mode 100644 packages/contentstack-export/src/export/modules-js/index.js delete mode 100644 packages/contentstack-export/src/export/modules-js/labels.js delete mode 100644 packages/contentstack-export/src/export/modules-js/locales.js delete mode 100644 packages/contentstack-export/src/export/modules-js/marketplace-apps.js delete mode 100644 packages/contentstack-export/src/export/modules-js/stack.js delete mode 100644 packages/contentstack-export/src/export/modules-js/webhooks.js delete mode 100644 packages/contentstack-export/src/export/modules-js/workflows.js delete mode 100755 packages/contentstack-import/src/import/modules-js/assets.js delete mode 100755 packages/contentstack-import/src/import/modules-js/content-types.js delete mode 100644 packages/contentstack-import/src/import/modules-js/custom-roles.js delete mode 100755 packages/contentstack-import/src/import/modules-js/entries.js delete mode 100755 packages/contentstack-import/src/import/modules-js/environments.js delete mode 100644 packages/contentstack-import/src/import/modules-js/extensions.js delete mode 100644 packages/contentstack-import/src/import/modules-js/global-fields.js delete mode 100644 packages/contentstack-import/src/import/modules-js/index.js delete mode 100644 packages/contentstack-import/src/import/modules-js/labels.js delete mode 100755 packages/contentstack-import/src/import/modules-js/locales.js delete mode 100644 packages/contentstack-import/src/import/modules-js/marketplace-apps.js delete mode 100644 packages/contentstack-import/src/import/modules-js/webhooks.js delete mode 100644 packages/contentstack-import/src/import/modules-js/workflows.js delete mode 100644 packages/contentstack-import/src/utils/log.ts diff --git a/packages/contentstack-export/src/commands/cm/stacks/export.ts b/packages/contentstack-export/src/commands/cm/stacks/export.ts index 21a4b78012..5304f4d059 100644 --- a/packages/contentstack-export/src/commands/cm/stacks/export.ts +++ b/packages/contentstack-export/src/commands/cm/stacks/export.ts @@ -19,7 +19,7 @@ import { import { ModuleExporter } from '../../../export'; import { Context, ExportConfig } from '../../../types'; -import { setupExportConfig, writeExportMetaFile } from '../../../utils'; +import { setupExportConfig } from '../../../utils'; export default class ExportCommand extends Command { static description: string = messageHandler.parse('Export content from a stack'); @@ -134,10 +134,6 @@ export default class ExportCommand extends Command { const managementAPIClient: ContentstackClient = await managementSDKClient(exportConfig); const moduleExporter = new ModuleExporter(managementAPIClient, exportConfig); await moduleExporter.start(); - - if (!exportConfig.branches?.length) { - writeExportMetaFile(exportConfig); - } log.success( `The content of the stack ${exportConfig.apiKey} has been exported successfully!`, exportConfig.context, diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 6823e2a534..dee5213920 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -1,7 +1,6 @@ import { DefaultConfig } from '../types'; const config: DefaultConfig = { - contentVersion: 2, versioning: false, host: 'https://api.contentstack.io/v3', developerHubUrls: { @@ -489,7 +488,6 @@ const config: DefaultConfig = { writeConcurrency: 5, developerHubBaseUrl: '', marketplaceAppEncryptionKey: 'nF2ejRQcTv', - onlyTSModules: ['taxonomies'], }; export default config; diff --git a/packages/contentstack-export/src/export/module-exporter.ts b/packages/contentstack-export/src/export/module-exporter.ts index a3fba04acd..90bc73f854 100644 --- a/packages/contentstack-export/src/export/module-exporter.ts +++ b/packages/contentstack-export/src/export/module-exporter.ts @@ -7,10 +7,9 @@ import { getBranchFromAlias, CLIProgressManager, } from '@contentstack/cli-utilities'; -import { setupBranches, setupExportDir, writeExportMetaFile } from '../utils'; import startModuleExport from './modules'; -import startJSModuleExport from './modules-js'; import { ExportConfig, Modules } from '../types'; +import { setupBranches, setupExportDir } from '../utils'; class ModuleExporter { private managementAPIClient: ContentstackClient; @@ -48,46 +47,55 @@ class ModuleExporter { } async exportByBranches(): Promise { - // loop through the branches and export it parallel - for (const [index, branch] of this.exportConfig.branches.entries()) { - try { - this.exportConfig.branchName = branch.uid; - this.stackAPIClient.stackHeaders.branch = branch.uid; - this.exportConfig.branchDir = path.join(this.exportConfig.exportDir, branch.uid); - - // Reset progress manager for each branch (except the first one which was initialized in export command) - if (index >= 0) { - CLIProgressManager.clearGlobalSummary(); - CLIProgressManager.initializeGlobalSummary( - `EXPORT-${branch.uid}`, - branch.uid, - `Exporting "${branch.uid}" branch content...`, - ); - } - - log.info(`Exporting content from branch ${branch.uid}`, this.exportConfig.context); - writeExportMetaFile(this.exportConfig, this.exportConfig.branchDir); - await this.export(); - - // Print branch-specific summary - if (index <= this.exportConfig.branches.length - 1) { - CLIProgressManager.printGlobalSummary(); - } - - log.success(`The content of branch ${branch.uid} has been exported successfully!`, this.exportConfig.context); - } catch (error) { - handleAndLogError( - error, - { ...this.exportConfig.context, branch: branch.uid }, - messageHandler.parse('FAILED_EXPORT_CONTENT_BRANCH', { branch: branch.uid }), - ); - throw new Error(messageHandler.parse('FAILED_EXPORT_CONTENT_BRANCH', { branch: branch.uid })); + let targetBranch; + + if (this.exportConfig.branchName) { + // User specified a branch - export only that branch + targetBranch = this.exportConfig.branches.find((branch) => branch.uid === this.exportConfig.branchName); + if (!targetBranch) { + throw new Error(`Branch '${this.exportConfig.branchName}' not found in available branches`); + } + } else { + // No specific branch mentioned - export only the main branch + targetBranch = this.exportConfig.branches.find((branch) => branch.uid === 'main'); + if (!targetBranch) { + throw new Error('No main branch or available branches found'); } } + + try { + this.exportConfig.branchName = targetBranch.uid; + this.stackAPIClient.stackHeaders.branch = targetBranch.uid; + this.exportConfig.branchDir = path.join(this.exportConfig.exportDir, targetBranch.uid); + + // Initialize progress manager for the target branch + CLIProgressManager.clearGlobalSummary(); + CLIProgressManager.initializeGlobalSummary( + `EXPORT-${targetBranch.uid}`, + targetBranch.uid, + `Exporting "${targetBranch.uid}" branch content...`, + ); + + log.info(`Exporting content from '${targetBranch.uid}' branch`, this.exportConfig.context); + await this.export(); + CLIProgressManager.printGlobalSummary(); + + log.success( + `The content of branch ${targetBranch.uid} has been exported successfully!`, + this.exportConfig.context, + ); + } catch (error) { + handleAndLogError( + error, + { ...this.exportConfig.context, branch: targetBranch?.uid }, + messageHandler.parse('FAILED_EXPORT_CONTENT_BRANCH', { branch: targetBranch?.uid }), + ); + throw new Error(messageHandler.parse('FAILED_EXPORT_CONTENT_BRANCH', { branch: targetBranch?.uid })); + } } async export() { - log.info(`Started to export content, version is ${this.exportConfig.contentVersion}`, this.exportConfig.context); + log.info(`Started to export content`, this.exportConfig.context); // checks for single module or all modules if (this.exportConfig.singleModuleExport) { return this.exportSingleModule(this.exportConfig.moduleName); @@ -99,22 +107,11 @@ class ModuleExporter { log.info(`Exporting module: ${moduleName}`, this.exportConfig.context); // export the modules by name // calls the module runner which inturn calls the module itself - if (this.exportConfig.contentVersion === 2) { - await startModuleExport({ - stackAPIClient: this.stackAPIClient, - exportConfig: this.exportConfig, - moduleName, - }); - } else { - //NOTE - new modules support only ts - if (this.exportConfig.onlyTSModules.indexOf(moduleName) === -1) { - await startJSModuleExport({ - stackAPIClient: this.stackAPIClient, - exportConfig: this.exportConfig, - moduleName, - }); - } - } + await startModuleExport({ + stackAPIClient: this.stackAPIClient, + exportConfig: this.exportConfig, + moduleName, + }); } async exportSingleModule(moduleName: Modules): Promise { diff --git a/packages/contentstack-export/src/export/modules-js/assets.js b/packages/contentstack-export/src/export/modules-js/assets.js deleted file mode 100644 index 798e6320fd..0000000000 --- a/packages/contentstack-export/src/export/modules-js/assets.js +++ /dev/null @@ -1,445 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const mkdirp = require('mkdirp'); -const path = require('path'); -const fs = require('fs'); -const Promise = require('bluebird'); -const _ = require('lodash'); -const chalk = require('chalk'); -const progress = require('progress-stream'); -const { HttpClient, configHandler, validateUids, sanitizePath, validateFileName } = require('@contentstack/cli-utilities'); -const { fileHelper, log, formatError } = require('../../utils'); -let { default: config } = require('../../config'); - -module.exports = class ExportAssets { - config; - bLimit; - vLimit; - invalidKeys; - folderJSONPath; - folderData = []; - assetsFolderPath; - assetContents = {}; - httpClient = HttpClient.create(); - assetConfig = config.modules.assets; - - constructor(exportConfig, stackAPIClient) { - this.stackAPIClient = stackAPIClient; - this.config = _.merge(config, exportConfig); - this.folderData = []; - this.assetContents = {}; - this.assetDownloadRetry = {}; - this.assetDownloadRetryLimit = 3; - this.invalidKeys = this.assetConfig.invalidKeys; - this.bLimit = this.assetConfig.batchLimit || 15; - this.vLimit = this.assetConfig.downloadLimit || this.config.fetchConcurrency || 3; - } - - start() { - const self = this; - this.assetsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.assetConfig.dirName); - this.assetContentsFile = path.resolve(this.assetsFolderPath, 'assets.json'); - this.folderJSONPath = path.resolve(this.assetsFolderPath, 'folders.json'); - - log(this.config, 'Starting assets export', 'success'); - - // Create asset folder - mkdirp.sync(this.assetsFolderPath); - - return new Promise((resolve, reject) => { - // TBD: getting all the assets should have optimized - return self - .getAssetCount() - .then((count) => { - const assetBatches = []; - - if (typeof count !== 'number' || count === 0) { - log(self.config, 'No assets found', 'success'); - return resolve(); - } - for (let i = 0; i <= count; i += self.bLimit) { - assetBatches.push(i); - } - - return Promise.map( - assetBatches, - (batch) => { - return self - .getAssetJSON(batch) - .then((assetsJSON) => { - return Promise.map( - assetsJSON, - (assetJSON) => { - if (self.assetConfig.downloadVersionAssets) { - return self - .getVersionedAssetJSON(assetJSON.uid, assetJSON._version) - .then(() => { - self.assetContents[assetJSON.uid] = assetJSON; - }) - .catch((error) => { - log( - self.config, - `Asset '${assetJSON.uid}' failed to download.\n ${formatError(error)}`, - 'error', - ); - log(self.config, error, 'error'); - }); - } else { - return self - .downloadAsset(assetJSON) - .then(() => { - self.assetContents[assetJSON.uid] = assetJSON; - }) - .catch((err) => { - log(self.config, `Asset '${assetJSON.uid}' download failed. ${formatError(err)}`, 'error'); - return err; - }); - } - }, - { concurrency: self.vLimit }, - ) - .then(() => { - log(self.config, 'Batch no ' + (batch + 1) + ' of assets is complete', 'success'); - // fileHelper.writeFileSync(this.assetContentsFile, self.assetContents) - }) - .catch((error) => { - log(self.config, `Asset batch ${batch + 1} failed to download`, 'error'); - log(self.config, formatError(error), 'error'); - log(self.config, error, 'error'); - }); - }) - .catch((error) => { - log(self.config, error, 'error'); - reject(error); - }); - }, - { concurrency: self.assetConfig.concurrencyLimit || 1 }, - ) - .then(() => { - fileHelper.writeFileSync(self.assetContentsFile, self.assetContents); - - return self - .exportFolders() - .then(() => { - log(self.config, chalk.green('Asset export completed successfully'), 'success'); - return resolve(); - }) - .catch((error) => { - log(self.config, error, 'error'); - reject(error); - }); - }) - .catch((error) => { - fileHelper.writeFileSync(self.assetContentsFile, self.assetContents); - log(self.config, `Asset export failed. ${formatError(error)}`, 'error'); - log(self.config, error, 'error'); - reject(error); - }); - }) - .catch((error) => { - log(self.config, error, 'error'); - reject(error); - }); - }); - } - - exportFolders() { - const self = this; - return new Promise((resolve, reject) => { - return self - .getAssetCount(true) - .then((fCount) => { - if (fCount === 0) { - log(self.config, 'No folders were found in the stack!', 'success'); - return resolve(); - } - - return self - .getFolderJSON(0, fCount) - .then(() => { - // asset folders have been successfully exported - log(self.config, 'Asset-folders have been successfully exported!', 'success'); - return resolve(); - }) - .catch((error) => { - log(self.config, `Error while exporting asset-folders!\n ${formatError(error)}`, 'error'); - return reject(error); - }); - }) - .catch((error) => { - log(self.config, error, 'error'); - // error while fetching asset folder count - return reject(error); - }); - }); - } - - getFolderJSON(skip, fCount) { - const self = this; - return new Promise((resolve, reject) => { - if (typeof skip !== 'number') { - skip = 0; - } - - if (skip >= fCount) { - fileHelper.writeFileSync(self.folderJSONPath, self.folderData); - return resolve(); - } - - const queryRequestObj = { - skip, - include_folders: true, - query: { is_dir: true }, - }; - - self.stackAPIClient - .asset() - .query(queryRequestObj) - .find() - .then((response) => { - skip += 100; - self.folderData.push(...response.items); - return self.getFolderJSON(skip, fCount).then(resolve).catch(reject); - }) - .catch((error) => reject(error)); - }); - } - - getAssetCount(folder) { - const self = this; - return new Promise((resolve, reject) => { - if (folder && typeof folder === 'boolean') { - const queryOptions = { - skip: 99999990, - include_count: true, - include_folders: true, - query: { is_dir: true }, - }; - self.stackAPIClient - .asset() - .query(queryOptions) - .find() - .then((asset) => { - return resolve(asset.count); - }) - .catch((error) => { - log(self.config, error, 'error'); - }); - } else { - const queryOptions = { skip: 99999990, include_count: true }; - self.stackAPIClient - .asset() - .query(queryOptions) - .find() - .then(({ count }) => resolve(count)) - .catch((error) => { - log(self.config, error, 'error'); - reject(error); - }); - } - }); - } - - getAssetJSON(skip) { - const self = this; - return new Promise((resolve, reject) => { - if (typeof skip !== 'number') { - skip = 0; - } - const queryRequestObj = { - skip: skip, - limit: self.bLimit, - include_publish_details: true, - except: { - BASE: self.invalidKeys, - }, - }; - - self.stackAPIClient - .asset() - .query(queryRequestObj) - .find() - .then(({ items }) => resolve(items)) - .catch((error) => { - log(self.config, error, 'error'); - return reject(); - }); - }); - } - - getVersionedAssetJSON(uid, version, bucket) { - const self = this; - const assetVersionInfo = bucket || []; - - return new Promise((resolve, reject) => { - if (self.assetDownloadRetry[uid + version] > self.assetDownloadRetryLimit) { - console.log('Reached max', self.assetDownloadRetry[uid + version]); - return reject(new Error('Asset Max download retry limit exceeded! ' + uid)); - } - - if (version <= 0) { - if(validateUids(uid)){ - const assetVersionInfoFile = path.resolve(sanitizePath(self.assetsFolderPath), sanitizePath(uid), '_contentstack_' + sanitizePath(uid) + '.json'); - fileHelper.writeFileSync(assetVersionInfoFile, assetVersionInfo); - return resolve(); - } - } - const queryrequestOption = { - version: version, - include_publish_details: true, - except: { - BASE: self.invalidKeys, - }, - }; - - self.stackAPIClient - .asset(uid) - .fetch(queryrequestOption) - .then((versionedAssetJSONResponse) => { - self - .downloadAsset(versionedAssetJSONResponse) - .then(() => { - assetVersionInfo.splice(0, 0, versionedAssetJSONResponse); - // Remove duplicates - assetVersionInfo = _.uniqWith(assetVersionInfo, _.isEqual); - self.getVersionedAssetJSON(uid, --version, assetVersionInfo).then(resolve).catch(reject); - }) - .catch(reject); - }) - .catch((error) => { - log(self.config, error, 'error'); - - if (error.status === 408) { - console.log('retrying', uid); - // retrying when timeout - self.assetDownloadRetry[uid + version] - ? ++self.assetDownloadRetry[uid + version] - : (self.assetDownloadRetry[uid + version] = 1); - return self.getVersionedAssetJSON(uid, version, assetVersionInfo).then(resolve).catch(reject); - } - reject(error); - }); - }); - } - - downloadAsset(asset) { - const self = this; - return new Promise(async (resolve, reject) => { - if(!validateUids(asset.uid) && !validateFileName(asset.filename)) { - reject(`UIDs not valid`) - } - const assetFolderPath = path.resolve(sanitizePath(self.assetsFolderPath), sanitizePath(asset.uid)); - const assetFilePath = path.resolve(sanitizePath(assetFolderPath), sanitizePath(asset.filename)); - - if (fs.existsSync(assetFilePath)) { - log( - self.config, - 'Skipping download of { title: ' + asset.filename + ', uid: ' + asset.uid + ' }, as they already exist', - 'success', - ); - return resolve(); - } - self.assetStream = { - url: self.config.securedAssets ? `${asset.url}?authtoken=${configHandler.get('authtoken')}` : asset.url, - }; - - fileHelper.makeDirectory(assetFolderPath); - const assetFileStream = fs.createWriteStream(assetFilePath); - self.assetStream.url = encodeURI(self.assetStream.url); - self.httpClient - .options({ responseType: 'stream' }) - .get(self.assetStream.url) - .then(({ data: assetStreamRequest }) => { - if (self.assetConfig.enableDownloadStatus) { - const str = progress({ - time: 5000, - length: assetStreamRequest.headers['content-length'], - }); - str.on('progress', (progressData) => { - console.log(`${asset.filename}: ${Math.round(progressData.percentage)}%`); - }); - assetStreamRequest.pipe(str).pipe(assetFileStream); - } - assetStreamRequest.pipe(assetFileStream); - }) - .catch((error) => { - log(self.config, error, 'error'); - reject(error); - }); - assetFileStream - .on('close', function () { - log(self.config, 'Downloaded ' + asset.filename + ': ' + asset.uid + ' successfully!', 'success'); - return resolve(); - }) - .on('error', (error) => { - log(self.config, `Download ${asset.filename}: ${asset.uid} failed!`, 'error'); - log(self.config, error, 'error'); - reject(error); - }); - }); - } - - getFolders() { - const self = this; - return new Promise((resolve, reject) => { - return self - .getAssetCount(true) - .then((count) => { - if (count === 0) { - log(self.config, 'No folders were found in the stack', 'success'); - return resolve(); - } - return self - .getFolderDetails(0, count) - .then(function () { - log(self.config, chalk.green('Exported asset-folders successfully!'), 'success'); - return resolve(); - }) - .catch(function (error) { - log(self.config, error, 'error'); - reject(error); - }); - }) - .catch(function (error) { - log(self.config, error, 'error'); - reject(error); - }); - }); - } - - getFolderDetails(skip, tCount) { - const self = this; - return new Promise((resolve, reject) => { - if (typeof skip !== 'number') { - skip = 0; - } - if (skip > tCount) { - fileHelper.writeFileSync(self.folderJSONPath, self.folderContents); - return resolve(); - } - const queryRequestObj = { - skip: skip, - include_folders: true, - query: { is_dir: true }, - }; - self.stackAPIClient - .asset() - .query(queryRequestObj) - .find() - .then((folderDetailsResponse) => { - for (let i in folderDetailsResponse.items) { - self.folderContents.push(folderDetailsResponse.items[i]); - } - skip += 100; - return self.getFolderDetails(skip, tCount).then(resolve).catch(reject); - }) - .catch((error) => { - log(self.config, error, 'error'); - }); - }); - } -}; diff --git a/packages/contentstack-export/src/export/modules-js/content-types.js b/packages/contentstack-export/src/export/modules-js/content-types.js deleted file mode 100644 index d5dcde84ee..0000000000 --- a/packages/contentstack-export/src/export/modules-js/content-types.js +++ /dev/null @@ -1,89 +0,0 @@ -const path = require('path'); -const chalk = require('chalk'); -const { fileHelper, executeTask, formatError, log } = require('../../utils'); -const { sanitizePath } = require('@contentstack/cli-utilities'); - -class ContentTypesExport { - constructor(exportConfig, stackAPIClient) { - this.stackAPIClient = stackAPIClient; - this.exportConfig = exportConfig; - this.contentTypesConfig = exportConfig.modules.content_types; - this.qs = { - include_count: true, - asc: 'updated_at', - limit: this.contentTypesConfig.limit, - include_global_field_schema: true, - }; - // If content type id is provided then use it as part of query - if (Array.isArray(this.exportConfig.contentTypes) && this.exportConfig.contentTypes.length > 0) { - this.qs.uid = { $in: this.exportConfig.contentTypes }; - } - this.contentTypesPath = path.resolve( - sanitizePath(exportConfig.data), - sanitizePath(exportConfig.branchName) || '', - sanitizePath(this.contentTypesConfig.dirName), - ); - this.contentTypes = []; - this.fetchConcurrency = this.contentTypesConfig.fetchConcurrency || this.exportConfig.fetchConcurrency; - this.writeConcurrency = this.contentTypesConfig.writeConcurrency || this.exportConfig.writeConcurrency; - } - - async start() { - try { - log(this.exportConfig, 'Starting content type export', 'success'); - await fileHelper.makeDirectory(this.contentTypesPath); - await this.getContentTypes(); - await this.writeContentTypes(this.contentTypes); - log(this.exportConfig, chalk.green('Content type(s) exported successfully'), 'success'); - } catch (error) { - log(this.exportConfig, `Failed to export content types ${formatError(error)}`, 'error'); - throw new Error('Failed to export content types'); - } - } - - async getContentTypes(skip = 0) { - if (skip) { - this.qs.skip = skip; - } - - const contentTypeSearchResponse = await this.stackAPIClient.contentType().query(this.qs).find(); - if (Array.isArray(contentTypeSearchResponse.items) && contentTypeSearchResponse.items.length > 0) { - let updatedContentTypes = this.sanitizeAttribs(contentTypeSearchResponse.items); - this.contentTypes.push(...updatedContentTypes); - - skip += this.contentTypesConfig.limit; - if (skip > contentTypeSearchResponse.count) { - return; - } - return await this.getContentTypes(skip); - } else { - log(this.exportConfig, 'No content types returned for the given query', 'info'); - } - } - - sanitizeAttribs(contentTypes) { - let updatedContentTypes = []; - contentTypes.forEach((contentType) => { - for (let key in contentType) { - if (this.contentTypesConfig.validKeys.indexOf(key) === -1) { - delete contentType[key]; - } - } - updatedContentTypes.push(contentType); - }); - return updatedContentTypes; - } - - async writeContentTypes(contentTypes) { - function write(contentType) { - return fileHelper.writeFile( - path.join(sanitizePath(this.contentTypesPath), `${sanitizePath (contentType.uid === 'schema' ? 'schema|1' : contentType.uid)}.json`), - contentType, - ); - } - await executeTask(contentTypes, write.bind(this), { concurrency: this.writeConcurrency }); - return fileHelper.writeFile(path.join(this.contentTypesPath, 'schema.json'), contentTypes); - } -} - -module.exports = ContentTypesExport; diff --git a/packages/contentstack-export/src/export/modules-js/custom-roles.js b/packages/contentstack-export/src/export/modules-js/custom-roles.js deleted file mode 100644 index b75681c9bd..0000000000 --- a/packages/contentstack-export/src/export/modules-js/custom-roles.js +++ /dev/null @@ -1,91 +0,0 @@ -'use strict'; - -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const { merge } = require('lodash'); -const { fileHelper, log, formatError } = require('../../utils'); -const { default: config } = require('../../config'); - -module.exports = class ExportCustomRoles { - roles = {}; - customRoles = {}; - EXISTING_ROLES = { - Admin: 1, - Developer: 1, - 'Content Manager': 1, - }; - rolesConfig = config.modules.customRoles; - - constructor(exportConfig, stackAPIClient) { - this.config = merge(config, exportConfig); - this.stackAPIClient = stackAPIClient; - } - - async start() { - const self = this; - - try { - log(this.config, 'Starting roles export', 'success'); - - const rolesFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.rolesConfig.dirName); - mkdirp.sync(rolesFolderPath); - const roles = await self.stackAPIClient.role().fetchAll({ include_rules: true, include_permissions: true }); - const customRoles = roles.items.filter((role) => !self.EXISTING_ROLES[role.name]); - if (!customRoles.length) { - log(self.config, 'No custom roles were found in the Stack', 'success'); - return; - } - await self.getCustomRolesLocales( - customRoles, - path.join(rolesFolderPath, self.rolesConfig.customRolesLocalesFileName), - self.stackAPIClient, - self.config, - ); - self.customRoles = {}; - customRoles.forEach((role) => { - log(self.config, `'${role.name}' role was exported successfully`, 'success'); - self.customRoles[role.uid] = role; - }); - fileHelper.writeFileSync(path.join(rolesFolderPath, self.rolesConfig.fileName), self.customRoles); - log(self.config, chalk.green('All the custom roles have been exported successfully'), 'success'); - } catch (error) { - if (error.statusCode === 401) { - log( - self.config, - 'You are not allowed to export roles, Unless you provide email and password in config', - 'error', - ); - return; - } - log(self.config, `Error occurred in exporting roles. ${formatError(error)}`, 'error'); - throw error; - } - } - - async getCustomRolesLocales(customRoles, customRolesLocalesFilepath, stackAPIClient, config) { - const localesMap = {}; - for (const role of customRoles) { - const rulesLocales = role.rules.find((rule) => rule.module === 'locale'); - if (rulesLocales.locales && rulesLocales.locales.length) { - rulesLocales.locales.forEach((locale) => { - localesMap[locale] = 1; - }); - } - } - if (Object.keys(localesMap).length) { - const locales = await stackAPIClient.locale().query({}).find(); - const sourceLocalesMap = {}; - for (const locale of locales.items) { - sourceLocalesMap[locale.uid] = locale; - } - for (const locale in localesMap) { - if (sourceLocalesMap[locale] !== undefined) { - delete sourceLocalesMap[locale]['stackHeaders']; - } - localesMap[locale] = sourceLocalesMap[locale]; - } - fileHelper.writeFileSync(customRolesLocalesFilepath, localesMap); - } - } -}; diff --git a/packages/contentstack-export/src/export/modules-js/entries.js b/packages/contentstack-export/src/export/modules-js/entries.js deleted file mode 100644 index a9a9627be7..0000000000 --- a/packages/contentstack-export/src/export/modules-js/entries.js +++ /dev/null @@ -1,200 +0,0 @@ -const path = require('path'); -const chalk = require('chalk'); -const { values } = require('lodash'); -const { executeTask, formatError, fileHelper, log } = require('../../utils'); -const { sanitizePath } = require('@contentstack/cli-utilities'); - -class EntriesExport { - constructor(exportConfig, stackAPIClient) { - this.stackAPIClient = stackAPIClient; - this.exportConfig = exportConfig; - this.entriesConfig = exportConfig.modules.entries; - this.entriesRootPath = path.resolve((sanitizePath(exportConfig.data)), sanitizePath(exportConfig.branchName || ''), sanitizePath(this.entriesConfig.dirName)); - this.localesFilePath = path.resolve( - sanitizePath(exportConfig.data), - sanitizePath(exportConfig.branchName || ''), - sanitizePath(exportConfig.modules.locales.dirName), - sanitizePath(exportConfig.modules.locales.fileName), - ); - this.schemaFilePath = path.resolve( - sanitizePath(exportConfig.data), - sanitizePath(exportConfig.branchName || ''), - sanitizePath(exportConfig.modules.content_types.dirName), - 'schema.json', - ); - this.fetchConcurrency = this.entriesConfig.fetchConcurrency || exportConfig.fetchConcurrency; - this.writeConcurrency = this.entriesConfig.writeConcurrency || exportConfig.writeConcurrency; - } - - async start() { - try { - log(this.exportConfig, 'Starting entries export', 'info'); - const locales = await fileHelper.readFile(this.localesFilePath); - const contentTypes = await fileHelper.readFile(this.schemaFilePath); - if (contentTypes.length === 0) { - log(this.exportConfig, 'No content types found to export entries', 'info'); - return; - } - const entryRequestOptions = this.createRequestObjects(locales, contentTypes); - for (let requestOption of entryRequestOptions) { - log( - this.exportConfig, - `Starting export of entries of content_type - ${requestOption.content_type} locale - ${requestOption.locale}`, - 'info', - ); - await fileHelper.makeDirectory(path.join(this.entriesRootPath, requestOption.content_type)); - const entries = await this.getEntries(requestOption); - let entriesFilePath = path.join( - this.entriesRootPath, - requestOption.content_type, - requestOption.locale + '.json', - ); - await fileHelper.writeLargeFile(entriesFilePath, entries); - log( - this.exportConfig, - `Exported entries of type '${requestOption.content_type}' locale '${requestOption.locale}'`, - 'success', - ); - if (this.exportConfig.versioning) { - log( - this.exportConfig, - `Started export versioned entries of type '${requestOption.content_type}' locale '${requestOption.locale}'`, - 'info', - ); - for (let entry of values(entries)) { - const versionedEntries = await this.getEntryByVersion( - { - ...requestOption, - uid: entry.uid, - }, - entry._version, - ); - let versionedEntryPath = path.join( - this.entriesRootPath, - requestOption.locale, - requestOption.content_type, - entry.uid, - ); - await fileHelper.makeDirectory(versionedEntryPath); - if (versionedEntries.length > 0) { - const write = (versionedEntry) => - fileHelper.writeFile( - path.join(sanitizePath(versionedEntryPath), 'version-' + sanitizePath(versionedEntry._version) + '.json'), - versionedEntry, - ); - await executeTask(versionedEntries, write.bind(this), { concurrency: this.writeConcurrency }); - log( - this.exportConfig, - `Exported versioned entries of type '${requestOption.content_type}' locale '${requestOption.locale}'`, - 'success', - ); - } - } - } - } - log(this.exportConfig, chalk.green('Entries exported successfully'), 'success'); - } catch (error) { - log(this.exportConfig, `Failed to export entries ${formatError(error)}`, 'error'); - throw new Error('Failed to export entries'); - } - } - - async getEntries(requestOptions, skip = 0, entries = {}) { - let requestObject = { - locale: requestOptions.locale, - skip, - limit: this.entriesConfig.limit, - include_count: true, - include_publish_details: true, - query: { - locale: requestOptions.locale, - }, - }; - - const entriesSearchResponse = await this.stackAPIClient - .contentType(requestOptions.content_type) - .entry() - .query(requestObject) - .find(); - - if (Array.isArray(entriesSearchResponse.items) && entriesSearchResponse.items.length > 0) { - // clean up attribs and add to parent entry list - this.sanitizeAttribs(entriesSearchResponse.items, entries); - skip += this.entriesConfig.limit || 100; - if (skip > entriesSearchResponse.count) { - return entries; - } - return await this.getEntries(requestOptions, skip, entries); - } - return entries; - } - - async getEntryByVersion(requestOptions, version, entries = []) { - const queryRequestObject = { - locale: requestOptions.locale, - except: { - BASE: this.entriesConfig.invalidKeys, - }, - version, - }; - const entryResponse = await this.stackAPIClient - .contentType(requestOptions.content_type) - .entry(requestOptions.uid) - .fetch(queryRequestObject); - entries.push(entryResponse); - if (--version > 0) { - return await this.getEntryByVersion(requestOptions, version, entries); - } - return entries; - } - - async getEntriesCount(requestOptions) { - let requestObject = { - locale: requestOptions.locale, - limit: 1, - include_count: true, - include_publish_details: true, - query: { - locale: requestOptions.locale, - }, - }; - - const entriesSearchResponse = await this.stackAPIClient - .contentType(requestOptions.content_type) - .entry() - .query(requestObject) - .find(); - - return entriesSearchResponse.count; - } - - createRequestObjects(locales, contentTypes) { - let requestObjects = []; - contentTypes.forEach((contentType) => { - if (Object.keys(locales).length !== 0) { - for (let locale in locales) { - requestObjects.push({ - content_type: contentType.uid, - locale: locales[locale].code, - }); - } - } - requestObjects.push({ - content_type: contentType.uid, - locale: this.exportConfig.master_locale.code, - }); - }); - - return requestObjects; - } - - sanitizeAttribs(entries, entriesList = {}) { - entries.forEach((entry) => { - this.entriesConfig.invalidKeys.forEach((key) => delete entry[key]); - entriesList[entry.uid] = entry; - }); - return entriesList; - } -} - -module.exports = EntriesExport; diff --git a/packages/contentstack-export/src/export/modules-js/environments.js b/packages/contentstack-export/src/export/modules-js/environments.js deleted file mode 100644 index 6c8af23acf..0000000000 --- a/packages/contentstack-export/src/export/modules-js/environments.js +++ /dev/null @@ -1,69 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const { merge } = require('lodash'); -const { fileHelper, log, formatError } = require('../../utils'); -module.exports = class ExportEnvironments { - config = {}; - master = {}; - environments = {}; - requestOptions = { - json: true, - qs: { - asc: 'updated_at', - include_count: true, - }, - }; - - constructor(exportConfig, stackAPIClient) { - this.config = merge(this.config, exportConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - const self = this; - const environmentConfig = self.config.modules.environments; - const environmentsFolderPath = path.resolve( - self.config.data, - self.config.branchName || '', - environmentConfig.dirName, - ); - - // Create folder for environments - fileHelper.makeDirectory(environmentsFolderPath); - log(this.config, 'Starting environment export', 'success'); - return new Promise(function (resolve, reject) { - self.stackAPIClient - .environment() - .query(self.requestOptions.qs) - .find() - .then((environmentResponse) => { - if (environmentResponse.items.length !== 0) { - for (let i = 0, total = environmentResponse.count; i < total; i++) { - const envUid = environmentResponse.items[i].uid; - self.master[envUid] = ''; - self.environments[envUid] = environmentResponse.items[i]; - delete self.environments[envUid].uid; - delete self.environments[envUid]['ACL']; - } - fileHelper.writeFileSync(path.join(environmentsFolderPath, environmentConfig.fileName), self.environments); - log(self.config, chalk.green('All the environments have been exported successfully'), 'success'); - return resolve(); - } - if (environmentResponse.items.length === 0) { - log(self.config, 'No environments found', 'success'); - resolve(); - } - }) - .catch((error) => { - log(self.config, `Environments export failed. ${formatError(error)}`, 'error'); - reject(error); - }); - }); - } -}; diff --git a/packages/contentstack-export/src/export/modules-js/extensions.js b/packages/contentstack-export/src/export/modules-js/extensions.js deleted file mode 100644 index 931b6f7c46..0000000000 --- a/packages/contentstack-export/src/export/modules-js/extensions.js +++ /dev/null @@ -1,66 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const mkdirp = require('mkdirp'); -const path = require('path'); -const chalk = require('chalk'); -const { merge } = require('lodash'); -const { formatError, log, fileHelper } = require('../../utils'); -const { default: config } = require('../../config'); - -module.exports = class ExportExtensions { - master = {}; - extensions = {}; - extensionConfig = config.modules.extensions; - queryRequestOptions = { - asc: 'updated_at', - include_count: true, - }; - - constructor(exportConfig, stackAPIClient) { - this.config = merge(config, exportConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - log(this.config, 'Starting extension export', 'success'); - - const self = this; - const extensionsFolderPath = path.resolve( - this.config.data, - this.config.branchName || '', - this.extensionConfig.dirName, - ); - // Create folder for extensions - mkdirp.sync(extensionsFolderPath); - return new Promise(function (resolve, reject) { - self.stackAPIClient - .extension() - .query(self.queryRequestOptions) - .find() - .then((extension) => { - if (extension.items.length !== 0) { - for (let i = 0, total = extension.count; i < total; i++) { - const extUid = extension.items[i].uid; - self.master[extUid] = ''; - self.extensions[extUid] = extension.items[i]; - delete self.extensions[extUid].uid; - delete self.extensions[extUid].SYS_ACL; - } - fileHelper.writeFileSync(path.join(extensionsFolderPath, self.extensionConfig.fileName), self.extensions); - log(self.config, chalk.green('All the extensions have been exported successfully'), 'success'); - return resolve(); - } - log(self.config, 'No extensions found', 'success'); - resolve(); - }) - .catch((error) => { - log(self.config, `Failed to export extensions. ${formatError(error)}`, 'error'); - reject(); - }); - }); - } -}; diff --git a/packages/contentstack-export/src/export/modules-js/global-fields.js b/packages/contentstack-export/src/export/modules-js/global-fields.js deleted file mode 100644 index ef32dc6457..0000000000 --- a/packages/contentstack-export/src/export/modules-js/global-fields.js +++ /dev/null @@ -1,121 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const { merge } = require('lodash'); -const { formatError, log, fileHelper } = require('../../utils'); -const { default: config } = require('../../config'); -const { sanitizePath } = require('@contentstack/cli-utilities'); - -module.exports = class ExportGlobalFields { - limit = 100; - config = {}; - global_fields = []; - master = {}; - globalfields = {}; - requestOptions = {}; - globalfieldsFolderPath; - globalfieldsConfig = config.modules.globalfields; - validKeys = config.modules.globalfields.validKeys; - - constructor(exportConfig, stackAPIClient) { - this.requestOptions = { - qs: { - skip: 0, - limit: this.limit, - asc: 'updated_at', - include_count: true, - }, - }; - this.config = merge(config, exportConfig); - this.stackAPIClient = stackAPIClient; - this.globalfieldsFolderPath = path.resolve( - sanitizePath(this.config.data), - sanitizePath(this.config.branchName || ''), - sanitizePath(this.globalfieldsConfig.dirName), - ); - } - - start() { - const self = this; - // Create folder for Global Fields - mkdirp.sync(self.globalfieldsFolderPath); - log(self.config, 'Starting Global Fields export', 'success'); - - return new Promise(function (resolve, reject) { - try { - return self - .getGlobalFields(0, self.config) - .then(function (result) { - if (!result) { - return self.writeGlobalFields().then(resolve).catch(reject); - } - return resolve(); - }) - .catch(reject); - } catch (error) { - log(self.config, error, 'error'); - return reject(error); - } - }); - } - - getGlobalFields(skip, globalFieldConfig) { - const self = this; - self.requestOptions.qs.skip = skip; - return new Promise(function (resolve, reject) { - self.stackAPIClient - .globalField() - .query(self.requestOptions.qs) - .find() - .then((globalFieldResponse) => { - try { - if (globalFieldResponse.items.length === 0) { - log(globalFieldConfig, 'No global fields found', 'success'); - return resolve('No Global Fields'); - } - globalFieldResponse.items.forEach(function (globalField) { - for (const key in globalField) { - if (self.validKeys.indexOf(key) === -1) { - delete globalField[key]; - } - } - self.global_fields.push(globalField); - }); - skip += self.limit; - if (skip >= globalFieldResponse.count) { - return resolve(); - } - return self.getGlobalFields(skip, globalFieldConfig).then(resolve).catch(reject); - } catch (error) { - log(globalFieldConfig, `Failed to export global-fields. ${formatError(error)}`, 'error'); - reject(error); - } - }) - .catch(reject); - }); - } - - writeGlobalFields() { - const self = this; - return new Promise(function (resolve, reject) { - try { - fileHelper.writeFileSync( - path.join(self.globalfieldsFolderPath, self.globalfieldsConfig.fileName), - self.global_fields, - ); - log(self.config, chalk.green('Global Fields export completed successfully'), 'success'); - - resolve(); - } catch (error) { - log(self.config, error, 'error'); - reject(error); - } - }); - } -}; diff --git a/packages/contentstack-export/src/export/modules-js/index.js b/packages/contentstack-export/src/export/modules-js/index.js deleted file mode 100644 index 23233448ad..0000000000 --- a/packages/contentstack-export/src/export/modules-js/index.js +++ /dev/null @@ -1,8 +0,0 @@ -async function startModuleExport(modulePayload) { - const { moduleName, exportConfig, stackAPIClient } = modulePayload; - const { default: ModuleRunner } = await import(`./${moduleName}.js`); - const moduleRunner = new ModuleRunner(exportConfig, stackAPIClient); - return moduleRunner.start(); -} - -module.exports = startModuleExport; diff --git a/packages/contentstack-export/src/export/modules-js/labels.js b/packages/contentstack-export/src/export/modules-js/labels.js deleted file mode 100644 index 9b6f4fba20..0000000000 --- a/packages/contentstack-export/src/export/modules-js/labels.js +++ /dev/null @@ -1,63 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const { merge } = require('lodash'); -const { formatError, log, fileHelper } = require('../../utils'); -const { default: config } = require('../../config'); - -module.exports = class ExportLabels { - labels = {}; - labelConfig = config.modules.labels; - - constructor(exportConfig, stackAPIClient) { - this.config = merge(config, exportConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - log(this.config, 'Starting labels export', 'success'); - const self = this; - const labelsFolderPath = path.resolve(config.data, this.config.branchName || '', self.labelConfig.dirName); - // Create locale folder - mkdirp.sync(labelsFolderPath); - return new Promise(function (resolve, reject) { - return self.stackAPIClient - .label() - .query() - .find() - .then((response) => { - if (response.items.length !== 0) { - response.items.forEach(function (label) { - log(self.config, `'${label.name}' label was exported successfully`, 'success'); - self.labels[label.uid] = label; - const deleteItems = self.config.modules.labels.invalidKeys; - deleteItems.forEach((e) => delete label[e]); - }); - log(self.config, chalk.green('All the labels have been exported successfully'), 'success'); - } else { - log(self.config, 'No labels found', 'success'); - } - fileHelper.writeFileSync(path.join(labelsFolderPath, self.labelConfig.fileName), self.labels); - resolve(); - }) - .catch(function (error) { - if (error.statusCode === 401) { - log( - self.config, - 'You are not allowed to export label, Unless you provide email and password in config', - 'error', - ); - return resolve(); - } - log(self.config, `Failed to export labels. ${formatError(error)}`, 'error'); - reject(); - }); - }); - } -}; diff --git a/packages/contentstack-export/src/export/modules-js/locales.js b/packages/contentstack-export/src/export/modules-js/locales.js deleted file mode 100644 index a377737d7b..0000000000 --- a/packages/contentstack-export/src/export/modules-js/locales.js +++ /dev/null @@ -1,71 +0,0 @@ -const path = require('path'); -const chalk = require('chalk'); -const { formatError, log, fileHelper } = require('../../utils'); -const { sanitizePath } = require('@contentstack/cli-utilities'); -class LocaleExport { - constructor(exportConfig, stackAPIClient) { - this.stackAPIClient = stackAPIClient; - this.exportConfig = exportConfig; - this.localeConfig = exportConfig.modules.locales; - this.masterLocaleConfig = exportConfig.modules.masterLocale; - this.qs = { - include_count: true, - asc: 'updated_at', - only: { - BASE: this.localeConfig.requiredKeys, - }, - }; - - this.localesPath = path.resolve(sanitizePath(exportConfig.data), sanitizePath(exportConfig.branchName || ''), sanitizePath(this.localeConfig.dirName)); - this.locales = {}; - this.masterLocale = {}; - this.fetchConcurrency = this.localeConfig.fetchConcurrency || this.exportConfig.fetchConcurrency; - this.writeConcurrency = this.localeConfig.writeConcurrency || this.exportConfig.writeConcurrency; - } - - async start() { - try { - log(this.exportConfig, 'Starting locale export', 'success'); - fileHelper.makeDirectory(this.localesPath); - await this.getLocales(); - await fileHelper.writeFile(path.join(this.localesPath, this.localeConfig.fileName), this.locales); - await fileHelper.writeFile(path.join(this.localesPath, this.masterLocaleConfig.fileName), this.masterLocale); - log(this.exportConfig, 'Completed locale export', 'success'); - } catch (error) { - log(this.exportConfig, `Failed to export locales. ${formatError(error)}`, 'error'); - throw new Error('Failed to export locales'); - } - } - - async getLocales(skip = 0) { - if (skip) { - this.qs.skip = skip; - } - let localesFetchResponse = await this.stackAPIClient.locale().query(this.qs).find(); - if (Array.isArray(localesFetchResponse.items) && localesFetchResponse.items.length > 0) { - this.sanitizeAttribs(localesFetchResponse.items); - skip += this.localeConfig.limit || 100; - if (skip > localesFetchResponse.count) { - return; - } - return await this.getLocales(skip); - } - } - - sanitizeAttribs(locales) { - locales.forEach((locale) => { - for (let key in locale) { - if (this.localeConfig.requiredKeys.indexOf(key) === -1) { - delete locale[key]; - } - } - if (locale.code === this.exportConfig.master_locale.code) { - this.masterLocale[locale.uid] = locale; - } else { - this.locales[locale.uid] = locale; - } - }); - } -} - -module.exports = LocaleExport; diff --git a/packages/contentstack-export/src/export/modules-js/marketplace-apps.js b/packages/contentstack-export/src/export/modules-js/marketplace-apps.js deleted file mode 100644 index ca4d7526f9..0000000000 --- a/packages/contentstack-export/src/export/modules-js/marketplace-apps.js +++ /dev/null @@ -1,172 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ -const _ = require('lodash'); -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const eachOf = require('async/eachOf'); -const { - cliux, - HttpClient, - NodeCrypto, - managementSDKClient, - HttpClientDecorator, - OauthDecorator, - isAuthenticated, -} = require('@contentstack/cli-utilities'); -const { default: config } = require('../../config'); -const { formatError, log, fileHelper } = require('../../utils'); -const { createNodeCryptoInstance } = require('../../utils'); - -module.exports = class ExportMarketplaceApps { - client; - config; - httpClient; - nodeCrypto; - marketplaceAppPath = null; - developerHubBaseUrl = null; - marketplaceAppConfig = config.modules.marketplace_apps; - - constructor(credentialConfig) { - this.config = _.merge(config, credentialConfig); - } - - async start() { - if (!isAuthenticated()) { - cliux.print( - 'WARNING!!! To export Marketplace apps, you must be logged in. Please check csdx auth:login --help to log in', - { color: 'yellow' }, - ); - return Promise.resolve(); - } - - this.developerHubBaseUrl = this.config.developerHubBaseUrl || (await getDeveloperHubUrl(this.config)); - this.appSdkAxiosInstance = await managementSDKClient({ - endpoint: this.developerHubBaseUrl, - }); - await this.getOrgUid(); - - const httpClient = new HttpClient(); - if (!this.config.auth_token) { - this.httpClient = new OauthDecorator(httpClient); - const headers = await this.httpClient.preHeadersCheck(this.config); - this.httpClient = this.httpClient.headers(headers); - } else { - this.httpClient = new HttpClientDecorator(httpClient); - this.httpClient.headers(this.config); - } - - log(this.config, 'Starting marketplace app export', 'success'); - this.marketplaceAppPath = path.resolve( - this.config.data, - this.config.branchName || '', - this.marketplaceAppConfig.dirName, - ); - mkdirp.sync(this.marketplaceAppPath); - - this.nodeCrypto = await createNodeCryptoInstance(config); - - return this.exportInstalledExtensions(); - } - - async getOrgUid() { - const tempAPIClient = await managementSDKClient({ host: this.config.host }); - const tempStackData = await tempAPIClient - .stack({ api_key: this.config.source_stack }) - .fetch() - .catch((error) => { - log(this.config, formatError(error), 'error'); - console.log(error); - }); - - if (tempStackData?.org_uid) { - this.config.org_uid = tempStackData.org_uid; - } - } - - async exportInstalledExtensions() { - const client = await managementSDKClient({ host: this.developerHubBaseUrl.split('://').pop() }); - const installedApps = (await this.getAllStackSpecificApps()) || []; - - if (!_.isEmpty(installedApps)) { - for (const [index, app] of _.entries(installedApps)) { - await this.getAppConfigurations(client, installedApps, [+index, app]); - } - - fileHelper.writeFileSync(path.join(this.marketplaceAppPath, this.marketplaceAppConfig.fileName), installedApps); - - log(this.config, chalk.green('All the marketplace apps have been exported successfully'), 'success'); - } else { - log(this.config, 'No marketplace apps found', 'success'); - } - } - - getAllStackSpecificApps(listOfApps = [], skip = 0) { - return this.appSdkAxiosInstance.axiosInstance - .get(`/installations?target_uids=${this.config.source_stack}&skip=${skip}`, { - headers: { - organization_uid: this.config.org_uid - }, - }) - .then(async ({ data }) => { - const { data: apps, count } = data; - - if (!this.nodeCrypto && _.find(apps, (app) => !_.isEmpty(app.configuration))) { - await this.createNodeCryptoInstance(); - } - - listOfApps.push( - ..._.map(apps, (app) => { - if (_.has(app, 'configuration')) { - app['configuration'] = this.nodeCrypto.encrypt(app.configuration || configuration); - } - - return app; - }), - ); - - if (count - (skip + 50) > 0) { - return await this.getAllStackSpecificApps(listOfApps, skip + 50); - } - - return listOfApps; - }) - .catch((error) => { - log(this.config, `Failed to export marketplace-apps. ${formatError(error)}`, 'error'); - }); - } - - async getAppConfigurations(sdkClient, installedApps, [index, appInstallation]) { - const appName = appInstallation.manifest.name; - log(this.config, `Exporting ${appName} app and it's config.`, 'success'); - - await sdkClient - .organization(this.config.org_uid) - .app(appInstallation.manifest.uid) - .installation(appInstallation.uid) - .installationData() - .then(async (result) => { - const { data, error } = result; - if (_.has(data, 'server_configuration')) { - if (!this.nodeCrypto && _.has(data, 'server_configuration')) { - await this.createNodeCryptoInstance(); - } - - if (!_.isEmpty(data.server_configuration)) { - installedApps[index]['server_configuration'] = this.nodeCrypto.encrypt(data.server_configuration); - log(this.config, `Exported ${appName} app and it's config.`, 'success'); - } else { - log(this.config, `Exported ${appName} app`, 'success'); - } - } else if (error) { - log(this.config, `Error on exporting ${appName} app and it's config.`, 'error'); - } - }) - .catch((err) => { - log(this.config, `Failed to export ${appName} app config ${formatError(err)}`, 'error'); - }); - } -}; diff --git a/packages/contentstack-export/src/export/modules-js/stack.js b/packages/contentstack-export/src/export/modules-js/stack.js deleted file mode 100644 index d585964c04..0000000000 --- a/packages/contentstack-export/src/export/modules-js/stack.js +++ /dev/null @@ -1,99 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const path = require('path'); -const mkdirp = require('mkdirp'); -const merge = require('lodash/merge'); -const { default: config } = require('../../config'); -const { managementSDKClient, isAuthenticated } = require('@contentstack/cli-utilities'); -const { log, fileHelper, formatError } = require('../../utils'); - -class ExportStack { - stackConfig = config.modules.stack; - - constructor(exportConfig, stackAPIClient) { - this.config = merge(config, exportConfig); - this.stackAPIClient = stackAPIClient; - this.requestOption = { - uri: this.config.host + this.config.apis.stacks, - headers: this.config.headers, - json: true, - }; - } - - async start() { - const self = this; - if (isAuthenticated()) { - const tempAPIClient = await managementSDKClient({ host: config.host }); - const tempStackData = await tempAPIClient - .stack({ api_key: self.config.source_stack }) - .fetch() - .catch((error) => { - }); - - if (tempStackData && tempStackData.org_uid) { - self.config.org_uid = tempStackData.org_uid; - self.config.sourceStackName = tempStackData.name; - } - } - - if (!self.config.preserveStackVersion && !self.config.hasOwnProperty('master_locale')) { - const apiDetails = { - limit: 100, - skip: 0, - include_count: true, - }; - return self.getLocales(apiDetails); - } else if (self.config.preserveStackVersion) { - log(self.config, 'Exporting stack details', 'success'); - let stackFolderPath = path.resolve(self.config.data, this.stackConfig.dirName); - let stackContentsFile = path.resolve(stackFolderPath, this.stackConfig.fileName); - - mkdirp.sync(stackFolderPath); - - return new Promise((resolve, reject) => { - return self.stackAPIClient - .fetch() - .then((response) => { - fileHelper.writeFile(stackContentsFile, response); - log(self.config, 'Exported stack details successfully!', 'success'); - return resolve(response); - }) - .catch(reject); - }); - } - } - - getLocales(apiDetails) { - const self = this; - return new Promise((resolve, reject) => { - const result = self.stackAPIClient.locale().query(apiDetails); - - result - .find() - .then((response) => { - const masterLocalObj = response.items.find((obj) => { - if (obj.fallback_locale === null) { - return obj; - } - }); - apiDetails.skip += apiDetails.limit; - if (masterLocalObj) { - return resolve(masterLocalObj); - } else if (apiDetails.skip <= response.count) { - return resolve(self.getLocales(apiDetails)); - } else { - return reject('Master locale not found'); - } - }) - .catch((error) => { - return reject(error); - }); - }); - } -} - -module.exports = ExportStack; diff --git a/packages/contentstack-export/src/export/modules-js/webhooks.js b/packages/contentstack-export/src/export/modules-js/webhooks.js deleted file mode 100644 index ec5e559a20..0000000000 --- a/packages/contentstack-export/src/export/modules-js/webhooks.js +++ /dev/null @@ -1,73 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const { merge } = require('lodash'); -const { default: config } = require('../../config'); -const { formatError, log, fileHelper } = require('../../utils'); - -// Create folder for environments -module.exports = class ExportWebhooks { - config; - master = {}; - webhooks = {}; - requestOptions = { - include_count: true, - asc: 'updated_at', - }; - webhooksConfig = config.modules.webhooks; - - constructor(exportConfig, stackAPIClient) { - this.config = merge(config, exportConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - log(this.config, 'Starting webhooks export', 'success'); - const self = this; - const webhooksFolderPath = path.resolve( - this.config.data, - this.config.branchName || '', - self.webhooksConfig.dirName, - ); - mkdirp.sync(webhooksFolderPath); - return new Promise((resolve, reject) => { - self.stackAPIClient - .webhook() - .fetchAll(self.requestOptions) - .then((webhooks) => { - if (webhooks.items.length !== 0) { - for (let i = 0, total = webhooks.count; i < total; i++) { - const webUid = webhooks.items[i].uid; - self.master[webUid] = ''; - self.webhooks[webUid] = webhooks.items[i]; - delete self.webhooks[webUid].uid; - delete self.webhooks[webUid].SYS_ACL; - } - fileHelper.writeFileSync(path.join(webhooksFolderPath, self.webhooksConfig.fileName), self.webhooks); - log(self.config, chalk.green('All the webhooks have been exported successfully'), 'success'); - return resolve(); - } - log(self.config, 'No webhooks found', 'success'); - resolve(); - }) - .catch(function (error) { - if (error.statusCode === 401) { - log( - self.config, - 'You are not allowed to export webhooks, Unless you provide email and password in config', - 'error', - ); - return resolve(); - } - log(self.config, `Failed to export webhooks. ${formatError(error)}`, 'error'); - reject('Failed to export webhooks'); - }); - }); - } -}; diff --git a/packages/contentstack-export/src/export/modules-js/workflows.js b/packages/contentstack-export/src/export/modules-js/workflows.js deleted file mode 100644 index e7cd7667ff..0000000000 --- a/packages/contentstack-export/src/export/modules-js/workflows.js +++ /dev/null @@ -1,102 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const { merge } = require('lodash'); -const { formatError, log, fileHelper } = require('../../utils'); -const { default: config } = require('../../config'); -module.exports = class ExportWorkFlows { - config; - workflows = {}; - workFlowConfig = config.modules.workflows; - - constructor(exportConfig, stackAPIClient) { - this.config = merge(config, exportConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - log(this.config, 'Starting workflow export', 'success'); - - const self = this; - const workflowsFolderPath = path.resolve( - this.config.data, - this.config.branchName || '', - this.workFlowConfig.dirName, - ); - mkdirp.sync(workflowsFolderPath); - - return new Promise(function (resolve, reject) { - return self.stackAPIClient - .workflow() - .fetchAll() - .then(async (response) => { - try { - if (response.items.length) { - await self.getWorkflowsData(self, response.items); - log(self.config, chalk.green('All the workflow have been exported successfully'), 'success'); - } - if (!response.items.length) { - log(self.config, 'No workflow were found in the Stack', 'success'); - } - fileHelper.writeFileSync(path.join(workflowsFolderPath, self.workFlowConfig.fileName), self.workflows); - resolve(); - } catch (error) { - log(self.config, formatError(error), 'error'); - reject(error); - } - }) - .catch(function (error) { - if (error.statusCode === 401) { - log( - self.config, - 'You are not allowed to export workflow, Unless you provide email and password in config', - 'error', - ); - return resolve(); - } - log(self.config, formatError(error), 'error'); - resolve(); - }); - }); - } - - async getWorkflowRoles(self, workflow) { - try { - for (const stage of workflow.workflow_stages) { - if (stage.SYS_ACL.roles.uids.length) { - for (let i = 0; i < stage.SYS_ACL.roles.uids.length; i++) { - const roleUid = stage.SYS_ACL.roles.uids[i]; - const roleData = await self.stackAPIClient - .role(roleUid) - .fetch({ include_rules: true, include_permissions: true }); - stage.SYS_ACL.roles.uids[i] = roleData; - } - } - } - } catch (error) { - log(self.config, `Error fetching roles in export workflows task. ${formatError(error)}`, 'error'); - throw new Error({ message: 'Error fetching roles in export workflows task.' }); - } - } - - async getWorkflowsData(self, workflows) { - try { - for (const workflow of workflows) { - log(self.config, workflow.name + ' workflow was exported successfully', 'success'); - await self.getWorkflowRoles(self, workflow); - self.workflows[workflow.uid] = workflow; - const deleteItems = config.modules.workflows.invalidKeys; - deleteItems.forEach((e) => delete workflow[e]); - } - } catch (error) { - log(self.config, `Error fetching workflow data in export workflows task. ${formatError(error)}`, 'error'); - throw error; - } - } -}; diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index c70a1de2dc..4345185b8c 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -5,7 +5,6 @@ interface AnyProperty { } export default interface DefaultConfig { - contentVersion: number; versioning: boolean; host: string; cdn?: string; @@ -215,5 +214,4 @@ export default interface DefaultConfig { writeConcurrency: number; developerHubBaseUrl: string; marketplaceAppEncryptionKey: string; - onlyTSModules: string[]; } diff --git a/packages/contentstack-export/src/utils/common-helper.ts b/packages/contentstack-export/src/utils/common-helper.ts index 0ece3d4205..8721244370 100644 --- a/packages/contentstack-export/src/utils/common-helper.ts +++ b/packages/contentstack-export/src/utils/common-helper.ts @@ -4,7 +4,6 @@ * MIT Licensed */ -import * as path from 'path'; import promiseLimit from 'promise-limit'; import { isAuthenticated, getLogPath, sanitizePath } from '@contentstack/cli-utilities'; @@ -79,12 +78,3 @@ export const executeTask = function ( }), ); }; - -// Note: we can add more useful details in meta file -export const writeExportMetaFile = (exportConfig: ExportConfig, metaFilePath?: string) => { - const exportMeta = { - contentVersion: exportConfig.contentVersion, - logsPath: getLogPath(), - }; - fsUtil.writeFile(path.join(sanitizePath(metaFilePath || exportConfig.exportDir), 'export-info.json'), exportMeta); -}; diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 1af842feda..8afa6d5115 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -436,7 +436,6 @@ const config: DefaultConfig = { getEncryptionKeyMaxRetry: 3, // useBackedupDir: '', // backupConcurrency: 10, - onlyTSModules: ['taxonomies', 'personalize', 'variant-entries', 'stack'], auditConfig: { noLog: false, // Skip logs printing on terminal skipConfirm: true, // Skip confirmation if any diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index 2a6879c091..37dd8644b0 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -4,15 +4,8 @@ import messages, { $t } from '@contentstack/cli-audit/lib/messages'; import { addLocale, cliux, ContentstackClient, log } from '@contentstack/cli-utilities'; import startModuleImport from './modules'; -import startJSModuleImport from './modules-js'; import { ImportConfig, Modules } from '../types'; -import { - backupHandler, - masterLocalDetails, - sanitizeStack, - setupBranchConfig, - executeImportPathLogic, -} from '../utils'; +import { backupHandler, masterLocalDetails, sanitizeStack, setupBranchConfig, executeImportPathLogic } from '../utils'; class ModuleImporter { private managementAPIClient: ContentstackClient; @@ -29,15 +22,14 @@ class ModuleImporter { } async start(): Promise { - if (!this.importConfig.management_token) { const stackDetails: Record = await this.stackAPIClient.fetch(); this.importConfig.stackName = stackDetails.name as string; this.importConfig.org_uid = stackDetails.org_uid as string; } - + await this.resolveImportPath(); - + await setupBranchConfig(this.importConfig, this.stackAPIClient); if (this.importConfig.branchAlias && this.importConfig.branchName) { this.stackAPIClient = this.managementAPIClient.stack({ @@ -83,7 +75,7 @@ class ModuleImporter { } async import() { - log.info(`Starting to import content version ${this.importConfig.contentVersion}`, this.importConfig.context); + log.info(`Starting to import`, this.importConfig.context); // checks for single module or all modules if (this.importConfig.singleModuleImport) { @@ -96,23 +88,11 @@ class ModuleImporter { log.info(`Starting import of ${moduleName} module`, this.importConfig.context); // import the modules by name // calls the module runner which inturn calls the module itself - // NOTE: Implement a mechanism to determine whether module is new or old - if (this.importConfig.contentVersion === 2) { - return startModuleImport({ - stackAPIClient: this.stackAPIClient, - importConfig: this.importConfig, - moduleName, - }); - } else { - //NOTE - new modules support only ts - if (this.importConfig.onlyTSModules.indexOf(moduleName) === -1) { - return startJSModuleImport({ - stackAPIClient: this.stackAPIClient, - importConfig: this.importConfig, - moduleName, - }); - } - } + return startModuleImport({ + stackAPIClient: this.stackAPIClient, + importConfig: this.importConfig, + moduleName, + }); } async importAllModules(): Promise { diff --git a/packages/contentstack-import/src/import/modules-js/assets.js b/packages/contentstack-import/src/import/modules-js/assets.js deleted file mode 100755 index df621297dd..0000000000 --- a/packages/contentstack-import/src/import/modules-js/assets.js +++ /dev/null @@ -1,498 +0,0 @@ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ -const fs = require('fs'); -const _ = require('lodash'); -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const Promise = require('bluebird'); -let { default: config } = require('../../config'); -const { fileHelper, log, uploadAssetHelper } = require('../../utils'); -const { sanitizePath, validateUids, validateFileName } = require('@contentstack/cli-utilities'); - -module.exports = class ImportAssets { - assets; - fails = []; - assetConfig; - mapperDirPath; - assetBatchLimit; - uidMapping = {}; - urlMapping = {}; - environmentPath; - assetBucket = []; - assetsFolderPath; - folderBucket = []; - folderDetails = []; - mappedFolderUids = {}; - - constructor(importConfig, stackAPIClient) { - this.config = _.merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - this.assetConfig = config.modules['assets-old']; - this.assetBatchLimit = - this.assetConfig.hasOwnProperty('assetBatchLimit') && typeof this.assetConfig.assetBatchLimit === 'number' - ? this.assetConfig.assetBatchLimit - : 2; - } - - start() { - let self = this; - log(self.config, 'Migrating assets', 'success'); - this.assetsFolderPath = path.join(this.config.data, this.assetConfig.dirName); - this.mapperDirPath = path.resolve(this.config.data, 'mapper', 'assets'); - this.environmentPath = path.resolve(this.config.data, 'environments', 'environments.json'); - this.uidMapperPath = path.join(this.mapperDirPath, 'uid-mapping.json'); - this.urlMapperPath = path.join(this.mapperDirPath, 'url-mapping.json'); - this.failsPath = path.join(this.mapperDirPath, 'fail.json'); - this.assets = fileHelper.readFileSync(path.join(this.assetsFolderPath, this.assetConfig.fileName)); - this.environment = fileHelper.readFileSync(this.environmentPath); - if (fs.existsSync(this.uidMapperPath)) { - this.uidMapping = fileHelper.readFileSync(this.uidMapperPath); - } - if (fs.existsSync(this.urlMapperPath)) { - this.urlMapping = fileHelper.readFileSync(this.urlMapperPath); - } - - mkdirp.sync(this.mapperDirPath); - - return new Promise(function (resolve, reject) { - if (self.assets === undefined || _.isEmpty(self.assets)) { - log(self.config, 'No Assets Found', 'success'); - return resolve({ empty: true }); - } - - let batches = []; - let assetUids = Object.keys(self.assets); - - for (let i = 0; i < assetUids.length; i += self.assetBatchLimit) { - batches.push(assetUids.slice(i, i + self.assetBatchLimit)); - } - - return self - .importFolders() - .then(function () { - return Promise.map( - batches, - async function (batch, index) { - return Promise.map( - batch, - function (assetUid) { - if (self.uidMapping.hasOwnProperty(assetUid)) { - log( - self.config, - 'Skipping upload of asset: ' + assetUid + '. Its mapped to: ' + self.uidMapping[assetUid], - 'success', - ); - // the asset has been already imported - return void 0; - } - if(!validateUids(assetUid)){ - reject(`UID Not Valid`) - } - let currentAssetFolderPath = path.join(sanitizePath(self.assetsFolderPath), sanitizePath(assetUid)); - if (fs.existsSync(currentAssetFolderPath)) { - // if this is true, means, the exported asset data is versioned - // hence, upload each asset with its version - if (self.config.versioning) { - return self.uploadVersionedAssets(assetUid, currentAssetFolderPath).catch(function (error) { - log(self.config, 'Asset upload failed \n' + error, 'error'); - }); - } - - let uidContainer = {}; - let urlContainer = {}; - if(!validateFileName(self.assets[assetUid].filename)){ - reject(`File Name Not Valid`) - } - let assetPath = path.resolve(sanitizePath(currentAssetFolderPath), sanitizePath(self.assets[assetUid].filename)); - - if (self.assets[assetUid].parent_uid && typeof self.assets[assetUid].parent_uid === 'string') { - if (self.mappedFolderUids.hasOwnProperty(self.assets[assetUid].parent_uid)) { - self.assets[assetUid].parent_uid = self.mappedFolderUids[self.assets[assetUid].parent_uid]; - } else { - log( - self.config, - `'${self.assets[assetUid].parent_uid}' parent_uid was not found! Thus, setting it as 'null'`, - 'error', - ); - } - } - - return self - .uploadAsset(assetPath, self.assets[assetUid], uidContainer, urlContainer) - .then(async function () { - self.uidMapping[assetUid] = uidContainer[assetUid]; - self.urlMapping[self.assets[assetUid].url] = urlContainer[self.assets[assetUid].url]; - - if (self.config.entriesPublish && self.assets[assetUid].publish_details.length > 0) { - let assetsUid = uidContainer[assetUid]; - try { - return await self.publish(assetsUid, self.assets[assetUid]); - } catch (error) { - return error; - } - } - // assetUid has been successfully uploaded - // log them onto /mapper/assets/success.json - }) - .catch(function (error) { - log(self.config, 'Asset upload failed \n' + error, 'error'); - return error; - // asset failed to upload - // log them onto /mapper/assets/fail.json - }); - } - log(self.config, `'${currentAssetFolderPath}' does not exist!`, 'error'); - }, - { concurrency: self.assetConfig.assetBatchLimit }, - ).then(function () { - fileHelper.writeFileSync(self.uidMapperPath, self.uidMapping); - fileHelper.writeFileSync(self.urlMapperPath, self.urlMapping); - // completed uploading assets - log(self.config, 'Completed asset import of batch no: ' + (index + 1), 'success'); - return index + 1; - // TODO: if there are failures, retry - }); - }, - { concurrency: 1 }, - ) - .then(function () { - let numberOfSuccessfulAssetUploads = Object.keys(self.uidMapping).length; - if (numberOfSuccessfulAssetUploads > 0) { - log( - self.config, - chalk.green(numberOfSuccessfulAssetUploads + ' assets uploaded successfully!'), - 'success', - ); - } - // TODO: if there are failures, retry - return resolve(); - }) - .catch(function (error) { - log(self.config, error, 'error'); - return reject(error); - }); - }) - .catch(function (error) { - log(self.config, error, 'error'); - return reject(error); - }); - }); - } - - uploadVersionedAssets(uid, assetFolderPath) { - let self = this; - return new Promise(function (resolve, reject) { - if(!validateUids(uid)){ - reject(`UID not valid`) - } - let versionedAssetMetadata = fileHelper.readFileSync( - path.join(sanitizePath(assetFolderPath), '_contentstack_' + sanitizePath(uid) + '.json'), - ); - // using last version, find asset's parent - let lastVersion = versionedAssetMetadata[versionedAssetMetadata.length - 1]; - - if (typeof lastVersion.parent_uid === 'string') { - if (self.mappedFolderUids.hasOwnProperty(lastVersion.parent_uid)) { - // update each version of that asset with the last version's parent_uid - versionedAssetMetadata.forEach(function (assetMetadata) { - assetMetadata.parent_uid = self.mappedFolderUids[lastVersion.parent_uid]; - }); - } else { - log(self.config, (lastVersion.parent_uid + " parent_uid was not found! Thus, setting it as 'null'", 'error')); - versionedAssetMetadata.forEach(function (assetMetadata) { - assetMetadata.parent_uid = null; - }); - } - } - let counter = 0; - let uidContainer = {}; - let urlContainer = {}; - let filesStreamed = []; - - return Promise.map( - versionedAssetMetadata, - function () { - let assetMetadata = versionedAssetMetadata[counter]; - if(!validateFileName(assetMetadata.filename)){ - reject(`File Name not valid`) - } - let assetPath = path.join(sanitizePath(assetFolderPath), sanitizePath(assetMetadata.filename)); - - if (++counter === 1) { - return self - .uploadAsset(assetPath, assetMetadata, uidContainer, urlContainer) - .then(function () { - filesStreamed.push(assetMetadata.filename); - }) - .catch((error) => { - log(self.config, error, 'error'); - reject(error); - }); - } - - return self - .updateAsset(assetPath, assetMetadata, filesStreamed, uidContainer, urlContainer) - .then(function () { - filesStreamed.push(assetMetadata.filename); - }) - .catch((error) => { - log(self.config, error, 'error'); - }); - }, - { concurrency: self.assetConfig.uploadAssetsConcurrency }, - ) - .then(function () { - self.uidMapping[uid] = uidContainer[uid]; - for (let url in urlContainer) { - self.urlMapping[url] = urlContainer[url]; - } - // completed uploading all the versions of the asset - return resolve(); - }) - .catch(function (error) { - // failed to upload asset - // write it on fail logs, but do not stop the process - log(self.config, 'Failed to upload asset\n' + error, 'error'); - return resolve(); - }); - }); - } - - updateAsset(assetPath, metadata, filesStreamed, _uidContainer, urlContainer) { - const self = this; - return new Promise(function (resolve, reject) { - let requestOption = {}; - if (filesStreamed && filesStreamed.indexOf(metadata.filename) !== -1) { - log(self.config, 'Skipping re-upload/streaming of ' + metadata.uid + '/' + metadata.filename, 'success'); - requestOption.formData = {}; - return resolve(); - } - - log(self.config, 'Streaming: ' + metadata.uid + '/' + metadata.filename, 'success'); - requestOption.formData = {}; - - if (metadata.hasOwnProperty('parent_uid') && typeof metadata.parent_uid === 'string') { - requestOption.formData['asset[parent_uid]'] = metadata.parent_uid; - } - - if (metadata.hasOwnProperty('description') && typeof metadata.description === 'string') { - requestOption.formData['asset[description]'] = metadata.description; - } - - if (metadata.hasOwnProperty('tags') && Array.isArray(metadata.tags)) { - requestOption.formData['asset[tags]'] = metadata.tags; - } - - if (metadata.hasOwnProperty('title') && typeof metadata.title === 'string') { - requestOption.formData['asset[title]'] = metadata.title; - } - - return uploadAssetHelper(self.config, requestOption, assetPath) - .then(function (response) { - urlContainer[metadata.url] = response.url; - return resolve(); - }) - .catch(function (error) { - log(self.config, error, 'error'); - return reject(error); - }); - }); - } - - uploadAsset(assetPath, metadata, uidContainer, urlContainer) { - const self = this; - return new Promise(function (resolve, reject) { - let requestOption = {}; - - if (metadata.hasOwnProperty('parent_uid') && typeof metadata.parent_uid === 'string') { - requestOption.parent_uid = metadata.parent_uid; - } - - if (metadata.hasOwnProperty('description') && typeof metadata.description === 'string') { - requestOption.description = metadata.description; - } - - // eslint-disable-next-line no-prototype-builtins - if (metadata.hasOwnProperty('tags') && Array.isArray(metadata.tags)) { - requestOption.tags = metadata.tags; - } - - if (metadata.hasOwnProperty('title') && typeof metadata.title === 'string') { - requestOption.title = metadata.title; - } - return uploadAssetHelper(self.config, requestOption, assetPath) - .then(function (response) { - uidContainer[metadata.uid] = response.uid; - urlContainer[metadata.url] = response.url; - return resolve(); - }) - .catch(function (error) { - log(self.config, error, 'error'); - return reject(error); - }); - }); - } - - importFolders() { - let self = this; - return new Promise(function (resolve, reject) { - let mappedFolderPath = path.resolve(self.config.data, 'mapper', 'assets', 'folder-mapping.json'); - self.folderDetails = fileHelper.readFileSync(path.resolve(self.assetsFolderPath, 'folders.json')); - - if (_.isEmpty(self.folderDetails)) { - log(self.config, 'No folders were found at: ' + path.join(self.assetsFolderPath, 'folders.json'), 'success'); - return resolve(); - } - let tree = self.buildTree(_.cloneDeep(self.folderDetails)); - let createdFolders = {}; - let createdFolderUids = []; - // if a few folders have already been created, skip re-creating them - if (fs.existsSync(mappedFolderPath)) { - createdFolders = fileHelper.readFileSync(mappedFolderPath); - // check if the read file has mapped objects - if (_.isPlainObject(createdFolders)) { - createdFolderUids = Object.keys(createdFolders); - } - } - self.buildFolderReqObjs(createdFolderUids, tree, null); - let idx = 0; - return Promise.map( - self.folderBucket, - function () { - let folder = self.folderBucket[idx]; - if (createdFolders.hasOwnProperty(folder.json.asset.parent_uid)) { - // replace old uid with new - folder.json.asset.parent_uid = createdFolders[folder.json.asset.parent_uid]; - } - return self.stackAPIClient - .asset() - .folder() - .create(folder.json) - .then((response) => { - log(self.config, "Created folder: '" + folder.json.asset.name + "'", 'success'); - // assigning newUid to oldUid - createdFolders[folder.oldUid] = response.uid; - fileHelper.writeFileSync(mappedFolderPath, createdFolders); - idx++; - }) - .catch(function (err) { - let error = JSON.parse(err.message); - if (error.errors.authorization || error.errors.api_key) { - log(self.config, 'Api_key or management_token is not valid', 'error'); - return reject(error); - } - - log(self.config, err, 'error'); - return error; - }); - }, - { concurrency: self.assetConfig.importFoldersConcurrency }, - ) - .then(function () { - self.mappedFolderUids = fileHelper.readFileSync(mappedFolderPath); - // completed creating folders - return resolve(); - }) - .catch(function (error) { - log(self.config, error, 'error'); - return reject(error); - }); - }); - } - - buildFolderReqObjs(createdFolderUids, tree, parent_uid) { - let self = this; - for (let leaf in tree) { - // if the folder is already created, skip - if (createdFolderUids.indexOf(leaf) !== -1) { - continue; - } - let folderObj = _.find(self.folderDetails, { uid: leaf }); - let requestOption = { - json: { - asset: { - name: folderObj.name, - parent_uid: parent_uid || null, - }, - }, - oldUid: leaf, - }; - self.folderBucket.push(requestOption); - if (Object.keys(tree[leaf]).length > 0) { - self.buildFolderReqObjs(createdFolderUids, tree[leaf], leaf); - } - } - } - - buildTree(coll) { - let tree = {}; - for (let i = 0; i < coll.length; i++) { - if (coll[i].parent_uid === null || !coll[i].hasOwnProperty('parent_uid')) { - tree[coll[i].uid] = {}; - coll.splice(i, 1); - i--; - } - } - this.findBranches(tree, _.keys(tree), coll); - return tree; - } - - findBranches(tree, branches, coll) { - let self = this; - _.forEach(branches, (branch) => { - for (const element of coll) { - if (branch === element.parent_uid) { - let childUid = element.uid; - tree[branch][childUid] = {}; - self.findBranches(tree[branch], [childUid], coll); - } - } - }); - } - - publish(assetUid, assetObject) { - let envId = []; - let self = this; - let locales = []; - let requestObject = { json: { asset: {} } }; - - return new Promise(function (resolve, reject) { - _.forEach(assetObject.publish_details, function (pubObject) { - if (self.environment.hasOwnProperty(pubObject.environment)) { - envId.push(self.environment[pubObject.environment].name); - let idx = _.indexOf(locales, pubObject.locale); - if (idx === -1) { - locales.push(pubObject.locale); - } - } - }); - requestObject.json.asset.environments = envId; - requestObject.json.asset.locales = locales; - return self.stackAPIClient - .asset(assetUid) - .publish({ publishDetails: requestObject.json.asset }) - .then(function () { - log(self.config, 'Asset ' + assetUid + ' published successfully', 'success'); - return resolve(); - }) - .catch(function (err) { - if (err && err.message) { - let error; - try { - error = JSON.parse(err.message); - } catch (cError) { - error = { errorMessage: err.message }; - } - log(self.config, 'Asset ' + assetUid + ' not published, ' + error.errorMessage, 'error'); - return reject(err); - } - - return reject(err); - }); - }); - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/content-types.js b/packages/contentstack-import/src/import/modules-js/content-types.js deleted file mode 100755 index 0c7e1df68a..0000000000 --- a/packages/contentstack-import/src/import/modules-js/content-types.js +++ /dev/null @@ -1,231 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const chalk = require('chalk'); -const { cloneDeep, find, findIndex } = require('lodash'); -const { fileHelper, log, executeTask, formatError, schemaTemplate, lookupExtension } = require('../../utils'); -const { sanitizePath } = require('@contentstack/cli-utilities'); - -class ContentTypesImport { - constructor(importConfig, stackAPIClient) { - this.stackAPIClient = stackAPIClient; - this.importConfig = importConfig; - this.contentTypeConfig = importConfig.modules.content_types; - this.globalFieldConfig = importConfig.modules.globalfields; - this.importConcurrency = this.contentTypeConfig.importConcurrency || this.importConfig.importConcurrency; - this.writeConcurrency = this.contentTypeConfig.writeConcurrency || this.importConfig.writeConcurrency; - this.contentTypesFolderPath = path.join(sanitizePath(this.importConfig.data), sanitizePath(this.contentTypeConfig.dirName)); - this.mapperFolderPath = path.join(sanitizePath(this.importConfig.data), 'mapper', 'content_types'); - this.existingContentTypesPath = path.join(sanitizePath(this.mapperFolderPath), 'success.json'); - this.globalFieldsFolderPath = path.resolve(sanitizePath(this.importConfig.data),sanitizePath(this.globalFieldConfig.dirName)); - this.globalFieldMapperFolderPath = path.join(sanitizePath(importConfig.data), 'mapper', 'global_fields', 'success.json'); - this.globalFieldPendingPath = path.join(sanitizePath(importConfig.data), 'mapper', 'global_fields', 'pending_global_fields.js'); - this.ignoredFilesInContentTypesFolder = new Map([ - ['__master.json', 'true'], - ['__priority.json', 'true'], - ['schema.json', 'true'], - ['.DS_Store', 'true'], - ]); - this.contentTypes = []; - this.existingContentTypesUIds = []; - this.titleToUIdMap = new Map(); - this.requestOptions = { - json: {}, - }; - this.fieldRules = []; - this.installedExtensions = []; - this.globalFields = []; - this.existingGlobalFields = []; - this.pendingGlobalFields = []; - } - - async start() { - try { - const appMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); - this.installedExtensions = ( - (await fileHelper.readFileSync(appMapperPath)) || { extension_uid: {} } - ).extension_uid; - // read content types - // remove content types already existing - if (fs.existsSync(this.existingContentTypesPath)) { - this.existingContentTypesUIds = fileHelper.readFileSync(this.existingContentTypesPath) || []; - this.existingContentTypesUIds = new Map(this.existingContentTypesUIds.map((id) => [id, 'true'])); - } - - const contentTypeFiles = fileHelper.readdirSync(this.contentTypesFolderPath); - for (let contentTypeName of contentTypeFiles) { - if (!this.ignoredFilesInContentTypesFolder.has(contentTypeName)) { - const contentTypePath = path.join(this.contentTypesFolderPath, contentTypeName); - const contentType = await fileHelper.readFile(contentTypePath); - if (!this.existingContentTypesUIds.length || !this.existingContentTypesUIds.has(contentType.uid)) { - this.contentTypes.push(await fileHelper.readFile(contentTypePath)); - } - } - } - - // seed content type - log(this.importConfig, 'Started to seed content types', 'info'); - await executeTask(this.contentTypes, this.seedContentType.bind(this), { concurrency: this.importConcurrency }); - log(this.importConfig, 'Created content types', 'success'); - - log(this.importConfig, 'Started to update content types with references', 'info'); - await executeTask(this.contentTypes, this.updateContentType.bind(this), { concurrency: this.importConcurrency }); - log(this.importConfig, 'Updated content types with references', 'success'); - - // global field update - this.pendingGlobalFields = fileHelper.readFileSync(this.globalFieldPendingPath); - if (Array.isArray(this.pendingGlobalFields) && this.pendingGlobalFields.length > 0) { - this.globalFields = fileHelper.readFileSync( - path.resolve(this.globalFieldsFolderPath, this.globalFieldConfig.fileName), - ); - this.existingGlobalFields = fileHelper.readFileSync(this.globalFieldMapperFolderPath); - try { - log(this.importConfig, 'Started to update pending global field with content type references', 'info'); - await executeTask(this.pendingGlobalFields, this.updateGlobalFields.bind(this), { - concurrency: this.importConcurrency, - }); - log(this.importConfig, 'Updated pending global fields with content type with references', 'success'); - } catch (error) { - log( - this.importConfig, - `Failed to updates global fields with content type reference ${formatError(error)}`, - 'error', - ); - } - } - - // write field rules - if (this.fieldRules.length > 0) { - try { - await fileHelper.writeFile(path.join(this.contentTypesFolderPath, 'field_rules_uid.json'), this.fieldRules); - } catch (error) { - log(this.importConfig, `Failed to write field rules ${formatError(error)}`, 'success'); - } - } - - log(this.importConfig, chalk.green('Content types imported successfully'), 'success'); - } catch (error) { - let message_content_type = ""; - if (error.request !== undefined && JSON.parse(error.request.data).content_type !== undefined) { - if (JSON.parse(error.request.data).content_type.uid) { - message_content_type = - ' Update the content type with content_type_uid - ' + JSON.parse(error.request.data).content_type.uid; - } else if (JSON.parse(error.request.data).content_type.title) { - message_content_type = - ' Update the content type with content_type_title - ' + JSON.parse(error.request.data).content_type.title; - } - error.errorMessage = error.errorMessage + message_content_type; - } - log(this.importConfig, formatError(error.errorMessage), 'error'); - log(this.importConfig, formatError(error), 'error'); - throw new Error('Failed to import content types'); - } - } - - async seedContentType(contentType) { - const body = cloneDeep(schemaTemplate); - body.content_type.uid = contentType.uid; - body.content_type.title = contentType.title; - const requestObject = cloneDeep(this.requestOptions); - requestObject.json = body; - - try { - await this.stackAPIClient.contentType().create(requestObject.json); - } catch (error) { - if (error.errorCode === 115 && (error.errors.uid || error.errors.title)) { - // content type uid already exists - // _.remove(self.contentTypes, { uid: uid }); - return true; - } - throw error; - } - } - - async updateContentType(contentType) { - if (typeof contentType !== 'object') return; - const requestObject = cloneDeep(this.requestOptions); - if (contentType.field_rules) { - contentType.field_rules = this.updateFieldRules(contentType); - if (!contentType.field_rules.length) { - delete contentType.field_rules; - } - this.fieldRules.push(contentType.uid); - } - - lookupExtension( - this.importConfig, - contentType.schema, - this.importConfig.preserveStackVersion, - this.installedExtensions, - ); - requestObject.json.content_type = contentType; - const contentTypeResponse = this.stackAPIClient.contentType(contentType.uid); - Object.assign(contentTypeResponse, cloneDeep(contentType)); - await contentTypeResponse.update(); - log(this.importConfig, contentType.uid + ' updated with references', 'success'); - } - - async updateGlobalFields(uid) { - const globalField = find(this.globalFields, { uid }); - if (globalField) { - lookupExtension( - this.importConfig, - globalField.schema, - this.importConfig.preserveStackVersion, - this.installedExtensions, - ); - let globalFieldObj = this.stackAPIClient.globalField(globalField.uid); - Object.assign(globalFieldObj, cloneDeep(globalField)); - try { - const globalFieldResponse = await globalFieldObj.update(); - const existingGlobalField = findIndex(this.existingGlobalFields, (existingGlobalFieldUId) => { - return globalFieldResponse.uid === existingGlobalFieldUId; - }); - - // Improve write the updated global fields once all updates are completed - this.existingGlobalFields.splice(existingGlobalField, 1, globalField); - await fileHelper.writeFile(this.globalFieldMapperFolderPath, this.existingGlobalFields).catch((error) => { - log(this.importConfig, `failed to write updated the global field ${uid} ${formatError(error)}`); - }); - log(this.importConfig, `Updated the global field ${uid} with content type references `); - return true; - } catch (error) { - log(this.importConfig, `failed to update the global field ${uid} ${formatError(error)}`); - } - } else { - log(this.importConfig, `Global field ${uid} does not exist, and hence failed to update.`); - } - } - - async mapUidToTitle() { - this.contentTypes.forEach((ct) => { - this.titleToUIdMap.set(ct.uid, ct.title); - }); - } - - updateFieldRules(contentType) { - const fieldDataTypeMap = {}; - for (let i = 0; i < contentType.schema.length; i++) { - const field = contentType.schema[i]; - fieldDataTypeMap[field.uid] = field.data_type; - } - const fieldRules = [...contentType.field_rules]; - let len = fieldRules.length; - // Looping backwards as we need to delete elements as we move. - for (let i = len - 1; i >= 0; i--) { - const conditions = fieldRules[i].conditions; - let isReference = false; - for (let j = 0; j < conditions.length; j++) { - const field = conditions[j].operand_field; - if (fieldDataTypeMap[field] === 'reference') { - isReference = true; - } - } - if (isReference) { - fieldRules.splice(i, 1); - } - } - return fieldRules; - } -} - -module.exports = ContentTypesImport; diff --git a/packages/contentstack-import/src/import/modules-js/custom-roles.js b/packages/contentstack-import/src/import/modules-js/custom-roles.js deleted file mode 100644 index c5a1c768e9..0000000000 --- a/packages/contentstack-import/src/import/modules-js/custom-roles.js +++ /dev/null @@ -1,168 +0,0 @@ -'use strict'; - -const mkdirp = require('mkdirp'); -const fs = require('fs'); -const path = require('path'); -const chalk = require('chalk'); -const { merge } = require('lodash'); -const { fileHelper, log, formatError } = require('../../utils'); -let { default: config } = require('../../config'); - -module.exports = class ImportCustomRoles { - fails = []; - labelUids = []; - customRolesUidMapper = {}; - customRolesConfig = config.modules.customRoles; - - constructor(importConfig, stackAPIClient) { - this.config = merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - } - - async start() { - const self = this; - log(this.config, chalk.white('Migrating custom-roles'), 'success'); - - let customRolesFolderPath = path.resolve(this.config.data, this.customRolesConfig.dirName); - let customRolesMapperPath = path.resolve(this.config.data, 'mapper', 'custom-roles'); - let entriesUidMapperFolderPath = path.resolve(this.config.data, 'mapper', 'entries'); - let customRolesFailsPath = path.resolve(this.config.data, 'custom-roles', 'fails.json'); - let environmentsUidMapperFolderPath = path.resolve(this.config.data, 'mapper', 'environments'); - let customRolesUidMapperPath = path.resolve(this.config.data, 'mapper', 'custom-roles', 'uid-mapping.json'); - let customRolesLocalesFilePath = path.resolve( - customRolesFolderPath, - this.customRolesConfig.customRolesLocalesFileName, - ); - - try { - self.customRoles = fileHelper.readFileSync(path.resolve(customRolesFolderPath, this.customRolesConfig.fileName)); - self.customRolesLocales = fileHelper.readFileSync(customRolesLocalesFilePath); - // Mapper file paths. - - if (fs.existsSync(customRolesMapperPath)) { - this.customRolesUidMapper = fileHelper.readFileSync(customRolesUidMapperPath) || {}; - } - - mkdirp.sync(customRolesMapperPath); - - if (!self.customRoles) { - log(self.config, chalk.white('No custom-roles found'), 'info'); - return; - } - self.customRolesUids = Object.keys(self.customRoles); - - self.localesUidMap = await getLocalesUidMap(self.stackAPIClient, self.config, self.customRolesLocales); - - self.environmentsUidMap={} - if (fs.existsSync(environmentsUidMapperFolderPath)) { - self.environmentsUidMap = fileHelper.readFileSync( - path.resolve(environmentsUidMapperFolderPath, 'uid-mapping.json'), - ); - } - self.entriesUidMap={} - if (fs.existsSync(entriesUidMapperFolderPath)) { - self.entriesUidMap = fileHelper.readFileSync(path.resolve(entriesUidMapperFolderPath, 'uid-mapping.json')); - } - - for (const uid of self.customRolesUids) { - const customRole = self.customRoles[uid]; - - if (uid in self.customRolesUidMapper) { - log( - self.config, - chalk.white(`The custom-role '${customRole.name}' already exists. Skipping it to avoid duplicates!`), - 'success', - ); - continue; - } - - try { - customRole.rules.forEach((rule) => { - const transformUids = getTransformUidsFactory(rule); - rule = transformUids(rule, self.environmentsUidMap, self.localesUidMap, self.entriesUidMap); - }); - // rules.branch is required to create custom roles. - const branchRuleExists = customRole.rules.find((rule) => rule.module === 'branch'); - if (!branchRuleExists) { - customRole.rules.push({ - module: 'branch', - branches: ['main'], - acl: { read: true }, - }); - } - const role = await self.stackAPIClient.role().create({ role: customRole }); - self.customRolesUidMapper[uid] = role; - fileHelper.writeFileSync(customRolesUidMapperPath, self.customRolesUidMapper); - } catch (error) { - self.fails.push(customRole); - - if (((error && error.errors && error.errors.name) || '').includes('is not a unique.')) { - log(self.config, `custom-role '${customRole.name}' already exists`, 'info'); - } else { - if (!(error && error.errors && error.errors.name)) { - log(self.config, `custom-role: ${customRole.name} already exists`, 'error'); - } else { - log(self.config, `custom-role: ${customRole.name} failed`, 'error'); - } - - log(self.config, formatError(error), 'error'); - } - } - } - log(self.config, chalk.green('Custom-roles have been imported successfully!'), 'success'); - } catch (error) { - fileHelper.writeFileSync(customRolesFailsPath, self.fails); - log(self.config, 'Custom-roles import failed', 'error'); - log(self.config, formatError(error), 'error'); - throw error; - } - } -}; - -const getTransformUidsFactory = (rule) => { - if (rule.module === 'environment') { - return environmentUidTransformer; - } else if (rule.module === 'locale') { - return localeUidTransformer; - } else if (rule.module === 'entry') { - return entryUidTransformer; - } else { - return noopTransformer; - } -}; - -const environmentUidTransformer = (rule, environmentsUidMap) => { - rule.environments = rule.environments.map((env) => environmentsUidMap[env]); - return rule; -}; - -const localeUidTransformer = (rule, environmentsUidMap, localesUidMap) => { - rule.locales = rule.locales.map((locale) => localesUidMap[locale]); - return rule; -}; - -const entryUidTransformer = (rule, environmentsUidMap, localesUidMap, entriesUidMap) => { - rule.entries = rule.entries.map((entry) => entriesUidMap[entry]); - return rule; -}; - -const noopTransformer = (rule) => { - return rule; -}; - -const getLocalesUidMap = async (stackAPIClient, config, sourceLocales) => { - const { items } = await stackAPIClient.locale().query().find(); - const [targetLocalesMap, sourceLocalesMap] = [{}, {}]; - - items.forEach((locale) => { - targetLocalesMap[locale.code] = locale.uid; - }); - for (const key in sourceLocales) { - sourceLocalesMap[sourceLocales[key].code] = key; - } - const localesUidMap = {}; - for (const key in sourceLocalesMap) { - localesUidMap[sourceLocalesMap[key]] = targetLocalesMap[key]; - } - return localesUidMap; -}; diff --git a/packages/contentstack-import/src/import/modules-js/entries.js b/packages/contentstack-import/src/import/modules-js/entries.js deleted file mode 100755 index 9583d447e7..0000000000 --- a/packages/contentstack-import/src/import/modules-js/entries.js +++ /dev/null @@ -1,1517 +0,0 @@ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const Promise = require('bluebird'); -const fs = require('fs'); -const path = require('path'); -const _ = require('lodash'); -const mkdirp = require('mkdirp'); -const chalk = require('chalk'); -const { - fileHelper, - log, - formatError, - lookupExtension, - suppressSchemaReference, - lookupAssets, - lookupEntries, -} = require('../../utils'); -const { default: config } = require('../../config'); -const { sanitizePath } = require('@contentstack/cli-utilities'); -const addlogs = log; -module.exports = class ImportEntries { - mappedAssetUidPath; - mappedAssetUrlPath; - entryMapperPath; - environmentPath; - entryUidMapperPath; - uniqueUidMapperPath; - modifiedSchemaPath; - createdEntriesWOUidPath; - failedWOPath; - masterLanguage; - reqConcurrency; - eConfig; - ePath; - ctPath; - lPath; - importConcurrency; - skipFiles = ['__master.json', '__priority.json', 'schema.json','.DS_Store']; - - constructor(importConfig, stackAPIClient) { - this.config = _.merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - this.mappedAssetUidPath = path.resolve(this.config.data, 'mapper', 'assets', 'uid-mapping.json'); - this.mappedAssetUrlPath = path.resolve(this.config.data, 'mapper', 'assets', 'url-mapping.json'); - - this.entryMapperPath = path.resolve(this.config.data, 'mapper', 'entries'); - this.environmentPath = path.resolve(this.config.data, 'environments', 'environments.json'); - mkdirp.sync(this.entryMapperPath); - - this.entryUidMapperPath = path.join(this.entryMapperPath, 'uid-mapping.json'); - this.uniqueUidMapperPath = path.join(this.entryMapperPath, 'unique-mapping.json'); - this.modifiedSchemaPath = path.join(this.entryMapperPath, 'modified-schemas.json'); - - this.createdEntriesWOUidPath = path.join(this.entryMapperPath, 'created-entries-wo-uid.json'); - this.failedWOPath = path.join(this.entryMapperPath, 'failedWO.json'); - - this.reqConcurrency = this.config.concurrency; - this.eConfig = this.config.modules.entries; - this.ePath = path.resolve(this.config.data, this.eConfig.dirName); - this.ctPath = path.resolve(this.config.data, this.config.modules.content_types.dirName); - this.lPath = path.resolve( - this.config.data, - this.config.modules.locales.dirName, - this.config.modules.locales.fileName, - ); - - this.importConcurrency = this.eConfig.importConcurrency || this.config.importConcurrency; - - // Object of Schemas, referred to by their content type uid - this.ctSchemas = {}; - // Array of content type uids, that have reference fields - this.refSchemas = []; - // map of content types uids and their json-rte fields - this.ctJsonRte = []; - // map of content types uids and their json-rte fields - this.ctJsonRteWithEntryRefs = []; - // Entry refs that are held back to resolve after all entries have been created - this.jsonRteEntryRefs = {}; - // Collection of entries, that were not created, as they already exist on Stack - this.createdEntriesWOUid = []; - // Collection of entry uids, mapped to the language they exist in - this.uniqueUids = {}; - // Map of old entry uid to new - this.mappedUids = {}; - // Entries that were created successfully - this.success = []; - // Entries that failed to get created OR updated - this.fails = []; - // List of installed extensions to replace uid - this.installedExtensions = []; - - let files = fs.readdirSync(this.ctPath); - this.environment = fileHelper.readFileSync(this.environmentPath); - for (let index in files) { - if (index) { - try { - if (this.skipFiles.indexOf(files[index]) === -1) { - if (files[index] != 'field_rules_uid.json') { - let schema = require(path.resolve(path.join(this.ctPath, files[index]))); - this.ctSchemas[schema.uid] = schema; - } - } - } catch (error) { - addlogs(this.config, `Failed to read the content types to import entries ${formatError(error)}`, 'error'); - process.exit(0); - } - } - } - } - - async start() { - let self = this; - this.masterLanguage = this.config.master_locale; - log(this.config, 'Migrating entries', 'success'); - let languages = fileHelper.readFileSync(this.lPath); - const appMapperPath = path.join(this.config.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); - this.installedExtensions = ((await fileHelper.readFileSync(appMapperPath)) || { extension_uid: {} }).extension_uid; - - return new Promise((resolve, reject) => { - let langs = [self.masterLanguage.code]; - for (let i in languages) { - if (i) { - langs.push(languages[i].code); - } - } - - // Step 1: Removes field rules from content type - // This allows to handle cases like self references and circular reference - // if mandatory reference fields are not filed in entries then avoids the error - // Also remove field visibility rules - return self - .supressFields() - .then(async () => { - log(this.config, 'Completed suppressing content type reference fields', 'success'); - let mappedAssetUids = fileHelper.readFileSync(this.mappedAssetUidPath) || {}; - let mappedAssetUrls = fileHelper.readFileSync(this.mappedAssetUrlPath) || {}; - - // Step 2: Iterate over available languages to create entries in each. - let counter = 0; - return Promise.map( - langs, - async () => { - let lang = langs[counter]; - if ( - (self.config.hasOwnProperty('onlylocales') && self.config.onlylocales.indexOf(lang) !== -1) || - !self.config.hasOwnProperty('onlylocales') - ) { - addlogs(self.config, `Starting to create entries ${lang} locale`, 'info'); - await self.createEntries(lang, mappedAssetUids, mappedAssetUrls); - log(this.config, 'Entries created successfully', 'info'); - try { - await self.getCreatedEntriesWOUid(); - } catch (error) { - addlogs( - self.config, - `Failed get the existing entries to update the mapper ${formatError(error)}, 'error`, - ); - } - log(this.config, 'Starting to update entries with references', 'info'); - await self.repostEntries(lang); - log(this.config, "Successfully imported '" + lang + "' entries!", 'success'); - counter++; - } else { - addlogs(self.config, `'${lang}' has not been configured for import, thus skipping it`, 'success'); - counter++; - } - }, - { - concurrency: 1, - }, - ).then(async () => { - // Step 3: Revert all the changes done in content type in step 1 - log(this.config, 'Restoring content type changes', 'info'); - await self.unSuppressFields(); - log(this.config, 'Removing entries from master language which got created by default', 'info'); - await self.removeBuggedEntries(); - log(this.config, 'Updating the field rules of content type', 'info'); - let ct_field_visibility_uid = fileHelper.readFileSync(path.join(this.ctPath + '/field_rules_uid.json')); - let ct_files = fs.readdirSync(this.ctPath); - if (ct_field_visibility_uid && ct_field_visibility_uid != 'undefined') { - for (const element of ct_field_visibility_uid) { - if (ct_files.indexOf(element + '.json') > -1) { - let schema = require(path.resolve(this.ctPath, element)); - try { - await self.field_rules_update(schema); - } catch (error) { - addlogs( - self.config, - `Failed to update the field rules for content type '${schema.uid}' ${formatError(error)}`, - 'error', - ); - } - } - } - } - log(this.config, chalk.green('Entries have been imported successfully!'), 'success'); - if (this.config.entriesPublish) { - log(this.config, chalk.green('Publishing entries'), 'success'); - return self - .publish(langs) - .then(() => { - log(this.config, chalk.green('All the entries have been published successfully'), 'success'); - return resolve(); - }) - .catch((error) => { - log(this.config, `Error in publishing entries ${formatError(error)}`, 'error'); - }); - } - return resolve(); - }); - }) - .catch((error) => { - log(self.config, formatError(error), 'error'); - reject('Failed import entries'); - }); - }); - } - - createEntries(lang, mappedAssetUids, mappedAssetUrls) { - let self = this; - return new Promise(async (resolve, reject) => { - let contentTypeUids = Object.keys(self.ctSchemas); - if (fs.existsSync(this.entryUidMapperPath)) { - self.mappedUids = await fileHelper.readLargeFile(this.entryUidMapperPath); - } - self.mappedUids = self.mappedUids || {}; - return Promise.map( - contentTypeUids, - async (ctUid) => { - let eLangFolderPath = path.join(this.entryMapperPath, lang); - let eLogFolderPath = path.join(this.entryMapperPath, lang, ctUid); - mkdirp.sync(eLogFolderPath); - // entry file path - let eFilePath = path.resolve(this.ePath, ctUid, lang + '.json'); - - // log created/updated entries - let successEntryLogPath = path.join(eLogFolderPath, 'success.json'); - let failedEntryLogPath = path.join(eLogFolderPath, 'fails.json'); - let createdEntriesPath = path.join(eLogFolderPath, 'created-entries.json'); - let createdEntries = {}; - - if (fs.existsSync(createdEntriesPath)) { - createdEntries = await fileHelper.readLargeFile(createdEntriesPath); - createdEntries = createdEntries || {}; - } - if (fs.existsSync(eFilePath)) { - let entries = await fileHelper.readLargeFile(eFilePath); - if (!_.isPlainObject(entries) || _.isEmpty(entries)) { - log( - this.config, - chalk.white("No entries were found for Content type:'" + ctUid + "' in '" + lang + "' language!"), - 'success', - ); - } else { - addlogs(this.config, `Creating entries for content type ${ctUid} in language ${lang} ...`, 'success'); - for (let eUid in entries) { - if (eUid) { - try { - // check ctUid in self.ctJsonRte array, if ct exists there... only then remove entry references for json rte - // also with json rte, api creates the json-rte field with the same uid as passed in the payload. - - if (self.ctJsonRte.indexOf(ctUid) > -1) { - entries[eUid] = self.removeUidsFromJsonRteFields(entries[eUid], self.ctSchemas[ctUid].schema); - } - - // remove entry references from json-rte fields - if (self.ctJsonRteWithEntryRefs.indexOf(ctUid) > -1) { - entries[eUid] = self.removeEntryRefsFromJSONRTE(entries[eUid], self.ctSchemas[ctUid].schema); - } - // will replace all old asset uid/urls with new ones - entries[eUid] = lookupAssets( - { - content_type: self.ctSchemas[ctUid], - entry: entries[eUid], - }, - mappedAssetUids, - mappedAssetUrls, - eLangFolderPath, - self.installedExtensions, - ); - } catch (error) { - addlogs(this.config, 'Failed to update entry while creating entry id ' + eUid); - addlogs(this.config, formatError(error), 'error'); - } - } - } - let eUids = Object.keys(entries); - let batches = []; - - let entryBatchLimit = this.eConfig.batchLimit || 10; - let batchSize = Math.round(entryBatchLimit / 3); - - // Run entry creation in batches of ~16~ entries - for (let i = 0; i < eUids.length; i += batchSize) { - batches.push(eUids.slice(i, i + batchSize)); - } - return Promise.map( - batches, - async (batch) => { - return Promise.map( - batch, - async (eUid, batchIndex) => { - // if entry is already created - if (createdEntries.hasOwnProperty(eUid)) { - log( - this.config, - 'Skipping ' + - JSON.stringify({ - content_type: ctUid, - locale: lang, - oldEntryUid: eUid, - newEntryUid: createdEntries[eUid], - }) + - ' as it is already created', - 'success', - ); - self.success[ctUid] = createdEntries[eUid]; - // if its a non-master language, i.e. the entry isn't present in the master language - if (lang !== this.masterLanguage.code) { - self.uniqueUids[eUid] = self.uniqueUids[eUid] || {}; - if (self.uniqueUids[eUid].locales) { - self.uniqueUids[eUid].locales.push(lang); - } else { - self.uniqueUids[eUid].locales = [lang]; - } - self.uniqueUids[eUid].content_type = ctUid; - } - return; - } - let requestObject = { - qs: { - locale: lang, - }, - json: { - entry: entries[eUid], - }, - }; - if (self.mappedUids.hasOwnProperty(eUid)) { - let entryToUpdate = self.stackAPIClient.contentType(ctUid).entry(self.mappedUids[eUid]); - Object.assign(entryToUpdate, _.cloneDeep(entries[eUid])); - return entryToUpdate - .update({ locale: entryToUpdate.locale }) - .then(async (entryResponse) => { - self.success[ctUid] = self.success[ctUid] || []; - self.success[ctUid].push(entries[eUid]); - if (!self.mappedUids.hasOwnProperty(eUid)) { - self.mappedUids[eUid] = entryResponse.uid; - createdEntries = entryResponse; - // if its a non-master language, i.e. the entry isn't present in the master language - if (lang !== this.masterLanguage.code) { - self.uniqueUids[eUid] = self.uniqueUids[eUid] || {}; - if (self.uniqueUids[eUid].locales) { - self.uniqueUids[eUid].locales.push(lang); - } else { - self.uniqueUids[eUid].locales = [lang]; - } - self.uniqueUids[eUid].content_type = ctUid; - } - } - }) - .catch((error) => { - log(this.config, `Failed to update an entry ${eUid} ${formatError(error)}`, 'error'); - self.fails.push({ - content_type: ctUid, - locale: lang, - entry: entries[eUid], - error: formatError(error), - }); - return error; - }); - } - delete requestObject.json.entry.publish_details; - return self.stackAPIClient - .contentType(ctUid) - .entry() - .create(requestObject.json, { locale: lang }) - .then(async (entryResponse) => { - self.success[ctUid] = self.success[ctUid] || []; - self.success[ctUid].push(entries[eUid]); - if (!self.mappedUids.hasOwnProperty(eUid)) { - self.mappedUids[eUid] = entryResponse.uid; - createdEntries = entryResponse; - // if its a non-master language, i.e. the entry isn't present in the master language - if (lang !== this.masterLanguage.code) { - self.uniqueUids[eUid] = self.uniqueUids[eUid] || {}; - if (self.uniqueUids[eUid].locales) { - self.uniqueUids[eUid].locales.push(lang); - } else { - self.uniqueUids[eUid].locales = [lang]; - } - self.uniqueUids[eUid].content_type = ctUid; - } - } - }) - .catch((error) => { - if (error.hasOwnProperty('errorCode') && error.errorCode === 119) { - if (error.errors.title) { - log( - this.config, - 'Entry ' + eUid + ' already exist, skip to avoid creating a duplicate entry', - 'error', - ); - } else { - log( - this.config, - `Failed to create an entry '${eUid}' ${formatError( - error, - )} Title of the failed entry: '${entries[eUid].title}'`, - 'error', - ); - } - self.createdEntriesWOUid.push({ - content_type: ctUid, - locale: lang, - entry: entries[eUid], - error: error, - }); - fileHelper.writeFileSync(this.createdEntriesWOUidPath, self.createdEntriesWOUid); - return; - } - // TODO: if status code: 422, check the reason - // 429 for rate limit - log( - this.config, - `Failed to create an entry '${eUid}' ${formatError(error)}. Title of the failed entry: '${ - entries[eUid].title - }'`, - 'error', - ); - self.fails.push({ - content_type: ctUid, - locale: lang, - entry: entries[eUid], - error: error, - }); - }); - // create/update 5 entries at a time - }, - { - concurrency: this.importConcurrency, - }, - ).then(() => { - fileHelper.writeFileSync(successEntryLogPath, self.success[ctUid]); - fileHelper.writeFileSync(failedEntryLogPath, self.fails[ctUid]); - fileHelper.writeFileSync(this.entryUidMapperPath, self.mappedUids); - fileHelper.writeFileSync(this.uniqueUidMapperPath, self.uniqueUids); - fileHelper.writeFileSync(createdEntriesPath, createdEntries); - }); - // process one batch at a time - }, - { - concurrency: 1, - }, - ).then(() => { - if (self.success && self.success[ctUid] && self.success[ctUid].length > 0) - log( - this.config, - self.success[ctUid].length + - ' entries created successfully in ' + - ctUid + - ' content type in ' + - lang + - ' locale!', - 'success', - ); - if (self.fails && self.fails[ctUid] && self.fails[ctUid].length > 0) - log( - this.config, - self.fails[ctUid].length + - ' entries failed to create in ' + - ctUid + - ' content type in ' + - lang + - ' locale!', - 'error', - ); - self.success[ctUid] = []; - self.fails[ctUid] = []; - }); - } - } else { - log( - this.config, - `Unable to find entry file path for '${ctUid}' content type!\nThe file '${eFilePath}' does not exist!`, - 'error', - ); - } - }, - { - concurrency: 1, - }, - ) - .then(() => { - log(this.config, chalk.green("Entries created successfully in '" + lang + "' language"), 'success'); - return resolve(); - }) - .catch((error) => { - let title = JSON.parse(error?.request?.data || '{}').entry?.title; - addlogs( - this.config, - chalk.red( - "Failed to create entries in '" + - lang + - "' language. " + - 'Title of the failed entry: ' + - `'${title || ''}'`, - ), - 'error', - ); - return reject(error); - }); - }); - } - getCreatedEntriesWOUid() { - let self = this; - return new Promise((resolve) => { - self.createdEntriesWOUid = fileHelper.readFileSync(this.createdEntriesWOUidPath); - self.failedWO = []; - if (_.isArray(self.createdEntriesWOUid) && self.createdEntriesWOUid.length > 0) { - return Promise.map( - self.createdEntriesWOUid, - (entry) => { - return self.fetchEntry(entry); - }, - { - concurrency: this.importConcurrency, - }, - ).then(() => { - fileHelper.writeFileSync(this.failedWOPath, self.failedWO); - log(this.config, 'Mapped entries without mapped uid successfully!', 'success'); - return resolve(); - }); - } - log(this.config, 'No entries without mapped uid found!', 'success'); - return resolve(); - }); - } - repostEntries(lang) { - let self = this; - return new Promise(async (resolve, reject) => { - let _mapped_ = await fileHelper.readLargeFile(path.join(this.entryMapperPath, 'uid-mapping.json')); - if (_.isPlainObject(_mapped_)) { - self.mappedUids = _.merge(_mapped_, self.mappedUids); - } - return Promise.map( - self.refSchemas, - async (ctUid) => { - let eFolderPath = path.join(this.entryMapperPath, lang, ctUid); - let eSuccessFilePath = path.join(eFolderPath, 'success.json'); - let eFilePath = path.resolve(this.ePath, ctUid, lang + '.json'); - let sourceStackEntries = await fileHelper.readLargeFile(eFilePath); - - if (!fs.existsSync(eSuccessFilePath)) { - log(this.config, 'Success file was not found at: ' + eSuccessFilePath, 'success'); - return; - } - - let entries = await fileHelper.readLargeFile(eSuccessFilePath, { type: 'array' }); // TBD LARGE - entries = entries || []; - if (entries.length === 0) { - log(this.config, "No entries were created to be updated in '" + lang + "' language!", 'success'); - return; - } - - // Keep track of entries that have their references updated - let refsUpdatedUids = fileHelper.readFileSync(path.join(eFolderPath, 'refsUpdatedUids.json')); - let refsUpdateFailed = fileHelper.readFileSync(path.join(eFolderPath, 'refsUpdateFailed.json')); - let schema = self.ctSchemas[ctUid]; - - let batches = []; - refsUpdatedUids = refsUpdatedUids || []; - refsUpdateFailed = refsUpdateFailed || []; - - // map reference uids @mapper/language/mapped-uids.json - // map failed reference uids @mapper/language/unmapped-uids.json - let refUidMapperPath = path.join(this.entryMapperPath, lang); - - addlogs(this.config, 'staring to update the entry for reposting'); - - entries = _.map(entries, (entry) => { - try { - let uid = entry.uid; - let updatedEntry; - - // restores json rte entry refs if they exist - if (self.ctJsonRte.indexOf(ctUid) > -1) { - // the entries stored in eSuccessFilePath, have the same uids as the entries from source data - updatedEntry = self.restoreJsonRteEntryRefs(entry, sourceStackEntries[entry.uid], schema.schema); - } else { - updatedEntry = entry; - } - - let _entry = lookupEntries( - { - content_type: schema, - entry: updatedEntry, - }, - _.clone(self.mappedUids), - refUidMapperPath, - ); - // if there's self references, the uid gets replaced - _entry.uid = uid; - return _entry; - } catch (error) { - addlogs( - this.config, - `Failed to update the entry '${uid}' references while reposting ${formatError(error)}`, - 'error', - ); - } - }); - - log(this.config, 'Starting the reposting process for entries'); - - const entryBatchLimit = this.eConfig.batchLimit || 10; - const batchSize = Math.round(entryBatchLimit / 3); - // Run entry creation in batches - for (let i = 0; i < entries.length; i += batchSize) { - batches.push(entries.slice(i, i + batchSize)); - } - return Promise.map( - batches, - async (batch, index) => { - return Promise.map( - batch, - async (entry) => { - entry.uid = self.mappedUids[entry.uid]; - if (refsUpdatedUids.indexOf(entry.uid) !== -1) { - log( - this.config, - 'Entry: ' + - entry.uid + - ' in Content Type: ' + - ctUid + - ' in lang: ' + - lang + - ' references fields are already updated.', - 'success', - ); - return; - } - - let promiseResult = new Promise((resolveUpdatedUids, rejectUpdatedUids) => { - let entryResponse = self.stackAPIClient.contentType(ctUid).entry(entry.uid); - Object.assign(entryResponse, entry); - delete entryResponse.publish_details; - return entryResponse - .update({ locale: lang }) - .then((response) => { - refsUpdatedUids.push(response.uid); - return resolveUpdatedUids(); - }) - .catch((error) => { - log( - this.config, - `Entry Uid '${entry.uid}' of Content Type '${ctUid}' failed to update in locale '${lang}'`, - 'error', - ); - - log(this.config, formatError(error), 'error'); - refsUpdateFailed.push({ - content_type: ctUid, - entry: entry, - locale: lang, - error: error, - }); - return rejectUpdatedUids(error); - }); - }); - await promiseResult; - }, - { - concurrency: this.importConcurrency, - }, - ) - .then(() => { - // batch completed successfully - fileHelper.writeFileSync(path.join(eFolderPath, 'success.json'), entries); - fileHelper.writeFileSync(path.join(eFolderPath, 'refsUpdatedUids.json'), refsUpdatedUids); - fileHelper.writeFileSync(path.join(eFolderPath, 'refsUpdateFailed.json'), refsUpdateFailed); - log(this.config, 'Completed re-post entries batch no: ' + (index + 1) + ' successfully!', 'success'); - }) - .catch((error) => { - // error while executing entry in batch - addlogs(this.config, `Failed re-post entries at batch no: '${index + 1}`, 'error'); - addlogs(this.config, formatError(error), 'error'); - // throw error; - }); - }, - { - concurrency: 1, - }, - ) - .then(() => { - // finished updating entries with references - log( - this.config, - "Imported entries of Content Type: '" + ctUid + "' in language: '" + lang + "' successfully!", - 'success', - ); - }) - .catch((error) => { - // error while updating entries with references - addlogs(this.config, `Failed re-post entries of content type '${ctUid}' locale '${lang}'`, 'error'); - addlogs(this.config, formatError(error), 'error'); - // throw error; - }); - }, - { - concurrency: 1, - }, - ) - .then(() => { - // completed updating entry references - log(this.config, chalk.green("Imported entries in '" + lang + "' language successfully!"), 'success'); - return resolve(); - }) - .catch((error) => { - // error while updating entry references - addlogs(this.config, chalk.red('Failed to re post entries in ' + lang + ' language'), 'error'); - return reject(error); - }); - }); - } - supressFields() { - // it should be spelled as suppressFields - log(this.config, 'Suppressing content type reference fields', 'success'); - let self = this; - return new Promise(async (resolve, reject) => { - let modifiedSchemas = []; - let suppressedSchemas = []; - - for (let uid in self.ctSchemas) { - if (uid) { - let contentTypeSchema = _.cloneDeep(self.ctSchemas[uid]); - let flag = { - suppressed: false, - references: false, - jsonRte: false, - jsonRteEmbeddedEntries: false, - }; - if (contentTypeSchema.field_rules) { - delete contentTypeSchema.field_rules; - } - - // Set mandatory or unique flag to false - suppressSchemaReference(contentTypeSchema.schema, flag); - // Check if suppress modified flag - if (flag.suppressed) { - suppressedSchemas.push(contentTypeSchema); - modifiedSchemas.push(self.ctSchemas[uid]); - } - - if (flag.references) { - self.refSchemas.push(uid); - } - - if (flag.jsonRte) { - self.ctJsonRte.push(uid); - if (flag.jsonRteEmbeddedEntries) { - self.ctJsonRteWithEntryRefs.push(uid); - // pushing ct uid to refSchemas, because - // repostEntries uses refSchemas content types for - // reposting entries - if (self.refSchemas.indexOf(uid) === -1) { - self.refSchemas.push(uid); - } - } - } - - // Replace extensions with new UID - lookupExtension( - this.config, - contentTypeSchema.schema, - this.config.preserveStackVersion, - self.installedExtensions, - ); - } - } - - // write modified schema in backup file - fileHelper.writeFileSync(this.modifiedSchemaPath, modifiedSchemas); - - return Promise.map( - suppressedSchemas, - async (schema) => { - let contentTypeResponse = self.stackAPIClient.contentType(schema.uid); - Object.assign(contentTypeResponse, _.cloneDeep(schema)); - return contentTypeResponse - .update() - .then((_updatedcontentType) => { - // empty function - }) - .catch((_error) => { - addlogs(this.config, formatError(_error), 'error'); - reject(`Failed suppress content type '${schema.uid}' reference fields`); - }); - // update 5 content types at a time - }, - { - // update reqConcurrency content types at a time - concurrency: this.importConcurrency, - }, - ) - .then(() => { - return resolve(); - }) - .catch((error) => { - log(this.config, formatError(error), 'error'); - return reject('Failed to suppress reference fields in content type'); - }); - }); - } - fetchEntry(query) { - let self = this; - return new Promise((resolve, _reject) => { - let requestObject = { - qs: { - query: { - title: query.entry.title, - }, - locale: query.locale, - }, - }; - - return self.stackAPIClient - .contentType(query.content_type) - .entry() - .query(requestObject.qs) - .find() - .then((response) => { - if (response.body.entries.length <= 0) { - log(this.config, 'Unable to map entry WO uid: ' + query.entry.uid, 'error'); - self.failedWO.push(query); - return resolve(); - } - self.mappedUids[query.entry.uid] = response.body.entries[0].uid; - let _ePath = path.join(sanitizePath(this.entryMapperPath), sanitizePath(query.locale), sanitizePath(query.content_type), 'success.json'); - let entries = fileHelper.readFileSync(_ePath); - entries.push(query.entry); - fileHelper.writeFileSync(_ePath, entries); - log( - this.config, - 'Completed mapping entry wo uid: ' + query.entry.uid + ': ' + response.body.entries[0].uid, - 'clientsuccess', - ); - return resolve(); - }) - .catch((_error) => { - return resolve(); - }); - }); - } - unSuppressFields() { - let self = this; - return new Promise(async (resolve, reject) => { - let modifiedSchemas = fileHelper.readFileSync(this.modifiedSchemaPath); - let modifiedSchemasUids = []; - let updatedExtensionUidsSchemas = []; - for (let uid in modifiedSchemas) { - if (uid) { - let _contentTypeSchema = _.cloneDeep(modifiedSchemas[uid]); - if (_contentTypeSchema.field_rules) { - delete _contentTypeSchema.field_rules; - } - - lookupExtension( - this.config, - _contentTypeSchema.schema, - this.config.preserveStackVersion, - self.installedExtensions, - ); - updatedExtensionUidsSchemas.push(_contentTypeSchema); - } - } - - return Promise.map( - updatedExtensionUidsSchemas, - async (schema) => { - let promise = new Promise((resolveContentType, rejectContentType) => { - self.stackAPIClient - .contentType(schema.uid) - .fetch() - .then((contentTypeResponse) => { - contentTypeResponse.schema = schema.schema; - contentTypeResponse - .update() - .then((_updatedcontentType) => { - modifiedSchemasUids.push(schema.uid); - log( - this.config, - chalk.white("Content type: '" + schema.uid + "' has been restored to its previous glory!"), - ); - return resolveContentType(); - }) - .catch((error) => { - addlogs(this.config, chalk.red('Failed to re-update ' + schema.uid), 'error'); - addlogs(this.config, error, 'error'); - return rejectContentType(error); - }); - }) - .catch((error) => { - log(this.config, error, 'error'); - return rejectContentType(error); - }); - }); - await promise; - }, - { - concurrency: this.reqConcurrency, - }, - ) - .then(() => { - for (let i = 0; i < modifiedSchemas.length; i++) { - if (modifiedSchemasUids.indexOf(modifiedSchemas[i].uid) !== -1) { - modifiedSchemas.splice(i, 1); - i--; - } - } - // re-write, in case some schemas failed to update - fileHelper.writeFileSync(this.modifiedSchemaPath, _.compact(modifiedSchemas)); - log(this.config, 'Re-modified content type schemas to their original form!', 'success'); - return resolve(); - }) - .catch((error) => { - // failed to update modified schemas back to their original form - return reject(error); - }); - }); - } - removeBuggedEntries() { - let self = this; - return new Promise((resolve, reject) => { - let entries = fileHelper.readFileSync(this.uniqueUidMapperPath); - let bugged = []; - let removed = []; - for (let uid in entries) { - if (entries[uid].locales.indexOf(this.masterLanguage.code) === -1) { - bugged.push({ - content_type: entries[uid].content_type, - uid: uid, - }); - } - } - - return Promise.map( - bugged, - (entry) => { - return self.stackAPIClient - .contentType(entry.content_type) - .entry(self.mappedUids[entry.uid]) - .delete({ locale: this.masterLanguage.code }) - .then(() => { - removed.push(self.mappedUids[entry.uid]); - log(this.config, 'Removed bugged entry from master ' + JSON.stringify(entry), 'success'); - }) - .catch((error) => { - addlogs(this.config, chalk.red('Failed to remove bugged entry from master language'), 'error'); - addlogs(this.config, formatError(error), 'error'); - }); - }, - { - concurrency: this.importConcurrency, - }, - ) - .then(() => { - for (let i = 0; i < bugged.length; i++) { - if (removed.indexOf(bugged[i].uid) !== -1) { - bugged.splice(i, 1); - i--; - } - } - - fileHelper.writeFileSync(path.join(this.entryMapperPath, 'removed-uids.json'), removed); - fileHelper.writeFileSync(path.join(this.entryMapperPath, 'pending-uids.json'), bugged); - - log(this.config, chalk.green('The stack has been eradicated from bugged entries!'), 'success'); - return resolve(); - }) - .catch((error) => { - // error while removing bugged entries from stack - addlogs(this.config, formatError(error), 'error'); - }); - }); - } - field_rules_update(schema) { - return new Promise((resolve, reject) => { - if (schema.field_rules) { - let fieldRuleLength = schema.field_rules.length; - const fieldDatatypeMap = {}; - for (let i = 0; i < schema.schema.length; i++) { - const field = schema.schema[i].uid; - fieldDatatypeMap[field] = schema.schema[i].data_type; - } - for (let k = 0; k < fieldRuleLength; k++) { - let fieldRuleConditionLength = schema.field_rules[k].conditions.length; - for (let i = 0; i < fieldRuleConditionLength; i++) { - if (fieldDatatypeMap[schema.field_rules[k].conditions[i].operand_field] === 'reference') { - let fieldRulesValue = schema.field_rules[k].conditions[i].value; - let fieldRulesArray = fieldRulesValue.split('.'); - let updatedValue = []; - for (const element of fieldRulesArray) { - let splitedFieldRulesValue = element; - let oldUid = fileHelper.readFileSync(path.join(this.entryUidMapperPath)); - if (oldUid.hasOwnProperty(splitedFieldRulesValue)) { - updatedValue.push(oldUid[splitedFieldRulesValue]); - } else { - updatedValue.push(element); - } - } - schema.field_rules[k].conditions[i].value = updatedValue.join('.'); - } - } - } - } else { - log(this.config, 'field_rules is not available', 'error'); - } - - this.stackAPIClient - .contentType(schema.uid) - .fetch() - .then((contentTypeResponse) => { - // Object.assign(ctObj, _.cloneDeep(schema)) - contentTypeResponse.field_rules = schema.field_rules; - return contentTypeResponse.update(); - }) - .then(() => { - return resolve(); - }) - .catch((error) => { - log(this.config, `failed to update the field rules ${formatError(error)}`); - return reject(error); - }); - }); - } - publish(langs) { - let self = this; - let requestObject = { - entry: {}, - }; - - let contentTypeUids = Object.keys(self.ctSchemas); - let entryMapper = fileHelper.readFileSync(this.entryUidMapperPath); - - return new Promise((resolve, reject) => { - return Promise.map( - langs, - (_lang, counter) => { - let lang = langs[counter]; - return Promise.map( - contentTypeUids, - async (ctUid) => { - let eFilePath = path.resolve(this.ePath, ctUid, lang + '.json'); - let entries = await fileHelper.readLargeFile(eFilePath); - if (entries === undefined) { - addlogs(this.config, `No entries were found for Content type: ${ctUid} in language: ${lang}`, 'info'); - } else { - let eUids = Object.keys(entries); - let batches = []; - let batchSize; - - if (eUids.length > 0) { - let entryBatchLimit = this.eConfig.batchLimit || 10; - batchSize = Math.round(entryBatchLimit / 3); - // Run entry creation in batches - for (let i = 0; i < eUids.length; i += batchSize) { - batches.push(eUids.slice(i, i + batchSize)); - } - } else { - return; - } - - return Promise.map( - batches, - async (batch, index) => { - return Promise.map( - batch, - async (eUid) => { - let entry = entries[eUid]; - let envId = []; - let locales = []; - if (entry.publish_details && entry.publish_details.length > 0) { - _.forEach(entries[eUid].publish_details, (pubObject) => { - if ( - self.environment.hasOwnProperty(pubObject.environment) && - _.indexOf(envId, self.environment[pubObject.environment].name) === -1 - ) { - envId.push(self.environment[pubObject.environment].name); - } - if (pubObject.locale) { - let idx = _.indexOf(locales, pubObject.locale); - if (idx === -1) { - locales.push(pubObject.locale); - } - } - }); - - let entryUid = entryMapper[eUid]; - if (entryUid) { - requestObject.entry.environments = envId; - requestObject.entry.locales = locales; - return new Promise((resolveEntryPublished, rejectEntryPublished) => { - self.stackAPIClient - .contentType(ctUid) - .entry(entryUid) - .publish({ publishDetails: requestObject.entry, locale: lang }) - // eslint-disable-next-line max-nested-callbacks - .then((result) => { - // addlogs(this.config, 'Entry ' + eUid + ' published successfully in ' + ctUid + ' content type', 'success') - addlogs( - this.config, - `Entry '${eUid}' published successfully in '${ctUid}' content type`, - 'success', - ); - return resolveEntryPublished(result); - // eslint-disable-next-line max-nested-callbacks - }) - .catch((err) => { - addlogs( - this.config, - `failed to publish entry '${eUid}' content type '${ctUid}' ${formatError(err)}`, - 'error', - ); - return resolveEntryPublished(''); - }); - }); - } - } else { - return {}; - } - }, - { - concurrency: 1, - }, - ) - .then(() => { - // empty function - }) - .catch((error) => { - // error while executing entry in batch - addlogs(this.config, formatError(error), 'error'); - addlogs(this.config, error, 'error'); - }); - }, - { - concurrency: 1, - }, - ) - .then(() => { - // addlogs(this.config, 'Entries published successfully in ' + ctUid + ' content type', 'success') - addlogs(this.config, `Entries published successfully in '${ctUid}' content type`, 'info'); - }) - .catch((error) => { - console.log(error); - addlogs( - this.config, - `failed to publish entry in content type '${ctUid}' ${formatError(error)}`, - 'error', - ); - }); - } - }, - { - concurrency: 1, - }, - ) - .then(() => { - // empty function - // log('Published entries successfully in ' +); - }) - .catch((error) => { - addlogs(this.config, `Failed to publish few entries in ${lang} ${formatError(error)}`, 'error'); - }); - }, - { - concurrency: 1, - }, - ) - .then(() => { - return resolve(); - }) - .catch((error) => { - addlogs(this.config, `Failed to publish entries ${formatError(error)}`, 'error'); - }); - }); - } - removeEntryRefsFromJSONRTE(entry, ctSchema) { - for (const element of ctSchema) { - switch (element.data_type) { - case 'blocks': { - if (entry[element.uid]) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e) => { - let key = Object.keys(e).pop(); - let subBlock = element.blocks.filter((block) => block.uid === key).pop(); - e[key] = this.removeEntryRefsFromJSONRTE(e[key], subBlock.schema); - return e; - }); - } - } - break; - } - case 'global_field': - case 'group': { - if (entry[element.uid]) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e) => { - e = this.removeEntryRefsFromJSONRTE(e, element.schema); - return e; - }); - } else { - entry[element.uid] = this.removeEntryRefsFromJSONRTE(entry[element.uid], element.schema); - } - } - break; - } - case 'json': { - const structuredPTag = '{"type":"p","attrs":{},"children":[{"text":""}]}'; - if (entry[element.uid] && element.field_metadata.rich_text_type) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((jsonRteData) => { - // repeated code from else block, will abstract later - let entryReferences = jsonRteData.children.filter((e) => this.doEntryReferencesExist(e)); - if (entryReferences.length > 0) { - jsonRteData.children = jsonRteData.children.filter((e) => !this.doEntryReferencesExist(e)); - if (jsonRteData.children.length === 0) { - jsonRteData.children.push(JSON.parse(structuredPTag)); - } - return jsonRteData; // return jsonRteData without entry references - } else { - return jsonRteData; // return jsonRteData as it is, because there are no entry references - } - }); - } else { - let entryReferences = entry[element.uid].children.filter((e) => this.doEntryReferencesExist(e)); - if (entryReferences.length > 0) { - entry[element.uid].children = entry[element.uid].children.filter( - (e) => !this.doEntryReferencesExist(e), - ); - if (entry[element.uid].children.length === 0) { - entry[element.uid].children.push(JSON.parse(structuredPTag)); - } - } - } - } - break; - } - } - } - return entry; - } - doEntryReferencesExist(element) { - // checks if the children of p element contain any references - // only checking one level deep, not recursive - - if (element.length) { - for (const item of element) { - if ((item.type === 'p' || item.type === 'a' || item.type === 'span') && item.children && item.children.length > 0) { - return this.doEntryReferencesExist(item.children); - } else if (this.isEntryRef(item)) { - return true; - } - } - } else { - if (this.isEntryRef(element)) { - return true; - } - - if ((element.type === 'p' || element.type === 'a' || element.type ==='span') && element.children && element.children.length > 0) { - return this.doEntryReferencesExist(element.children); - } - } - return false; - } - restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema) { - let mappedAssetUids = fileHelper.readFileSync(this.mappedAssetUidPath) || {}; - let mappedAssetUrls = fileHelper.readFileSync(this.mappedAssetUrlPath) || {}; - for (const element of ctSchema) { - switch (element.data_type) { - case 'blocks': { - if (entry[element.uid]) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e, eIndex) => { - let key = Object.keys(e).pop(); - let subBlock = element.blocks.filter((block) => block.uid === key).pop(); - let sourceStackElement = sourceStackEntry[element.uid][eIndex][key]; - e[key] = this.restoreJsonRteEntryRefs(e[key], sourceStackElement, subBlock.schema); - return e; - }); - } - } - break; - } - case 'global_field': - case 'group': { - if (entry[element.uid]) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e, eIndex) => { - let sourceStackElement = sourceStackEntry[element.uid][eIndex]; - e = this.restoreJsonRteEntryRefs(e, sourceStackElement, element.schema); - return e; - }); - } else { - let sourceStackElement = sourceStackEntry[element.uid]; - entry[element.uid] = this.restoreJsonRteEntryRefs(entry[element.uid], sourceStackElement, element.schema); - } - } - break; - } - case 'json': { - if (entry[element.uid] && element.field_metadata.rich_text_type) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((field, index) => { - // i am facing a Maximum call stack exceeded issue, - // probably because of this loop operation - - let entryRefs = sourceStackEntry[element.uid][index].children - .map((e, i) => { - return { index: i, value: e }; - }) - .filter((e) => this.doEntryReferencesExist(e.value)) - .map((e) => { - // commenting the line below resolved the maximum call stack exceeded issue - // e.value = this.setDirtyTrue(e.value) - this.setDirtyTrue(e.value); - return e; - }) - .map((e) => { - // commenting the line below resolved the maximum call stack exceeded issue - // e.value = this.resolveAssetRefsInEntryRefsForJsonRte(e, mappedAssetUids, mappedAssetUrls) - this.resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls); - return e; - }); - - if (entryRefs.length > 0) { - entryRefs.forEach((entryRef) => { - field.children.splice(entryRef.index, 0, entryRef.value); - }); - } - return field; - }); - } else { - let entryRefs = sourceStackEntry[element.uid].children - .map((e, index) => { - return { index: index, value: e }; - }) - .filter((e) => this.doEntryReferencesExist(e.value)) - .map((e) => { - this.setDirtyTrue(e.value); - return e; - }) - .map((e) => { - this.resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls); - return e; - }); - - if (entryRefs.length > 0) { - entryRefs.forEach((entryRef) => { - if (!_.isEmpty(entry[element.uid]) && entry[element.uid].children) { - entry[element.uid].children.splice(entryRef.index, 0, entryRef.value); - } - }); - } - } - } - break; - } - } - } - return entry; - } - isEntryRef(element) { - return element.type === 'reference' && element.attrs.type === 'entry'; - } - removeUidsFromJsonRteFields(entry, ctSchema) { - for (const element of ctSchema) { - switch (element.data_type) { - case 'blocks': { - if (entry[element.uid]) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e) => { - let key = Object.keys(e).pop(); - let subBlock = element.blocks.filter((block) => block.uid === key).pop(); - e[key] = this.removeUidsFromJsonRteFields(e[key], subBlock.schema); - return e; - }); - } - } - break; - } - case 'global_field': - case 'group': { - if (entry[element.uid]) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e) => { - e = this.removeUidsFromJsonRteFields(e, element.schema); - return e; - }); - } else { - entry[element.uid] = this.removeUidsFromJsonRteFields(entry[element.uid], element.schema); - } - } - break; - } - case 'json': { - if (entry[element.uid] && element.field_metadata.rich_text_type) { - if (element.multiple) { - entry[element.uid] = entry[element.uid].map((jsonRteData) => { - delete jsonRteData.uid; // remove uid - - if (_.isObject(jsonRteData.attrs)) { - jsonRteData.attrs.dirty = true; - } - - if (!_.isEmpty(jsonRteData.children)) { - jsonRteData.children = _.map(jsonRteData.children, (child) => this.removeUidsFromChildren(child)); - } - - return jsonRteData; - }); - } else { - delete entry[element.uid].uid; // remove uid - if (entry[element.uid] && _.isObject(entry[element.uid].attrs)) { - entry[element.uid].attrs.dirty = true; - } - if (entry[element.uid] && !_.isEmpty(entry[element.uid].children)) { - entry[element.uid].children = _.map(entry[element.uid].children, (child) => - this.removeUidsFromChildren(child), - ); - } - } - } - break; - } - } - } - return entry; - } - removeUidsFromChildren(children) { - if (children.length && children.length > 0) { - return children.map((child) => { - if (child.type && child.type.length > 0) { - delete child.uid; // remove uid - - if (_.isObject(child.attrs)) { - child.attrs.dirty = true; - } - } - if (child.children && child.children.length > 0) { - child.children = this.removeUidsFromChildren(child.children); - } - return child; - }); - } else { - if (children.type && children.type.length > 0) { - delete children.uid; // remove uid - if (_.isObject(children.attrs)) { - children.attrs.dirty = true; - } - } - if (children.children && children.children.length > 0) { - children.children = this.removeUidsFromChildren(children.children); - } - return children; - } - } - setDirtyTrue(jsonRteChild) { - // also removing uids in this function - if (jsonRteChild.type) { - if (_.isObject(jsonRteChild.attrs)) { - jsonRteChild.attrs['dirty'] = true; - } - delete jsonRteChild.uid; - - if (jsonRteChild.children && jsonRteChild.children.length > 0) { - jsonRteChild.children = jsonRteChild.children.map((subElement) => this.setDirtyTrue(subElement)); - } - } - return jsonRteChild; - } - resolveAssetRefsInEntryRefsForJsonRte(jsonRteChild, mappedAssetUids, mappedAssetUrls) { - if (jsonRteChild.type) { - if (jsonRteChild.attrs.type === 'asset') { - let assetUrl; - if (mappedAssetUids[jsonRteChild.attrs['asset-uid']]) { - jsonRteChild.attrs['asset-uid'] = mappedAssetUids[jsonRteChild.attrs['asset-uid']]; - } - - if (jsonRteChild.attrs['display-type'] !== 'link') { - assetUrl = jsonRteChild.attrs['asset-link']; - } else { - assetUrl = jsonRteChild.attrs['href']; - } - - if (mappedAssetUrls[assetUrl]) { - if (jsonRteChild.attrs['display-type'] !== 'link') { - jsonRteChild.attrs['asset-link'] = mappedAssetUrls[assetUrl]; - } else { - jsonRteChild.attrs['href'] = mappedAssetUrls[assetUrl]; - } - } - } - - if (jsonRteChild.children && jsonRteChild.children.length > 0) { - jsonRteChild.children = jsonRteChild.children.map((subElement) => - this.resolveAssetRefsInEntryRefsForJsonRte(subElement, mappedAssetUids, mappedAssetUrls), - ); - } - } - - return jsonRteChild; - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/environments.js b/packages/contentstack-import/src/import/modules-js/environments.js deleted file mode 100755 index 3da28de070..0000000000 --- a/packages/contentstack-import/src/import/modules-js/environments.js +++ /dev/null @@ -1,102 +0,0 @@ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const fs = require('fs'); -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const Promise = require('bluebird'); -const { isEmpty, merge } = require('lodash'); - -const { readFileSync, writeFileSync } = require('../../utils/file-helper'); -const { log } = require('../../utils/logger'); -const { formatError } = require('../../utils'); -const config = require('../../config').default; - -module.exports = class ImportEnvironments { - fails = []; - success = []; - envUidMapper = {}; - fetchConcurrency = config.modules.environments.concurrency || config.fetchConcurrency || 2; - - constructor(importConfig, stackAPIClient) { - this.config = importConfig; - this.stackAPIClient = stackAPIClient; - } - - start() { - log(this.config, 'Migrating environment', 'success'); - - const self = this; - let environmentConfig = config.modules.environments; - let environmentsFolderPath = path.resolve(this.config.data, environmentConfig.dirName); - let envMapperPath = path.resolve(this.config.data, 'mapper', 'environments'); - let envUidMapperPath = path.resolve(this.config.data, 'mapper', 'environments', 'uid-mapping.json'); - let envSuccessPath = path.resolve(this.config.data, 'environments', 'success.json'); - let envFailsPath = path.resolve(this.config.data, 'environments', 'fails.json'); - self.environments = readFileSync(path.resolve(environmentsFolderPath, environmentConfig.fileName)); - - if (fs.existsSync(envUidMapperPath)) { - self.envUidMapper = readFileSync(envUidMapperPath); - self.envUidMapper = self.envUidMapper || {}; - } - - mkdirp.sync(envMapperPath); - return new Promise(function (resolve, reject) { - if (self.environments === undefined || isEmpty(self.environments)) { - log(self.config, chalk.yellow('No Environment Found'), 'success'); - return resolve({ empty: true }); - } - - let envUids = Object.keys(self.environments); - return Promise.map( - envUids, - function (envUid) { - let env = self.environments[envUid]; - if (!self.envUidMapper.hasOwnProperty(envUid)) { - let requestOption = { environment: env }; - - return self.stackAPIClient - .environment() - .create(requestOption) - .then((environment) => { - self.success.push(environment.items); - self.envUidMapper[envUid] = environment.uid; - writeFileSync(envUidMapperPath, self.envUidMapper); - }) - .catch(function (err) { - let error = JSON.parse(err.message); - - if (error.errors.name) { - log(self.config, `Environment '${env.name}' already exists`, 'error'); - } else { - log( - config, - `Environment '${env.name}' failed to be import\n ${JSON.stringify(error.errors)}`, - 'error', - ); - } - }); - } else { - // the environment has already been created - log(config, `The environment '${env.name}' already exists. Skipping it to avoid duplicates!`, 'success'); - } - }, - { concurrency: self.fetchConcurrency }, - ) - .then(function () { - writeFileSync(envSuccessPath, self.success); - log(self.config, chalk.green('Environments have been imported successfully!'), 'success'); - resolve(''); - }) - .catch(function (error) { - writeFileSync(envFailsPath, self.fails); - log(self.config, `Failed to import environment ${formatError(error)}`, 'error'); - reject(error); - }); - }); - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/extensions.js b/packages/contentstack-import/src/import/modules-js/extensions.js deleted file mode 100644 index 47f1031b9c..0000000000 --- a/packages/contentstack-import/src/import/modules-js/extensions.js +++ /dev/null @@ -1,100 +0,0 @@ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const mkdirp = require('mkdirp'); -const fs = require('fs'); -const path = require('path'); -const Promise = require('bluebird'); -const chalk = require('chalk'); -const { isEmpty, merge } = require('lodash'); -const { default: config } = require('../../config'); -const { fileHelper, log, formatError } = require('../../utils'); - -module.exports = class ImportExtensions { - fails = []; - success = []; - extUidMapper = {}; - extensionsConfig = config.modules.extensions; - reqConcurrency = config.concurrency || config.fetchConcurrency || 1; - - constructor(importConfig, stackAPIClient) { - this.config = merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - log(this.config, chalk.white('Migrating extensions'), 'success'); - - const self = this; - let extensionsFolderPath = path.resolve(this.config.data, this.extensionsConfig.dirName); - let extMapperPath = path.resolve(this.config.data, 'mapper', 'extensions'); - let extUidMapperPath = path.resolve(this.config.data, 'mapper/extensions', 'uid-mapping.json'); - let extSuccessPath = path.resolve(this.config.data, 'extensions', 'success.json'); - let extFailsPath = path.resolve(this.config.data, 'extensions', 'fails.json'); - this.extensions = fileHelper.readFileSync(path.resolve(extensionsFolderPath, this.extensionsConfig.fileName)); - if (fs.existsSync(extUidMapperPath)) { - self.extUidMapper = fileHelper.readFileSync(extUidMapperPath); - self.extUidMapper = self.extUidMapper || {}; - } - - mkdirp.sync(extMapperPath); - - return new Promise(function (resolve, reject) { - if (self.extensions === undefined || isEmpty(self.extensions)) { - log(self.config, chalk.white('No Extensions Found'), 'success'); - return resolve({ empty: true }); - } - let extUids = Object.keys(self.extensions); - return Promise.map( - extUids, - function (extUid) { - let ext = self.extensions[extUid]; - if (!self.extUidMapper.hasOwnProperty(extUid)) { - return self.stackAPIClient - .extension() - .create({ extension: ext }) - .then((response) => { - self.success.push(response); - self.extUidMapper[extUid] = response.uid; - fileHelper.writeFileSync(extUidMapperPath, self.extUidMapper); - }) - .catch(function (err) { - let error = JSON.parse(err.message); - self.fails.push(ext); - if (error.errors.title) { - log(self.config, `Extension '${ext.title}' already exists`, 'error'); - } else { - log(self.config, "Extension: '" + ext.title + "' failed to import" + formatError(error), 'error'); - } - }); - } - // the extensions has already been created - log( - config, - chalk.white("The extension: '" + ext.title + "' already exists. Skipping it to avoid duplicates!"), - 'success', - ); - // import 2 extensions at a time - }, - { - concurrency: self.reqConcurrency, - }, - ) - .then(function () { - // extensions have imported successfully - fileHelper.writeFileSync(extSuccessPath, self.success); - log(self.config, chalk.green('Extensions have been imported successfully!'), 'success'); - resolve(); - }) - .catch(function (error) { - // error while importing extensions - fileHelper.writeFileSync(extFailsPath, self.fails); - log(self.config, `Extension import failed ${formatError(error)}`, 'error'); - reject(error); - }); - }); - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/global-fields.js b/packages/contentstack-import/src/import/modules-js/global-fields.js deleted file mode 100644 index 12ea4a4d86..0000000000 --- a/packages/contentstack-import/src/import/modules-js/global-fields.js +++ /dev/null @@ -1,123 +0,0 @@ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -let fs = require('fs'); -let path = require('path'); -let chalk = require('chalk'); -let mkdirp = require('mkdirp'); -let Promise = require('bluebird'); -const { isEmpty, merge } = require('lodash'); -let { default: config } = require('../../config'); -const { fileHelper, log, formatError, lookupExtension, removeReferenceFields } = require('../../utils'); - -global._globalField_pending = []; - -module.exports = class ImportGlobalFields { - fails = []; - success = []; - snipUidMapper = {}; - installedExtensions = []; - reqConcurrency = config.concurrency || config.fetchConcurrency || 1; - - constructor(importConfig, stackAPIClient) { - this.config = merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - } - - async start() { - log(this.config, chalk.white('Migrating global-fields'), 'success'); - - let self = this; - let globalfieldsConfig = config.modules.globalfields; - let globalfieldsFolderPath = path.resolve(this.config.data, globalfieldsConfig.dirName); - let globalfieldsMapperPath = path.resolve(this.config.data, 'mapper', 'global_fields'); - let globalfieldsUidMapperPath = path.resolve(this.config.data, 'mapper', 'global_fields', 'uid-mapping.json'); - let globalfieldsSuccessPath = path.resolve(this.config.data, 'mapper', 'global_fields', 'success.json'); - let globalFieldsPending = path.resolve(this.config.data, 'mapper', 'global_fields', 'pending_global_fields.js'); - let globalfieldsFailsPath = path.resolve(this.config.data, 'mapper', 'global_fields', 'fails.json'); - self.globalfields = fileHelper.readFileSync(path.resolve(globalfieldsFolderPath, globalfieldsConfig.fileName)); - const appMapperPath = path.join(this.config.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); - this.installedExtensions = ((await fileHelper.readFileSync(appMapperPath)) || { extension_uid: {} }).extension_uid; - if (fs.existsSync(globalfieldsUidMapperPath)) { - self.snipUidMapper = fileHelper.readFileSync(globalfieldsUidMapperPath); - self.snipUidMapper = self.snipUidMapper || {}; - } - - if (!fs.existsSync(globalfieldsMapperPath)) { - mkdirp.sync(globalfieldsMapperPath); - } - - return new Promise(function (resolve, reject) { - if (self.globalfields === undefined || isEmpty(self.globalfields)) { - log(self.config, chalk.white('No globalfields Found'), 'success'); - fileHelper.writeFileSync(globalFieldsPending, _globalField_pending); - return resolve({ empty: true }); - } - let snipUids = Object.keys(self.globalfields); - return Promise.map( - snipUids, - async function (snipUid) { - let flag = { supressed: false }; - let snip = self.globalfields[snipUid]; - lookupExtension(self.config, snip.schema, self.config.preserveStackVersion, self.installedExtensions); - await removeReferenceFields(snip.schema, flag, self.stackAPIClient); - - if (flag.supressed) { - // eslint-disable-next-line no-undef - _globalField_pending.push(snip.uid); - } - - if (!self.snipUidMapper.hasOwnProperty(snipUid)) { - let requestOption = { global_field: snip }; - return self.stackAPIClient - .globalField() - .create(requestOption) - .then((globalField) => { - self.success.push(globalField.items); - let global_field_uid = globalField.uid; - self.snipUidMapper[snipUid] = globalField.items; - fileHelper.writeFileSync(globalfieldsUidMapperPath, self.snipUidMapper); - log(self.config, chalk.green('Global field ' + global_field_uid + ' created successfully'), 'success'); - }) - .catch(function (err) { - let error = JSON.parse(err.message); - if (error.errors.title) { - // eslint-disable-next-line no-undef - log(self.config, `Globalfield '${snip.uid} already exists'`, 'error'); - } else { - log(self.config, chalk.red(`Globalfield '${snip.title}' failed to import. ${formatError(error)}`), 'error'); - } - - self.fails.push(snip); - }); - } else { - // globalfields has already been created - log( - self.config, - chalk.white('The globalfields already exists. Skipping it to avoid duplicates!'), - 'success', - ); - } - // import 2 globalfields at a time - }, - { concurrency: self.reqConcurrency }, - ) - .then(function () { - // globalfields have imported successfully - fileHelper.writeFileSync(globalfieldsSuccessPath, self.success); - fileHelper.writeFileSync(globalFieldsPending, _globalField_pending); - log(self.config, chalk.green('globalfields have been imported successfully!'), 'success'); - return resolve(); - }) - .catch(function (err) { - let error = JSON.parse(err); - fileHelper.writeFileSync(globalfieldsFailsPath, self.fails); - log(self.config, `Globalfields import failed. ${formatError(err)}`, 'error'); - return reject(error); - }); - }); - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/index.js b/packages/contentstack-import/src/import/modules-js/index.js deleted file mode 100644 index e652960fa3..0000000000 --- a/packages/contentstack-import/src/import/modules-js/index.js +++ /dev/null @@ -1,6 +0,0 @@ -export default async function startModuleImport(modulePayload) { - const { moduleName, importConfig, stackAPIClient } = modulePayload; - const { default: ModuleRunner } = await import(`./${moduleName}`); - const moduleRunner = new ModuleRunner(importConfig, stackAPIClient); - return moduleRunner.start(); -} diff --git a/packages/contentstack-import/src/import/modules-js/labels.js b/packages/contentstack-import/src/import/modules-js/labels.js deleted file mode 100644 index b685f78d23..0000000000 --- a/packages/contentstack-import/src/import/modules-js/labels.js +++ /dev/null @@ -1,172 +0,0 @@ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const Promise = require('bluebird'); -const { existsSync } = require('fs'); -const { isEmpty, merge } = require('lodash'); -let { default: config } = require('../../config'); -const { fileHelper, log, formatError } = require('../../utils'); - -module.exports = class ImportLabels { - config; - fails = []; - success = []; - labelUids = []; - labelsFolderPath; - labelUidMapper = {}; - labelConfig = config.modules.labels; - reqConcurrency = config.concurrency || config.fetchConcurrency || 1; - - constructor(importConfig, stackAPIClient) { - this.config = merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - const self = this; - log(this.config, chalk.white('Migrating labels'), 'success'); - let labelMapperPath = path.resolve(this.config.data, 'mapper', 'labels'); - let labelFailsPath = path.resolve(this.config.data, 'labels', 'fails.json'); - let labelSuccessPath = path.resolve(this.config.data, 'labels', 'success.json'); - this.labelsFolderPath = path.resolve(this.config.data, this.labelConfig.dirName); - let labelUidMapperPath = path.resolve(this.config.data, 'mapper', 'labels', 'uid-mapping.json'); - - if (existsSync(labelUidMapperPath)) { - this.labelUidMapper = fileHelper.readFileSync(labelUidMapperPath); - this.labelUidMapper = this.labelUidMapper || {}; - } - - self.labels = fileHelper.readFileSync(path.resolve(this.labelsFolderPath, this.labelConfig.fileName)); - - mkdirp.sync(labelMapperPath); - - return new Promise(function (resolve, reject) { - if (self.labels == undefined || isEmpty(self.labels)) { - log(self.config, chalk.white('No Label Found'), 'success'); - return resolve({ empty: true }); - } - self.labelUids = Object.keys(self.labels); - return Promise.map( - self.labelUids, - function (labelUid) { - let label = self.labels[labelUid]; - - if (label.parent.length != 0) { - delete label['parent']; - } - - if (!self.labelUidMapper.hasOwnProperty(labelUid)) { - let requestOption = { label: label }; - - return self.stackAPIClient - .label() - .create(requestOption) - .then(function (response) { - self.labelUidMapper[labelUid] = response; - }) - .catch(function (error) { - self.fails.push(label); - if (error.errors.name) { - log(self.config, `Label '${label.name}' already exist`, 'error'); - } else { - log(self.config, `Label '${label.name}' failed to be imported\n`, 'error'); - } - }); - } else { - // the label has already been created - log( - self.config, - chalk.white(`The label '${label.name}' already exists. Skipping it to avoid duplicates!'`), - 'success', - ); - } - // import 1 labels at a time - }, - { concurrency: self.reqConcurrency }, - ) - .then(function () { - fileHelper.writeFileSync(labelUidMapperPath, self.labelUidMapper); - // eslint-disable-next-line no-undef - return self - .updateLabels() - .then(function () { - fileHelper.writeFileSync(labelSuccessPath, self.success); - log(self.config, chalk.green('Labels have been imported successfully!'), 'success'); - return resolve(); - }) - .catch(function (error) { - log(self.config, self.config, `Failed to import label, ${formatError(error)}`, 'error'); - // eslint-disable-next-line no-console - return reject(error); - }); - }) - .catch(function (error) { - // error while importing labels - fileHelper.writeFileSync(labelUidMapperPath, self.labelUidMapper); - fileHelper.writeFileSync(labelFailsPath, self.fails); - log(self.config, `Failed to import label, ${formatError(error)}`, 'error'); - return reject(error); - }); - }); - } - - updateLabels() { - const self = this; - return new Promise(function (resolve, reject) { - let labelsObj = fileHelper.readFileSync(path.resolve(self.labelsFolderPath, self.labelConfig.fileName)); - return Promise.map( - self.labelUids, - function (labelUid) { - let label = labelsObj[labelUid]; - if (self.labelUidMapper.hasOwnProperty(labelUid)) { - let newLabelUid = self.labelUidMapper[labelUid]; - if (label.parent.length > 0) { - let parentUids = label.parent; - for (let i = 0; i < parentUids.length; i++) { - if (self.labelUidMapper.hasOwnProperty(parentUids[i])) { - label.parent[i] = self.labelUidMapper[parentUids[i]].uid; - } - } - return self.stackAPIClient - .label(newLabelUid.uid) - .fetch() - .then(function (response) { - //Object.assign(response, _.cloneDeep(label)) - response.parent = label.parent; - response - .update() - .then((result) => { - self.success.push(result); - }) - .catch((error) => { - log(self.config, formatError(error), 'error'); - return reject(error); - }); - }) - .catch(function (error) { - log(self.config, formatError(error), 'error'); - return reject(error); - }); - } - } - }, - { - concurrency: self.reqConcurrency, - }, - ) - .then(function () { - return resolve(); - }) - .catch(function (error) { - // eslint-disable-next-line no-console - return reject(error); - }); - }); - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/locales.js b/packages/contentstack-import/src/import/modules-js/locales.js deleted file mode 100755 index 1b2d12828c..0000000000 --- a/packages/contentstack-import/src/import/modules-js/locales.js +++ /dev/null @@ -1,213 +0,0 @@ -/* eslint-disable no-prototype-builtins */ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -let fs = require('fs'); -let path = require('path'); -let chalk = require('chalk'); -let mkdirp = require('mkdirp'); -let Promise = require('bluebird'); -let { isEmpty, merge, cloneDeep } = require('lodash'); -const { cliux } = require('@contentstack/cli-utilities'); -let { default: config } = require('../../config'); -const { fileHelper, log, formatError } = require('../../utils'); -module.exports = class ImportLanguages { - constructor(importConfig, stackAPIClient) { - this.config = merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - this.fails = []; - this.success = []; - this.langUidMapper = {}; - this.masterLanguage = importConfig.master_locale?.code; - this.langConfig = importConfig.modules.locales; - this.sourceMasterLangConfig = config.modules.masterLocale; - this.reqConcurrency = importConfig.concurrency || importConfig.fetchConcurrency || 1; - } - - start() { - log(this.config, 'Migrating languages', 'success'); - - const self = this; - let langMapperPath = path.resolve(this.config.data, 'mapper', 'languages'); - let langFolderPath = path.resolve(this.config.data, this.langConfig.dirName); - let langFailsPath = path.resolve(this.config.data, 'mapper', 'languages', 'fails.json'); - let langUidMapperPath = path.resolve(this.config.data, 'mapper', 'languages', 'uid-mapper.json'); - self.langSuccessPath = path.resolve(this.config.data, 'mapper', 'languages', 'success.json'); - self.languages = fileHelper.readFileSync(path.resolve(langFolderPath, this.langConfig.fileName)); - self.sourceMasterLanguages = fileHelper.readFileSync( - path.resolve(langFolderPath, this.sourceMasterLangConfig.fileName), - ); - mkdirp.sync(langMapperPath); - - if (fs.existsSync(langUidMapperPath)) { - self.langUidMapper = fileHelper.readFileSync(langUidMapperPath); - self.langUidMapper = self.langUidMapper || {}; - } - - return new Promise(async function (resolve, reject) { - if (self.languages === undefined || isEmpty(self.languages)) { - log(self.config, chalk.white('No Languages Found'), 'success'); - return resolve({ empty: true }); - } - - let sourceMasterLangDetails = self.sourceMasterLanguages && Object.values(self.sourceMasterLanguages); - if ( - sourceMasterLangDetails && - sourceMasterLangDetails[0] && - sourceMasterLangDetails[0]['code'] && - self.masterLanguage['code'] === sourceMasterLangDetails[0]['code'] - ) { - await self.checkAndUpdateMasterLocaleName(sourceMasterLangDetails).catch((error) => { - log(self.config, formatError(error), 'warn'); - }); - } - - let langUids = Object.keys(self.languages); - return Promise.map( - langUids, - (langUid) => { - let lang = self.languages[langUid]; - if (!self.langUidMapper.hasOwnProperty(langUid) && lang.code !== self.masterLanguage) { - let requestOption = { - locale: { - code: lang.code, - name: lang.name, - }, - }; - - return self.stackAPIClient - .locale() - .create(requestOption) - .then((locale) => { - self.success.push(locale.items); - self.langUidMapper[langUid] = locale.uid; - fileHelper.writeFileSync(langUidMapperPath, self.langUidMapper); - }) - .catch(function (err) { - let error = JSON.parse(err.message); - if (error.hasOwnProperty('errorCode') && error.errorCode === 247) { - if(error?.errors?.code){ - log(self.config, error.errors.code[0], 'error'); - }else{ - log(self.config, err, 'error'); - } - return err; - } - self.fails.push(lang); - log(self.config, `Language '${lang.code}' failed to import\n`, 'error'); - log(self.config, formatError(err), 'error'); - }); - } else { - // the language has already been created - log(self.config, `The language '${lang.code}' already exists.`, 'error'); - } - - return Promise.resolve(); - // import 2 languages at a time - }, - { concurrency: self.reqConcurrency }, - ) - .then(function () { - // languages have imported successfully - return self - .updateLocales(langUids) - .then(() => { - fileHelper.writeFileSync(self.langSuccessPath, self.success); - log(self.config, chalk.green('Languages have been imported successfully!'), 'success'); - resolve(); - }) - .catch(function (error) { - log(self.config, formatError(error), 'error'); - reject(error); - }); - }) - .catch(function (error) { - // error while importing languages - fileHelper.writeFileSync(langFailsPath, self.fails); - log(self.config, `Language import failed. ${formatError(error)}`, 'error'); - reject('failed to import Languages'); - }); - }); - } - - updateLocales(langUids) { - let self = this; - return new Promise(function (resolve, reject) { - Promise.all( - langUids.map(async (langUid) => { - let lang = {}; - let requireKeys = self.config.modules.locales.requiredKeys; - let _lang = self.languages[langUid]; - requireKeys.forEach((e) => { - lang[e] = _lang[e]; - }); - let langobj = self.stackAPIClient.locale(lang.code); - Object.assign(langobj, cloneDeep(lang)); - langobj.update().then(() => { - // empty function - }); - }), - ) - .then(resolve) - .catch((error) => { - log(self.config, formatError(error), 'error'); - reject(error); - }); - }); - } - - checkAndUpdateMasterLocaleName(sourceMasterLangDetails) { - let self = this; - return new Promise(async function (resolve, reject) { - let masterLangDetails = await self.stackAPIClient - .locale(self.masterLanguage['code']) - .fetch() - .catch((error) => { - log(self.config, formatError(error), 'warn'); - }); - if ( - masterLangDetails && - masterLangDetails['name'] && - sourceMasterLangDetails[0]['name'] && - masterLangDetails['name'].toString().toUpperCase() !== - sourceMasterLangDetails[0]['name'].toString().toUpperCase() - ) { - cliux.print('WARNING!!! The master language name for the source and destination is different.', { - color: 'yellow', - }); - cliux.print(`Old Master language name: ${masterLangDetails['name']}`, { color: 'red' }); - cliux.print(`New Master language name: ${sourceMasterLangDetails[0]['name']}`, { color: 'green' }); - let confirm = await cliux.inquire({ - type: 'confirm', - message: 'Are you sure you want to update name of master language?', - name: 'confirmation', - }); - - if (confirm) { - let languid = sourceMasterLangDetails[0] && sourceMasterLangDetails[0]['uid']; - let lang = self.sourceMasterLanguages[languid]; - if (!lang) return reject('Locale not found.!'); - const langObj = self.stackAPIClient.locale(lang.code); - Object.assign(langObj, { name: lang.name }); - langObj - .update() - .then(() => { - fileHelper.writeFileSync(self.langSuccessPath, self.success); - log(self.config, chalk.green('Master Languages name have been updated successfully!'), 'success'); - resolve(); - }) - .catch(function (error) { - reject(error); - }); - } else { - resolve(); - } - } else { - resolve(); - } - }); - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/marketplace-apps.js b/packages/contentstack-import/src/import/modules-js/marketplace-apps.js deleted file mode 100644 index e2cb4d60fa..0000000000 --- a/packages/contentstack-import/src/import/modules-js/marketplace-apps.js +++ /dev/null @@ -1,557 +0,0 @@ -/*! - * Contentstack Export - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ -const fs = require('fs'); -const _ = require('lodash'); -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const { - cliux, - HttpClient, - NodeCrypto, - managementSDKClient, - isAuthenticated, - HttpClientDecorator, - OauthDecorator, -} = require('@contentstack/cli-utilities'); - -const { - log, - formatError, - fileHelper: { readFileSync, writeFile }, -} = require('../../utils'); -const { trace } = require('../../utils/log'); -const { default: config } = require('../../config'); -const { getAllStackSpecificApps } = require('../../utils/marketplace-app-helper'); - -module.exports = class ImportMarketplaceApps { - client; - httpClient; - appOriginalName; - appUidMapping = {}; - appNameMapping = {}; - marketplaceApps = []; - installationUidMapping = {}; - developerHubBaseUrl = null; - marketplaceAppFolderPath = ''; - marketplaceAppConfig = config.modules.marketplace_apps; - - constructor(importConfig, stackAPIClient) { - this.config = _.merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - } - - async start() { - this.mapperDirPath = path.resolve(this.config.data, 'mapper', 'marketplace_apps'); - this.uidMapperPath = path.join(this.mapperDirPath, 'uid-mapping.json'); - this.marketplaceAppFolderPath = path.resolve(this.config.data, this.marketplaceAppConfig.dirName); - this.marketplaceApps = readFileSync( - path.resolve(this.marketplaceAppFolderPath, this.marketplaceAppConfig.fileName), - ); - - if (_.isEmpty(this.marketplaceApps)) { - return Promise.resolve(); - } else if (!isAuthenticated()) { - cliux.print( - '\nWARNING!!! To import Marketplace apps, you must be logged in. Please check csdx auth:login --help to log in\n', - { color: 'yellow' }, - ); - return Promise.resolve(); - } - - this.developerHubBaseUrl = this.config.developerHubBaseUrl || (await getDeveloperHubUrl(this.config)); - this.config.developerHubBaseUrl = this.developerHubBaseUrl; - this.client = await managementSDKClient({ endpoint: this.developerHubBaseUrl }); - this.appSdkAxiosInstance = await managementSDKClient({ - host: this.developerHubBaseUrl.split('://').pop() - }); - await this.getOrgUid(); - - const httpClient = new HttpClient(); - if (!this.config.auth_token) { - this.httpClient = new OauthDecorator(httpClient); - const headers = await this.httpClient.preHeadersCheck(this.config); - this.httpClient = this.httpClient.headers(headers); - } else { - this.httpClient = new HttpClientDecorator(httpClient); - this.httpClient.headers(this.config); - } - - if (!fs.existsSync(this.mapperDirPath)) { - mkdirp.sync(this.mapperDirPath); - } - - return this.startInstallation(); - } - - async getOrgUid() { - const tempAPIClient = await managementSDKClient({ host: this.config.host }); - const tempStackData = await tempAPIClient - .stack({ api_key: this.config.target_stack }) - .fetch() - .catch((error) => { - console.log(error); - }); - - if (tempStackData?.org_uid) { - this.config.org_uid = tempStackData.org_uid; - } - } - - async getAndValidateEncryptionKey(defaultValue, retry = 1) { - let appConfig = _.find( - this.marketplaceApps, - ({ configuration, server_configuration }) => !_.isEmpty(configuration) || !_.isEmpty(server_configuration), - ); - - if (!appConfig) { - return defaultValue; - } - - const encryptionKey = await cliux.inquire({ - type: 'input', - name: 'name', - default: defaultValue, - validate: (key) => { - if (!key) return "Encryption key can't be empty."; - - return true; - }, - message: 'Enter Marketplace app configurations encryption key', - }); - - try { - appConfig = !_.isEmpty(appConfig.configuration) ? appConfig.configuration : appConfig.server_configuration; - this.nodeCrypto = new NodeCrypto({ encryptionKey }); - this.nodeCrypto.decrypt(appConfig); - } catch (error) { - if (retry < this.config.getEncryptionKeyMaxRetry && error.code === 'ERR_OSSL_EVP_BAD_DECRYPT') { - cliux.print( - `Provided encryption key is not valid or your data might be corrupted.! attempt(${retry}/${this.config.getEncryptionKeyMaxRetry})`, - { color: 'red' }, - ); - // NOTE max retry limit is 3 - return this.getAndValidateEncryptionKey(encryptionKey, retry + 1); - } else { - cliux.print( - `Maximum retry limit exceeded. Closing the process, please try again.! attempt(${retry}/${this.config.getEncryptionKeyMaxRetry})`, - { color: 'red' }, - ); - process.exit(1); - } - } - - return encryptionKey; - } - - /** - * @method startInstallation - * @returns {Promise} - */ - async startInstallation() { - const cryptoArgs = {}; - - if (this.config.marketplaceAppEncryptionKey) { - cryptoArgs['encryptionKey'] = this.config.marketplaceAppEncryptionKey; - } - - if (this.config.forceStopMarketplaceAppsPrompt) { - cryptoArgs['encryptionKey'] = this.config.marketplaceAppEncryptionKey; - this.nodeCrypto = new NodeCrypto(cryptoArgs); - } else { - await this.getAndValidateEncryptionKey(this.config.marketplaceAppEncryptionKey); - } - - // NOTE install all private apps which is not available for stack. - await this.handleAllPrivateAppsCreationProcess(); - const installedApps = await getAllStackSpecificApps(this.config); - - log(this.config, 'Starting marketplace app installation', 'success'); - - for (let app of this.marketplaceApps) { - await this.installApps(app, installedApps); - } - - const uidMapper = await this.generateUidMapper(); - await writeFile(this.uidMapperPath, { - app_uid: this.appUidMapping, - extension_uid: uidMapper || {}, - installation_uid: this.installationUidMapping, - }); - } - - async generateUidMapper() { - const listOfNewMeta = []; - const listOfOldMeta = []; - const extensionUidMap = {}; - const allInstalledApps = (await getAllStackSpecificApps(this.config)) || []; - - for (const app of this.marketplaceApps) { - listOfOldMeta.push(..._.map(app?.ui_location?.locations, 'meta').flat()); - } - for (const app of allInstalledApps) { - listOfNewMeta.push(..._.map(app?.ui_location?.locations, 'meta').flat()); - } - for (const { extension_uid, name, path, uid, data_type } of _.filter(listOfOldMeta, 'name')) { - const meta = - _.find(listOfNewMeta, { name, path }) || - _.find(listOfNewMeta, { name: this.appNameMapping[name], path }) || - _.find(listOfNewMeta, { name, uid, data_type }); - - if (meta) { - extensionUidMap[extension_uid] = meta.extension_uid; - } - } - - return extensionUidMap; - } - - /** - * @method handleAllPrivateAppsCreationProcess - * @param {Object} options - * @returns {Promise} - */ - async handleAllPrivateAppsCreationProcess() { - const privateApps = _.filter(this.marketplaceApps, { manifest: { visibility: 'private' } }); - - if (_.isEmpty(privateApps)) { - return Promise.resolve(); - } - - await this.getConfirmationToCreateApps(privateApps); - - log(this.config, 'Starting developer hub private apps re-creation', 'success'); - - for (let app of privateApps) { - // NOTE keys can be passed to install new app in the developer hub - app.manifest = _.pick(app.manifest, ['uid', 'name', 'description', 'icon', 'target_type', 'webhook', 'oauth']); - this.appOriginalName = app.manifest.name; - await this.createPrivateApps({ - oauth: app.oauth, - webhook: app.webhook, - ui_location: app.ui_location, - ...app.manifest, - }); - } - - this.appOriginalName = undefined; - } - - async getConfirmationToCreateApps(privateApps) { - if (!this.config.forceStopMarketplaceAppsPrompt) { - if ( - !(await cliux.confirm( - chalk.yellow( - `WARNING!!! The listed apps are private apps that are not available in the destination stack: \n\n${_.map( - privateApps, - ({ manifest: { name } }, index) => `${String(index + 1)}) ${name}`, - ).join('\n')}\n\nWould you like to re-create the private app and then proceed with the installation? (y/n)`, - ), - )) - ) { - if ( - await cliux.confirm( - chalk.yellow( - `\nWARNING!!! Canceling the app re-creation may break the content type and entry import. Would you like to proceed without re-create the private app? (y/n)`, - ), - ) - ) { - return Promise.resolve(true); - } - - if ( - !(await cliux.confirm( - chalk.yellow('\nWould you like to re-create the private app and then proceed with the installation? (y/n)'), - )) - ) { - process.exit(); - } - } - } - } - - async createPrivateApps(app, uidCleaned = false, appSuffix = 1) { - let locations = app?.ui_location?.locations; - - if (!uidCleaned && !_.isEmpty(locations)) { - app.ui_location.locations = this.updateManifestUILocations(locations, 'uid'); - } else if (uidCleaned && !_.isEmpty(locations)) { - app.ui_location.locations = this.updateManifestUILocations(locations, 'name', appSuffix); - } - - if (app.name > 20) { - app.name = app.name.slice(0, 20); - } - - const response = await this.client - .organization(this.config.org_uid) - .app() - .create(_.omit(app, ['uid'])) - .catch((error) => error); - - return this.appCreationCallback(app, response, appSuffix); - } - - async appCreationCallback(app, response, appSuffix) { - const { statusText, message } = response || {}; - - if (message) { - if (_.toLower(statusText) === 'conflict') { - return this.handleNameConflict(app, appSuffix); - } else { - log(this.config, formatError(message), 'error'); - - if (this.config.forceStopMarketplaceAppsPrompt) return Promise.resolve(); - - if ( - await cliux.confirm( - chalk.yellow( - 'WARNING!!! The above error may have an impact if the failed app is referenced in entries/content type. Would you like to proceed? (y/n)', - ), - ) - ) { - Promise.resolve(); - } else { - process.exit(); - } - } - } else if (response.uid) { - // NOTE new app installation - log(this.config, `${response.name} app created successfully.!`, 'success'); - this.appUidMapping[app.uid] = response.uid; - this.appNameMapping[this.appOriginalName] = response.name; - } - } - - async handleNameConflict(app, appSuffix) { - const appName = this.config.forceStopMarketplaceAppsPrompt - ? this.getAppName(app.name, appSuffix) - : await cliux.inquire({ - type: 'input', - name: 'name', - validate: this.validateAppName, - default: this.getAppName(app.name, appSuffix), - message: `${app.name} app already exist. Enter a new name to create an app.?`, - }); - app.name = appName; - - return this.createPrivateApps(app, true, appSuffix + 1); - } - - updateManifestUILocations(locations, type = 'uid', appSuffix = 1) { - switch (type) { - case 'uid': - return _.map(locations, (location) => { - if (location.meta) { - location.meta = _.map(location.meta, (meta) => _.omit(meta, ['uid'])); - } - - return location; - }); - case 'name': - return _.map(locations, (location) => { - if (location.meta) { - location.meta = _.map(location.meta, (meta) => { - if (meta.name) { - const name = `${_.first(_.split(meta.name, '◈'))}◈${appSuffix}`; - - if (!this.appNameMapping[this.appOriginalName]) { - this.appNameMapping[this.appOriginalName] = name; - } - - meta.name = name; - } - - return meta; - }); - } - - return location; - }); - } - } - - getAppName(name, appSuffix = 1) { - if (name.length >= 19) name = name.slice(0, 18); - - name = `${_.first(_.split(name, '◈'))}◈${appSuffix}`; - - return name; - } - - /** - * @method installApps - * - * @param {Record} app - * @param {Record[]} installedApps - * @returns {Promise} - */ - async installApps(app, installedApps) { - let updateParam; - let installation; - const { configuration, server_configuration } = app; - const currentStackApp = _.find(installedApps, { manifest: { uid: app.manifest.uid } }); - - if (!currentStackApp) { - // NOTE install new app - installation = await this.client - .organization(this.config.org_uid) - .app(this.appUidMapping[app.manifest.uid] || app.manifest.uid) - .install({ targetUid: this.config.target_stack, targetType: 'stack' }) - .catch((error) => error); - - if (installation.installation_uid) { - let appName = this.appNameMapping[app.manifest.name] - ? this.appNameMapping[app.manifest.name] - : app.manifest.name; - log(this.config, `${appName} app installed successfully.!`, 'success'); - await this.makeRedirectUrlCall(installation, app.manifest.name); - this.installationUidMapping[app.uid] = installation.installation_uid; - updateParam = { manifest: app.manifest, ...installation, configuration, server_configuration }; - } else if (installation.message) { - trace(installation, 'error', true); - log(this.config, formatError(installation.message), 'success'); - await this.confirmToCloseProcess(installation); - } - } else if (!_.isEmpty(configuration) || !_.isEmpty(server_configuration)) { - log(this.config, `${app.manifest.name} is already installed`, 'success'); - updateParam = await this.ifAppAlreadyExist(app, currentStackApp); - } - - if (!this.appUidMapping[app.manifest.uid]) { - this.appUidMapping[app.manifest.uid] = currentStackApp ? currentStackApp.manifest.uid : app.manifest.uid; - } - - // NOTE update configurations - if (updateParam && (!_.isEmpty(updateParam.configuration) || !_.isEmpty(updateParam.server_configuration))) { - await this.updateAppsConfig(updateParam); - } - } - - async makeRedirectUrlCall(response, appName) { - if (response.redirect_url) { - log(this.config, `${appName} - OAuth api call started.!`, 'info'); - await new HttpClient({ maxRedirects: 20, maxBodyLength: Infinity }) - .get(response.redirect_url) - .then(async ({ response }) => { - if (_.includes([501, 403], response.status)) { - trace(response, 'error', true); // NOTE Log complete stack and hide on UI - log(this.config, `${appName} - ${response.statusText}, OAuth api call failed.!`, 'error'); - log(this.config, formatError(response), 'error'); - await this.confirmToCloseProcess({ message: response.data }); - } else { - log(this.config, `${appName} - OAuth api call completed.!`, 'success'); - } - }) - .catch((error) => { - trace(error, 'error', true); - if (_.includes([501, 403], error.status)) { - log(this.config, formatError(error), 'error'); - } - }); - } - } - - async ifAppAlreadyExist(app, currentStackApp) { - let updateParam; - const { - manifest: { name }, - configuration, - server_configuration, - } = app; - - if (!_.isEmpty(configuration) || !_.isEmpty(server_configuration)) { - cliux.print( - `\nWARNING!!! The ${name} app already exists and it may have its own configuration. But the current app you install has its own configuration which is used internally to manage content.\n`, - { color: 'yellow' }, - ); - - const configOption = this.config.forceStopMarketplaceAppsPrompt - ? 'Update it with the new configuration.' - : await cliux.inquire({ - choices: [ - 'Update it with the new configuration.', - 'Do not update the configuration (WARNING!!! If you do not update the configuration, there may be some issues with the content which you import).', - 'Exit', - ], - type: 'list', - name: 'value', - message: 'Choose the option to proceed', - }); - - if (configOption === 'Exit') { - process.exit(); - } else if (configOption === 'Update it with the new configuration.') { - updateParam = { manifest: app.manifest, ...currentStackApp, configuration, server_configuration }; - } - } - - return updateParam; - } - - async confirmToCloseProcess(installation) { - cliux.print(`\nWARNING!!! ${formatError(installation.message)}\n`, { color: 'yellow' }); - - if (!this.config.forceStopMarketplaceAppsPrompt) { - if ( - !(await cliux.confirm( - chalk.yellow( - 'WARNING!!! The above error may have an impact if the failed app is referenced in entries/content type. Would you like to proceed? (y/n)', - ), - )) - ) { - process.exit(); - } - } - } - - /** - * @method updateAppsConfig - * @param {Object<{ data, app }>} param - * @returns {Promise} - */ - updateAppsConfig(app) { - const payload = {}; - const { uid, configuration, server_configuration } = app; - - if (!_.isEmpty(configuration)) { - payload['configuration'] = this.nodeCrypto.decrypt(configuration); - } - if (!_.isEmpty(server_configuration)) { - payload['server_configuration'] = this.nodeCrypto.decrypt(server_configuration); - } - - if (_.isEmpty(app) || _.isEmpty(payload) || !uid) { - return Promise.resolve(); - } - return this.appSdkAxiosInstance.axiosInstance - .put(`${this.developerHubBaseUrl}/installations/${uid}`, payload, { - headers: { - organization_uid: this.config.org_uid - }, - }) - .then(({ data }) => { - if (data.message) { - trace(data, 'error', true); - log(this.config, formatError(data.message), 'success'); - } else { - log(this.config, `${app.manifest.name} app config updated successfully.!`, 'success'); - } - }) - .catch((error) => { - trace(data, 'error', true); - log(this.config, formatError(error), 'error') - }); - } - - validateAppName(name) { - if (name.length < 3 || name.length > 20) { - return 'The app name should be within 3-20 characters long.'; - } - - return true; - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/webhooks.js b/packages/contentstack-import/src/import/modules-js/webhooks.js deleted file mode 100644 index fb5c573e47..0000000000 --- a/packages/contentstack-import/src/import/modules-js/webhooks.js +++ /dev/null @@ -1,106 +0,0 @@ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const mkdirp = require('mkdirp'); -const fs = require('fs'); -const path = require('path'); -const Promise = require('bluebird'); -const chalk = require('chalk'); -const { isEmpty, merge } = require('lodash'); -let { default: config } = require('../../config'); -const { fileHelper, log, formatError } = require('../../utils'); - -module.exports = class ImportWebhooks { - config; - fails = []; - success = []; - webUidMapper = {}; - webhooksConfig = config.modules.webhooks; - reqConcurrency = config.concurrency || config.fetchConcurrency; - - constructor(importConfig, stackAPIClient) { - this.config = merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - log(this.config, chalk.white('Migrating webhooks'), 'success'); - - const self = this; - - let webMapperPath = path.resolve(this.config.data, 'mapper', 'webhooks'); - let webFailsPath = path.resolve(this.config.data, 'mapper', 'webhooks', 'fails.json'); - let webSuccessPath = path.resolve(this.config.data, 'mapper', 'webhooks', 'success.json'); - let webUidMapperPath = path.resolve(this.config.data, 'mapper', 'webhooks', 'uid-mapping.json'); - - let webhooksFolderPath = path.resolve(this.config.data, this.webhooksConfig.dirName); - this.webhooks = fileHelper.readFileSync(path.resolve(webhooksFolderPath, this.webhooksConfig.fileName)); - - if (fs.existsSync(webUidMapperPath)) { - self.webUidMapper = fileHelper.readFileSync(webUidMapperPath); - self.webUidMapper = self.webUidMapper || {}; - } - - mkdirp.sync(webMapperPath); - - return new Promise(function (resolve, reject) { - if (self.webhooks == undefined || isEmpty(self.webhooks)) { - log(self.config, chalk.white('No Webhooks Found'), 'success'); - return resolve({ empty: true }); - } - - let webUids = Object.keys(self.webhooks); - return Promise.map( - webUids, - function (webUid) { - let web = self.webhooks[webUid]; - if (self.config.importWebhookStatus !== 'current' || self.config.importWebhookStatus === 'disable') { - web.disabled = true; - } - - if (!self.webUidMapper.hasOwnProperty(webUid)) { - let requestOption = { json: { webhook: web } }; - - return self.stackAPIClient - .webhook() - .create(requestOption.json) - .then(function (response) { - self.success.push(response); - self.webUidMapper[webUid] = response.uid; - fileHelper.writeFileSync(webUidMapperPath, self.webUidMapper); - }) - .catch(function (err) { - let error = JSON.parse(err.message); - self.fails.push(web); - log(self.config, `Webhooks '${web.name}' failed to be import.\n ${formatError(error)}`, 'error'); - }); - } else { - // the webhooks has already been created - log( - self.config, - chalk.white("The Webhooks: '" + web.name + "' already exists. Skipping it to avoid duplicates!"), - 'success', - ); - } - // import 2 webhooks at a time - }, - { concurrency: self.reqConcurrency }, - ) - .then(function () { - // webhooks have imported successfully - fileHelper.writeFileSync(webSuccessPath, self.success); - log(self.config, chalk.green('Webhooks have been imported successfully!'), 'success'); - return resolve(); - }) - .catch(function (error) { - // error while importing environments - fileHelper.writeFileSync(webFailsPath, self.fails); - log(self.config, `Webhooks import failed. ${formatError(error)}`, 'error'); - return reject(error); - }); - }); - } -}; diff --git a/packages/contentstack-import/src/import/modules-js/workflows.js b/packages/contentstack-import/src/import/modules-js/workflows.js deleted file mode 100644 index 494dbc3c41..0000000000 --- a/packages/contentstack-import/src/import/modules-js/workflows.js +++ /dev/null @@ -1,200 +0,0 @@ -/*! - * Contentstack Import - * Copyright (c) 2024 Contentstack LLC - * MIT Licensed - */ - -const fs = require('fs'); -const path = require('path'); -const chalk = require('chalk'); -const mkdirp = require('mkdirp'); -const Promise = require('bluebird'); -const { isEmpty, merge, filter, map, cloneDeep, find } = require('lodash'); - -let { default: config } = require('../../config'); -const { fileHelper, log, formatError } = require('../../utils'); - -module.exports = class importWorkflows { - fails = []; - success = []; - workflowUidMapper = {}; - workflowConfig = config.modules.workflows; - reqConcurrency = config.concurrency || config.fetchConcurrency || 1; - - constructor(importConfig, stackAPIClient) { - this.config = merge(config, importConfig); - this.stackAPIClient = stackAPIClient; - } - - start() { - log(this.config, chalk.white('Migrating workflows'), 'success'); - - let self = this; - let workflowMapperPath = path.resolve(this.config.data, 'mapper', 'workflows'); - let workflowFailsPath = path.resolve(this.config.data, 'workflows', 'fails.json'); - let workflowSuccessPath = path.resolve(this.config.data, 'workflows', 'success.json'); - let workflowUidMapperPath = path.resolve(this.config.data, 'mapper', 'workflows', 'uid-mapping.json'); - let workflowFolderPath = path.resolve(this.config.data, this.workflowConfig.dirName); - - self.workflows = fileHelper.readFileSync(path.resolve(workflowFolderPath, this.workflowConfig.fileName)); - - if (fs.existsSync(workflowUidMapperPath)) { - this.workflowUidMapper = fileHelper.readFileSync(workflowUidMapperPath); - this.workflowUidMapper = this.workflowUidMapper || {}; - } - - mkdirp.sync(workflowMapperPath); - - return new Promise(function (resolve, reject) { - if (self.workflows == undefined || isEmpty(self.workflows)) { - log(self.config, chalk.white('No workflow Found'), 'success'); - return resolve({ empty: true }); - } - self.workflowsUids = Object.keys(self.workflows); - return Promise.map( - self.workflowsUids, - async function (workflowUid) { - let workflow = self.workflows[workflowUid]; - - if (!self.workflowUidMapper.hasOwnProperty(workflowUid)) { - const roleNameMap = {}; - const workflowStages = workflow.workflow_stages; - const oldWorkflowStages = cloneDeep(workflow.workflow_stages); - const roles = await self.stackAPIClient.role().fetchAll(); - - for (const role of roles.items) { - roleNameMap[role.name] = role.uid; - } - - for (const stage of workflowStages) { - delete stage.uid; - - if (!isEmpty(stage.next_available_stages)) { - stage.next_available_stages = ['$all']; - } - - if (stage.SYS_ACL.users.uids.length && stage.SYS_ACL.users.uids[0] !== '$all') { - stage.SYS_ACL.users.uids = ['$all']; - } - - if (stage.SYS_ACL.roles.uids.length) { - try { - for (let i = 0; i < stage.SYS_ACL.roles.uids.length; i++) { - const roleData = stage.SYS_ACL.roles.uids[i]; - if (!roleNameMap[roleData.name]) { - // rules.branch is required to create custom roles. - const branchRuleExists = roleData.rules.find((rule) => rule.module === 'branch'); - if (!branchRuleExists) { - roleData.rules.push({ - module: 'branch', - branches: ['main'], - acl: { read: true }, - }); - } - - const role = await self.stackAPIClient.role().create({ role: roleData }); - stage.SYS_ACL.roles.uids[i] = role.uid; - roleNameMap[roleData.name] = role.uid; - } else { - stage.SYS_ACL.roles.uids[i] = roleNameMap[roleData.name]; - } - } - } catch (error) { - log(self.config, `Error while importing workflows roles. ${formatError(error)}`, 'error'); - reject({ message: 'Error while importing workflows roles' }); - } - } - } - - if (workflow.admin_users !== undefined) { - log(self.config, chalk.yellow('We are skipping import of `Workflow superuser(s)` from workflow'), 'info'); - delete workflow.admin_users; - } - // One branch is required to create workflow. - if (!workflow.branches) { - workflow.branches = ['main']; - } - - return self.stackAPIClient - .workflow() - .create({ workflow }) - .then(async function (response) { - if ( - !isEmpty(filter(oldWorkflowStages, ({ next_available_stages }) => !isEmpty(next_available_stages))) - ) { - let updateRresponse = await self - .updateNextAvailableStagesUid(response, response.workflow_stages, oldWorkflowStages) - .catch((error) => { - log(self.config, `Workflow '${workflow.name}' update failed.`, 'error'); - log(self.config, error, 'error'); - }); - - if (updateRresponse) response = updateRresponse; - } - - self.workflowUidMapper[workflowUid] = response; - fileHelper.writeFileSync(workflowUidMapperPath, self.workflowUidMapper); - }) - .catch(function (error) { - self.fails.push(workflow); - if (error.errors.name) { - log(self.config, `workflow '${workflow.name}' already exist`, 'error'); - } else if (error.errors['workflow_stages.0.users']) { - log( - self.config, - "Failed to import Workflows as you've specified certain roles in the Stage transition and access rules section. We currently don't import roles to the stack.", - 'error', - ); - } else { - log(self.config, `Workflow '${workflow.name}' failed.`, 'error'); - } - }); - } else { - // the workflow has already been created - log( - self.config, - chalk.white(`The Workflows ${workflow.name} already exists. Skipping it to avoid duplicates!`), - 'success', - ); - } - // import 1 workflows at a time - }, - { concurrency: self.reqConcurrency }, - ) - .then(function () { - fileHelper.writeFileSync(workflowSuccessPath, self.success); - log(self.config, chalk.green('Workflows have been imported successfully!'), 'success'); - resolve(); - }) - .catch(function (error) { - fileHelper.writeFileSync(workflowFailsPath, self.fails); - log(self.config, `Workflows import failed. ${formatError(error)}`, 'error'); - return reject(error); - }); - }); - } - - updateNextAvailableStagesUid(workflow, newWorkflowStages, oldWorkflowStages) { - newWorkflowStages = map(newWorkflowStages, (newStage, index) => { - const oldStage = oldWorkflowStages[index]; - if (!isEmpty(oldStage.next_available_stages)) { - newStage.next_available_stages = map(oldStage.next_available_stages, (stageUid) => { - if (stageUid === '$all') return stageUid; - const stageName = find(oldWorkflowStages, { uid: stageUid })?.name; - return find(newWorkflowStages, { name: stageName })?.uid; - }).filter((val) => val); - } - - return newStage; - }); - - const updateWorkflow = this.stackAPIClient.workflow(workflow.uid); - Object.assign(updateWorkflow, { - name: workflow.name, - branches: workflow.branches, - workflow_stages: newWorkflowStages, - content_types: workflow.content_types - }); - return updateWorkflow.update(); - } -}; diff --git a/packages/contentstack-import/src/import/modules/marketplace-apps.ts b/packages/contentstack-import/src/import/modules/marketplace-apps.ts index 21fdb54778..46b8daf3c3 100644 --- a/packages/contentstack-import/src/import/modules/marketplace-apps.ts +++ b/packages/contentstack-import/src/import/modules/marketplace-apps.ts @@ -20,7 +20,6 @@ import { handleAndLogError, } from '@contentstack/cli-utilities'; -import { trace } from '../../utils/log'; import { askEncryptionKey, getLocationName } from '../../utils/interactive'; import { ModuleClassParams, MarketplaceAppsConfig, ImportConfig, Installation, Manifest } from '../../types'; import { @@ -538,7 +537,6 @@ export default class ImportMarketplaceApps extends BaseClass { return this.createPrivateApp(updatedApp, appSuffix + 1, true); } else { this.progressManager?.tick(false, `${app.name}`, message, PROCESS_NAMES.CREATE_APPS); - trace(response, 'error', true); log.error(formatError(message), this.importConfig.context); if (this.importConfig.forceStopMarketplaceAppsPrompt) { @@ -700,7 +698,7 @@ export default class ImportMarketplaceApps extends BaseClass { .setConfiguration(this.nodeCrypto.decrypt(configuration)) .then(({ data }: any) => { if (data?.message) { - trace(data, 'error', true); + log.debug(data, this.importConfig.context); log.info(formatError(data.message), this.importConfig.context); } else { log.success(`${appName} app config updated successfully.!`, this.importConfig.context); @@ -708,7 +706,7 @@ export default class ImportMarketplaceApps extends BaseClass { } }) .catch((error: any) => { - trace(error, 'error', true); + log.debug(error, this.importConfig.context); log.error(formatError(error), this.importConfig.context); log.debug(`Configuration update failed for: ${appName}`, this.importConfig.context); }); @@ -722,7 +720,7 @@ export default class ImportMarketplaceApps extends BaseClass { .setServerConfig(this.nodeCrypto.decrypt(server_configuration)) .then(({ data }: any) => { if (data?.message) { - trace(data, 'error', true); + log.debug(data, this.importConfig.context); log.error(formatError(data.message), this.importConfig.context); } else { log.success(`${appName} app server config updated successfully.!`, this.importConfig.context); @@ -730,7 +728,7 @@ export default class ImportMarketplaceApps extends BaseClass { } }) .catch((error: any) => { - trace(error, 'error', true); + log.debug(error, this.importConfig.context); log.error(formatError(error), this.importConfig.context); log.debug(`Server configuration update failed for: ${appName}`, this.importConfig.context); }); diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index fb2d16fd66..05d98eb991 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -185,7 +185,6 @@ export default interface DefaultConfig { getEncryptionKeyMaxRetry: number; createBackupDir?: string; overwriteSupportedModules: string[]; - onlyTSModules: string[]; auditConfig?: { noLog?: boolean; // Skip logs printing on terminal skipConfirm?: boolean; // Skip confirmation if any diff --git a/packages/contentstack-import/src/types/import-config.ts b/packages/contentstack-import/src/types/import-config.ts index 2c4c9a0006..86db5668d1 100644 --- a/packages/contentstack-import/src/types/import-config.ts +++ b/packages/contentstack-import/src/types/import-config.ts @@ -50,7 +50,6 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { authtoken?: string; destinationStackName?: string; org_uid?: string; - contentVersion: number; replaceExisting?: boolean; skipExisting?: boolean; skipAudit?: boolean; diff --git a/packages/contentstack-import/src/utils/backup-handler.ts b/packages/contentstack-import/src/utils/backup-handler.ts index 3d101608f2..825c792d1c 100755 --- a/packages/contentstack-import/src/utils/backup-handler.ts +++ b/packages/contentstack-import/src/utils/backup-handler.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import { copy } from 'fs-extra'; import { cliux, sanitizePath, log } from '@contentstack/cli-utilities'; -import { fileHelper, trace } from './index'; +import { fileHelper } from './index'; import { ImportConfig } from '../types'; export default async function backupHandler(importConfig: ImportConfig): Promise { @@ -59,7 +59,6 @@ export default async function backupHandler(importConfig: ImportConfig): Promise return new Promise((resolve, reject) => { return copy(sourceDir, backupDirPath, (error: any) => { if (error) { - trace(error, 'error', true); return reject(error); } diff --git a/packages/contentstack-import/src/utils/import-path-resolver.ts b/packages/contentstack-import/src/utils/import-path-resolver.ts index dccc45bfa6..ebd3a2a4e4 100644 --- a/packages/contentstack-import/src/utils/import-path-resolver.ts +++ b/packages/contentstack-import/src/utils/import-path-resolver.ts @@ -1,9 +1,10 @@ import * as path from 'path'; import { log } from '@contentstack/cli-utilities'; -import { fileExistsSync, readFile } from './file-helper'; -import { askBranchSelection } from './interactive'; -import { ImportConfig } from '../types'; + import defaultConfig from '../config'; +import { ImportConfig } from '../types'; +import { askBranchSelection } from './interactive'; +import { fileExistsSync, readFile } from './file-helper'; /** * Selects a branch from directory structure when multiple branches are found @@ -55,8 +56,8 @@ export const selectBranchFromDirectory = async (contentDir: string): Promise<{ b return { branchPath: selectedBranchPath }; } } catch (error) { - log.error(`Error selecting branch directory from directory structure: ${error}`); - throw error; + log.error(`Error selecting branch directory from directory structure: ${error}`); + throw error; } }; @@ -123,7 +124,10 @@ export const resolveImportPath = async (importConfig: ImportConfig, stackAPIClie * @param importConfig - The import configuration object * @param resolvedPath - The resolved path */ -export const updateImportConfigWithResolvedPath = async (importConfig: ImportConfig, resolvedPath: string): Promise => { +export const updateImportConfigWithResolvedPath = async ( + importConfig: ImportConfig, + resolvedPath: string, +): Promise => { log.debug(`Updating import config with resolved path: ${resolvedPath}`); if (!fileExistsSync(resolvedPath)) { @@ -137,18 +141,8 @@ export const updateImportConfigWithResolvedPath = async (importConfig: ImportCon importConfig.data = resolvedPath; - const exportInfoPath = path.join(resolvedPath, 'export-info.json'); - if (fileExistsSync(exportInfoPath)) { - const exportInfo = await readFile(exportInfoPath); - importConfig.contentVersion = exportInfo?.contentVersion || 2; - log.debug(`Content version set to ${importConfig.contentVersion} from ${exportInfoPath}`); - } else { - importConfig.contentVersion = 1; - log.debug(`No export-info.json found at ${exportInfoPath}, setting content version to 1`); - } - log.debug( - `Import config updated - contentDir: ${importConfig.contentDir}, branchDir: ${importConfig.branchDir}, data: ${importConfig.data}, contentVersion: ${importConfig.contentVersion}`, + `Import config updated - contentDir: ${importConfig.contentDir}, branchDir: ${importConfig.branchDir}, data: ${importConfig.data},`, ); }; diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index b8ee3594fa..49ab1146be 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -31,6 +31,5 @@ export { restoreJsonRteEntryRefs, } from './entries-helper'; export * from './common-helper'; -export * from './log'; export { lookUpTaxonomy, lookUpTerms } from './taxonomies-helper'; export { MODULE_CONTEXTS, MODULE_NAMES, PROCESS_NAMES, PROCESS_STATUS } from './constants'; diff --git a/packages/contentstack-import/src/utils/log.ts b/packages/contentstack-import/src/utils/log.ts deleted file mode 100644 index aac6d63457..0000000000 --- a/packages/contentstack-import/src/utils/log.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { join } from 'path'; -import { LogEntry } from 'winston/index'; -import { Logger, pathValidator, sanitizePath } from '@contentstack/cli-utilities'; -import { LogsType, MessageType } from '@contentstack/cli-utilities/lib/logger'; - -import { ImportConfig } from '../types'; - -let logger: Logger; - -export function isImportConfig(config: ImportConfig | MessageType): config is ImportConfig { - return (config as ImportConfig).data !== undefined && (config as ImportConfig)?.contentVersion !== undefined; -} - -export function log(entry: LogEntry): void; -export function log(error: MessageType, logType: LogsType): void; -export function log(error: MessageType, logType: 'error', hidden: boolean): void; -export function log(entryOrMessage: MessageType, logType?: LogsType, hidden?: boolean): Logger | void { - logger = initLogger(); - - if (logType === 'error') { - logger.log(entryOrMessage, logType, hidden); - } else { - logger.log(entryOrMessage, logType); - } -} - -export function initLogger(config?: ImportConfig | undefined) { - if (!logger) { - const basePath = pathValidator(join(sanitizePath(config?.cliLogsPath ?? process.cwd()), 'logs', 'import')); - logger = new Logger(Object.assign(config ?? {}, { basePath })); - } - - return logger; -} - -export { logger }; - -export const trace = log; diff --git a/packages/contentstack-import/src/utils/marketplace-app-helper.ts b/packages/contentstack-import/src/utils/marketplace-app-helper.ts index 5061722e59..97dece11af 100644 --- a/packages/contentstack-import/src/utils/marketplace-app-helper.ts +++ b/packages/contentstack-import/src/utils/marketplace-app-helper.ts @@ -14,7 +14,6 @@ import { handleAndLogError } from '@contentstack/cli-utilities'; -import { trace } from '../utils/log'; import { ImportConfig, Installation } from '../types'; import { formatError } from '../utils'; import { getAppName, askAppName, selectConfiguration } from '../utils/interactive'; @@ -36,7 +35,6 @@ export const getAllStackSpecificApps = async ( .fetchAll({ target_uids: config.target_stack, skip }) .catch((error) => { handleAndLogError(error) - trace(error, 'error', true); }); if (collection) { @@ -157,15 +155,12 @@ export const makeRedirectUrlCall = async (response: any, appName: string, config .then(async ({ response }: any) => { if (includes([501, 403], response.status)) { log.error(`OAuth API call failed for ${appName}: ${response.statusText}`); - trace(response, 'error', true); await confirmToCloseProcess(response.data, config); } else { log.success(`OAuth API call completed successfully for app: ${appName}`); } }) .catch((error) => { - trace(error, 'error', true); - if (includes([501, 403], error.status)) { handleAndLogError(error); } diff --git a/packages/contentstack-variants/src/types/export-config.ts b/packages/contentstack-variants/src/types/export-config.ts index d947086649..61d3e512e7 100644 --- a/packages/contentstack-variants/src/types/export-config.ts +++ b/packages/contentstack-variants/src/types/export-config.ts @@ -33,7 +33,6 @@ export type masterLocale = { export interface DefaultConfig { context: Context; - contentVersion: number; versioning: boolean; host: string; cdn?: string; @@ -233,7 +232,6 @@ export interface DefaultConfig { writeConcurrency: number; developerHubBaseUrl: string; marketplaceAppEncryptionKey: string; - onlyTSModules: string[]; } export interface ExportConfig extends DefaultConfig { diff --git a/packages/contentstack-variants/src/types/import-config.ts b/packages/contentstack-variants/src/types/import-config.ts index d46f32f098..78b27edf85 100644 --- a/packages/contentstack-variants/src/types/import-config.ts +++ b/packages/contentstack-variants/src/types/import-config.ts @@ -79,7 +79,6 @@ export interface ImportConfig extends ImportDefaultConfig, AnyProperty { authtoken?: string; destinationStackName?: string; org_uid?: string; - contentVersion: number; replaceExisting?: boolean; skipExisting?: boolean; stackName?: string; From ce7dc688b7bca7465a8f4c45889314ebef2488c8 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 1 Oct 2025 19:48:55 +0530 Subject: [PATCH 2/2] fix: Cannot convert undefined or null to object --- .../src/export/modules/custom-roles.ts | 4 ++-- .../src/export/modules/environments.ts | 6 +++--- .../src/export/modules/extensions.ts | 6 +++--- .../contentstack-export/src/export/modules/labels.ts | 4 ++-- .../src/export/modules/locales.ts | 12 ++++++------ .../src/export/modules/marketplace-apps.ts | 2 +- .../src/export/modules/taxonomies.ts | 8 ++++---- .../src/export/modules/webhooks.ts | 6 +++--- .../src/export/modules/workflows.ts | 4 ++-- .../contentstack-import/src/import/modules/assets.ts | 4 ++-- .../src/import/modules/content-types.ts | 4 ++-- .../src/import/modules/entries.ts | 2 +- .../src/import/modules/extensions.ts | 2 +- .../src/import/modules/global-fields.ts | 6 +++--- .../src/import/modules/locales.ts | 2 +- .../src/import/modules/taxonomies.ts | 8 ++++---- 16 files changed, 40 insertions(+), 40 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/custom-roles.ts b/packages/contentstack-export/src/export/modules/custom-roles.ts index 19ec0417a0..48bb96fdcc 100644 --- a/packages/contentstack-export/src/export/modules/custom-roles.ts +++ b/packages/contentstack-export/src/export/modules/custom-roles.ts @@ -104,7 +104,7 @@ export default class ExportCustomRoles extends BaseClass { progress.completeProcess(PROCESS_NAMES.PROCESS_MAPPINGS, true); log.debug( - `Custom roles export completed. Total custom roles: ${Object.keys(this.customRoles).length}`, + `Custom roles export completed. Total custom roles: ${Object.keys(this.customRoles || {}).length}`, this.exportConfig.context, ); this.completeProgress(true); @@ -177,7 +177,7 @@ export default class ExportCustomRoles extends BaseClass { this.progressManager?.tick(true, `locale: ${locale.name}`, null, PROCESS_NAMES.FETCH_LOCALES); } - log.debug(`Mapped ${Object.keys(this.sourceLocalesMap).length} locales`, this.exportConfig.context); + log.debug(`Mapped ${Object.keys(this.sourceLocalesMap || {}).length} locales`, this.exportConfig.context); } async getCustomRolesLocales() { diff --git a/packages/contentstack-export/src/export/modules/environments.ts b/packages/contentstack-export/src/export/modules/environments.ts index ee46f46e73..68961f3e17 100644 --- a/packages/contentstack-export/src/export/modules/environments.ts +++ b/packages/contentstack-export/src/export/modules/environments.ts @@ -57,7 +57,7 @@ export default class ExportEnvironments extends BaseClass { progress.updateStatus('Fetching environments...'); await this.getEnvironments(); - log.debug(`Retrieved ${Object.keys(this.environments).length} environments`, this.exportConfig.context); + log.debug(`Retrieved ${Object.keys(this.environments || {}).length} environments`, this.exportConfig.context); if (this.environments === undefined || isEmpty(this.environments)) { log.info(messageHandler.parse('ENVIRONMENT_NOT_FOUND'), this.exportConfig.context); @@ -66,7 +66,7 @@ export default class ExportEnvironments extends BaseClass { log.debug(`Writing environments to: ${environmentsFilePath}`, this.exportConfig.context); fsUtil.writeFile(environmentsFilePath, this.environments); log.success( - messageHandler.parse('ENVIRONMENT_EXPORT_COMPLETE', Object.keys(this.environments).length), + messageHandler.parse('ENVIRONMENT_EXPORT_COMPLETE', Object.keys(this.environments || {}).length), this.exportConfig.context, ); } @@ -130,7 +130,7 @@ export default class ExportEnvironments extends BaseClass { } log.debug( - `Sanitization complete. Total environments processed: ${Object.keys(this.environments).length}`, + `Sanitization complete. Total environments processed: ${Object.keys(this.environments || {}).length}`, this.exportConfig.context, ); } diff --git a/packages/contentstack-export/src/export/modules/extensions.ts b/packages/contentstack-export/src/export/modules/extensions.ts index 00b52d0f6e..7665aa30c9 100644 --- a/packages/contentstack-export/src/export/modules/extensions.ts +++ b/packages/contentstack-export/src/export/modules/extensions.ts @@ -58,7 +58,7 @@ export default class ExportExtensions extends BaseClass { progress.updateStatus('Fetching extensions...'); await this.getExtensions(); - log.debug(`Retrieved ${Object.keys(this.extensions).length} extensions`, this.exportConfig.context); + log.debug(`Retrieved ${Object.keys(this.extensions || {}).length} extensions`, this.exportConfig.context); if (this.extensions === undefined || isEmpty(this.extensions)) { log.info(messageHandler.parse('EXTENSION_NOT_FOUND'), this.exportConfig.context); @@ -67,7 +67,7 @@ export default class ExportExtensions extends BaseClass { log.debug(`Writing extensions to: ${extensionsFilePath}`, this.exportConfig.context); fsUtil.writeFile(extensionsFilePath, this.extensions); log.success( - messageHandler.parse('EXTENSION_EXPORT_COMPLETE', Object.keys(this.extensions).length), + messageHandler.parse('EXTENSION_EXPORT_COMPLETE', Object.keys(this.extensions || {}).length), this.exportConfig.context, ); } @@ -131,7 +131,7 @@ export default class ExportExtensions extends BaseClass { } log.debug( - `Sanitization complete. Total extensions processed: ${Object.keys(this.extensions).length}`, + `Sanitization complete. Total extensions processed: ${Object.keys(this.extensions || {}).length}`, this.exportConfig.context, ); } diff --git a/packages/contentstack-export/src/export/modules/labels.ts b/packages/contentstack-export/src/export/modules/labels.ts index 50a3aa7b56..aa9edab2bf 100644 --- a/packages/contentstack-export/src/export/modules/labels.ts +++ b/packages/contentstack-export/src/export/modules/labels.ts @@ -58,7 +58,7 @@ export default class ExportLabels extends BaseClass { progress.updateStatus('Fetching labels...'); await this.getLabels(); - log.debug(`Retrieved ${Object.keys(this.labels).length} labels`, this.exportConfig.context); + log.debug(`Retrieved ${Object.keys(this.labels || {}).length} labels`, this.exportConfig.context); if (this.labels === undefined || isEmpty(this.labels)) { log.info(messageHandler.parse('LABELS_NOT_FOUND'), this.exportConfig.context); @@ -67,7 +67,7 @@ export default class ExportLabels extends BaseClass { log.debug(`Writing labels to: ${labelsFilePath}`, this.exportConfig.context); fsUtil.writeFile(labelsFilePath, this.labels); log.success( - messageHandler.parse('LABELS_EXPORT_COMPLETE', Object.keys(this.labels).length), + messageHandler.parse('LABELS_EXPORT_COMPLETE', Object.keys(this.labels || {}).length), this.exportConfig.context, ); } diff --git a/packages/contentstack-export/src/export/modules/locales.ts b/packages/contentstack-export/src/export/modules/locales.ts index ad528194f1..58cc3960ee 100644 --- a/packages/contentstack-export/src/export/modules/locales.ts +++ b/packages/contentstack-export/src/export/modules/locales.ts @@ -74,8 +74,8 @@ export default class LocaleExport extends BaseClass { progress.updateStatus('Fetching locale definitions...'); await this.getLocales(); log.debug( - `Retrieved ${Object.keys(this.locales).length} locales and ${ - Object.keys(this.masterLocale).length + `Retrieved ${Object.keys(this.locales || {}).length} locales and ${ + Object.keys(this.masterLocale || {}).length } master locales`, this.exportConfig.context, ); @@ -89,8 +89,8 @@ export default class LocaleExport extends BaseClass { log.success( messageHandler.parse( 'LOCALES_EXPORT_COMPLETE', - Object.keys(this.locales).length, - Object.keys(this.masterLocale).length, + Object.keys(this.locales || {}).length, + Object.keys(this.masterLocale || {}).length, ), this.exportConfig.context, ); @@ -156,8 +156,8 @@ export default class LocaleExport extends BaseClass { }); log.debug( - `Sanitization complete. Master locales: ${Object.keys(this.masterLocale).length}, Regular locales: ${ - Object.keys(this.locales).length + `Sanitization complete. Master locales: ${Object.keys(this.masterLocale || {}).length}, Regular locales: ${ + Object.keys(this.locales || {}).length }`, this.exportConfig.context, ); diff --git a/packages/contentstack-export/src/export/modules/marketplace-apps.ts b/packages/contentstack-export/src/export/modules/marketplace-apps.ts index d303b9a981..9aa5854ef4 100644 --- a/packages/contentstack-export/src/export/modules/marketplace-apps.ts +++ b/packages/contentstack-export/src/export/modules/marketplace-apps.ts @@ -232,7 +232,7 @@ export default class ExportMarketplaceApps extends BaseClass { fsUtil.writeFile(marketplaceAppsFilePath, this.installedApps); log.success( - messageHandler.parse('MARKETPLACE_APPS_EXPORT_COMPLETE', Object.keys(this.installedApps).length), + messageHandler.parse('MARKETPLACE_APPS_EXPORT_COMPLETE', Object.keys(this.installedApps || {}).length), this.exportConfig.context, ); } diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index f98fe8ab57..54b7b555e9 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -79,7 +79,7 @@ export default class ExportTaxonomies extends BaseClass { await this.getAllTaxonomies(); progress.completeProcess(PROCESS_NAMES.FETCH_TAXONOMIES, true); - const actualTaxonomyCount = Object.keys(this.taxonomies)?.length; + const actualTaxonomyCount = Object.keys(this.taxonomies || {})?.length; log.debug( `Found ${actualTaxonomyCount} taxonomies to export (API reported ${totalCount})`, this.exportConfig.context, @@ -105,7 +105,7 @@ export default class ExportTaxonomies extends BaseClass { log.info('No taxonomies found to export detailed information', this.exportConfig.context); } - const taxonomyCount = Object.keys(this.taxonomies).length; + const taxonomyCount = Object.keys(this.taxonomies || {}).length; log.success(messageHandler.parse('TAXONOMY_EXPORT_COMPLETE', taxonomyCount), this.exportConfig.context); this.completeProgress(true); } catch (error) { @@ -178,7 +178,7 @@ export default class ExportTaxonomies extends BaseClass { } log.debug( - `Sanitization complete. Total taxonomies processed: ${Object.keys(this.taxonomies).length}`, + `Sanitization complete. Total taxonomies processed: ${Object.keys(this.taxonomies || {}).length}`, this.exportConfig.context, ); } @@ -189,7 +189,7 @@ export default class ExportTaxonomies extends BaseClass { */ async exportTaxonomies(): Promise { log.debug( - `Exporting ${Object.keys(this.taxonomies)?.length} taxonomies with detailed information`, + `Exporting ${Object.keys(this.taxonomies || {})?.length} taxonomies with detailed information`, this.exportConfig.context, ); diff --git a/packages/contentstack-export/src/export/modules/webhooks.ts b/packages/contentstack-export/src/export/modules/webhooks.ts index 354527982f..26f3d40232 100644 --- a/packages/contentstack-export/src/export/modules/webhooks.ts +++ b/packages/contentstack-export/src/export/modules/webhooks.ts @@ -55,7 +55,7 @@ export default class ExportWebhooks extends BaseClass { progress.updateStatus('Fetching webhooks...'); await this.getWebhooks(); - log.debug(`Retrieved ${Object.keys(this.webhooks).length} webhooks`, this.exportConfig.context); + log.debug(`Retrieved ${Object.keys(this.webhooks || {}).length} webhooks`, this.exportConfig.context); if (this.webhooks === undefined || isEmpty(this.webhooks)) { log.info(messageHandler.parse('WEBHOOK_NOT_FOUND'), this.exportConfig.context); @@ -64,7 +64,7 @@ export default class ExportWebhooks extends BaseClass { log.debug(`Writing webhooks to: ${webhooksFilePath}`, this.exportConfig.context); fsUtil.writeFile(webhooksFilePath, this.webhooks); log.success( - messageHandler.parse('WEBHOOK_EXPORT_COMPLETE', Object.keys(this.webhooks).length), + messageHandler.parse('WEBHOOK_EXPORT_COMPLETE', Object.keys(this.webhooks || {}).length), this.exportConfig.context, ); } @@ -130,7 +130,7 @@ export default class ExportWebhooks extends BaseClass { } log.debug( - `Sanitization complete. Total webhooks processed: ${Object.keys(this.webhooks).length}`, + `Sanitization complete. Total webhooks processed: ${Object.keys(this.webhooks || {}).length}`, this.exportConfig.context, ); } diff --git a/packages/contentstack-export/src/export/modules/workflows.ts b/packages/contentstack-export/src/export/modules/workflows.ts index 3e38886662..1107b87cfa 100644 --- a/packages/contentstack-export/src/export/modules/workflows.ts +++ b/packages/contentstack-export/src/export/modules/workflows.ts @@ -58,7 +58,7 @@ export default class ExportWorkFlows extends BaseClass { progress.updateStatus('Fetching workflow definitions...'); await this.getWorkflows(); - log.debug(`Retrieved ${Object.keys(this.workflows).length} workflows`, this.exportConfig.context); + log.debug(`Retrieved ${Object.keys(this.workflows || {}).length} workflows`, this.exportConfig.context); if (this.workflows === undefined || isEmpty(this.workflows)) { log.info(messageHandler.parse('WORKFLOW_NOT_FOUND'), this.exportConfig.context); @@ -67,7 +67,7 @@ export default class ExportWorkFlows extends BaseClass { log.debug(`Writing workflows to: ${workflowsFilePath}`, this.exportConfig.context); fsUtil.writeFile(workflowsFilePath, this.workflows); log.success( - messageHandler.parse('WORKFLOW_EXPORT_COMPLETE', Object.keys(this.workflows).length), + messageHandler.parse('WORKFLOW_EXPORT_COMPLETE', Object.keys(this.workflows || {}).length), this.exportConfig.context, ); } diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 7009dcba50..e5625e1a3c 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -297,12 +297,12 @@ export default class ImportAssets extends BaseClass { if (!isVersion) { if (!isEmpty(this.assetsUidMap)) { - const uidMappingCount = Object.keys(this.assetsUidMap).length; + const uidMappingCount = Object.keys(this.assetsUidMap || {}).length; log.debug(`Writing ${uidMappingCount} UID mappings`, this.importConfig.context); this.fs.writeFile(this.assetUidMapperPath, this.assetsUidMap); } if (!isEmpty(this.assetsUrlMap)) { - const urlMappingCount = Object.keys(this.assetsUrlMap).length; + const urlMappingCount = Object.keys(this.assetsUrlMap || {}).length; log.debug(`Writing ${urlMappingCount} URL mappings`, this.importConfig.context); this.fs.writeFile(this.assetUrlMapperPath, this.assetsUrlMap); } diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index 667137ec5d..bb82baaf02 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -429,8 +429,8 @@ export default class ContentTypesImport extends BaseClass { log.debug( `Analysis complete: ${this.cTs?.length} content types, ${this.gFs?.length} global fields, ${ this.pendingGFs?.length - } pending GFs, ${Object.keys(this.installedExtensions)?.length} extensions, ${ - Object.keys(this.taxonomies)?.length + } pending GFs, ${Object.keys(this.installedExtensions || {})?.length} extensions, ${ + Object.keys(this.taxonomies || {})?.length } taxonomies`, this.importConfig.context, ); diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 002712c613..beeee5a90b 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -366,7 +366,7 @@ export default class EntriesImport extends BaseClass { private async processEntryPublishing(): Promise { log.info('Starting entry publishing process', this.importConfig.context); this.envs = fileHelper.readFileSync(this.envPath); - log.debug(`Loaded ${Object.keys(this.envs).length} environments for publishing`, this.importConfig.context); + log.debug(`Loaded ${Object.keys(this.envs || {}).length} environments for publishing`, this.importConfig.context); const entryRequestOptions = this.populateEntryCreatePayload(); for (let entryRequestOption of entryRequestOptions) { diff --git a/packages/contentstack-import/src/import/modules/extensions.ts b/packages/contentstack-import/src/import/modules/extensions.ts index 723bf878cf..eb205d792d 100644 --- a/packages/contentstack-import/src/import/modules/extensions.ts +++ b/packages/contentstack-import/src/import/modules/extensions.ts @@ -367,7 +367,7 @@ export default class ImportExtensions extends BaseClass { return [0]; } - const count = Object.keys(this.extensions).length; + const count = Object.keys(this.extensions || {}).length; log.debug(`Loaded ${count} extension items from file`, this.importConfig.context); return [count]; }); diff --git a/packages/contentstack-import/src/import/modules/global-fields.ts b/packages/contentstack-import/src/import/modules/global-fields.ts index e98b955432..bc13201eaa 100644 --- a/packages/contentstack-import/src/import/modules/global-fields.ts +++ b/packages/contentstack-import/src/import/modules/global-fields.ts @@ -144,7 +144,7 @@ export default class ImportGlobalFields extends BaseClass { async seedGFs(): Promise { log.debug('Starting global fields seeding process', this.importConfig.context); - const gfsToSeed = Array.isArray(this.gFs) ? this.gFs.length : Object.keys(this.gFs).length; + const gfsToSeed = Array.isArray(this.gFs) ? this.gFs.length : Object.keys(this.gFs || {}).length; log.debug(`Seeding ${gfsToSeed} global fields`, this.importConfig.context); const onSuccess = ({ response: globalField, apiData: { uid } = undefined }: any) => { @@ -231,7 +231,7 @@ export default class ImportGlobalFields extends BaseClass { async updateGFs(): Promise { log.debug('Starting Update process', this.importConfig.context); - const gfsToUpdate = Array.isArray(this.gFs) ? this.gFs.length : Object.keys(this.gFs).length; + const gfsToUpdate = Array.isArray(this.gFs) ? this.gFs.length : Object.keys(this.gFs || {}).length; log.debug(`Updating ${gfsToUpdate} global fields`, this.importConfig.context); const onSuccess = ({ response: globalField, apiData: { uid } = undefined }: any) => { @@ -424,7 +424,7 @@ export default class ImportGlobalFields extends BaseClass { return [0]; } - const count = Array.isArray(this.gFs) ? this.gFs?.length : Object.keys(this.gFs)?.length; + const count = Array.isArray(this.gFs) ? this.gFs?.length : Object.keys(this.gFs || {})?.length; log.debug(`Loaded ${count} global field items from file`, this.importConfig.context); return [count]; }); diff --git a/packages/contentstack-import/src/import/modules/locales.ts b/packages/contentstack-import/src/import/modules/locales.ts index 080b459cd5..a7c3bc4ee1 100644 --- a/packages/contentstack-import/src/import/modules/locales.ts +++ b/packages/contentstack-import/src/import/modules/locales.ts @@ -245,7 +245,7 @@ export default class ImportLocales extends BaseClass { if (fileHelper.fileExistsSync(this.langUidMapperPath)) { this.langUidMapper = fsUtil.readFile(this.langUidMapperPath) || {}; - const langUidCount = Object.keys(this.langUidMapper).length; + const langUidCount = Object.keys(this.langUidMapper || {}).length; log.debug(`Loaded existing language UID data: ${langUidCount} items`, this.config.context); } else { log.debug('No existing language UID mappings found', this.config.context); diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 1516d6f115..fc25786d81 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -193,10 +193,10 @@ export default class ImportTaxonomies extends BaseClass { createSuccessAndFailedFile() { log.debug('Creating success and failed files for taxonomies and terms', this.importConfig.context); - const createdTaxCount = Object.keys(this.createdTaxonomies)?.length; - const failedTaxCount = Object.keys(this.failedTaxonomies)?.length; - const createdTermsCount = Object.keys(this.createdTerms)?.length; - const failedTermsCount = Object.keys(this.failedTerms)?.length; + const createdTaxCount = Object.keys(this.createdTaxonomies || {})?.length; + const failedTaxCount = Object.keys(this.failedTaxonomies || {})?.length; + const createdTermsCount = Object.keys(this.createdTerms || {})?.length; + const failedTermsCount = Object.keys(this.failedTerms || {})?.length; log.debug( `Summary - Created taxonomies: ${createdTaxCount}, Failed taxonomies: ${failedTaxCount}`,