From ff43e2e66c124fc4c2153b9a10fb19c45cf6eda7 Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Thu, 3 Oct 2024 04:20:39 +0530 Subject: [PATCH 1/2] added test --- api/src/services/migration.service.ts | 10 +- api/src/services/projects.service.ts | 14 +- api/src/services/sitecore.service.ts | 44 ++--- api/src/utils/content-type-creator.utils.ts | 25 ++- api/src/utils/entries-field-creator.utils.ts | 130 +++++++++++-- api/src/utils/test-folder-creator.utils.ts | 176 ++++++++++++++++++ cli/packages/contentstack-audit/README.md | 14 +- .../src/import/modules/entries.ts | 11 +- .../contentstack-import/src/utils/log.ts | 2 +- .../contentstack-import/src/utils/logger.ts | 6 +- .../src/utils/marketplace-app-helper.ts | 6 +- .../src/utils/taxonomies-helper.ts | 2 +- .../migration-sitecore/libs/contenttypes.js | 4 +- 13 files changed, 361 insertions(+), 83 deletions(-) create mode 100644 api/src/utils/test-folder-creator.utils.ts diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 990fd5bd..5fd2a55b 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -15,11 +15,11 @@ import shell from 'shelljs' import path from "path"; import AuthenticationModel from "../models/authentication.js"; import { siteCoreService } from "./sitecore.service.js"; -import { fileURLToPath } from 'url'; import { copyDirectory } from '../utils/index.js' import { v4 } from "uuid"; import { setLogFilePath } from "../server.js"; import { mkdirp } from 'mkdirp'; +import { testFolderCreator } from "../utils/test-folder-creator.utils.js"; @@ -233,13 +233,10 @@ const runCli = async (rg: string, user_id: string, project: any) => { .find({ region: regionPresent, user_id }) .value(); if (userData?.authtoken && project?.destination_stack_id) { - // Manually define __filename and __dirname - const __filename = fileURLToPath(import.meta.url); - const dirPath = path.join(path.dirname(__filename), '..', '..'); - const sourcePath = path.join(dirPath, 'sitecoreMigrationData', project?.destination_stack_id); + const sourcePath = path.join(process.cwd(), 'sitecoreMigrationData', project?.destination_stack_id); const backupPath = path.join(process.cwd(), 'migration-data', `${project?.destination_stack_id}_${v4().slice(0, 4)}`); await copyDirectory(sourcePath, backupPath); - const loggerPath = path.join(backupPath, 'logs', 'import', 'combine.log'); + const loggerPath = path.join(backupPath, 'logs', 'import', 'success.log'); createDirectoryAndFile(loggerPath); await setLogFilePath(loggerPath); shell.cd(path.join(process.cwd(), '..', 'cli', 'packages', 'contentstack')); @@ -272,6 +269,7 @@ const fieldMapping = async (req: Request): Promise => { await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.destination_stack_id }); await siteCoreService?.createLocale(req, project?.destination_stack_id); await siteCoreService?.createVersionFile(project?.destination_stack_id); + await testFolderCreator?.({ destinationStackId: project?.destination_stack_id }); await runCli(region, user_id, project); } } diff --git a/api/src/services/projects.service.ts b/api/src/services/projects.service.ts index 30e83276..be28fd0d 100644 --- a/api/src/services/projects.service.ts +++ b/api/src/services/projects.service.ts @@ -122,7 +122,7 @@ const createProject = async (req: Request) => { uid: '', label: '', master_locale: '', - created_at: '', + created_at: '', isNewStack: false }, mapperKeys: {} @@ -542,11 +542,13 @@ const fileformatConfirmation = async (req: Request) => { true )) as number; - ProjectModelLowdb.update((data: any) => { - data.projects[projectIndex].legacy_cms.file_format_confirmation = - fileformat_confirmation; - data.projects[projectIndex].updated_at = new Date().toISOString(); - }); + if (!fileformat_confirmation) { + ProjectModelLowdb.update((data: any) => { + data.projects[projectIndex].legacy_cms.file_format_confirmation = + fileformat_confirmation; + data.projects[projectIndex].updated_at = new Date().toISOString(); + }); + } return { status: HTTP_CODES.OK, diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index b77c31f5..4fe9fa45 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -122,23 +122,23 @@ const cretaeAssets = async ({ packagePath, baseDir }: any) => { } catch (err) { console.error("🚀 ~ file: assets.js:52 ~ xml_folder?.forEach ~ err:", err) } + allAssetJSON[mestaData?.uid] = { + urlPath: `/assets/${mestaData?.uid}`, + uid: mestaData?.uid, + content_type: mestaData?.content_type, + file_size: mestaData.size, + tags: [], + filename: `${jsonAsset?.item?.$?.name}.${mestaData?.extension}`, + is_dir: false, + parent_uid: null, + title: jsonAsset?.item?.$?.name, + publish_details: [], + assetPath + } + allAssetJSON[mestaData?.uid].parent_uid = '2146b0cee522cc3a38d' } else { - console.info("asstes id not found.") - } - allAssetJSON[mestaData?.uid] = { - urlPath: `/assets/${mestaData?.uid}`, - uid: mestaData?.uid, - content_type: mestaData?.content_type, - file_size: mestaData.size, - tags: [], - filename: `${jsonAsset?.item?.$?.name}.${mestaData?.extension}`, - is_dir: false, - parent_uid: null, - title: jsonAsset?.item?.$?.name, - publish_details: [], - assetPath + console.info('blob is not there for this asstes', mestaData?.uid, '.') } - allAssetJSON[mestaData?.uid].parent_uid = '2146b0cee522cc3a38d' } } } @@ -191,7 +191,6 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', } } } - for await (const ctType of contentTypes) { const entryPresent: any = entriesData?.find((item: any) => uidCorrector({ uid: item?.template }) === ctType?.contentstackUid) if (entryPresent) { @@ -201,7 +200,7 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', const entryLocale: any = {}; if (typeof LOCALE_MAPPER?.masterLocale === 'object' && LOCALE_MAPPER?.masterLocale !== null && LOCALE_MAPPER?.masterLocale?.[master_locale] === locale) { newLocale = Object?.keys(LOCALE_MAPPER?.masterLocale)?.[0]; - Object.entries(entryPresent?.locale?.[locale] || {}).forEach(async ([uid, entry]: any) => { + Object.entries(entryPresent?.locale?.[locale] || {}).map(async ([uid, entry]: any) => { const entryObj: any = {}; entryObj.uid = uid; for await (const field of entry?.fields?.field ?? []) { @@ -214,28 +213,31 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', entryObj[fsc?.contentstackFieldUid] = `/${entry?.meta?.key}`; } if (getLastKey(fsc?.uid) === field?.$?.key) { - const content: any = await entriesFieldCreator({ field: fsc, content: field?.content, idCorrector, allAssetJSON }) + const content: any = await entriesFieldCreator({ field: fsc, content: field?.content, idCorrector, allAssetJSON, contentTypes, entriesData, locale }) entryObj[fsc?.contentstackFieldUid] = content; } } } } - entryLocale[uid] = unflatten(entryObj) ?? {}; + if (Object.keys?.(entryObj)?.length > 1) { + entryLocale[uid] = unflatten(entryObj) ?? {}; + } }); } - const fileMeta = { "1": `${locale}.json` }; + const fileMeta = { "1": `${newLocale}.json` }; const entryPath = path.join( process.cwd(), entrySave, ctType?.contentstackUid, newLocale ); - await writeFiles(entryPath, fileMeta, entryLocale, locale) + await writeFiles(entryPath, fileMeta, entryLocale, newLocale) } } else { console.info('Entries missing for', ctType?.contentstackUid) } } + return true; } catch (err) { console.error("🚀 ~ createEntry ~ err:", err) } diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index c08f40ee..1e0ffce4 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -86,17 +86,28 @@ const convertToSchemaFormate = ({ field, advanced = true }: any) => { case 'json': { return { "data_type": "json", - "display_name": field?.name, + "display_name": field?.title ?? field?.uid, "uid": field?.uid, "field_metadata": { "allow_json_rte": true, + "embed_entry": false, + "description": "", + "default_value": "", + "multiline": false, "rich_text_type": "advanced", + "options": [] }, - "reference_to": [], - "non_localizable": false, + "format": "", + "error_messages": { + "format": "" + }, + "reference_to": [ + "sys_assets" + ], "multiple": false, - "mandatory": false, - "unique": false + "non_localizable": false, + "unique": false, + "mandatory": false } // return { // "display_name": name, @@ -316,7 +327,7 @@ const convertToSchemaFormate = ({ field, advanced = true }: any) => { "display_name": field?.title, "uid": field?.uid, "data_type": "text", - "mandatory": true, + "mandatory": false, "unique": true, "field_metadata": { "_default": true @@ -409,7 +420,7 @@ export const contenTypeMaker = async ({ contentType, destinationStackId }: any) "field_metadata": {}, "schema": [], "uid": item?.contentstackFieldUid, - "multiple": true, + "multiple": false, "mandatory": false, "unique": false } diff --git a/api/src/utils/entries-field-creator.utils.ts b/api/src/utils/entries-field-creator.utils.ts index 5ea4494d..7f0cb287 100644 --- a/api/src/utils/entries-field-creator.utils.ts +++ b/api/src/utils/entries-field-creator.utils.ts @@ -3,6 +3,20 @@ import { JSDOM } from "jsdom"; import { htmlToJson } from '@contentstack/json-rte-serializer'; import { HTMLToJSON } from 'html-to-json-parser'; +const append = "a"; + +function startsWithNumber(str: string) { + return /^\d/.test(str); +} + +const uidCorrector = ({ uid }: any) => { + if (startsWithNumber(uid)) { + return `${append}_${_.replace(uid, new RegExp("[ -]", "g"), '_')?.toLowerCase()}` + } + return _.replace(uid, new RegExp("[ -]", "g"), '_')?.toLowerCase() +} + + const attachJsonRte = ({ content = "" }: any) => { const dom = new JSDOM(content); @@ -160,7 +174,7 @@ const findAssestInJsoRte = (jsonValue: any, allAssetJSON: any, idCorrector: any) -export const entriesFieldCreator = async ({ field, content, idCorrector, allAssetJSON }: any) => { +export const entriesFieldCreator = async ({ field, content, idCorrector, allAssetJSON, contentTypes, entriesData, locale }: any) => { switch (field?.ContentstackFieldType) { case 'multi_line_text': @@ -174,19 +188,27 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse } case 'dropdown': { - if (content?.includes('{')) { - return idCorrector({ id: content }); + const isOptionPresent = field?.advanced?.options?.find((ops: any) => ops?.key === content || ops?.value === content); + if (isOptionPresent) { + if (field?.advanced?.Multiple) { + if (!isOptionPresent?.key) { + return isOptionPresent + } + return isOptionPresent; + } + return isOptionPresent?.value ?? null; } else { - const isOptionPresent = field?.advanced?.options?.find((ops: any) => ops?.key === content || ops?.value === content); - if (isOptionPresent) { - if (isOptionPresent?.value?.includes('{')) { - return { key: isOptionPresent?.key, value: content } - } else if (!isOptionPresent?.key) { - return { value: content } + if (field?.advanced?.Default_value) { + const isOptionDefaultValue = field?.advanced?.options?.find((ops: any) => ops?.key === field?.advanced?.Default_value || ops?.value === field?.advanced?.Default_value); + if (field?.advanced?.Multiple) { + if (!isOptionDefaultValue?.key) { + return isOptionDefaultValue + } + return isOptionDefaultValue; } - return { key: isOptionPresent?.key, value: content } + return isOptionDefaultValue?.value ?? null; } else { - return { value: field?.advanced?.Default_value } + return field?.advanced?.Default_value; } } } @@ -202,19 +224,19 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse const fileData = attachJsonRte({ content }); fileData?.children?.forEach((item: any) => { if (item?.attrs?.['redactor-attributes']?.mediaid) { - // const assetUid = idCorrector({ id: item?.attrs?.['redactor-attributes']?.mediaid }); - console.info(''); + const assetUid = idCorrector({ id: item?.attrs?.['redactor-attributes']?.mediaid }); + return allAssetJSON?.[assetUid] ?? null; } else { - console.info(item?.attrs) + console.info('more', item?.attrs) } }) - return content; + return null; } //need to change this case 'link': { - const linkType: any = htmlConverter({ content }) - let obj: any = { title: '', url: '' }; + const linkType: any = await htmlConverter({ content }) + let obj: any = { title: '', href: '' }; if (typeof linkType === 'string') { const parseData = JSON?.parse?.(linkType); if (parseData?.type === 'div') { @@ -222,7 +244,7 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse if (item?.type === 'link') { obj = { title: item?.attributes?.id, - url: item?.attributes?.url ?? '' + href: item?.attributes?.url ?? '' } } }) @@ -231,8 +253,78 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse return obj; } + case 'reference': { + const refs: any = []; + if (field?.refrenceTo?.length) { + field?.refrenceTo?.forEach((entry: any) => { + const templatePresent = entriesData?.find((tel: any) => uidCorrector({ uid: tel?.template }) === entry); + content?.split('|')?.forEach((id: string) => { + const entryid = templatePresent?.locale?.[locale]?.[idCorrector({ id })]; + if (entryid) { + refs?.push({ + "uid": idCorrector({ id }), + "_content_type_uid": entry + }) + } else { + // console.info("no entry for following id", id) + } + }) + }) + } else { + console.info('test ====>'); + } + return refs; + } + + case 'text': { + return content; + } + + case 'global_field': { + const globalFieldsSchema = contentTypes?.find?.((gfd: any) => + gfd?.contentstackUid === field?.contentstackFieldUid && gfd?.type === 'global_field' + ); + if (globalFieldsSchema?.fieldMapping) { + const mainSchema = []; + const group: any = {}; + globalFieldsSchema?.fieldMapping?.forEach((item: any) => { + if (item?.ContentstackFieldType === 'group') { + group[item?.contentstackFieldUid] = { ...item, fieldMapping: [] }; + } else { + const groupSchema = group[item?.contentstackFieldUid?.split('.')?.[0]]; + if (groupSchema) { + group?.[groupSchema?.contentstackFieldUid]?.fieldMapping?.push(item); + } else { + mainSchema?.push(item); + } + } + }); + mainSchema?.push(group); + const obj: any = {}; + mainSchema?.forEach(async (field: any) => { + if (field?.['uid']) { + obj[field?.contentstackFieldUid] = await entriesFieldCreator({ field, content }); + } else { + Object?.values(field)?.forEach((item: any) => { + if (item?.ContentstackFieldType === 'group') { + item?.fieldMapping?.forEach(async (ele: any) => { + obj[ele?.contentstackFieldUid] = await entriesFieldCreator({ field: ele, content }); + }) + } + }) + } + }) + return await obj; + } + break; + } + + case 'boolean': { + return typeof content === 'string' && content === '1' ? true : false; + } + default: { - console.info(field?.ContentstackFieldType); + console.info(field?.ContentstackFieldType, 'umesh'); return content; } } diff --git a/api/src/utils/test-folder-creator.utils.ts b/api/src/utils/test-folder-creator.utils.ts new file mode 100644 index 00000000..98cff864 --- /dev/null +++ b/api/src/utils/test-folder-creator.utils.ts @@ -0,0 +1,176 @@ +import path from 'path'; +import fs from 'fs'; +import read from 'fs-readdir-recursive'; + + +async function writeOneFile(indexPath: string, fileMeta: any) { + fs.writeFile(indexPath, JSON.stringify(fileMeta), (err) => { + if (err) { + console.error('Error writing file: 3', err); + } + }); +} + +async function writeFiles(entryPath: string, fileMeta: any, entryLocale: any, locale: string) { + try { + const indexPath = path.join(entryPath, 'index.json'); + const localePath = path.join(entryPath, `${locale}.json`); + fs.access(entryPath, async (err) => { + if (err) { + fs.mkdir(entryPath, { recursive: true }, async (err) => { + if (err) { + console.error('Error writing file: 2', err); + } else { + await writeOneFile(indexPath, fileMeta) + await writeOneFile(localePath, entryLocale) + } + }); + } else { + await writeOneFile(indexPath, fileMeta) + await writeOneFile(localePath, entryLocale) + } + }); + } catch (error) { + console.error('Error writing files:', error); + } +} + +const saveContent = async (ct: any, contentSave: string) => { + try { + // Check if the directory exists + await fs.promises.access(contentSave).catch(async () => { + // If the directory doesn't exist, create it + await fs.promises.mkdir(contentSave, { recursive: true }); + }); + // Write the individual content to its own file + const filePath = path.join(process.cwd(), contentSave, `${ct?.uid}.json`); + await fs.promises.writeFile(filePath, JSON.stringify(ct)); + // Append the content to schema.json + const schemaFilePath = path.join(process.cwd(), contentSave, 'schema.json'); + let schemaData = []; + try { + // Read existing schema.json file if it exists + const schemaFileContent = await fs.promises.readFile(schemaFilePath, 'utf8'); + schemaData = JSON.parse(schemaFileContent); + } catch (readError: any) { + if (readError?.code !== 'ENOENT') { + throw readError; // rethrow if it's not a "file not found" error + } + } + // Append new content to schemaData + schemaData.push(ct); + // Write the updated schemaData back to schema.json + await fs.promises.writeFile(schemaFilePath, JSON.stringify(schemaData, null, 2)); + + } catch (err) { + console.error("Error:", err); + } + +} + + +async function cleanDirectory(folderPath: string, foldersToKeep: any[]): Promise { + try { + // Ensure we're only working with the first 10 folders to keep + const foldersToKeepLimited = foldersToKeep.map((item: any) => item?.uid); + + // Read all items (files and folders) in the directory + const itemsInDirectory = await fs.promises.readdir(folderPath, { withFileTypes: true }); + + // Loop through all items in the directory + for (const item of itemsInDirectory) { + const itemPath = path.join(folderPath, item?.name); + + // Check if the item is a directory and if it's not in the list of folders to keep + if (item?.isDirectory() && !foldersToKeepLimited?.includes(item?.name)) { + // Delete the folder and its contents + await fs.promises.rm(itemPath, { recursive: true, force: true }); + console.info(`Deleted folder: ${item.name}`); + } + } + console.info("Cleanup completed!"); + } catch (err) { + console.error(`Error while cleaning directory: ${(err as Error).message}`); + } +} + + +async function deleteFolderAsync(folderPath: string): Promise { + try { + await fs.promises.rm(folderPath, { recursive: true, force: true }); + console.info(`Folder ${folderPath} deleted successfully.`); + } catch (err) { + console.error(`Error while deleting folder: ${(err as Error).message}`); + } +} + + + +const sortAssets = async (baseDir: string) => { + const assetsPath = path.join(process.cwd(), baseDir, 'assets'); + const assetsFilesPath = path.join(assetsPath, 'files'); + const assetsJson = JSON.parse(await fs.promises.readFile(path.join(assetsPath, 'index.json'), 'utf8')); + const sortAsset = Object?.values?.(assetsJson)?.slice(0, 10); + const assetsMeta: any = {}; + sortAsset?.forEach((item: any) => { + assetsMeta[item?.uid] = item; + }) + await cleanDirectory(assetsFilesPath, sortAsset); + await fs.promises.writeFile(path.join(assetsPath, 'index.json'), JSON?.stringify?.(assetsMeta)); +} + +const sortContentType = async (baseDir: string, finalData: any) => { + const contentTypePath: string = path.join(process.cwd(), baseDir, 'content_types'); + const contentSave = path.join(baseDir, 'content_types'); + const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, 'schema.json'), 'utf8')); + const contentTypes: any = []; + finalData?.forEach((ct: any) => { + const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType) + contentTypes?.push(findCtData); + }) + await deleteFolderAsync(contentTypePath); + for await (const ctItem of contentTypes) { + await saveContent(ctItem, contentSave); + } +} + +export const testFolderCreator = async ({ destinationStackId }: any) => { + const baseDir = path.join('sitecoreMigrationData', destinationStackId); + const entryDelete = path.join(process.cwd(), baseDir, 'entries'); + const entrySave = path.join(baseDir, 'entries'); + const entriesPath = path.join(process.cwd(), baseDir, 'entries'); + const allData = []; + for await (const filePath of read(entriesPath)) { + if (!filePath?.endsWith('index.json')) { + const entryData = await JSON.parse(await fs.promises.readFile(path.join(entriesPath, filePath), 'utf8')); + if (Object?.keys?.(entryData)?.length) { + const ct = filePath?.split?.('/')?.[0]; + const locale = filePath?.split?.('/')?.[1]; + allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length, entryData, filePath, locale }) + } + } + } + const sortData = allData.sort((a, b) => b?.count - a?.count).slice?.(1, 4); + const finalData: any = []; + sortData.forEach((et: any) => { + const entryObj: any = {}; + const ctData = Object?.values?.(et?.entryData)?.splice?.(0, 5); + ctData?.forEach((entItem: any) => { + entryObj[entItem?.uid] = entItem; + }) + finalData?.push({ contentType: et?.contentType, entryObj, locale: et?.locale }); + }); + await sortAssets(baseDir); + await sortContentType(baseDir, finalData); + await deleteFolderAsync(entryDelete); + for await (const entry of finalData) { + const fileMeta = { "1": `${entry?.locale}.json` }; + const entryPath = path.join( + process.cwd(), + entrySave, + entry?.contentType, + entry?.locale + ); + await writeFiles(entryPath, fileMeta, entry?.entryObj, entry?.locale); + } +} \ No newline at end of file diff --git a/cli/packages/contentstack-audit/README.md b/cli/packages/contentstack-audit/README.md index 23d84184..4d874d45 100644 --- a/cli/packages/contentstack-audit/README.md +++ b/cli/packages/contentstack-audit/README.md @@ -267,7 +267,7 @@ EXAMPLES $ csdx plugins ``` -_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.8/src/commands/plugins/index.ts)_ +_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.10/src/commands/plugins/index.ts)_ ## `csdx plugins:add PLUGIN` @@ -341,7 +341,7 @@ EXAMPLES $ csdx plugins:inspect myplugin ``` -_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.8/src/commands/plugins/inspect.ts)_ +_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.10/src/commands/plugins/inspect.ts)_ ## `csdx plugins:install PLUGIN` @@ -390,7 +390,7 @@ EXAMPLES $ csdx plugins:install someuser/someplugin ``` -_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.8/src/commands/plugins/install.ts)_ +_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.10/src/commands/plugins/install.ts)_ ## `csdx plugins:link PATH` @@ -420,7 +420,7 @@ EXAMPLES $ csdx plugins:link myplugin ``` -_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.8/src/commands/plugins/link.ts)_ +_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.10/src/commands/plugins/link.ts)_ ## `csdx plugins:remove [PLUGIN]` @@ -461,7 +461,7 @@ FLAGS --reinstall Reinstall all plugins after uninstalling. ``` -_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.8/src/commands/plugins/reset.ts)_ +_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.10/src/commands/plugins/reset.ts)_ ## `csdx plugins:uninstall [PLUGIN]` @@ -489,7 +489,7 @@ EXAMPLES $ csdx plugins:uninstall myplugin ``` -_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.8/src/commands/plugins/uninstall.ts)_ +_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.10/src/commands/plugins/uninstall.ts)_ ## `csdx plugins:unlink [PLUGIN]` @@ -533,5 +533,5 @@ DESCRIPTION Update installed plugins. ``` -_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.8/src/commands/plugins/update.ts)_ +_See code: [@oclif/plugin-plugins](https://github.com/oclif/plugin-plugins/blob/v5.4.10/src/commands/plugins/update.ts)_ diff --git a/cli/packages/contentstack-import/src/import/modules/entries.ts b/cli/packages/contentstack-import/src/import/modules/entries.ts index 299e0621..de125f17 100644 --- a/cli/packages/contentstack-import/src/import/modules/entries.ts +++ b/cli/packages/contentstack-import/src/import/modules/entries.ts @@ -187,8 +187,7 @@ export default class EntriesImport extends BaseClass { await this.publishEntries(entryRequestOption).catch((error) => { log( this.importConfig, - `Error in publishing entries of ${entryRequestOption.cTUid} in locale ${ - entryRequestOption.locale + `Error in publishing entries of ${entryRequestOption.cTUid} in locale ${entryRequestOption.locale } ${formatError(error)}`, 'error', ); @@ -360,7 +359,7 @@ export default class EntriesImport extends BaseClass { if (!isMasterLocale && !additionalInfo[entry.uid]?.isLocalized) { this.autoCreatedEntries.push({ cTUid, locale, entryUid: response.uid }); } - this.entriesUidMapper[entry.uid] = response.uid; + this.entriesUidMapper[entry?.uid] = response.uid; entry.sourceEntryFilePath = path.join(sanitizePath(basePath), sanitizePath(additionalInfo.entryFileName)); // stores source file path temporarily entry.entryOldUid = entry.uid; // stores old uid temporarily entriesCreateFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true }); @@ -463,7 +462,7 @@ export default class EntriesImport extends BaseClass { delete entry.publish_details; // checking the entry is a localized one or not if (!isMasterLocale && this.entriesUidMapper.hasOwnProperty(entry.uid)) { - const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); + const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry?.uid]); Object.assign(entryResponse, cloneDeep(entry), { uid: this.entriesUidMapper[entry.uid] }); apiOptions.apiData = entryResponse; apiOptions.additionalInfo[entryResponse.uid] = { @@ -513,7 +512,7 @@ export default class EntriesImport extends BaseClass { const onSuccess = ({ response, apiData: entry, additionalInfo }: any) => { log(this.importConfig, `Replaced entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info'); - this.entriesUidMapper[entry.uid] = response.uid; + this.entriesUidMapper[entry?.uid] = response.uid; entriesReplaceFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true }); }; const onReject = ({ error, apiData: { uid, title } }: any) => { @@ -956,7 +955,7 @@ export default class EntriesImport extends BaseClass { } = { environments: [], locales: [], - entryUid: this.entriesUidMapper[entry.uid], + entryUid: this.entriesUidMapper[entry?.uid], }; if (entry.publish_details && entry.publish_details?.length > 0) { forEach(entry.publish_details, (pubObject) => { diff --git a/cli/packages/contentstack-import/src/utils/log.ts b/cli/packages/contentstack-import/src/utils/log.ts index 00ab0e00..aac6d634 100644 --- a/cli/packages/contentstack-import/src/utils/log.ts +++ b/cli/packages/contentstack-import/src/utils/log.ts @@ -1,5 +1,5 @@ import { join } from 'path'; -import { LogEntry } from 'winston'; +import { LogEntry } from 'winston/index'; import { Logger, pathValidator, sanitizePath } from '@contentstack/cli-utilities'; import { LogsType, MessageType } from '@contentstack/cli-utilities/lib/logger'; diff --git a/cli/packages/contentstack-import/src/utils/logger.ts b/cli/packages/contentstack-import/src/utils/logger.ts index a79810ba..07dd0924 100644 --- a/cli/packages/contentstack-import/src/utils/logger.ts +++ b/cli/packages/contentstack-import/src/utils/logger.ts @@ -92,6 +92,7 @@ function init(_logPath: string) { logger = winston.createLogger({ transports: [ new winston.transports.File(successTransport), + new winston.transports.File(combineTransport), new winston.transports.Console({ format: winston.format.combine( winston.format.simple(), @@ -105,6 +106,7 @@ function init(_logPath: string) { errorLogger = winston.createLogger({ transports: [ new winston.transports.File(errorTransport), + new winston.transports.File(combineTransport), new winston.transports.Console({ level: 'error', format: winston.format.combine( @@ -128,10 +130,6 @@ function init(_logPath: string) { ], levels: myCustomLevels.levels, }); - - // Change file permissions for combine.log to read/write for everyone - const combineLogPath = path.join(sanitizePath(logsDir), 'combine.log'); - fs.chmodSync(combineLogPath, 0o666); // Read/write for everyone } return { diff --git a/cli/packages/contentstack-import/src/utils/marketplace-app-helper.ts b/cli/packages/contentstack-import/src/utils/marketplace-app-helper.ts index 22ca1959..702a1a93 100644 --- a/cli/packages/contentstack-import/src/utils/marketplace-app-helper.ts +++ b/cli/packages/contentstack-import/src/utils/marketplace-app-helper.ts @@ -13,10 +13,10 @@ import { } from '@contentstack/cli-utilities'; import { log } from './logger'; -import { trace } from './log'; +import { trace } from '../utils/log'; import { ImportConfig, Installation } from '../types'; -import { formatError } from '.'; -import { getAppName, askAppName, selectConfiguration } from './interactive'; +import { formatError } from '../utils'; +import { getAppName, askAppName, selectConfiguration } from '../utils/interactive'; export const getAllStackSpecificApps = async ( config: ImportConfig, diff --git a/cli/packages/contentstack-import/src/utils/taxonomies-helper.ts b/cli/packages/contentstack-import/src/utils/taxonomies-helper.ts index e6923e7a..9095cd37 100644 --- a/cli/packages/contentstack-import/src/utils/taxonomies-helper.ts +++ b/cli/packages/contentstack-import/src/utils/taxonomies-helper.ts @@ -2,7 +2,7 @@ * taxonomy lookup */ import find from 'lodash/find'; -import { log } from '.'; +import { log } from './'; import { ImportConfig } from '../types'; /** diff --git a/upload-api/migration-sitecore/libs/contenttypes.js b/upload-api/migration-sitecore/libs/contenttypes.js index 113adb61..2731d6aa 100644 --- a/upload-api/migration-sitecore/libs/contenttypes.js +++ b/upload-api/migration-sitecore/libs/contenttypes.js @@ -158,7 +158,7 @@ const ContentTypeSchema = ({ type, name, uid, default_value = "", id, choices = "contentstackFieldUid": uid, "ContentstackFieldType": "dropdown", "backupFieldType": "dropdown", - "advanced": { options: choices, Default_value: default_value !== "" ? default_value : null } + "advanced": { options: choices, Default_value: default_value !== "" ? default_value : null, Multiple: false } } } case "Image": { @@ -246,7 +246,7 @@ const ContentTypeSchema = ({ type, name, uid, default_value = "", id, choices = "contentstackFieldUid": uid, "ContentstackFieldType": "dropdown", "backupFieldType": "dropdown", - "advanced": { options: choices, Default_value: default_value !== "" ? default_value : null } + "advanced": { options: choices, Default_value: default_value !== "" ? default_value : null, Multiple: false } } } break; From ba894766d0d57fe653c62324faddc053fc5377c2 Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Thu, 3 Oct 2024 20:02:08 +0530 Subject: [PATCH 2/2] code added --- api/src/routes/migration.routes.ts | 2 +- api/src/services/migration.service.ts | 55 ++++++++++++++++----------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/api/src/routes/migration.routes.ts b/api/src/routes/migration.routes.ts index 78af53a8..ac0ce8f6 100644 --- a/api/src/routes/migration.routes.ts +++ b/api/src/routes/migration.routes.ts @@ -41,7 +41,7 @@ router.post( * @returns {Promise} - A promise that resolves when the test stack is deleted. */ router.post( - "/create-test-stack/:projectId", + "/create-test-stack/:orgId/:projectId", asyncRouter(migrationController.createTestStack) ); diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 5fd2a55b..e7af202b 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -7,7 +7,7 @@ import https from "../utils/https.utils.js"; import { LoginServiceType } from "../models/types.js"; import getAuthtoken from "../utils/auth.utils.js"; import logger from "../utils/logger.js"; -import { HTTP_TEXTS, HTTP_CODES, CS_REGIONS } from "../constants/index.js"; +import { HTTP_TEXTS, HTTP_CODES, CS_REGIONS, LOCALE_MAPPER } from "../constants/index.js"; import { ExceptionFunction } from "../utils/custom-errors.utils.js"; import { fieldAttacher } from "../utils/field-attacher.utils.js"; import ProjectModelLowdb from "../models/project-lowdb.js"; @@ -35,7 +35,11 @@ const createTestStack = async (req: Request): Promise => { const srcFun = "createTestStack"; const orgId = req?.params?.orgId; const projectId = req?.params?.projectId; - const { token_payload, name, description, master_locale } = req.body; + const { token_payload } = req.body; + const description = 'This is a system-generated test stack.' + const name = 'Test'; + const master_locale = Object?.keys?.(LOCALE_MAPPER?.masterLocale)?.[0]; + try { const authtoken = await getAuthtoken( @@ -44,8 +48,9 @@ const createTestStack = async (req: Request): Promise => { ); await ProjectModelLowdb.read(); - const projectData = ProjectModelLowdb.chain.get("projects").value(); - const testStackCount = projectData[0]?.test_stacks?.length + 1; + const projectData: any = ProjectModelLowdb.chain.get("projects").find({ id: projectId }).value(); + console.info("🚀 ~ createTestStack ~ projectData:", projectData) + const testStackCount = projectData?.test_stacks?.length + 1; const newName = name + "-" + testStackCount; const [err, res] = await safePromise( @@ -88,11 +93,10 @@ const createTestStack = async (req: Request): Promise => { .get("projects") .findIndex({ id: projectId }) .value(); - console.info(index); if (index > -1) { ProjectModelLowdb.update((data: any) => { - data.projects[index].current_test_stack_id = res.data.stack.uid; - data.projects[index].test_stacks.push(res.data.stack.uid); + data.projects[index].current_test_stack_id = res?.data?.stack?.api_key; + data.projects[index].test_stacks.push({ stackUid: res?.data?.stack?.api_key, isMigrated: false }); }); } return { @@ -224,7 +228,7 @@ function createDirectoryAndFile(filePath: string) { } -const runCli = async (rg: string, user_id: string, project: any) => { +const runCli = async (rg: string, user_id: string, stack_uid: any) => { try { const regionPresent = CS_REGIONS?.find((item: string) => item === rg) ?? 'NA'; await AuthenticationModel.read(); @@ -232,9 +236,9 @@ const runCli = async (rg: string, user_id: string, project: any) => { .get("users") .find({ region: regionPresent, user_id }) .value(); - if (userData?.authtoken && project?.destination_stack_id) { - const sourcePath = path.join(process.cwd(), 'sitecoreMigrationData', project?.destination_stack_id); - const backupPath = path.join(process.cwd(), 'migration-data', `${project?.destination_stack_id}_${v4().slice(0, 4)}`); + if (userData?.authtoken && stack_uid) { + const sourcePath = path.join(process.cwd(), 'sitecoreMigrationData', stack_uid); + const backupPath = path.join(process.cwd(), 'migration-data', `${stack_uid}_${v4().slice(0, 4)}`); await copyDirectory(sourcePath, backupPath); const loggerPath = path.join(backupPath, 'logs', 'import', 'success.log'); createDirectoryAndFile(loggerPath); @@ -246,7 +250,7 @@ const runCli = async (rg: string, user_id: string, project: any) => { cliLogger(region); const login = shell.exec(`node bin/run login -a ${userData?.authtoken} -e ${userData?.email}`); cliLogger(login); - const exportData = shell.exec(`node bin/run cm:stacks:import -k ${project?.destination_stack_id} -d ${sourcePath} --backup-dir=${backupPath} --yes`, { async: true }); + const exportData = shell.exec(`node bin/run cm:stacks:import -k ${stack_uid} -d ${sourcePath} --backup-dir=${backupPath} --yes`, { async: true }); cliLogger(exportData); } else { console.info('user not found.') @@ -259,18 +263,23 @@ const runCli = async (rg: string, user_id: string, project: any) => { const fieldMapping = async (req: Request): Promise => { const { orgId, projectId } = req?.params ?? {}; const { region, user_id } = req?.body?.token_payload ?? {}; - const project = ProjectModelLowdb.chain - .get("projects") - .find({ id: projectId }) - .value(); - if (project?.extract_path && project?.destination_stack_id) { + const project = ProjectModelLowdb.chain.get("projects").find({ id: projectId }).value(); + if (project?.extract_path && project?.current_test_stack_id) { const packagePath = project?.extract_path; - const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.destination_stack_id }); - await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.destination_stack_id }); - await siteCoreService?.createLocale(req, project?.destination_stack_id); - await siteCoreService?.createVersionFile(project?.destination_stack_id); - await testFolderCreator?.({ destinationStackId: project?.destination_stack_id }); - await runCli(region, user_id, project); + const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.current_test_stack_id }); + await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.current_test_stack_id }); + await siteCoreService?.createLocale(req, project?.current_test_stack_id); + await siteCoreService?.createVersionFile(project?.current_test_stack_id); + await testFolderCreator?.({ destinationStackId: project?.current_test_stack_id }); + await runCli(region, user_id, project?.current_test_stack_id); + // const projectIndex = ProjectModelLowdb.chain.get("projects").findIndex({ id: projectId }).value(); + // if (projectIndex > -1) { + // ProjectModelLowdb.update((data: any) => { + // const index = data.projects[projectIndex].test_stacks.findIndex((item: any) => item?.stackUid === project?.current_test_stack_id); + // console.info("🚀 ~ ProjectModelLowdb.update ~ index:", index) + // data.projects[projectIndex].current_test_stack_id = ''; + // }); + // } } }