From 3e114e7fbfb86604065c66c5fa0f129531b77501 Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Thu, 6 Mar 2025 02:28:37 +0530 Subject: [PATCH] fixed test migartion --- api/src/services/contentful.service.ts | 51 +++++++++++++-------- api/src/services/contentful/jsonRTE.ts | 2 +- api/src/services/migration.service.ts | 9 ++-- api/src/utils/content-type-creator.utils.ts | 2 +- api/src/utils/test-folder-creator.utils.ts | 39 ++++++++-------- 5 files changed, 58 insertions(+), 45 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 8d533e64..db542422 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -10,7 +10,7 @@ import { JSDOM } from "jsdom"; import { jsonToHtml, jsonToMarkdown, htmlToJson } from '@contentstack/json-rte-serializer'; -import { CHUNK_SIZE, MIGRATION_DATA_CONFIG } from "../constants/index.js"; +import { CHUNK_SIZE, MIGRATION_DATA_CONFIG, LOCALE_MAPPER } from "../constants/index.js"; import { Locale } from "../models/types.js"; import jsonRTE from "./contentful/jsonRTE.js"; import { getAllLocales, getLogMessage } from "../utils/index.js"; @@ -234,7 +234,7 @@ const processField = ( return assetsData; } else { const id = lang_value?.sys?.id; - if (id in assetId) { + if (assetId && id in assetId) { return assetId?.[id] } return null; @@ -253,7 +253,7 @@ const processField = ( return refs; } const id = lang_value?.sys?.id; - if (id in entryId) return [[entryId?.[id]]]; + if (entryId && id in entryId) return [[entryId?.[id]]]; return null; } @@ -436,11 +436,11 @@ const saveAsset = async ( const publishDetails: { environment: any; version: number; locale: any }[] = []; const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); - const environmentsId = await readFile(path.join(DATA, destination_stack_id, ENVIRONMENTS_DIR_NAME), ENVIRONMENTS_FILE_NAME); + const environmentsId = await readFile(path.join(DATA, destination_stack_id, ENVIRONMENTS_DIR_NAME), ENVIRONMENTS_FILE_NAME) ?? {}; const localeId = await readFile(path.join(DATA, destination_stack_id, LOCALE_DIR_NAME), LOCALE_CF_LANGUAGE); if (assets.fields.file && assets.fields.title) { - Object.values(environmentsId).forEach((env: any) => { + Object?.values?.(environmentsId)?.length && Object.values(environmentsId).forEach((env: any) => { if (env?.name === assets?.sys?.environment?.sys?.id) { Object.values(localeId).forEach((locale: any) => { publishDetails.push({ @@ -696,6 +696,18 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string, } }; +const mapLocales = ({ masterLocale, locale, locales }: any) => { + if (locales?.masterLocale?.[masterLocale ?? ''] === locale) { + return Object?.keys(locales?.masterLocale)?.[0] + } + for (const [key, value] of Object?.entries?.(locales) ?? {}) { + if (typeof value !== 'object' && value === locale) { + return key; + } + } + return locale.toLowerCase(); +} + /** * Creates and processes entries from a given package file and saves them to the destination stack directory. * @@ -722,7 +734,8 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string, * * @throws Will log errors encountered during file reading, processing, or writing of entries. */ -const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any): Promise => { +const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string): Promise => { + console.info("🚀 ~ createEntry ~ master_locale:", master_locale) const srcFunc = 'createEntry'; try { const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME); @@ -735,7 +748,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje if (entries && entries.length > 0) { const assetId = await readFile(assetsSave, ASSETS_SCHEMA_FILE) ?? []; const entryId = await readFile(path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME), REFERENCES_FILE_NAME); - const environmentsId = await readFile(environmentSave, ENVIRONMENTS_FILE_NAME); + const environmentsId = await readFile(environmentSave, ENVIRONMENTS_FILE_NAME) ?? {}; const displayField: { [key: string]: any } = {} content.map((item: any) => { displayField[item.name.toLowerCase().replace(/[^a-z0-9]+/g, "_")] = @@ -781,13 +794,15 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje }); const pathName = getDisplayName(name, displayField); locales.forEach((locale) => { - const publishDetails = Object?.values?.(environmentsId) + const localeCode = mapLocales({ masterLocale: master_locale, locale, locales: LOCALE_MAPPER }); + console.info("🚀 ~ locales.forEach ~ localeCode:", localeCode); + const publishDetails = Object?.values?.(environmentsId)?.length ? Object?.values?.(environmentsId) .filter((env: any) => env?.name === environment_id) ?.map((env: any) => ({ environment: env?.uid, version: 1, - locale: locale.toLowerCase(), - })); + locale: locale?.toLowerCase(), + })) : []; const title = entryData[name][locale][id][pathName] || ""; const urlTitle = title @@ -814,21 +829,18 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje ); }); }); - return entryData; }, {} ); - const writePromises = []; - - for (const [newKey, values] of Object.entries(result)) { + for await (const [newKey, values] of Object.entries(result)) { const currentCT = contentTypes?.find((ct: any) => ct?.otherCmsUid === newKey); const ctName = currentCT?.contentstackUid in mapperKeys ? mapperKeys?.[currentCT?.contentstackUid] : (currentCT?.contentstackUid ?? newKey.replace(/([A-Z])/g, "_$1").toLowerCase()); - for (const [localeKey, localeValues] of Object.entries( + for await (const [localeKey, localeValues] of Object.entries( values as { [key: string]: any } )) { - const chunks = await makeChunks(localeValues); + const chunks = makeChunks(localeValues); for (const [entryKey, entryValue] of Object.entries(localeValues)) { const message = getLogMessage( srcFunc, @@ -843,14 +855,13 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje entriesSave, ctName, localeKey.toLowerCase() ); - for (const [chunkId, chunkData] of Object.entries(chunks)) { + for await (const [chunkId, chunkData] of Object.entries(chunks)) { refs[chunkIndex++] = `${chunkId}-entries.json`; - writePromises.push(writeFile(filePath, `${chunkId}-entries.json`, chunkData)) + await writeFile(filePath, `${chunkId}-entries.json`, chunkData) } - writePromises.push(writeFile(filePath, ENTRIES_MASTER_FILE, refs)); + await writeFile(filePath, ENTRIES_MASTER_FILE, refs); } } - await Promise.all(writePromises); } else { const message = getLogMessage( srcFunc, diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 90d168b9..0d7da75c 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -250,7 +250,7 @@ function parseBlockReference(obj: any, lang?: LangType, destination_stack_id?: S function parseInlineReference(obj: any, lang?: LangType, destination_stack_id?: StackId): any { const entryId: { [key: string]: any } = destination_stack_id && readFile(path.join(process.cwd(), DATA, destination_stack_id, RTE_REFERENCES_DIR_NAME, RTE_REFERENCES_FILE_NAME)); - const entry = Object.entries(entryId).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue[obj.data.target.sys.id]); + const entry = entryId && Object.entries(entryId).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue[obj.data.target.sys.id]); if (entry) { const [arrayKey, arrayValue] = entry; diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 5b7a6c78..c7d4fad2 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -25,6 +25,7 @@ import { extensionService } from "./extension.service.js"; + /** * Creates a test stack. * @@ -259,7 +260,7 @@ const startTestMigration = async (req: Request): Promise => { await contentfulService?.createWebhooks(file_path, project?.current_test_stack_id, projectId); await contentfulService?.createEnvironment(file_path, project?.current_test_stack_id, projectId); await contentfulService?.createAssets(file_path, project?.current_test_stack_id, projectId, true); - await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys); + await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale); await contentfulService?.createVersionFile(project?.current_test_stack_id, projectId); break; } @@ -299,8 +300,8 @@ const startMigration = async (req: Request): Promise => { await customLogger(projectId, project?.destination_stack_id, 'info', message); await setLogFilePath(loggerPath); const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.destination_stack_id, region, user_id }); - await marketPlaceAppService?.createAppManifest({ orgId, destinationStackId: project?.current_test_stack_id, region, userId: user_id }); - await extensionService?.createExtension({ destinationStackId: project?.current_test_stack_id }); + await marketPlaceAppService?.createAppManifest({ orgId, destinationStackId: project?.destination_stack_id, region, userId: user_id }); + await extensionService?.createExtension({ destinationStackId: project?.destination_stack_id }); switch (cms) { case CMS.SITECORE_V8: case CMS.SITECORE_V9: @@ -337,7 +338,7 @@ const startMigration = async (req: Request): Promise => { await contentfulService?.createWebhooks(file_path, project?.destination_stack_id, projectId); await contentfulService?.createEnvironment(file_path, project?.destination_stack_id, projectId); await contentfulService?.createAssets(file_path, project?.destination_stack_id, projectId); - await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys); + await contentfulService?.createEntry(file_path, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale); await contentfulService?.createVersionFile(project?.destination_stack_id, projectId); break; } diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index 290130ca..66424018 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -748,7 +748,7 @@ export const contenTypeMaker = async ({ contentType, destinationStackId, project if (currentCt?.uid) { ct = await mergeTwoCts(ct, currentCt); } - if (ct?.uid) { + if (ct?.uid && ct?.schema?.length) { if (contentType?.type === 'global_field') { const globalSave = path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, GLOBAL_FIELDS_DIR_NAME); const message = getLogMessage(srcFunc, `Global Field ${ct?.uid} has been successfully Transformed.`, {}); diff --git a/api/src/utils/test-folder-creator.utils.ts b/api/src/utils/test-folder-creator.utils.ts index cdbafae4..f3d95b32 100644 --- a/api/src/utils/test-folder-creator.utils.ts +++ b/api/src/utils/test-folder-creator.utils.ts @@ -8,7 +8,6 @@ const { ENTRIES_DIR_NAME, ASSETS_DIR_NAME, ASSETS_SCHEMA_FILE, - CONTENT_TYPES_DIR_NAME, CONTENT_TYPES_SCHEMA_FILE, ENTRIES_MASTER_FILE, GLOBAL_FIELDS_DIR_NAME, @@ -240,22 +239,24 @@ const sortGlobalField = async (baseDir: string, finalData: any) => { } } -const sortContentType = async (baseDir: string, finalData: any) => { - const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME); - const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME); - const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8')); - await sortGlobalField(baseDir, finalData); - const contentTypes: any = []; - for await (const ct of finalData) { - const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType); - await lookForReference(findCtData, finalData); - contentTypes?.push(findCtData); - } - await deleteFolderAsync(contentTypePath); - for await (const ctItem of contentTypes) { - await saveContent(ctItem, contentSave); - } -} +//this code can be used in feature + +// const sortContentType = async (baseDir: string, finalData: any) => { +// const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME); +// const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME); +// const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8')); +// await sortGlobalField(baseDir, finalData); +// const contentTypes: any = []; +// for await (const ct of finalData) { +// const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType); +// await lookForReference(findCtData, finalData); +// contentTypes?.push(findCtData); +// } +// await deleteFolderAsync(contentTypePath); +// for await (const ctItem of contentTypes) { +// await saveContent(ctItem, contentSave); +// } +// } @@ -275,7 +276,7 @@ export const testFolderCreator = async ({ destinationStackId }: any) => { const pathParts = normalizedPath.split(path.sep); const ct = pathParts?.[0]; // First directory const locale = pathParts?.[1]; // Second directory - allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length, entryData, filePath, locale }) + allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length ?? 0, entryData, filePath, locale }) } } } @@ -292,7 +293,7 @@ export const testFolderCreator = async ({ destinationStackId }: any) => { finalData?.push({ contentType: et?.contentType, entryObj, locale: et?.locale }); }); await sortAssets(baseDir); - await sortContentType(baseDir, finalData); + // await sortContentType(baseDir, finalData); await deleteFolderAsync(entryDelete); for await (const entry of finalData) { const fileMeta = { "1": `${entry?.locale}.json` };