Skip to content

fixed test migartion #503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 6, 2025
Merged

fixed test migartion #503

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 31 additions & 20 deletions api/src/services/contentful.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { JSDOM } from "jsdom";
import { jsonToHtml, jsonToMarkdown, htmlToJson } from '@contentstack/json-rte-serializer';


import { CHUNK_SIZE, MIGRATION_DATA_CONFIG } from "../constants/index.js";
import { CHUNK_SIZE, MIGRATION_DATA_CONFIG, LOCALE_MAPPER } from "../constants/index.js";
import { Locale } from "../models/types.js";
import jsonRTE from "./contentful/jsonRTE.js";
import { getAllLocales, getLogMessage } from "../utils/index.js";
Expand Down Expand Up @@ -234,7 +234,7 @@ const processField = (
return assetsData;
} else {
const id = lang_value?.sys?.id;
if (id in assetId) {
if (assetId && id in assetId) {
return assetId?.[id]
}
return null;
Expand All @@ -253,7 +253,7 @@ const processField = (
return refs;
}
const id = lang_value?.sys?.id;
if (id in entryId) return [[entryId?.[id]]];
if (entryId && id in entryId) return [[entryId?.[id]]];
return null;
}

Expand Down Expand Up @@ -436,11 +436,11 @@ const saveAsset = async (
const publishDetails: { environment: any; version: number; locale: any }[] =
[];
const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME);
const environmentsId = await readFile(path.join(DATA, destination_stack_id, ENVIRONMENTS_DIR_NAME), ENVIRONMENTS_FILE_NAME);
const environmentsId = await readFile(path.join(DATA, destination_stack_id, ENVIRONMENTS_DIR_NAME), ENVIRONMENTS_FILE_NAME) ?? {};
const localeId = await readFile(path.join(DATA, destination_stack_id, LOCALE_DIR_NAME), LOCALE_CF_LANGUAGE);

if (assets.fields.file && assets.fields.title) {
Object.values(environmentsId).forEach((env: any) => {
Object?.values?.(environmentsId)?.length && Object.values(environmentsId).forEach((env: any) => {
if (env?.name === assets?.sys?.environment?.sys?.id) {
Object.values(localeId).forEach((locale: any) => {
publishDetails.push({
Expand Down Expand Up @@ -696,6 +696,18 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string,
}
};

const mapLocales = ({ masterLocale, locale, locales }: any) => {
if (locales?.masterLocale?.[masterLocale ?? ''] === locale) {
return Object?.keys(locales?.masterLocale)?.[0]
}
for (const [key, value] of Object?.entries?.(locales) ?? {}) {
if (typeof value !== 'object' && value === locale) {
return key;
}
}
return locale.toLowerCase();
}

/**
* Creates and processes entries from a given package file and saves them to the destination stack directory.
*
Expand All @@ -722,7 +734,8 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string,
*
* @throws Will log errors encountered during file reading, processing, or writing of entries.
*/
const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any): Promise<void> => {
const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string): Promise<void> => {
console.info("🚀 ~ createEntry ~ master_locale:", master_locale)
const srcFunc = 'createEntry';
try {
const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME);
Expand All @@ -735,7 +748,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
if (entries && entries.length > 0) {
const assetId = await readFile(assetsSave, ASSETS_SCHEMA_FILE) ?? [];
const entryId = await readFile(path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME), REFERENCES_FILE_NAME);
const environmentsId = await readFile(environmentSave, ENVIRONMENTS_FILE_NAME);
const environmentsId = await readFile(environmentSave, ENVIRONMENTS_FILE_NAME) ?? {};
const displayField: { [key: string]: any } = {}
content.map((item: any) => {
displayField[item.name.toLowerCase().replace(/[^a-z0-9]+/g, "_")] =
Expand Down Expand Up @@ -781,13 +794,15 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
});
const pathName = getDisplayName(name, displayField);
locales.forEach((locale) => {
const publishDetails = Object?.values?.(environmentsId)
const localeCode = mapLocales({ masterLocale: master_locale, locale, locales: LOCALE_MAPPER });
console.info("🚀 ~ locales.forEach ~ localeCode:", localeCode);
const publishDetails = Object?.values?.(environmentsId)?.length ? Object?.values?.(environmentsId)
.filter((env: any) => env?.name === environment_id)
?.map((env: any) => ({
environment: env?.uid,
version: 1,
locale: locale.toLowerCase(),
}));
locale: locale?.toLowerCase(),
})) : [];

const title = entryData[name][locale][id][pathName] || "";
const urlTitle = title
Expand All @@ -814,21 +829,18 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
);
});
});

return entryData;
},
{}
);
const writePromises = [];

for (const [newKey, values] of Object.entries(result)) {
for await (const [newKey, values] of Object.entries(result)) {
const currentCT = contentTypes?.find((ct: any) => ct?.otherCmsUid === newKey);
const ctName = currentCT?.contentstackUid in mapperKeys ?
mapperKeys?.[currentCT?.contentstackUid] : (currentCT?.contentstackUid ?? newKey.replace(/([A-Z])/g, "_$1").toLowerCase());
for (const [localeKey, localeValues] of Object.entries(
for await (const [localeKey, localeValues] of Object.entries(
values as { [key: string]: any }
)) {
const chunks = await makeChunks(localeValues);
const chunks = makeChunks(localeValues);
for (const [entryKey, entryValue] of Object.entries(localeValues)) {
const message = getLogMessage(
srcFunc,
Expand All @@ -843,14 +855,13 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
entriesSave,
ctName, localeKey.toLowerCase()
);
for (const [chunkId, chunkData] of Object.entries(chunks)) {
for await (const [chunkId, chunkData] of Object.entries(chunks)) {
refs[chunkIndex++] = `${chunkId}-entries.json`;
writePromises.push(writeFile(filePath, `${chunkId}-entries.json`, chunkData))
await writeFile(filePath, `${chunkId}-entries.json`, chunkData)
}
writePromises.push(writeFile(filePath, ENTRIES_MASTER_FILE, refs));
await writeFile(filePath, ENTRIES_MASTER_FILE, refs);
}
}
await Promise.all(writePromises);
} else {
const message = getLogMessage(
srcFunc,
Expand Down
2 changes: 1 addition & 1 deletion api/src/services/contentful/jsonRTE.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ function parseBlockReference(obj: any, lang?: LangType, destination_stack_id?: S

function parseInlineReference(obj: any, lang?: LangType, destination_stack_id?: StackId): any {
const entryId: { [key: string]: any } = destination_stack_id && readFile(path.join(process.cwd(), DATA, destination_stack_id, RTE_REFERENCES_DIR_NAME, RTE_REFERENCES_FILE_NAME));
const entry = Object.entries(entryId).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue[obj.data.target.sys.id]);
const entry = entryId && Object.entries(entryId).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue[obj.data.target.sys.id]);

if (entry) {
const [arrayKey, arrayValue] = entry;
Expand Down
9 changes: 5 additions & 4 deletions api/src/services/migration.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { extensionService } from "./extension.service.js";




/**
* Creates a test stack.
*
Expand Down Expand Up @@ -259,7 +260,7 @@ const startTestMigration = async (req: Request): Promise<any> => {
await contentfulService?.createWebhooks(file_path, project?.current_test_stack_id, projectId);
await contentfulService?.createEnvironment(file_path, project?.current_test_stack_id, projectId);
await contentfulService?.createAssets(file_path, project?.current_test_stack_id, projectId, true);
await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys);
await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale);
await contentfulService?.createVersionFile(project?.current_test_stack_id, projectId);
break;
}
Expand Down Expand Up @@ -299,8 +300,8 @@ const startMigration = async (req: Request): Promise<any> => {
await customLogger(projectId, project?.destination_stack_id, 'info', message);
await setLogFilePath(loggerPath);
const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.destination_stack_id, region, user_id });
await marketPlaceAppService?.createAppManifest({ orgId, destinationStackId: project?.current_test_stack_id, region, userId: user_id });
await extensionService?.createExtension({ destinationStackId: project?.current_test_stack_id });
await marketPlaceAppService?.createAppManifest({ orgId, destinationStackId: project?.destination_stack_id, region, userId: user_id });
await extensionService?.createExtension({ destinationStackId: project?.destination_stack_id });
switch (cms) {
case CMS.SITECORE_V8:
case CMS.SITECORE_V9:
Expand Down Expand Up @@ -337,7 +338,7 @@ const startMigration = async (req: Request): Promise<any> => {
await contentfulService?.createWebhooks(file_path, project?.destination_stack_id, projectId);
await contentfulService?.createEnvironment(file_path, project?.destination_stack_id, projectId);
await contentfulService?.createAssets(file_path, project?.destination_stack_id, projectId);
await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys);
await contentfulService?.createEntry(file_path, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale);
await contentfulService?.createVersionFile(project?.destination_stack_id, projectId);
break;
}
Expand Down
2 changes: 1 addition & 1 deletion api/src/utils/content-type-creator.utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,7 +748,7 @@ export const contenTypeMaker = async ({ contentType, destinationStackId, project
if (currentCt?.uid) {
ct = await mergeTwoCts(ct, currentCt);
}
if (ct?.uid) {
if (ct?.uid && ct?.schema?.length) {
if (contentType?.type === 'global_field') {
const globalSave = path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, GLOBAL_FIELDS_DIR_NAME);
const message = getLogMessage(srcFunc, `Global Field ${ct?.uid} has been successfully Transformed.`, {});
Expand Down
39 changes: 20 additions & 19 deletions api/src/utils/test-folder-creator.utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ const {
ENTRIES_DIR_NAME,
ASSETS_DIR_NAME,
ASSETS_SCHEMA_FILE,
CONTENT_TYPES_DIR_NAME,
CONTENT_TYPES_SCHEMA_FILE,
ENTRIES_MASTER_FILE,
GLOBAL_FIELDS_DIR_NAME,
Expand Down Expand Up @@ -240,22 +239,24 @@ const sortGlobalField = async (baseDir: string, finalData: any) => {
}
}

const sortContentType = async (baseDir: string, finalData: any) => {
const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME);
const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME);
const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8'));
await sortGlobalField(baseDir, finalData);
const contentTypes: any = [];
for await (const ct of finalData) {
const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType);
await lookForReference(findCtData, finalData);
contentTypes?.push(findCtData);
}
await deleteFolderAsync(contentTypePath);
for await (const ctItem of contentTypes) {
await saveContent(ctItem, contentSave);
}
}
//this code can be used in feature

// const sortContentType = async (baseDir: string, finalData: any) => {
// const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME);
// const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME);
// const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8'));
// await sortGlobalField(baseDir, finalData);
// const contentTypes: any = [];
// for await (const ct of finalData) {
// const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType);
// await lookForReference(findCtData, finalData);
// contentTypes?.push(findCtData);
// }
// await deleteFolderAsync(contentTypePath);
// for await (const ctItem of contentTypes) {
// await saveContent(ctItem, contentSave);
// }
// }



Expand All @@ -275,7 +276,7 @@ export const testFolderCreator = async ({ destinationStackId }: any) => {
const pathParts = normalizedPath.split(path.sep);
const ct = pathParts?.[0]; // First directory
const locale = pathParts?.[1]; // Second directory
allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length, entryData, filePath, locale })
allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length ?? 0, entryData, filePath, locale })
}
}
}
Expand All @@ -292,7 +293,7 @@ export const testFolderCreator = async ({ destinationStackId }: any) => {
finalData?.push({ contentType: et?.contentType, entryObj, locale: et?.locale });
});
await sortAssets(baseDir);
await sortContentType(baseDir, finalData);
// await sortContentType(baseDir, finalData);
await deleteFolderAsync(entryDelete);
for await (const entry of finalData) {
const fileMeta = { "1": `${entry?.locale}.json` };
Expand Down