From 2405c2c40a6377a2169b63d3b5e786ccee3a27e2 Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Wed, 9 Oct 2024 19:25:16 +0530 Subject: [PATCH 1/3] intial code --- ui/src/pages/Migration/index.tsx | 255 ++++++++++++----------- ui/src/services/api/migration.service.ts | 9 + 2 files changed, 140 insertions(+), 124 deletions(-) diff --git a/ui/src/pages/Migration/index.tsx b/ui/src/pages/Migration/index.tsx index e1d46747..ee78ee11 100644 --- a/ui/src/pages/Migration/index.tsx +++ b/ui/src/pages/Migration/index.tsx @@ -6,10 +6,10 @@ import { cbModal, Notification } from '@contentstack/venus-components'; // Redux files import { RootState } from '../../store'; -import { updateMigrationData, updateNewMigrationData } from '../../store/slice/migrationDataSlice'; +import { updateMigrationData, updateNewMigrationData } from '../../store/slice/migrationDataSlice'; // Services -import { getMigrationData, updateCurrentStepData, updateLegacyCMSData, updateDestinationStack, updateAffixData, fileformatConfirmation, updateFileFormatData, affixConfirmation, updateStackDetails, getExistingContentTypes, getExistingGlobalFields } from '../../services/api/migration.service'; +import { getMigrationData, updateCurrentStepData, updateLegacyCMSData, updateDestinationStack, updateAffixData, fileformatConfirmation, updateFileFormatData, affixConfirmation, updateStackDetails, getExistingContentTypes, getExistingGlobalFields, startMigration } from '../../services/api/migration.service'; import { getCMSDataFromFile } from '../../cmsData/cmsSelector'; // Utilities @@ -59,9 +59,9 @@ const Migration = () => { const stepperRef = useRef(null); const legacyCMSRef = useRef(null); - const selectedOrganisation = useSelector((state: RootState)=>state?.authentication?.selectedOrganisation); - const newMigrationData = useSelector((state:RootState)=> state?.migration?.newMigrationData); - const organisationsList = useSelector((state:RootState)=>state?.authentication?.organisationsList); + const selectedOrganisation = useSelector((state: RootState) => state?.authentication?.selectedOrganisation); + const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); + const organisationsList = useSelector((state: RootState) => state?.authentication?.organisationsList); const saveRef = useRef(null); @@ -69,17 +69,17 @@ const Migration = () => { fetchData(); }, [params?.stepId, params?.projectId, selectedOrganisation?.value]); - useEffect(()=> { + useEffect(() => { dispatch(updateNewMigrationData({ ...newMigrationData, - isprojectMapped : isProjectMapper - + isprojectMapped: isProjectMapper + })); - - },[isProjectMapper]); - // Function to get exisiting content types list - const fetchExistingContentTypes = async () => { + }, [isProjectMapper]); + + // Function to get exisiting content types list + const fetchExistingContentTypes = async () => { const { data, status } = await getExistingContentTypes(projectId); if (status === 201) { return data?.contentTypes; @@ -121,7 +121,7 @@ const Migration = () => { migration_steps_heading: data?.migration_steps_heading, settings: data?.settings })); - + await fetchProjectData(); const stepIndex = data?.all_steps?.findIndex((step: IFlowStep) => `${step?.name}` === params?.stepId); setCurrentStepIndex(stepIndex !== -1 ? stepIndex : 0); @@ -129,60 +129,60 @@ const Migration = () => { //Fetch project data const fetchProjectData = async () => { - if (isEmptyString(selectedOrganisation?.value) || isEmptyString(params?.projectId)) return; + if (isEmptyString(selectedOrganisation?.value) || isEmptyString(params?.projectId)) return; - const data = await getMigrationData(selectedOrganisation?.value, params?.projectId || ''); - if (data) { - setIsLoading(false); - setProjectData(data?.data); - } - setIsProjectMapper(true); - const projectData = data?.data; + const data = await getMigrationData(selectedOrganisation?.value, params?.projectId || ''); + if (data) { + setIsLoading(false); + setProjectData(data?.data); + } + setIsProjectMapper(true); + const projectData = data?.data; - const legacyCmsData:ILegacyCMSComponent = await getCMSDataFromFile(CS_ENTRIES.LEGACY_CMS); + const legacyCmsData: ILegacyCMSComponent = await getCMSDataFromFile(CS_ENTRIES.LEGACY_CMS); - const selectedCmsData: ICMSType = validateArray(legacyCmsData?.all_cms) - ? legacyCmsData?.all_cms?.find((cms: ICMSType) => cms?.cms_id === projectData?.legacy_cms?.cms) ?? DEFAULT_CMS_TYPE - : DEFAULT_CMS_TYPE; + const selectedCmsData: ICMSType = validateArray(legacyCmsData?.all_cms) + ? legacyCmsData?.all_cms?.find((cms: ICMSType) => cms?.cms_id === projectData?.legacy_cms?.cms) ?? DEFAULT_CMS_TYPE + : DEFAULT_CMS_TYPE; - const selectedFileFormatData: ICardType | undefined = validateArray( - selectedCmsData?.allowed_file_formats - ) - ? selectedCmsData.allowed_file_formats?.find( + const selectedFileFormatData: ICardType | undefined = validateArray( + selectedCmsData?.allowed_file_formats + ) + ? selectedCmsData.allowed_file_formats?.find( (cms: ICardType) => cms?.fileformat_id === projectData?.legacy_cms?.file_format ) - : defaultCardType; - - - const selectedOrganisationData = validateArray(organisationsList) - ? organisationsList?.find((org: IDropDown) => org?.value === projectData?.org_id) - : selectedOrganisation; - - let selectedStackData: IDropDown = { - value: projectData?.destination_stack_id, - label: '', - master_locale: '', - locales: [], - created_at: '', - isNewStack: false - }; - - selectedStackData = { - label: projectData?.stackDetails?.label, - value: projectData?.stackDetails?.value, - master_locale: projectData?.stackDetails?. master_locale, - created_at: projectData?.stackDetails?.created_at, - locales:[], - isNewStack: projectData?.stackDetails?.isNewStack - }; + : defaultCardType; + + + const selectedOrganisationData = validateArray(organisationsList) + ? organisationsList?.find((org: IDropDown) => org?.value === projectData?.org_id) + : selectedOrganisation; + + let selectedStackData: IDropDown = { + value: projectData?.destination_stack_id, + label: '', + master_locale: '', + locales: [], + created_at: '', + isNewStack: false + }; - const existingContentTypes = await fetchExistingContentTypes(); - const existingGlobalFields = await fetchExistingGlobalFields(); - - const stackLink = `${CS_URL[projectData?.region]}/stack/${projectData?.current_test_stack_id}/dashboard`; - - const projectMapper = { - ...newMigrationData, + selectedStackData = { + label: projectData?.stackDetails?.label, + value: projectData?.stackDetails?.value, + master_locale: projectData?.stackDetails?.master_locale, + created_at: projectData?.stackDetails?.created_at, + locales: [], + isNewStack: projectData?.stackDetails?.isNewStack + }; + + const existingContentTypes = await fetchExistingContentTypes(); + const existingGlobalFields = await fetchExistingGlobalFields(); + + const stackLink = `${CS_URL[projectData?.region]}/stack/${projectData?.current_test_stack_id}/dashboard`; + + const projectMapper = { + ...newMigrationData, legacy_cms: { ...newMigrationData?.legacy_cms, selectedCms: selectedCmsData, @@ -200,7 +200,7 @@ const Migration = () => { }, isValidated: projectData?.legacy_cms?.is_fileValid }, - isFileFormatCheckboxChecked: true, + isFileFormatCheckboxChecked: true, isRestictedKeywordCheckboxChecked: true, projectStatus: projectData?.status, currentStep: -1, @@ -208,7 +208,7 @@ const Migration = () => { destination_stack: { selectedOrg: selectedOrganisationData, selectedStack: selectedStackData, - stackArray:[] + stackArray: [] }, content_mapping: { isDropDownChanged: false, @@ -231,58 +231,58 @@ const Migration = () => { }; - const createStepper = (projectData: MigrationResponse,handleStepChange: (currentStep: number) => void) => { + const createStepper = (projectData: MigrationResponse, handleStepChange: (currentStep: number) => void) => { const steps = [ { data: , - id:'1', - title:'Legacy CMS' + ref={legacyCMSRef} + legacyCMSData={projectData?.legacy_cms} + handleStepChange={handleStepChange} + isCompleted={isCompleted} + handleOnAllStepsComplete={handleOnAllStepsComplete} />, + id: '1', + title: 'Legacy CMS' }, { data: , - id:'2', - title:'Destination Stack' + destination_stack={projectData?.destination_stack_id} + org_id={projectData?.org_id} + projectData={projectData} + handleStepChange={handleStepChange} + isCompleted={isCompleted} + handleOnAllStepsComplete={handleOnAllStepsComplete} />, + id: '2', + title: 'Destination Stack' }, { data: , - id:'3', - title:'Content Mapping' + id: '3', + title: 'Content Mapping' }, { data: , - id:'4', - title:'Test Migration' + id: '4', + title: 'Test Migration' }, { data: , - id:'5', - title:'Migration Execution' + id: '5', + title: 'Migration Execution' } - ] - return steps; + ] + return steps; } const handleClick = () => { // Call handleStepChange function - const x : string | undefined= params.stepId - const currentStep : number = parseInt(x || ''); - stepperRef?.current?.handleStepChange(currentStep-1); + const x: string | undefined = params.stepId + const currentStep: number = parseInt(x || ''); + stepperRef?.current?.handleStepChange(currentStep - 1); }; - const handleStepChange = (currentStep: number) => { + const handleStepChange = (currentStep: number) => { if (stepperRef?.current) { - stepperRef.current.handleStepChange(currentStep-1); + stepperRef.current.handleStepChange(currentStep - 1); } }; @@ -292,7 +292,7 @@ const Migration = () => { }; // handle on proceed to destination stack - const handleOnClickLegacyCms = async (event: MouseEvent ) => { + const handleOnClickLegacyCms = async (event: MouseEvent) => { setIsLoading(true); if (isCompleted) { @@ -305,17 +305,17 @@ const Migration = () => { await updateAffixData(selectedOrganisation?.value, projectId, { affix: newMigrationData?.legacy_cms?.affix }); await fileformatConfirmation(selectedOrganisation?.value, projectId, { fileformat_confirmation: true - }); + }); await affixConfirmation(selectedOrganisation?.value, projectId, { affix_confirmation: true }); await updateFileFormatData(selectedOrganisation?.value, projectId, { - file_format: newMigrationData?.legacy_cms?.selectedCms?.allowed_file_formats[0]?.fileformat_id?.toString() , + file_format: newMigrationData?.legacy_cms?.selectedCms?.allowed_file_formats[0]?.fileformat_id?.toString(), file_path: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.localPath, is_fileValid: newMigrationData?.legacy_cms?.uploadedFile?.isValidated, is_localPath: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.isLocalPath, - awsDetails:{ + awsDetails: { awsRegion: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.awsData?.awsRegion, bucketName: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.awsData?.bucketName, buketKey: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.awsData?.buketKey @@ -334,9 +334,9 @@ const Migration = () => { setIsLoading(false); if (legacyCMSRef?.current) { - const currentIndex = legacyCMSRef?.current?.getInternalActiveStepIndex() + 1; + const currentIndex = legacyCMSRef?.current?.getInternalActiveStepIndex() + 1; let result; - switch (currentIndex ) { + switch (currentIndex) { case 0: result = 'CMS'; break; @@ -356,25 +356,25 @@ const Migration = () => { } } - + }; // handle on proceed to content mapping const handleOnClickDestinationStack = async (event: MouseEvent) => { setIsLoading(true); - if(isCompleted && !isEmptyString(newMigrationData?.destination_stack?.selectedStack?.value)){ + if (isCompleted && !isEmptyString(newMigrationData?.destination_stack?.selectedStack?.value)) { event?.preventDefault(); //Update Data in backend await updateDestinationStack(selectedOrganisation?.value, projectId, { stack_api_key: newMigrationData?.destination_stack?.selectedStack?.value }); - await updateStackDetails(selectedOrganisation?.value, projectId,{ - label:newMigrationData?.destination_stack?.selectedStack?.label, - value:newMigrationData?.destination_stack?.selectedStack?.value, - master_locale:newMigrationData?.destination_stack?.selectedStack?.master_locale, - created_at:newMigrationData?.destination_stack?.selectedStack?.created_at, + await updateStackDetails(selectedOrganisation?.value, projectId, { + label: newMigrationData?.destination_stack?.selectedStack?.label, + value: newMigrationData?.destination_stack?.selectedStack?.value, + master_locale: newMigrationData?.destination_stack?.selectedStack?.master_locale, + created_at: newMigrationData?.destination_stack?.selectedStack?.created_at, isNewStack: newMigrationData?.destination_stack?.selectedStack?.isNewStack }) handleStepChange(2); @@ -384,7 +384,7 @@ const Migration = () => { const url = `/projects/${projectId}/migration/steps/3`; navigate(url, { replace: true }); } - } else{ + } else { setIsLoading(false); Notification({ notificationContent: { text: 'Please select a stack to proceed further' }, @@ -396,32 +396,32 @@ const Migration = () => { const handleOnClickContentMapper = async (event: MouseEvent) => { setIsModalOpen(true); - if(newMigrationData?.content_mapping?.isDropDownChanged) { + if (newMigrationData?.content_mapping?.isDropDownChanged) { return cbModal({ component: (props: ModalObj) => ( - { - const url = `/projects/${projectId}/migration/steps/4`; - navigate(url, { replace: true }); - - await updateCurrentStepData(selectedOrganisation.value, projectId); - handleStepChange(3); - }} + const url = `/projects/${projectId}/migration/steps/4`; + navigate(url, { replace: true }); + + await updateCurrentStepData(selectedOrganisation.value, projectId); + handleStepChange(3); + }} dropdownStateChange={changeDropdownState} - /> + /> ), modalProps: { - size: 'xsmall', - shouldCloseOnOverlayClick: false + size: 'xsmall', + shouldCloseOnOverlayClick: false } - }); + }); } - else{ + else { event.preventDefault(); const url = `/projects/${projectId}/migration/steps/4`; navigate(url, { replace: true }); @@ -442,7 +442,7 @@ const Migration = () => { handleStepChange(4); } - const changeDropdownState = () =>{ + const changeDropdownState = () => { const newMigrationDataObj: INewMigration = { ...newMigrationData, content_mapping: { ...newMigrationData?.content_mapping, isDropDownChanged: false } @@ -451,17 +451,24 @@ const Migration = () => { dispatch(updateNewMigrationData((newMigrationDataObj))); } + const handleOnClickExecutionMigration = async () => { + setIsLoading(false); + + await startMigration(selectedOrganisation.value, projectId) + } + const handleOnClickFunctions = [ - handleOnClickLegacyCms, + handleOnClickLegacyCms, handleOnClickDestinationStack, handleOnClickContentMapper, - handleOnClickTestMigration + handleOnClickTestMigration, + handleOnClickExecutionMigration ]; - + return (
- {projectData && - + {projectData && + }
diff --git a/ui/src/services/api/migration.service.ts b/ui/src/services/api/migration.service.ts index f13ee155..ea164008 100644 --- a/ui/src/services/api/migration.service.ts +++ b/ui/src/services/api/migration.service.ts @@ -311,4 +311,13 @@ export const createTestMigration = async (orgId: string, projectId: string) => { } catch (error) { return error; } +}; + +export const startMigration = async (orgId: string, projectId: string) => { + try { + return await postCall( + `${API_VERSION}/migration/start/${orgId}/${projectId}`, {}, options); + } catch (error) { + return error; + } }; \ No newline at end of file From 600178824b752f221cb0142ad18883d3f58e3ee6 Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Fri, 11 Oct 2024 15:14:47 +0530 Subject: [PATCH 2/3] intial code --- api/src/services/sitecore.service.ts | 147 ++++++++++++++----- api/src/utils/content-type-creator.utils.ts | 18 +++ api/src/utils/entries-field-creator.utils.ts | 2 +- 3 files changed, 129 insertions(+), 38 deletions(-) diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 4fe9fa45..0a36e98d 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -6,6 +6,9 @@ import _ from 'lodash'; import { LOCALE_MAPPER } from '../constants/index.js'; import { entriesFieldCreator, unflatten } from '../utils/entries-field-creator.utils.js'; import { orgService } from './org.service.js'; +import { getLogMessage } from '../utils/index.js'; +import logger from '../utils/logger.js'; + const append = "a"; @@ -78,6 +81,7 @@ const uidCorrector = ({ uid }: any) => { } const cretaeAssets = async ({ packagePath, baseDir }: any) => { + const srcFunc = 'cretaeAssets'; const assetsSave = path.join(baseDir, 'assets'); const allAssetJSON: any = {}; const folderName: any = path.join(packagePath, 'items', 'master', 'sitecore', 'media library'); @@ -121,6 +125,14 @@ const cretaeAssets = async ({ packagePath, baseDir }: any) => { , assets) } catch (err) { console.error("🚀 ~ file: assets.js:52 ~ xml_folder?.forEach ~ err:", err) + logger.error( + getLogMessage( + srcFunc, + `Not able to read the asset"${jsonAsset?.item?.$?.name}(${mestaData?.uid})".`, + {}, + err + ) + ) } allAssetJSON[mestaData?.uid] = { urlPath: `/assets/${mestaData?.uid}`, @@ -135,9 +147,22 @@ const cretaeAssets = async ({ packagePath, baseDir }: any) => { publish_details: [], assetPath } + logger.info( + getLogMessage( + srcFunc, + `Asset "${jsonAsset?.item?.$?.name}" has been successfully transformed.`, + {} + ) + ) allAssetJSON[mestaData?.uid].parent_uid = '2146b0cee522cc3a38d' } else { - console.info('blob is not there for this asstes', mestaData?.uid, '.') + logger.error( + getLogMessage( + srcFunc, + `Asset "${jsonAsset?.item?.$?.name}" blob is not there for this asstes.`, + {} + ) + ) } } } @@ -164,6 +189,7 @@ const cretaeAssets = async ({ packagePath, baseDir }: any) => { const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', destinationStackId }: { packagePath: any; contentTypes: any; master_locale?: string, destinationStackId: string }) => { try { + const srcFunc = 'createEntry'; const baseDir = path.join('sitecoreMigrationData', destinationStackId); const entrySave = path.join(baseDir, 'entries'); const allAssetJSON: any = await cretaeAssets({ packagePath, baseDir }); @@ -192,6 +218,13 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', } } for await (const ctType of contentTypes) { + logger.info( + getLogMessage( + srcFunc, + `Transforming entries of Content Type ${ctType?.contentstackUid} has begun.`, + {} + ) + ) const entryPresent: any = entriesData?.find((item: any) => uidCorrector({ uid: item?.template }) === ctType?.contentstackUid) if (entryPresent) { const locales: any = Object?.keys(entryPresent?.locale); @@ -221,6 +254,13 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', } if (Object.keys?.(entryObj)?.length > 1) { entryLocale[uid] = unflatten(entryObj) ?? {}; + logger.info( + getLogMessage( + srcFunc, + `Entry title "${entryObj?.title}"(${ctType?.contentstackUid}) in the ${newLocale} locale has been successfully transformed.`, + {} + ) + ) } }); } @@ -234,6 +274,13 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', await writeFiles(entryPath, fileMeta, entryLocale, newLocale) } } else { + logger.error( + getLogMessage( + srcFunc, + `No entries found for the content type ${ctType?.contentstackUid}.`, + {} + ) + ) console.info('Entries missing for', ctType?.contentstackUid) } } @@ -244,45 +291,71 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', } const createLocale = async (req: any, destinationStackId: string) => { - const baseDir = path.join('sitecoreMigrationData', destinationStackId); - const localeSave = path.join(baseDir, 'locale'); - const allLocalesResp = await orgService.getLocales(req) - const masterLocale = Object?.keys?.(LOCALE_MAPPER?.masterLocale)?.[0]; - const msLocale: any = {}; - const uid = uuidv4(); - msLocale[uid] = { - "code": masterLocale, - "fallback_locale": null, - "uid": uid, - "name": allLocalesResp?.data?.locales?.[masterLocale] ?? '' - } - const allLocales: any = {}; - for (const [key, value] of Object.entries(LOCALE_MAPPER)) { - const localeUid = uuidv4(); - if (key !== 'masterLocale' && typeof value === 'string') { - allLocales[localeUid] = { - "code": value, - "fallback_locale": masterLocale, - "uid": localeUid, - "name": allLocalesResp?.data?.locales?.[value] ?? '' - } + const srcFunc = 'createLocale'; + try { + const baseDir = path.join('sitecoreMigrationData', destinationStackId); + const localeSave = path.join(baseDir, 'locale'); + const allLocalesResp = await orgService.getLocales(req) + const masterLocale = Object?.keys?.(LOCALE_MAPPER?.masterLocale)?.[0]; + const msLocale: any = {}; + const uid = uuidv4(); + msLocale[uid] = { + "code": masterLocale, + "fallback_locale": null, + "uid": uid, + "name": allLocalesResp?.data?.locales?.[masterLocale] ?? '' } - } - const masterPath = path.join(localeSave, 'master-locale.json'); - const allLocalePath = path.join(localeSave, 'locales.json'); - fs.access(localeSave, async (err) => { - if (err) { - fs.mkdir(localeSave, { recursive: true }, async (err) => { - if (!err) { - await writeOneFile(masterPath, msLocale); - await writeOneFile(allLocalePath, allLocales); + logger.info( + getLogMessage( + srcFunc, + `Master locale ${masterLocale} has been successfully transformed.`, + {} + ) + ) + const allLocales: any = {}; + for (const [key, value] of Object.entries(LOCALE_MAPPER)) { + const localeUid = uuidv4(); + if (key !== 'masterLocale' && typeof value === 'string') { + allLocales[localeUid] = { + "code": value, + "fallback_locale": masterLocale, + "uid": localeUid, + "name": allLocalesResp?.data?.locales?.[value] ?? '' } - }) - } else { - await writeOneFile(masterPath, msLocale); - await writeOneFile(allLocalePath, allLocales); + logger.info( + getLogMessage( + srcFunc, + `locale ${value} has been successfully transformed.`, + {} + ) + ) + } } - }) + const masterPath = path.join(localeSave, 'master-locale.json'); + const allLocalePath = path.join(localeSave, 'locales.json'); + fs.access(localeSave, async (err) => { + if (err) { + fs.mkdir(localeSave, { recursive: true }, async (err) => { + if (!err) { + await writeOneFile(masterPath, msLocale); + await writeOneFile(allLocalePath, allLocales); + } + }) + } else { + await writeOneFile(masterPath, msLocale); + await writeOneFile(allLocalePath, allLocales); + } + }) + } catch (err) { + logger.error( + getLogMessage( + srcFunc, + `error while Createing the locales.`, + {}, + err + ) + ) + } } const createVersionFile = async (destinationStackId: string) => { diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index 1e0ffce4..d76c3432 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -1,5 +1,8 @@ import fs from 'fs'; import path from 'path'; +import logger from './logger.js'; +import { getLogMessage } from './index.js'; + interface Group { data_type: string; display_name?: string; // Assuming item?.contentstackField might be undefined @@ -406,6 +409,7 @@ const writeGlobalField = async (schema: any, globalSave: string) => { }; export const contenTypeMaker = async ({ contentType, destinationStackId }: any) => { + const srcFunc = 'contenTypeMaker'; const ct: ContentType = { title: contentType?.contentstackTitle, uid: contentType?.contentstackUid, @@ -451,9 +455,23 @@ export const contenTypeMaker = async ({ contentType, destinationStackId }: any) if (ct?.uid) { if (contentType?.type === 'global_field') { const globalSave = path.join('sitecoreMigrationData', destinationStackId, 'global_fields'); + logger.info( + getLogMessage( + srcFunc, + `Global Field ${ct?.uid} has been successfully Transformed.`, + {} + ) + ) await writeGlobalField(ct, globalSave); } else { const contentSave = path.join('sitecoreMigrationData', destinationStackId, 'content_types'); + logger.info( + getLogMessage( + srcFunc, + `ContentType ${ct?.uid} has been successfully Transformed.`, + {} + ) + ) await saveContent(ct, contentSave); } } else { diff --git a/api/src/utils/entries-field-creator.utils.ts b/api/src/utils/entries-field-creator.utils.ts index 7f0cb287..68098ebb 100644 --- a/api/src/utils/entries-field-creator.utils.ts +++ b/api/src/utils/entries-field-creator.utils.ts @@ -324,7 +324,7 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse } default: { - console.info(field?.ContentstackFieldType, 'umesh'); + console.info(field?.ContentstackFieldType, 'field missing'); return content; } } From 7e284027710dccdc4c24c1362c9c68569065875d Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Fri, 18 Oct 2024 22:12:40 +0530 Subject: [PATCH 3/3] refactor code --- api/src/config/prod.config.ts | 2 +- api/src/models/FieldMapper.ts | 3 +- api/src/models/contentTypesMapper-lowdb.ts | 3 +- api/src/models/project-lowdb.ts | 2 +- api/src/server.ts | 68 ++++++----- api/src/services/contentMapper.service.ts | 18 +-- api/src/services/migration.service.ts | 24 +++- api/src/services/runCli.service.ts | 10 +- api/src/services/sitecore.service.ts | 113 ++++++++---------- api/src/utils/content-type-creator.utils.ts | 23 ++-- api/src/utils/custom-logger.utils.ts | 82 +++++++++++++ api/src/utils/field-attacher.utils.ts | 2 +- api/src/utils/index.ts | 27 +++-- api/src/utils/watch.utils.ts | 39 ++++++ .../Actions/LoadOrganisation.tsx | 10 +- upload-api/.env | 2 +- upload-api/src/controllers/sitecore/index.ts | 1 + 17 files changed, 283 insertions(+), 146 deletions(-) create mode 100644 api/src/utils/custom-logger.utils.ts create mode 100644 api/src/utils/watch.utils.ts diff --git a/api/src/config/prod.config.ts b/api/src/config/prod.config.ts index a873b12a..07f9ef2a 100644 --- a/api/src/config/prod.config.ts +++ b/api/src/config/prod.config.ts @@ -14,5 +14,5 @@ export const prodConfig = { GCP_NA: "https://gcp-na-app.contentstack.com/#!", }, LOG_FILE_PATH: - process.platform === "win32" ? ".\\combine.log" : "./combine.log", // Replace with the actual path to your log file + process.platform === "win32" ? ".\\sample.log" : "./sample.log", // Replace with the actual path to your log file }; diff --git a/api/src/models/FieldMapper.ts b/api/src/models/FieldMapper.ts index aedea35e..4f93770a 100644 --- a/api/src/models/FieldMapper.ts +++ b/api/src/models/FieldMapper.ts @@ -1,5 +1,6 @@ import { JSONFile } from "lowdb/node"; import LowWithLodash from "../utils/lowdb-lodash.utils.js"; +import path from "path"; /** * Represents the advanced configuration options for a field mapper. @@ -44,7 +45,7 @@ const defaultData: FieldMapper = { field_mapper: [] }; * Represents the database instance for the FieldMapper model. */ const db = new LowWithLodash( - new JSONFile("database/field-mapper.json"), + new JSONFile(path.join(process.cwd(), "database", "field-mapper.json")), defaultData ); diff --git a/api/src/models/contentTypesMapper-lowdb.ts b/api/src/models/contentTypesMapper-lowdb.ts index cf7c667f..e4163818 100644 --- a/api/src/models/contentTypesMapper-lowdb.ts +++ b/api/src/models/contentTypesMapper-lowdb.ts @@ -1,4 +1,5 @@ import { JSONFile } from "lowdb/node"; +import path from 'path'; import LowWithLodash from "../utils/lowdb-lodash.utils.js"; /** @@ -80,7 +81,7 @@ const defaultData: ContentTypeMapperDocument = { ContentTypesMappers: [] }; * Represents the database instance for the content types mapper. */ const db = new LowWithLodash( - new JSONFile("database/contentTypesMapper.json"), + new JSONFile(path.join(process.cwd(), "database", 'contentTypesMapper.json')), defaultData ); diff --git a/api/src/models/project-lowdb.ts b/api/src/models/project-lowdb.ts index 693899ba..6279d4c8 100644 --- a/api/src/models/project-lowdb.ts +++ b/api/src/models/project-lowdb.ts @@ -63,7 +63,7 @@ interface Project { test_stacks: []; current_test_stack_id: string; legacy_cms: LegacyCMS; - content_mapper: []; + content_mapper: any[]; execution_log: [ExecutionLog]; created_at: string; updated_at: string; diff --git a/api/src/server.ts b/api/src/server.ts index 5ae0274b..157dd0e9 100644 --- a/api/src/server.ts +++ b/api/src/server.ts @@ -19,14 +19,15 @@ import migrationRoutes from "./routes/migration.routes.js"; import chokidar from "chokidar"; import { Server } from "socket.io"; import fs from "fs"; +import path from "path"; // Initialize file watcher for the log file const watcher = chokidar.watch(config.LOG_FILE_PATH, { usePolling: true, // Enables polling to detect changes in all environments - interval: 100, // Poll every 100ms (you can adjust this if needed) + interval: 1, // Poll every 100ms (you can adjust this if needed) awaitWriteFinish: { // Wait for file to finish being written before triggering - stabilityThreshold: 500, // Time to wait before considering the file stable - pollInterval: 100, // Interval at which to poll for file stability + stabilityThreshold: 1, // Time to wait before considering the file stable + pollInterval: 1, // Interval at which to poll for file stability }, persistent: true, // Keeps watching the file even after initial change }); // Initialize with initial log path @@ -34,15 +35,32 @@ const watcher = chokidar.watch(config.LOG_FILE_PATH, { let io: Server; // Socket.IO server instance // Dynamically change the log file path and update the watcher -export async function setLogFilePath(path: string) { - console.info(`Setting new log file path: ${path}`); - - // Stop watching the old log file - watcher.unwatch(config.LOG_FILE_PATH); - - // Update the config and start watching the new log file - config.LOG_FILE_PATH = path; - watcher.add(path); +export async function setLogFilePath(newPath: string) { + try { + // Ensure the new log file path is absolute and valid + const absolutePath = path.resolve(newPath); + console.info(`Attempting to set new log file path: ${absolutePath}`); + // Check if the new log file exists + // Stop watching the old log file + if (config.LOG_FILE_PATH) { + console.info(`Stopping watcher for previous log file: ${config.LOG_FILE_PATH}`); + watcher.unwatch(config.LOG_FILE_PATH); + } + // Update the config to use the new log file path + config.LOG_FILE_PATH = absolutePath; + + // Start watching the new log file + console.info(`Starting watcher for new log file: ${absolutePath}`); + watcher.add(absolutePath); + + } catch (error: any) { + console.error(`Failed to set new log file path: ${error.message}`); + // Optional: fallback to default or previous log file if the new path is invalid + if (config.LOG_FILE_PATH) { + console.info(`Reverting to previous log file path: ${config.LOG_FILE_PATH}`); + watcher.add(config.LOG_FILE_PATH); // Re-watch previous log file + } + } } try { @@ -99,26 +117,20 @@ try { }); // Emit initial log file content to connected clients - - // File watcher for log file changes - watcher.on("change", (path) => { + // File watcher for log file changes + watcher.on("change", async (path) => { console.info(`File changed: ${path}`); // Read the updated file content - fs.readFile(path, "utf8", (err, data) => { - if (err) { - logger.error(`Error reading log file: ${err}`); - return; - } - try { - // Emit the updated log content to connected clients - io.emit("logUpdate", data); - } catch (error) { - logger.error(`Error emitting log data: ${error}`); - } - }); + try { + const data = await fs.promises.readFile(path, "utf8") + // Emit the updated log content to connected clients + io.emit("logUpdate", data); + } catch (error) { + logger.error(`Error emitting log data: ${error}`); + } }); - })(); + } catch (e) { logger.error("Error while starting the server!"); logger.error(e); diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index e13275e6..7d6ea6c4 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -34,6 +34,7 @@ const putTestData = async (req: Request) => { const projectId = req.params.projectId; const contentTypes = req.body.contentTypes; + await FieldMapperModel.read(); /* @@ -61,7 +62,7 @@ const putTestData = async (req: Request) => { }); await ContentTypesMapperModelLowdb.read(); - const contentIds: string[] = []; + const contentIds: any[] = []; /* this code snippet is iterating over an array called contentTypes and @@ -70,7 +71,7 @@ const putTestData = async (req: Request) => { and the generated id values are pushed into the contentIds array. */ const contentType = contentTypes.map((item: any) => { - const id = item?.id.replace(/[{}]/g, "")?.toLowerCase() || uuidv4(); + const id = item?.id?.replace(/[{}]/g, "")?.toLowerCase() || uuidv4(); item.id = id; contentIds.push(id); return { ...item, id, projectId }; @@ -88,11 +89,12 @@ const putTestData = async (req: Request) => { .get("projects") .findIndex({ id: projectId }) .value(); - if (index > -1) { - ProjectModelLowdb.update((data: any) => { - data.projects[index].content_mapper = contentIds; - data.projects[index].extract_path = req?.body?.extractPath; - }); + if (index > -1 && contentIds?.length) { + ProjectModelLowdb.data.projects[index].content_mapper = contentIds; + ProjectModelLowdb.data.projects[index].extract_path = req?.body?.extractPath; + ProjectModelLowdb.write(); + } else { + throw new BadRequestError(HTTP_TEXTS.CONTENT_TYPE_NOT_FOUND); } const pData = ProjectModelLowdb.chain @@ -900,7 +902,7 @@ const getSingleGlobalField = async (req: Request) => { }, }) ); - + if (err) return { data: err.response.data, diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 9d58ac7f..c886b213 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -1,4 +1,5 @@ import { Request } from "express"; +import path from "path"; import ProjectModelLowdb from "../models/project-lowdb.js"; import { config } from "../config/index.js"; import { safePromise, getLogMessage } from "../utils/index.js"; @@ -12,6 +13,9 @@ import { fieldAttacher } from "../utils/field-attacher.utils.js"; import { siteCoreService } from "./sitecore.service.js"; import { testFolderCreator } from "../utils/test-folder-creator.utils.js"; import { utilsCli } from './runCli.service.js'; +import customLogger from "../utils/custom-logger.utils.js"; +import { setLogFilePath } from "../server.js"; + @@ -208,12 +212,16 @@ const startTestMigration = async (req: Request): Promise => { const project = ProjectModelLowdb.chain.get("projects").find({ id: projectId }).value(); const packagePath = project?.extract_path; if (packagePath && project?.current_test_stack_id) { + const loggerPath = path.join(process.cwd(), 'logs', projectId, `${project?.current_test_stack_id}.log`); + const message = getLogMessage('startTestMigration', 'Starting Test Migration...', {}); + await customLogger(projectId, project?.current_test_stack_id, 'info', message); + await setLogFilePath(loggerPath); const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.current_test_stack_id }); - await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.current_test_stack_id }); - await siteCoreService?.createLocale(req, project?.current_test_stack_id); + await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.current_test_stack_id, projectId }); + await siteCoreService?.createLocale(req, project?.current_test_stack_id, projectId); await siteCoreService?.createVersionFile(project?.current_test_stack_id); await testFolderCreator?.({ destinationStackId: project?.current_test_stack_id }); - await utilsCli?.runCli(region, user_id, project?.current_test_stack_id, projectId, true); + await utilsCli?.runCli(region, user_id, project?.current_test_stack_id, projectId, true, loggerPath); } } @@ -230,11 +238,15 @@ const startMigration = async (req: Request): Promise => { const project = ProjectModelLowdb.chain.get("projects").find({ id: projectId }).value(); const packagePath = project?.extract_path; if (packagePath && project?.destination_stack_id) { + const loggerPath = path.join(process.cwd(), 'logs', projectId, `${project?.destination_stack_id}.log`); + const message = getLogMessage('startTestMigration', 'Starting Migration...', {}); + await customLogger(projectId, project?.destination_stack_id, 'info', message); + await setLogFilePath(loggerPath); const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.destination_stack_id }); - await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.destination_stack_id }); - await siteCoreService?.createLocale(req, project?.destination_stack_id); + await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.destination_stack_id, projectId }); + await siteCoreService?.createLocale(req, project?.destination_stack_id, projectId); await siteCoreService?.createVersionFile(project?.destination_stack_id); - await utilsCli?.runCli(region, user_id, project?.destination_stack_id, projectId); + await utilsCli?.runCli(region, user_id, project?.destination_stack_id, projectId, false, loggerPath); } } diff --git a/api/src/services/runCli.service.ts b/api/src/services/runCli.service.ts index 94c9ee6b..93959845 100644 --- a/api/src/services/runCli.service.ts +++ b/api/src/services/runCli.service.ts @@ -2,11 +2,12 @@ import path from "path"; import fs from 'fs'; import shell from 'shelljs' import { v4 } from "uuid"; -import { setLogFilePath } from "../server.js"; import { copyDirectory, createDirectoryAndFile } from '../utils/index.js' import { CS_REGIONS } from "../constants/index.js"; import ProjectModelLowdb from "../models/project-lowdb.js"; import AuthenticationModel from "../models/authentication.js"; +import watchLogs from "../utils/watch.utils.js"; +import { setLogFilePath } from "../server.js"; const addCustomMessageInCliLogs = async (loggerPath: string, level: string = 'info', message: string) => { try { @@ -22,8 +23,7 @@ const addCustomMessageInCliLogs = async (loggerPath: string, level: string = 'in } } - -export const runCli = async (rg: string, user_id: string, stack_uid: any, projectId: string, isTest = false) => { +export const runCli = async (rg: string, user_id: string, stack_uid: any, projectId: string, isTest = false, transformePath: string) => { try { const regionPresent = CS_REGIONS?.find((item: string) => item === rg) ?? 'NA'; await AuthenticationModel.read(); @@ -36,10 +36,10 @@ export const runCli = async (rg: string, user_id: string, stack_uid: any, projec const backupPath = path.join(process.cwd(), 'migration-data', `${stack_uid}_${v4().slice(0, 4)}`); await copyDirectory(sourcePath, backupPath); const loggerPath = path.join(backupPath, 'logs', 'import', 'success.log'); - createDirectoryAndFile(loggerPath); + await createDirectoryAndFile(loggerPath, transformePath); await setLogFilePath(loggerPath); + await watchLogs(loggerPath, transformePath); shell.cd(path.join(process.cwd(), '..', 'cli', 'packages', 'contentstack')); - shell.exec('pwd'); shell.exec(`node bin/run config:set:region ${regionPresent}`); shell.exec(`node bin/run login -a ${userData?.authtoken} -e ${userData?.email}`); const importData = shell.exec(`node bin/run cm:stacks:import -k ${stack_uid} -d ${sourcePath} --backup-dir=${backupPath} --yes`, { async: true }); diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 0a36e98d..f4de0dea 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -7,7 +7,7 @@ import { LOCALE_MAPPER } from '../constants/index.js'; import { entriesFieldCreator, unflatten } from '../utils/entries-field-creator.utils.js'; import { orgService } from './org.service.js'; import { getLogMessage } from '../utils/index.js'; -import logger from '../utils/logger.js'; +import customLogger from '../utils/custom-logger.utils.js'; const append = "a"; @@ -80,7 +80,7 @@ const uidCorrector = ({ uid }: any) => { return _.replace(uid, new RegExp("[ -]", "g"), '_')?.toLowerCase() } -const cretaeAssets = async ({ packagePath, baseDir }: any) => { +const cretaeAssets = async ({ packagePath, baseDir, destinationStackId, projectId }: any) => { const srcFunc = 'cretaeAssets'; const assetsSave = path.join(baseDir, 'assets'); const allAssetJSON: any = {}; @@ -125,14 +125,13 @@ const cretaeAssets = async ({ packagePath, baseDir }: any) => { , assets) } catch (err) { console.error("🚀 ~ file: assets.js:52 ~ xml_folder?.forEach ~ err:", err) - logger.error( - getLogMessage( - srcFunc, - `Not able to read the asset"${jsonAsset?.item?.$?.name}(${mestaData?.uid})".`, - {}, - err - ) + const message = getLogMessage( + srcFunc, + `Not able to read the asset"${jsonAsset?.item?.$?.name}(${mestaData?.uid})".`, + {}, + err ) + await customLogger(projectId, destinationStackId, 'error', message); } allAssetJSON[mestaData?.uid] = { urlPath: `/assets/${mestaData?.uid}`, @@ -147,22 +146,20 @@ const cretaeAssets = async ({ packagePath, baseDir }: any) => { publish_details: [], assetPath } - logger.info( - getLogMessage( - srcFunc, - `Asset "${jsonAsset?.item?.$?.name}" has been successfully transformed.`, - {} - ) + const message = getLogMessage( + srcFunc, + `Asset "${jsonAsset?.item?.$?.name}" has been successfully transformed.`, + {} ) + await customLogger(projectId, destinationStackId, 'info', message); allAssetJSON[mestaData?.uid].parent_uid = '2146b0cee522cc3a38d' } else { - logger.error( - getLogMessage( - srcFunc, - `Asset "${jsonAsset?.item?.$?.name}" blob is not there for this asstes.`, - {} - ) + const message = getLogMessage( + srcFunc, + `Asset "${jsonAsset?.item?.$?.name}" blob is not there for this asstes.`, + {} ) + await customLogger(projectId, destinationStackId, 'error', message); } } } @@ -187,12 +184,12 @@ const cretaeAssets = async ({ packagePath, baseDir }: any) => { return allAssetJSON; } -const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', destinationStackId }: { packagePath: any; contentTypes: any; master_locale?: string, destinationStackId: string }) => { +const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', destinationStackId, projectId }: { packagePath: any; contentTypes: any; master_locale?: string, destinationStackId: string, projectId: string }) => { try { const srcFunc = 'createEntry'; const baseDir = path.join('sitecoreMigrationData', destinationStackId); const entrySave = path.join(baseDir, 'entries'); - const allAssetJSON: any = await cretaeAssets({ packagePath, baseDir }); + const allAssetJSON: any = await cretaeAssets({ packagePath, baseDir, destinationStackId, projectId }); const folderName: any = path.join(packagePath, 'items', 'master', 'sitecore', 'content'); const entriesData: any = []; if (fs.existsSync(folderName)) { @@ -218,13 +215,12 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', } } for await (const ctType of contentTypes) { - logger.info( - getLogMessage( - srcFunc, - `Transforming entries of Content Type ${ctType?.contentstackUid} has begun.`, - {} - ) + const message = getLogMessage( + srcFunc, + `Transforming entries of Content Type ${ctType?.contentstackUid} has begun.`, + {} ) + await customLogger(projectId, destinationStackId, 'info', message); const entryPresent: any = entriesData?.find((item: any) => uidCorrector({ uid: item?.template }) === ctType?.contentstackUid) if (entryPresent) { const locales: any = Object?.keys(entryPresent?.locale); @@ -254,13 +250,12 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', } if (Object.keys?.(entryObj)?.length > 1) { entryLocale[uid] = unflatten(entryObj) ?? {}; - logger.info( - getLogMessage( - srcFunc, - `Entry title "${entryObj?.title}"(${ctType?.contentstackUid}) in the ${newLocale} locale has been successfully transformed.`, - {} - ) + const message = getLogMessage( + srcFunc, + `Entry title "${entryObj?.title}"(${ctType?.contentstackUid}) in the ${newLocale} locale has been successfully transformed.`, + {} ) + await customLogger(projectId, destinationStackId, 'info', message) } }); } @@ -274,13 +269,12 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', await writeFiles(entryPath, fileMeta, entryLocale, newLocale) } } else { - logger.error( - getLogMessage( - srcFunc, - `No entries found for the content type ${ctType?.contentstackUid}.`, - {} - ) + const message = getLogMessage( + srcFunc, + `No entries found for the content type ${ctType?.contentstackUid}.`, + {} ) + await customLogger(projectId, destinationStackId, 'error', message) console.info('Entries missing for', ctType?.contentstackUid) } } @@ -290,7 +284,7 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', } } -const createLocale = async (req: any, destinationStackId: string) => { +const createLocale = async (req: any, destinationStackId: string, projectId: string) => { const srcFunc = 'createLocale'; try { const baseDir = path.join('sitecoreMigrationData', destinationStackId); @@ -305,13 +299,12 @@ const createLocale = async (req: any, destinationStackId: string) => { "uid": uid, "name": allLocalesResp?.data?.locales?.[masterLocale] ?? '' } - logger.info( - getLogMessage( - srcFunc, - `Master locale ${masterLocale} has been successfully transformed.`, - {} - ) + const message = getLogMessage( + srcFunc, + `Master locale ${masterLocale} has been successfully transformed.`, + {} ) + await customLogger(projectId, destinationStackId, 'info', message); const allLocales: any = {}; for (const [key, value] of Object.entries(LOCALE_MAPPER)) { const localeUid = uuidv4(); @@ -322,13 +315,12 @@ const createLocale = async (req: any, destinationStackId: string) => { "uid": localeUid, "name": allLocalesResp?.data?.locales?.[value] ?? '' } - logger.info( - getLogMessage( - srcFunc, - `locale ${value} has been successfully transformed.`, - {} - ) + const message = getLogMessage( + srcFunc, + `locale ${value} has been successfully transformed.`, + {} ) + await customLogger(projectId, destinationStackId, 'info', message); } } const masterPath = path.join(localeSave, 'master-locale.json'); @@ -347,14 +339,13 @@ const createLocale = async (req: any, destinationStackId: string) => { } }) } catch (err) { - logger.error( - getLogMessage( - srcFunc, - `error while Createing the locales.`, - {}, - err - ) + const message = getLogMessage( + srcFunc, + `error while Createing the locales.`, + {}, + err ) + await customLogger(projectId, destinationStackId, 'error', message); } } diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index d76c3432..12c9a709 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import path from 'path'; import logger from './logger.js'; import { getLogMessage } from './index.js'; +import customLogger from './custom-logger.utils.js'; interface Group { data_type: string; @@ -50,7 +51,7 @@ const arrangGroups = ({ schema }: any) => { } const convertToSchemaFormate = ({ field, advanced = true }: any) => { - // console.info("🚀 ~ convertToSchemaFormate ~ field:", field) + switch (field?.ContentstackFieldType) { case 'single_line_text': { return { @@ -408,7 +409,7 @@ const writeGlobalField = async (schema: any, globalSave: string) => { } }; -export const contenTypeMaker = async ({ contentType, destinationStackId }: any) => { +export const contenTypeMaker = async ({ contentType, destinationStackId, projectId }: any) => { const srcFunc = 'contenTypeMaker'; const ct: ContentType = { title: contentType?.contentstackTitle, @@ -455,23 +456,13 @@ export const contenTypeMaker = async ({ contentType, destinationStackId }: any) if (ct?.uid) { if (contentType?.type === 'global_field') { const globalSave = path.join('sitecoreMigrationData', destinationStackId, 'global_fields'); - logger.info( - getLogMessage( - srcFunc, - `Global Field ${ct?.uid} has been successfully Transformed.`, - {} - ) - ) + const message = getLogMessage(srcFunc, `Global Field ${ct?.uid} has been successfully Transformed.`, {}); + await customLogger(projectId, destinationStackId, 'info', message); await writeGlobalField(ct, globalSave); } else { const contentSave = path.join('sitecoreMigrationData', destinationStackId, 'content_types'); - logger.info( - getLogMessage( - srcFunc, - `ContentType ${ct?.uid} has been successfully Transformed.`, - {} - ) - ) + const message = getLogMessage(srcFunc, `ContentType ${ct?.uid} has been successfully Transformed.`, {}); + await customLogger(projectId, destinationStackId, 'info', message); await saveContent(ct, contentSave); } } else { diff --git a/api/src/utils/custom-logger.utils.ts b/api/src/utils/custom-logger.utils.ts new file mode 100644 index 00000000..01a365e4 --- /dev/null +++ b/api/src/utils/custom-logger.utils.ts @@ -0,0 +1,82 @@ +import fs from 'fs'; +import path from "path"; +import { createLogger, format, transports } from "winston"; +import logger from './logger.js'; + + +const fileExists = async (path: string): Promise => { + try { + await fs.promises.access(path); + return true; // Path exists + } catch (error) { + return false; // Path does not exist + } +} + +//Logger for custom logs +/** + * The logger instance used for logging messages. + */ +const customLogger = async (projectId: string, apiKey: string, level: string, message: string) => { + try { + const logDir = path.join(process.cwd(), 'logs', projectId); + const logFilePath = path.join(logDir, `${apiKey}.log`); + // Ensure log directory exists, using async/await with fs.promises + if (!fs.existsSync(logDir)) { + await fs.promises.mkdir(logDir, { recursive: true }); + } + + if (!fs.existsSync(logFilePath)) { + // If the file does not exist, create it and write an initial log entry + fs.promises.writeFile(logFilePath, 'Log file created\n', { flag: 'a' }); // 'a' flag for appending + console.info(`Log file created and initial entry written: ${logFilePath}`); + } + // Create a logger instance with a file transport + const log = createLogger({ + level: 'info', + format: format.combine(format.timestamp(), format.json()), + transports: [ + // Write logs to a file named after the apiKey + new transports.File({ filename: logFilePath }), + ], + }); + + // Handle the logging levels dynamically + switch (level) { + case 'error': { + log.error(message); // Log to file + logger.error(message); // Log to console/logger + break; + } + case 'warn': { + log.warn(message); + logger.warn(message); + break; + } + case 'info': { + log.info(message); + logger.info(message); + break; + } + case 'debug': { + log.debug(message); + logger.debug(message); + break; + } + default: { + log.info(message); // Default to info level + logger.info(message); + } + } + + if (await fileExists(logFilePath)) { + return; // Exit function if log file exists after logging + } else { + console.error(`Log file was not created.`); + } + } catch (error: any) { + console.error(`Failed to log message: ${error.message}`); + } +}; + +export default customLogger; \ No newline at end of file diff --git a/api/src/utils/field-attacher.utils.ts b/api/src/utils/field-attacher.utils.ts index 40459d52..41c48ca5 100644 --- a/api/src/utils/field-attacher.utils.ts +++ b/api/src/utils/field-attacher.utils.ts @@ -27,7 +27,7 @@ export const fieldAttacher = async ({ projectId, orgId, destinationStackId }: an return field; }) } - await contenTypeMaker({ contentType, destinationStackId }) + await contenTypeMaker({ contentType, destinationStackId, projectId }) contentTypes?.push?.(contentType); } } diff --git a/api/src/utils/index.ts b/api/src/utils/index.ts index 71940cb5..17fb91de 100644 --- a/api/src/utils/index.ts +++ b/api/src/utils/index.ts @@ -77,16 +77,21 @@ export async function copyDirectory(srcDir: string, destDir: string): Promise { + try { + const transformeLogs = await fs.promises.readFile(sourceFile, 'utf8'); + const logMessage = transformeLogs + '\n'; // Append a newline after the log content + await fs.promises.appendFile(destinationFile, logMessage); + console.info(`Merged logs from ${sourceFile} to ${destinationFile}`); + } catch (error: any) { + console.error(`Error merging logs: ${error.message}`); + } +}; + +// Function to watch the log file +const watchLogs = async (sourceFile: string, destinationFile: string) => { + // Initialize watcher + const watcher = chokidar.watch(sourceFile, { + persistent: true, + ignoreInitial: true, // Do not trigger on the existing state + awaitWriteFinish: { + stabilityThreshold: 100, // Wait 1 millisecond before triggering (for fast writes) + pollInterval: 100 // Check for changes every 1 ms + }, + }); + // Event listener for file changes + watcher.on('change', async (path: string) => { + console.info(`File ${path} has been changed`); + // Update the destination file when the source file changes + await mergeLogs(destinationFile, sourceFile); // Await the mergeLogs function + }); + // Handle errors + watcher.on('error', (error) => { + console.error(`Watcher error: ${error}`); + }); +}; + +export default watchLogs; \ No newline at end of file diff --git a/ui/src/components/DestinationStack/Actions/LoadOrganisation.tsx b/ui/src/components/DestinationStack/Actions/LoadOrganisation.tsx index fc33d2c4..dc5ad4b4 100644 --- a/ui/src/components/DestinationStack/Actions/LoadOrganisation.tsx +++ b/ui/src/components/DestinationStack/Actions/LoadOrganisation.tsx @@ -17,8 +17,8 @@ interface LoadOrganisationProps { const LoadOrganisation = (props: LoadOrganisationProps) => { /**** ALL HOOKS HERE ****/ - const newMigrationData = useSelector((state:RootState)=>state?.migration?.newMigrationData); - const selectedOrganisation = useSelector((state:RootState)=>state?.authentication?.selectedOrganisation); + const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); + const selectedOrganisation = useSelector((state: RootState) => state?.authentication?.selectedOrganisation); const dispatch = useDispatch(); @@ -29,7 +29,7 @@ const LoadOrganisation = (props: LoadOrganisationProps) => { const setNewMigrationData = (data: INewMigration) => { dispatch(updateNewMigrationData((data))); }; - + /**** ALL USEEffects HERE ****/ @@ -48,10 +48,10 @@ const LoadOrganisation = (props: LoadOrganisationProps) => { } }); }, []); - + return (
-