From deff74625ea55c7bf22439eafc4d72b8fc9e1faa Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 16:16:04 +0530 Subject: [PATCH 01/19] workflow changes --- .github/workflows/repo-sync.yml | 263 ++++++++++++++-------- upload-api/.DS_Store | Bin 8196 -> 8196 bytes upload-api/migration-contentful/.DS_Store | Bin 6148 -> 6148 bytes 3 files changed, 169 insertions(+), 94 deletions(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index d20041f6..3ee14aa5 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -8,109 +8,184 @@ on: env: BRANCH : dev # Change this to your default branch if needed - SITECORE_REPO: contentstack-expert-services/migration-tool-sitecore - CONTENTFUL_REPO: contentstack-expert-services/migration-tool-contentful - WORDPRESS_REPO: contentstack-expert-services/migration-tool-wordpress - RSYNC_SITECORE_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts/' --exclude 'src/services/wordpress.service.ts/'" # Exclude contentful and wordpress services - RSYNC_CONTENTFUL_EXCLUDES: "--exclude 'src/services/wordpress.service.ts/' --exclude 'src/services/sitecore.service.ts/'" # Exclude wordpress and sitecore services - RSYNC_WORDPRESS_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts/' --exclude 'src/services/sitecore.service.ts/'" # Exclude contentful and sitecore services - + SITECORE_REPO: contentstack-expert-services/migration-tool-sitecore # Change this to your sitecore repo + CONTENTFUL_REPO: contentstack-expert-services/migration-tool-contentful # Change this to your contentful repo + WORDPRESS_REPO: contentstack-expert-services/migration-tool-wordpress # Change this to your wordpress repo + RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts/' --exclude 'src/services/wordpress.service.ts/'" # Include only sitecore services + RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/wordpress.service.ts/' --exclude 'src/services/sitecore.service.ts/'" # Include only contentful services + RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts/' --exclude 'src/services/sitecore.service.ts/'" # Include only wordpress services + RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-contentful/' --exclude 'migration-wordpress/' --exclude 'controllers/wordpress/' --exclude 'models/contentful.json' --exclude 'models/wordpress.json' --exclude 'services/contentful' --exclude 'validators/aem' --exclude 'validators/contentful' --exclude 'validators/wordpress'" # Include only sitecore upload-api + RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-sitecore/' --exclude 'migration-wordpress/' --exclude 'controllers/sitecore/' --exclude 'models/wordpress.json' --exclude 'validators/aem' --exclude 'validators/sitecore' --exclude 'validators/wordpress'" # Include only contentful upload-api + RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-sitecore/' --exclude 'migration-contentful/' --exclude 'controllers/sitecore/' --exclude 'models/contentful.json' --exclude 'validators/aem' --exclude 'validators/sitecore' --exclude 'validators/contentful'" # Include only wordpress upload-api jobs: sync-on-merge: runs-on: ubuntu-latest permissions: - contents: write - pull-requests: write + contents: write + pull-requests: write steps: - - name: Check if PR was merged - id: check_merge - run: | - if [ "${{ github.event.pull_request.merged }}" = "true" ]; then - echo "PR was merged. Proceeding with sync." - echo "merged=true" >> $GITHUB_ENV - else - echo "PR was closed without merging. Skipping sync." - echo "merged=false" >> $GITHUB_ENV - fi - shell: bash + - name: Check if PR was merged + id: check_merge + run: | + if [ "${{ github.event.pull_request.merged }}" = "true" ]; then + echo "PR was merged. Proceeding with sync." + echo "merged=true" >> $GITHUB_ENV + else + echo "PR was closed without merging. Skipping sync." + echo "merged=false" >> $GITHUB_ENV + fi + shell: bash + + - name: Checkout migration-v2 + if: env.merged == 'true' + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch full history to compare changes + + - name: Detect changed files + if: env.merged == 'true' + id: file_changes + uses: dorny/paths-filter@v3 + with: + filters: | + api: + - 'api/**' + cli: + - 'cli/**' + ui: + - 'ui/**' + upload-api: + - 'upload-api/**' + index: + - 'index.js' + + - name: Setup Git + if: env.merged == 'true' + run: | + git config --global user.name "github-actions" + git config --global user.email "github-actions@github.com" + + - name: Sync changes to sitecore-repo (if applicable) + if: | + env.merged == 'true' && + ( + steps.file_changes.outputs.api == 'true' || + steps.file_changes.outputs.cli == 'true' || + steps.file_changes.outputs.ui == 'true' || + steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.index == 'true' + ) + run: | + git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.SITECORE_REPO }}.git + cd migration-tool-sitecore + git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} + rsync -av --delete $RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES ../api/ ./api/ + rsync -av --delete ../cli/ ./cli/ + rsync -av --delete ../ui/ ./ui/ + rsync -av --delete $RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES ../upload-api/ ./upload-api/ + git add . + git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" + git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} - - name: Checkout migration-v2 - if: env.merged == 'true' - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch full history to compare changes + - name: Create PR in sitecore-repo + if: | + env.merged == 'true' && + ( + steps.file_changes.outputs.api == 'true' || + steps.file_changes.outputs.cli == 'true' || + steps.file_changes.outputs.ui == 'true' || + steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.index == 'true' + ) + run: | + gh pr create --repo ${{ env.SITECORE_REPO }} \ + --title "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" \ + --body "This PR syncs changes from migration-v2:\n${{ github.event.pull_request.html_url }}" \ + --head sync-from-migration-v2-${{ github.event.pull_request.number }} \ + --base ${{ env.BRANCH }} + env: + GH_TOKEN: ${{ secrets.GH_PAT }} - - name: Detect changed files - if: env.merged == 'true' - id: file_changes - uses: dorny/paths-filter@v3 - with: - filters: | - api: - - 'api/**' - cli: - - 'cli/**' - ui: - - 'ui/**' - upload-api: - - 'upload-api/src/**' - migration-sitecore: - - 'upload-api/migration-sitecore/**' - migration-contentful: - - 'upload-api/migration-contentful/**' - migration-wordpress: - - 'upload-api/migration-wordpress/**' - index: - - 'index.js' + - name: Sync changes to wordpress-repo (if applicable) + if: | + env.merged == 'true' && + ( + steps.file_changes.outputs.api == 'true' || + steps.file_changes.outputs.cli == 'true' || + steps.file_changes.outputs.ui == 'true' || + steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.index == 'true' + ) + run: | + git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.CONTENTFUL_REPO }}.git + cd migration-tool-contentful + git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} + rsync -av --delete $RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES ../api/ ./api/ + rsync -av --delete ../cli/ ./cli/ + rsync -av --delete ../ui/ ./ui/ + rsync -av --delete $RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES ../upload-api/ ./upload-api/ + git add . + git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" + git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} - - name: Setup Git - if: env.merged == 'true' - run: | - git config --global user.name "github-actions" - git config --global user.email "github-actions@github.com" + - name: Create PR in contentful-repo + if: | + env.merged == 'true' && + ( + steps.file_changes.outputs.api == 'true' || + steps.file_changes.outputs.cli == 'true' || + steps.file_changes.outputs.ui == 'true' || + steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.index == 'true' + ) + run: | + gh pr create --repo ${{ env.CONTENTFUL_REPO }} \ + --title "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" \ + --body "This PR syncs changes from migration-v2:\n${{ github.event.pull_request.html_url }}" \ + --head sync-from-migration-v2-${{ github.event.pull_request.number }} \ + --base ${{ env.BRANCH }} + env: + GH_TOKEN: ${{ secrets.GH_PAT }} - - name: Sync changes to sitecore-repo (if applicable) - if: | - env.merged == 'true' && - ( - steps.file_changes.outputs.api == 'true' || - steps.file_changes.outputs.cli == 'true' || - steps.file_changes.outputs.ui == 'true' || - steps.file_changes.outputs.upload-api == 'true' || - steps.file_changes.outputs.migration-sitecore == 'true' || - steps.file_changes.outputs.index == 'true' - ) - run: | - git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.SITECORE_REPO }}.git - cd migration-tool-sitecore - git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} - rsync -av --delete $RSYNC_SITECORE_EXCLUDES ../api/ ./api/ - rsync -av --delete ../cli/ ./cli/ - rsync -av --delete ../ui/ ./ui/ - rsync -av --delete ../upload-api/src ./upload-api/src - rsync -av --delete ../upload-api/migration-sitecore ./upload-api/migration-sitecore - git add . - git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" - git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} + - name: Sync changes to wordpress-repo (if applicable) + if: | + env.merged == 'true' && + ( + steps.file_changes.outputs.api == 'true' || + steps.file_changes.outputs.cli == 'true' || + steps.file_changes.outputs.ui == 'true' || + steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.index == 'true' + ) + run: | + git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.WORDPRESS_REPO }}.git + cd migration-tool-wordpress + git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} + rsync -av --delete $RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES ../api/ ./api/ + rsync -av --delete ../cli/ ./cli/ + rsync -av --delete ../ui/ ./ui/ + rsync -av --delete $RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES ../upload-api/ ./upload-api/ + git add . + git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" + git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} - - name: Create PR in sitecore-repo - if: | - env.merged == 'true' && - ( - steps.file_changes.outputs.api == 'true' || - steps.file_changes.outputs.cli == 'true' || - steps.file_changes.outputs.ui == 'true' || - steps.file_changes.outputs.upload-api == 'true' || - steps.file_changes.outputs.migration-sitecore == 'true' || - steps.file_changes.outputs.index == 'true' - ) - run: | - gh pr create --repo ${{ env.SITECORE_REPO }} \ - --title "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" \ - --body "This PR syncs changes from migration-v2:\n${{ github.event.pull_request.html_url }}" \ - --head sync-from-migration-v2-${{ github.event.pull_request.number }} \ - --base ${{ env.BRANCH }} - env: - GH_TOKEN: ${{ secrets.GH_PAT }} + - name: Create PR in wordpress-repo + if: | + env.merged == 'true' && + ( + steps.file_changes.outputs.api == 'true' || + steps.file_changes.outputs.cli == 'true' || + steps.file_changes.outputs.ui == 'true' || + steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.index == 'true' + ) + run: | + gh pr create --repo ${{ env.WORDPRESS_REPO }} \ + --title "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" \ + --body "This PR syncs changes from migration-v2:\n${{ github.event.pull_request.html_url }}" \ + --head sync-from-migration-v2-${{ github.event.pull_request.number }} \ + --base ${{ env.BRANCH }} + env: + GH_TOKEN: ${{ secrets.GH_PAT }} diff --git a/upload-api/.DS_Store b/upload-api/.DS_Store index 7303c1852a661cba1456d949fe0fbd70e41f01d4..bf6a9fb002cbcc512463143246ca9e712a228136 100644 GIT binary patch delta 19 acmZp1XmQvuO^``7dvc7R&gLV64txMbI|g(B delta 19 acmZp1XmQvuO^`|9)#MmKoy|uC9ryrA`UdU* diff --git a/upload-api/migration-contentful/.DS_Store b/upload-api/migration-contentful/.DS_Store index f3a870ba4b65e70e7b8f28fe8b52985ee3fad216..110a0f239218bf604c4394c0d516938590d52330 100644 GIT binary patch delta 26 icmZoMXffEZj)f_idGZDp2{sMM!YAB&HlJfL69xcw;RwS3 delta 23 fcmZoMXffEZj)f_%Y4QdZiODZmL^j`H(GUgzY9$D5 From 47eb209be598d45479b82b2b75123dcfe2b97ec1 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 16:33:38 +0530 Subject: [PATCH 02/19] workflow changes --- .github/workflows/repo-sync.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 3ee14aa5..2d1fe9e1 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -81,10 +81,10 @@ jobs: git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.SITECORE_REPO }}.git cd migration-tool-sitecore git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} - rsync -av --delete $RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES ../api/ ./api/ + rsync -av --delete ${{ env.RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete $RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES ../upload-api/ ./upload-api/ + rsync -av --delete ${{ env.RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} @@ -122,10 +122,10 @@ jobs: git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.CONTENTFUL_REPO }}.git cd migration-tool-contentful git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} - rsync -av --delete $RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES ../api/ ./api/ + rsync -av --delete ${{ env.RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete $RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES ../upload-api/ ./upload-api/ + rsync -av --delete ${{ env.RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} @@ -163,10 +163,10 @@ jobs: git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.WORDPRESS_REPO }}.git cd migration-tool-wordpress git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} - rsync -av --delete $RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES ../api/ ./api/ + rsync -av --delete ${{ env.RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete $RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES ../upload-api/ ./upload-api/ + rsync -av --delete ${{ env.RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} From 7e6efd49eac958773ad774fecfd5e64929cf92ea Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 16:44:29 +0530 Subject: [PATCH 03/19] workflow changes test --- api/src/services/contentful.service.ts | 1 + api/src/services/contentful/jsonRTE.ts | 1 + api/src/services/sitecore.service.ts | 1 + api/src/services/wordpress.service.ts | 1 + index.js | 1 + upload-api/migration-contentful/config/index.json | 1 + upload-api/migration-sitecore/constants/index.js | 1 + upload-api/migration-wordpress/config/index.json | 2 -- upload-api/src/config/index.ts | 1 + 9 files changed, 8 insertions(+), 2 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 8d533e64..e7d247fb 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -331,6 +331,7 @@ const processField = ( break; } } + }; // Helper function to clean up brackets in non-numeric lang_value diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 90d168b9..34d054c3 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -314,6 +314,7 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack }; } + function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 90be5c59..9cb5737b 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -12,6 +12,7 @@ import customLogger from '../utils/custom-logger.utils.js'; const append = "a"; + const baseDirName = MIGRATION_DATA_CONFIG.DATA const { ENTRIES_DIR_NAME, diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index ee400ea0..6a659397 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -15,6 +15,7 @@ const { JSDOM } = jsdom; const virtualConsole = new jsdom.VirtualConsole(); // Get the current file's path const __filename = fileURLToPath(import.meta.url); + // Get the current directory const __dirname = path.dirname(__filename); diff --git a/index.js b/index.js index 92e49eb6..8993572b 100644 --- a/index.js +++ b/index.js @@ -33,6 +33,7 @@ const envContents = { 'Upload-API': uploadAPIEnvContent, }; + // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index 3c2a1243..975eb765 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -4,4 +4,5 @@ "contentful": "contentfulSchema" }, "fileName": "en-us.json" + } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index e7268fb1..d355f817 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -42,6 +42,7 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } + module.exports = { MIGRATION_DATA_CONFIG }; \ No newline at end of file diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index 454ee3a2..661544d9 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -47,8 +47,6 @@ "fileName": "en-us.json", "masterfile": "posts.json" - - } } } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index 1682e698..3d9d5784 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -13,6 +13,7 @@ export default { buketKey: 'project/package 45.zip' }, + localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' }; From eb2d4e0a64628d4a1023ddcad665547269588744 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:02:30 +0530 Subject: [PATCH 04/19] workflow yml --- .github/workflows/repo-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 2d1fe9e1..7695e977 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -81,7 +81,7 @@ jobs: git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.SITECORE_REPO }}.git cd migration-tool-sitecore git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} - rsync -av --delete ${{ env.RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ + rsync -av --delete ${{ env.RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ rsync -av --delete ${{ env.RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ From 93b31ed2a660bb3501cdd39be45d2819f279824d Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:06:57 +0530 Subject: [PATCH 05/19] workflow yml test --- api/src/services/contentful.service.ts | 1 + api/src/services/contentful/jsonRTE.ts | 1 + api/src/services/sitecore.service.ts | 1 + api/src/services/wordpress.service.ts | 1 + index.js | 1 + upload-api/migration-contentful/config/index.json | 1 + upload-api/migration-sitecore/constants/index.js | 1 + upload-api/migration-wordpress/config/index.json | 1 + upload-api/src/config/index.ts | 1 + 9 files changed, 9 insertions(+) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index e7d247fb..9b57f006 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -331,6 +331,7 @@ const processField = ( break; } } + }; diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 34d054c3..c1c52bfd 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -315,6 +315,7 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack } + function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 9cb5737b..d3780579 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -13,6 +13,7 @@ import customLogger from '../utils/custom-logger.utils.js'; const append = "a"; + const baseDirName = MIGRATION_DATA_CONFIG.DATA const { ENTRIES_DIR_NAME, diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index 6a659397..4fb9d05b 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -16,6 +16,7 @@ const virtualConsole = new jsdom.VirtualConsole(); // Get the current file's path const __filename = fileURLToPath(import.meta.url); + // Get the current directory const __dirname = path.dirname(__filename); diff --git a/index.js b/index.js index 8993572b..0188709e 100644 --- a/index.js +++ b/index.js @@ -34,6 +34,7 @@ const envContents = { }; + // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index 975eb765..9d1da548 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -4,5 +4,6 @@ "contentful": "contentfulSchema" }, "fileName": "en-us.json" + } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index d355f817..da480f0c 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -42,6 +42,7 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } + module.exports = { MIGRATION_DATA_CONFIG diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index 661544d9..8f0806c3 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -47,6 +47,7 @@ "fileName": "en-us.json", "masterfile": "posts.json" + } } } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index 3d9d5784..ad26b141 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -13,6 +13,7 @@ export default { buketKey: 'project/package 45.zip' }, + localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' From e0b5388f1462b0cf3f192ea455cfd17881322708 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:14:24 +0530 Subject: [PATCH 06/19] workflow yml test --- .github/workflows/repo-sync.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 7695e977..8390d55f 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -11,12 +11,12 @@ env: SITECORE_REPO: contentstack-expert-services/migration-tool-sitecore # Change this to your sitecore repo CONTENTFUL_REPO: contentstack-expert-services/migration-tool-contentful # Change this to your contentful repo WORDPRESS_REPO: contentstack-expert-services/migration-tool-wordpress # Change this to your wordpress repo - RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts/' --exclude 'src/services/wordpress.service.ts/'" # Include only sitecore services - RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/wordpress.service.ts/' --exclude 'src/services/sitecore.service.ts/'" # Include only contentful services - RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts/' --exclude 'src/services/sitecore.service.ts/'" # Include only wordpress services - RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-contentful/' --exclude 'migration-wordpress/' --exclude 'controllers/wordpress/' --exclude 'models/contentful.json' --exclude 'models/wordpress.json' --exclude 'services/contentful' --exclude 'validators/aem' --exclude 'validators/contentful' --exclude 'validators/wordpress'" # Include only sitecore upload-api - RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-sitecore/' --exclude 'migration-wordpress/' --exclude 'controllers/sitecore/' --exclude 'models/wordpress.json' --exclude 'validators/aem' --exclude 'validators/sitecore' --exclude 'validators/wordpress'" # Include only contentful upload-api - RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-sitecore/' --exclude 'migration-contentful/' --exclude 'controllers/sitecore/' --exclude 'models/contentful.json' --exclude 'validators/aem' --exclude 'validators/sitecore' --exclude 'validators/contentful'" # Include only wordpress upload-api + RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts' --exclude 'src/services/wordpress.service.ts'" # Include only sitecore services + RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/wordpress.service.ts' --exclude 'src/services/sitecore.service.ts'" # Include only contentful services + RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts' --exclude 'src/services/sitecore.service.ts'" # Include only wordpress services + RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-contentful/' --exclude 'migration-wordpress/' --exclude 'controllers/wordpress/' --exclude 'models/contentful.json' --exclude 'models/wordpress.json' --exclude 'services/contentful/' --exclude 'validators/aem/' --exclude 'validators/contentful/' --exclude 'validators/wordpress/'" # Include only sitecore upload-api + RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-sitecore/' --exclude 'migration-wordpress/' --exclude 'controllers/sitecore/' --exclude 'models/wordpress.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/wordpress/'" # Include only contentful upload-api + RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-sitecore/' --exclude 'migration-contentful/' --exclude 'controllers/sitecore/' --exclude 'models/contentful.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/contentful/'" # Include only wordpress upload-api jobs: sync-on-merge: runs-on: ubuntu-latest From 503f5c0f252e42a5721582973cb1fd3dc598f654 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:17:37 +0530 Subject: [PATCH 07/19] workflow yml test --- api/src/services/contentful.service.ts | 1 + api/src/services/contentful/jsonRTE.ts | 1 + api/src/services/sitecore.service.ts | 1 - api/src/services/wordpress.service.ts | 1 + index.js | 1 - upload-api/migration-contentful/config/index.json | 1 + upload-api/migration-sitecore/constants/index.js | 1 + upload-api/migration-wordpress/config/index.json | 1 + upload-api/src/config/index.ts | 1 + 9 files changed, 7 insertions(+), 2 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 9b57f006..97c223d3 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -332,6 +332,7 @@ const processField = ( } } + }; diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index c1c52bfd..f4bf92cd 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -316,6 +316,7 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack + function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index d3780579..9cb5737b 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -13,7 +13,6 @@ import customLogger from '../utils/custom-logger.utils.js'; const append = "a"; - const baseDirName = MIGRATION_DATA_CONFIG.DATA const { ENTRIES_DIR_NAME, diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index 4fb9d05b..cd8b4b52 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -17,6 +17,7 @@ const virtualConsole = new jsdom.VirtualConsole(); const __filename = fileURLToPath(import.meta.url); + // Get the current directory const __dirname = path.dirname(__filename); diff --git a/index.js b/index.js index 0188709e..8993572b 100644 --- a/index.js +++ b/index.js @@ -34,7 +34,6 @@ const envContents = { }; - // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index 9d1da548..3583969a 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -5,5 +5,6 @@ }, "fileName": "en-us.json" + } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index da480f0c..442b7035 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -43,6 +43,7 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } + module.exports = { MIGRATION_DATA_CONFIG diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index 8f0806c3..454ee3a2 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -47,6 +47,7 @@ "fileName": "en-us.json", "masterfile": "posts.json" + } } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index ad26b141..f5659b48 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -14,6 +14,7 @@ export default { }, + localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' From 09dd4c583f6303165364c72c08da10a39aad5b68 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:41:33 +0530 Subject: [PATCH 08/19] workflow CHANGES --- .github/workflows/repo-sync.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 8390d55f..41d4c112 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -58,6 +58,12 @@ jobs: - 'ui/**' upload-api: - 'upload-api/**' + migration-sitecore: + - 'upload-api/migration-sitecore/**' + migration-contentful: + - 'upload-api/migration-contentful/**' + migration-wordpress: + - 'upload-api/migration-wordpress/**' index: - 'index.js' @@ -75,6 +81,7 @@ jobs: steps.file_changes.outputs.cli == 'true' || steps.file_changes.outputs.ui == 'true' || steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.migration-sitecore == 'true' || steps.file_changes.outputs.index == 'true' ) run: | @@ -85,6 +92,7 @@ jobs: rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ rsync -av --delete ${{ env.RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ + rsync -av --delete ../upload-api/migration-sitecore ./upload-api/migration-sitecore git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} @@ -97,6 +105,7 @@ jobs: steps.file_changes.outputs.cli == 'true' || steps.file_changes.outputs.ui == 'true' || steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.migration-sitecore == 'true' || steps.file_changes.outputs.index == 'true' ) run: | @@ -116,6 +125,7 @@ jobs: steps.file_changes.outputs.cli == 'true' || steps.file_changes.outputs.ui == 'true' || steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.migration-contentful == 'true' || steps.file_changes.outputs.index == 'true' ) run: | @@ -126,6 +136,7 @@ jobs: rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ rsync -av --delete ${{ env.RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ + rsync -av --delete ../upload-api/migration-contentful ./upload-api/migration-contentful git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} @@ -138,6 +149,7 @@ jobs: steps.file_changes.outputs.cli == 'true' || steps.file_changes.outputs.ui == 'true' || steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.migration-contentful == 'true' || steps.file_changes.outputs.index == 'true' ) run: | @@ -157,6 +169,7 @@ jobs: steps.file_changes.outputs.cli == 'true' || steps.file_changes.outputs.ui == 'true' || steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.migration-wordpress == 'true' || steps.file_changes.outputs.index == 'true' ) run: | @@ -167,6 +180,7 @@ jobs: rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ rsync -av --delete ${{ env.RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ + rsync -av --delete ../upload-api/migration-wordpress ./upload-api/migration-wordpress git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} @@ -179,6 +193,7 @@ jobs: steps.file_changes.outputs.cli == 'true' || steps.file_changes.outputs.ui == 'true' || steps.file_changes.outputs.upload-api == 'true' || + steps.file_changes.outputs.migration-wordpress == 'true' || steps.file_changes.outputs.index == 'true' ) run: | From 3aca46a8c146935f3d35ae7c7867d1a64fc3d534 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:48:23 +0530 Subject: [PATCH 09/19] workflow testing --- api/src/services/contentful.service.ts | 1 - api/src/services/contentful/jsonRTE.ts | 1 - api/src/services/sitecore.service.ts | 1 - api/src/services/wordpress.service.ts | 1 - index.js | 1 + upload-api/migration-contentful/config/index.json | 1 - upload-api/migration-sitecore/constants/index.js | 1 - upload-api/migration-wordpress/config/index.json | 1 - upload-api/src/config/index.ts | 1 - upload-api/src/models/contentful.json | 1 + upload-api/src/models/wordpress.json | 1 + upload-api/src/services/contentful/index.ts | 1 + upload-api/src/validators/contentful/index.ts | 1 + upload-api/src/validators/sitecore/index.ts | 1 + upload-api/src/validators/wordpress/index.ts | 1 + 15 files changed, 7 insertions(+), 8 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 97c223d3..7f568dca 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -333,7 +333,6 @@ const processField = ( } - }; // Helper function to clean up brackets in non-numeric lang_value diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index f4bf92cd..c1c52bfd 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -316,7 +316,6 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack - function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 9cb5737b..90be5c59 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -12,7 +12,6 @@ import customLogger from '../utils/custom-logger.utils.js'; const append = "a"; - const baseDirName = MIGRATION_DATA_CONFIG.DATA const { ENTRIES_DIR_NAME, diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index cd8b4b52..4fb9d05b 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -17,7 +17,6 @@ const virtualConsole = new jsdom.VirtualConsole(); const __filename = fileURLToPath(import.meta.url); - // Get the current directory const __dirname = path.dirname(__filename); diff --git a/index.js b/index.js index 8993572b..0188709e 100644 --- a/index.js +++ b/index.js @@ -34,6 +34,7 @@ const envContents = { }; + // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index 3583969a..ea84a37f 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -6,5 +6,4 @@ "fileName": "en-us.json" - } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index 442b7035..da480f0c 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -43,7 +43,6 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } - module.exports = { MIGRATION_DATA_CONFIG diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index 454ee3a2..8f0806c3 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -47,7 +47,6 @@ "fileName": "en-us.json", "masterfile": "posts.json" - } } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index f5659b48..ad26b141 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -14,7 +14,6 @@ export default { }, - localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' diff --git a/upload-api/src/models/contentful.json b/upload-api/src/models/contentful.json index 5ff2ce71..441a70ba 100644 --- a/upload-api/src/models/contentful.json +++ b/upload-api/src/models/contentful.json @@ -31,4 +31,5 @@ "name": "roles", "required": "false" } + } diff --git a/upload-api/src/models/wordpress.json b/upload-api/src/models/wordpress.json index b0f90bfd..52e3e0f3 100644 --- a/upload-api/src/models/wordpress.json +++ b/upload-api/src/models/wordpress.json @@ -23,4 +23,5 @@ "name": "wp\\:attachment_url", "required": "false" } + } diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index 9ca2c637..cbd5d6c7 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -7,6 +7,7 @@ import { Config } from '../../models/types'; const { extractContentTypes, createInitialMapper } = require('migration-contentful'); + const createContentfulMapper = async ( projectId: string | string[], app_token: string | string[], diff --git a/upload-api/src/validators/contentful/index.ts b/upload-api/src/validators/contentful/index.ts index 16a7e681..a0711634 100644 --- a/upload-api/src/validators/contentful/index.ts +++ b/upload-api/src/validators/contentful/index.ts @@ -5,6 +5,7 @@ interface JsonData { [key: string]: any; } + //function to validate json file data function contentfulValidator(data: string): boolean { let jsonData: JsonData; diff --git a/upload-api/src/validators/sitecore/index.ts b/upload-api/src/validators/sitecore/index.ts index 551c50e6..69ecb291 100644 --- a/upload-api/src/validators/sitecore/index.ts +++ b/upload-api/src/validators/sitecore/index.ts @@ -9,6 +9,7 @@ interface props { data: items; } + async function sitecoreValidator({ data }: props) { try { let templates: any[] = []; diff --git a/upload-api/src/validators/wordpress/index.ts b/upload-api/src/validators/wordpress/index.ts index 86570598..2a758e99 100644 --- a/upload-api/src/validators/wordpress/index.ts +++ b/upload-api/src/validators/wordpress/index.ts @@ -1,6 +1,7 @@ import Config from '../../models/wordpress.json'; import * as Cheerio from 'cheerio'; + //function to validate xml file data function wordpressValidator(data: string): boolean { try { From 665a3e5f809b05c9771b7f1fd88d12bee858bc4e Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 18:09:10 +0530 Subject: [PATCH 10/19] workflow test --- api/src/services/contentful.service.ts | 1 - api/src/services/contentful/jsonRTE.ts | 1 - api/src/services/sitecore.service.ts | 1 - api/src/services/wordpress.service.ts | 1 - index.js | 1 - upload-api/migration-contentful/config/index.json | 2 -- upload-api/migration-sitecore/constants/index.js | 1 - upload-api/migration-wordpress/config/index.json | 1 - upload-api/src/config/index.ts | 1 - upload-api/src/models/contentful.json | 1 - upload-api/src/models/wordpress.json | 1 - upload-api/src/services/contentful/index.ts | 1 - upload-api/src/validators/contentful/index.ts | 1 - upload-api/src/validators/sitecore/index.ts | 1 - upload-api/src/validators/wordpress/index.ts | 1 - upload-api/tsconfig.json | 1 + 16 files changed, 1 insertion(+), 16 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 7f568dca..c853eb40 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -332,7 +332,6 @@ const processField = ( } } - }; // Helper function to clean up brackets in non-numeric lang_value diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index c1c52bfd..34d054c3 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -315,7 +315,6 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack } - function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 90be5c59..6d403a3d 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -9,7 +9,6 @@ import { orgService } from './org.service.js'; import { getLogMessage } from '../utils/index.js'; import customLogger from '../utils/custom-logger.utils.js'; - const append = "a"; const baseDirName = MIGRATION_DATA_CONFIG.DATA diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index 4fb9d05b..6a659397 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -16,7 +16,6 @@ const virtualConsole = new jsdom.VirtualConsole(); // Get the current file's path const __filename = fileURLToPath(import.meta.url); - // Get the current directory const __dirname = path.dirname(__filename); diff --git a/index.js b/index.js index 0188709e..8993572b 100644 --- a/index.js +++ b/index.js @@ -34,7 +34,6 @@ const envContents = { }; - // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index ea84a37f..3c2a1243 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -4,6 +4,4 @@ "contentful": "contentfulSchema" }, "fileName": "en-us.json" - - } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index da480f0c..3496c92d 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -43,7 +43,6 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } - module.exports = { MIGRATION_DATA_CONFIG }; \ No newline at end of file diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index 8f0806c3..661544d9 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -47,7 +47,6 @@ "fileName": "en-us.json", "masterfile": "posts.json" - } } } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index ad26b141..7c80dc03 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -14,7 +14,6 @@ export default { }, - localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' }; diff --git a/upload-api/src/models/contentful.json b/upload-api/src/models/contentful.json index 441a70ba..5ff2ce71 100644 --- a/upload-api/src/models/contentful.json +++ b/upload-api/src/models/contentful.json @@ -31,5 +31,4 @@ "name": "roles", "required": "false" } - } diff --git a/upload-api/src/models/wordpress.json b/upload-api/src/models/wordpress.json index 52e3e0f3..b0f90bfd 100644 --- a/upload-api/src/models/wordpress.json +++ b/upload-api/src/models/wordpress.json @@ -23,5 +23,4 @@ "name": "wp\\:attachment_url", "required": "false" } - } diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index cbd5d6c7..9ca2c637 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -7,7 +7,6 @@ import { Config } from '../../models/types'; const { extractContentTypes, createInitialMapper } = require('migration-contentful'); - const createContentfulMapper = async ( projectId: string | string[], app_token: string | string[], diff --git a/upload-api/src/validators/contentful/index.ts b/upload-api/src/validators/contentful/index.ts index a0711634..16a7e681 100644 --- a/upload-api/src/validators/contentful/index.ts +++ b/upload-api/src/validators/contentful/index.ts @@ -5,7 +5,6 @@ interface JsonData { [key: string]: any; } - //function to validate json file data function contentfulValidator(data: string): boolean { let jsonData: JsonData; diff --git a/upload-api/src/validators/sitecore/index.ts b/upload-api/src/validators/sitecore/index.ts index 69ecb291..551c50e6 100644 --- a/upload-api/src/validators/sitecore/index.ts +++ b/upload-api/src/validators/sitecore/index.ts @@ -9,7 +9,6 @@ interface props { data: items; } - async function sitecoreValidator({ data }: props) { try { let templates: any[] = []; diff --git a/upload-api/src/validators/wordpress/index.ts b/upload-api/src/validators/wordpress/index.ts index 2a758e99..86570598 100644 --- a/upload-api/src/validators/wordpress/index.ts +++ b/upload-api/src/validators/wordpress/index.ts @@ -1,7 +1,6 @@ import Config from '../../models/wordpress.json'; import * as Cheerio from 'cheerio'; - //function to validate xml file data function wordpressValidator(data: string): boolean { try { diff --git a/upload-api/tsconfig.json b/upload-api/tsconfig.json index dbb6d8bb..f57491a8 100644 --- a/upload-api/tsconfig.json +++ b/upload-api/tsconfig.json @@ -44,6 +44,7 @@ // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + /* JavaScript Support */ "allowJs": true /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */, // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ From 75881108bc0e900ae31c99b91c189d7fd1bce91d Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 18:20:46 +0530 Subject: [PATCH 11/19] workflow changes --- .github/workflows/repo-sync.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 41d4c112..81d04df1 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -14,9 +14,9 @@ env: RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts' --exclude 'src/services/wordpress.service.ts'" # Include only sitecore services RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/wordpress.service.ts' --exclude 'src/services/sitecore.service.ts'" # Include only contentful services RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts' --exclude 'src/services/sitecore.service.ts'" # Include only wordpress services - RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-contentful/' --exclude 'migration-wordpress/' --exclude 'controllers/wordpress/' --exclude 'models/contentful.json' --exclude 'models/wordpress.json' --exclude 'services/contentful/' --exclude 'validators/aem/' --exclude 'validators/contentful/' --exclude 'validators/wordpress/'" # Include only sitecore upload-api - RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-sitecore/' --exclude 'migration-wordpress/' --exclude 'controllers/sitecore/' --exclude 'models/wordpress.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/wordpress/'" # Include only contentful upload-api - RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES: "--exclude 'migration-sitecore/' --exclude 'migration-contentful/' --exclude 'controllers/sitecore/' --exclude 'models/contentful.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/contentful/'" # Include only wordpress upload-api + RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/wordpress/' --exclude 'models/contentful.json' --exclude 'models/wordpress.json' --exclude 'services/contentful/' --exclude 'validators/aem/' --exclude 'validators/contentful/' --exclude 'validators/wordpress/'" # Include only sitecore upload-api + RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/sitecore/' --exclude 'models/wordpress.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/wordpress/'" # Include only contentful upload-api + RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/sitecore/' --exclude 'models/contentful.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/contentful/'" # Include only wordpress upload-api jobs: sync-on-merge: runs-on: ubuntu-latest @@ -57,7 +57,7 @@ jobs: ui: - 'ui/**' upload-api: - - 'upload-api/**' + - 'upload-api/src/**' migration-sitecore: - 'upload-api/migration-sitecore/**' migration-contentful: @@ -88,6 +88,7 @@ jobs: git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.SITECORE_REPO }}.git cd migration-tool-sitecore git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} + git pull origin ${{ env.BRANCH }} rsync -av --delete ${{ env.RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ @@ -132,6 +133,7 @@ jobs: git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.CONTENTFUL_REPO }}.git cd migration-tool-contentful git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} + git pull origin ${{ env.BRANCH }} rsync -av --delete ${{ env.RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ @@ -176,6 +178,7 @@ jobs: git clone https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ env.WORDPRESS_REPO }}.git cd migration-tool-wordpress git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }} + git pull origin ${{ env.BRANCH }} rsync -av --delete ${{ env.RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ From 600156de764dcffb51cd012b67dbfd812ca1d18f Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 18:35:16 +0530 Subject: [PATCH 12/19] workflow checking --- api/src/services/contentful.service.ts | 1 - api/src/services/contentful/jsonRTE.ts | 1 - api/src/services/sitecore.service.ts | 1 + api/src/services/wordpress.service.ts | 1 - index.js | 1 + upload-api/migration-contentful/config/index.json | 1 + upload-api/migration-sitecore/constants/index.js | 1 + upload-api/migration-wordpress/config/index.json | 1 + upload-api/src/config/index.ts | 1 - upload-api/src/models/contentful.json | 1 + upload-api/src/models/wordpress.json | 1 + upload-api/src/services/contentful/index.ts | 1 + upload-api/src/validators/contentful/index.ts | 1 + upload-api/src/validators/sitecore/index.ts | 1 + upload-api/src/validators/wordpress/index.ts | 1 + upload-api/tsconfig.json | 1 - 16 files changed, 11 insertions(+), 5 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index c853eb40..8d533e64 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -331,7 +331,6 @@ const processField = ( break; } } - }; // Helper function to clean up brackets in non-numeric lang_value diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 34d054c3..90d168b9 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -314,7 +314,6 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack }; } - function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 6d403a3d..90be5c59 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -9,6 +9,7 @@ import { orgService } from './org.service.js'; import { getLogMessage } from '../utils/index.js'; import customLogger from '../utils/custom-logger.utils.js'; + const append = "a"; const baseDirName = MIGRATION_DATA_CONFIG.DATA diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index 6a659397..f48641a9 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -10,7 +10,6 @@ import customLogger from "../utils/custom-logger.utils.js"; import { getLogMessage } from "../utils/index.js"; import { Advanced } from "../models/FieldMapper.js"; - const { JSDOM } = jsdom; const virtualConsole = new jsdom.VirtualConsole(); // Get the current file's path diff --git a/index.js b/index.js index 8993572b..0188709e 100644 --- a/index.js +++ b/index.js @@ -34,6 +34,7 @@ const envContents = { }; + // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index 3c2a1243..975eb765 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -4,4 +4,5 @@ "contentful": "contentfulSchema" }, "fileName": "en-us.json" + } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index 3496c92d..da480f0c 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -43,6 +43,7 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } + module.exports = { MIGRATION_DATA_CONFIG }; \ No newline at end of file diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index 661544d9..8f0806c3 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -47,6 +47,7 @@ "fileName": "en-us.json", "masterfile": "posts.json" + } } } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index 7c80dc03..1682e698 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -13,7 +13,6 @@ export default { buketKey: 'project/package 45.zip' }, - localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' }; diff --git a/upload-api/src/models/contentful.json b/upload-api/src/models/contentful.json index 5ff2ce71..441a70ba 100644 --- a/upload-api/src/models/contentful.json +++ b/upload-api/src/models/contentful.json @@ -31,4 +31,5 @@ "name": "roles", "required": "false" } + } diff --git a/upload-api/src/models/wordpress.json b/upload-api/src/models/wordpress.json index b0f90bfd..52e3e0f3 100644 --- a/upload-api/src/models/wordpress.json +++ b/upload-api/src/models/wordpress.json @@ -23,4 +23,5 @@ "name": "wp\\:attachment_url", "required": "false" } + } diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index 9ca2c637..cbd5d6c7 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -7,6 +7,7 @@ import { Config } from '../../models/types'; const { extractContentTypes, createInitialMapper } = require('migration-contentful'); + const createContentfulMapper = async ( projectId: string | string[], app_token: string | string[], diff --git a/upload-api/src/validators/contentful/index.ts b/upload-api/src/validators/contentful/index.ts index 16a7e681..a0711634 100644 --- a/upload-api/src/validators/contentful/index.ts +++ b/upload-api/src/validators/contentful/index.ts @@ -5,6 +5,7 @@ interface JsonData { [key: string]: any; } + //function to validate json file data function contentfulValidator(data: string): boolean { let jsonData: JsonData; diff --git a/upload-api/src/validators/sitecore/index.ts b/upload-api/src/validators/sitecore/index.ts index 551c50e6..69ecb291 100644 --- a/upload-api/src/validators/sitecore/index.ts +++ b/upload-api/src/validators/sitecore/index.ts @@ -9,6 +9,7 @@ interface props { data: items; } + async function sitecoreValidator({ data }: props) { try { let templates: any[] = []; diff --git a/upload-api/src/validators/wordpress/index.ts b/upload-api/src/validators/wordpress/index.ts index 86570598..2a758e99 100644 --- a/upload-api/src/validators/wordpress/index.ts +++ b/upload-api/src/validators/wordpress/index.ts @@ -1,6 +1,7 @@ import Config from '../../models/wordpress.json'; import * as Cheerio from 'cheerio'; + //function to validate xml file data function wordpressValidator(data: string): boolean { try { diff --git a/upload-api/tsconfig.json b/upload-api/tsconfig.json index f57491a8..dbb6d8bb 100644 --- a/upload-api/tsconfig.json +++ b/upload-api/tsconfig.json @@ -44,7 +44,6 @@ // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ - /* JavaScript Support */ "allowJs": true /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */, // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ From 6821ac9a4df7c4310267c56b34c09046ee572291 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 18:39:55 +0530 Subject: [PATCH 13/19] workflow checking --- .github/workflows/repo-sync.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 81d04df1..4d7c2ef1 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -92,7 +92,7 @@ jobs: rsync -av --delete ${{ env.RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete ${{ env.RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ + rsync -av --delete ${{ env.RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src ./upload-api/src rsync -av --delete ../upload-api/migration-sitecore ./upload-api/migration-sitecore git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" @@ -137,7 +137,7 @@ jobs: rsync -av --delete ${{ env.RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete ${{ env.RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ + rsync -av --delete ${{ env.RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src ./upload-api/src rsync -av --delete ../upload-api/migration-contentful ./upload-api/migration-contentful git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" @@ -182,7 +182,7 @@ jobs: rsync -av --delete ${{ env.RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete ${{ env.RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/ ./upload-api/ + rsync -av --delete ${{ env.RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src ./upload-api/src rsync -av --delete ../upload-api/migration-wordpress ./upload-api/migration-wordpress git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" From c84bd18ecee2a164f8958f0b38ea40e32fa27402 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 18:42:15 +0530 Subject: [PATCH 14/19] workflow test --- api/src/services/contentful.service.ts | 1 + api/src/services/contentful/jsonRTE.ts | 1 + api/src/services/sitecore.service.ts | 1 - api/src/services/wordpress.service.ts | 1 + index.js | 2 -- upload-api/migration-contentful/config/index.json | 1 - upload-api/migration-sitecore/constants/index.js | 1 - upload-api/migration-wordpress/config/index.json | 2 -- upload-api/src/config/index.ts | 1 + upload-api/src/models/contentful.json | 1 - upload-api/src/models/wordpress.json | 1 - upload-api/src/services/contentful/index.ts | 1 - upload-api/src/validators/contentful/index.ts | 1 - upload-api/src/validators/sitecore/index.ts | 1 - upload-api/src/validators/wordpress/index.ts | 1 - 15 files changed, 4 insertions(+), 13 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 8d533e64..4a866ca4 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -333,6 +333,7 @@ const processField = ( } }; + // Helper function to clean up brackets in non-numeric lang_value const cleanBrackets = (lang_value: any) => { const myJSON = JSON.stringify(lang_value); diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 90d168b9..34d054c3 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -314,6 +314,7 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack }; } + function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 90be5c59..6d403a3d 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -9,7 +9,6 @@ import { orgService } from './org.service.js'; import { getLogMessage } from '../utils/index.js'; import customLogger from '../utils/custom-logger.utils.js'; - const append = "a"; const baseDirName = MIGRATION_DATA_CONFIG.DATA diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index f48641a9..668c2a68 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -11,6 +11,7 @@ import { getLogMessage } from "../utils/index.js"; import { Advanced } from "../models/FieldMapper.js"; const { JSDOM } = jsdom; + const virtualConsole = new jsdom.VirtualConsole(); // Get the current file's path const __filename = fileURLToPath(import.meta.url); diff --git a/index.js b/index.js index 0188709e..92e49eb6 100644 --- a/index.js +++ b/index.js @@ -33,8 +33,6 @@ const envContents = { 'Upload-API': uploadAPIEnvContent, }; - - // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index 975eb765..3c2a1243 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -4,5 +4,4 @@ "contentful": "contentfulSchema" }, "fileName": "en-us.json" - } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index da480f0c..3496c92d 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -43,7 +43,6 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } - module.exports = { MIGRATION_DATA_CONFIG }; \ No newline at end of file diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index 8f0806c3..d5ef93c8 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -46,8 +46,6 @@ "dirName": "posts", "fileName": "en-us.json", "masterfile": "posts.json" - - } } } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index 1682e698..3d9d5784 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -13,6 +13,7 @@ export default { buketKey: 'project/package 45.zip' }, + localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' }; diff --git a/upload-api/src/models/contentful.json b/upload-api/src/models/contentful.json index 441a70ba..5ff2ce71 100644 --- a/upload-api/src/models/contentful.json +++ b/upload-api/src/models/contentful.json @@ -31,5 +31,4 @@ "name": "roles", "required": "false" } - } diff --git a/upload-api/src/models/wordpress.json b/upload-api/src/models/wordpress.json index 52e3e0f3..b0f90bfd 100644 --- a/upload-api/src/models/wordpress.json +++ b/upload-api/src/models/wordpress.json @@ -23,5 +23,4 @@ "name": "wp\\:attachment_url", "required": "false" } - } diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index cbd5d6c7..9ca2c637 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -7,7 +7,6 @@ import { Config } from '../../models/types'; const { extractContentTypes, createInitialMapper } = require('migration-contentful'); - const createContentfulMapper = async ( projectId: string | string[], app_token: string | string[], diff --git a/upload-api/src/validators/contentful/index.ts b/upload-api/src/validators/contentful/index.ts index a0711634..16a7e681 100644 --- a/upload-api/src/validators/contentful/index.ts +++ b/upload-api/src/validators/contentful/index.ts @@ -5,7 +5,6 @@ interface JsonData { [key: string]: any; } - //function to validate json file data function contentfulValidator(data: string): boolean { let jsonData: JsonData; diff --git a/upload-api/src/validators/sitecore/index.ts b/upload-api/src/validators/sitecore/index.ts index 69ecb291..551c50e6 100644 --- a/upload-api/src/validators/sitecore/index.ts +++ b/upload-api/src/validators/sitecore/index.ts @@ -9,7 +9,6 @@ interface props { data: items; } - async function sitecoreValidator({ data }: props) { try { let templates: any[] = []; diff --git a/upload-api/src/validators/wordpress/index.ts b/upload-api/src/validators/wordpress/index.ts index 2a758e99..86570598 100644 --- a/upload-api/src/validators/wordpress/index.ts +++ b/upload-api/src/validators/wordpress/index.ts @@ -1,7 +1,6 @@ import Config from '../../models/wordpress.json'; import * as Cheerio from 'cheerio'; - //function to validate xml file data function wordpressValidator(data: string): boolean { try { From 1d60ffe7ee0bc3a5238969db8b3d13adf6abf2e2 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 18:51:36 +0530 Subject: [PATCH 15/19] workflow changes --- .github/workflows/repo-sync.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 4d7c2ef1..e9c8c441 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -92,8 +92,8 @@ jobs: rsync -av --delete ${{ env.RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete ${{ env.RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src ./upload-api/src - rsync -av --delete ../upload-api/migration-sitecore ./upload-api/migration-sitecore + rsync -av --delete ${{ env.RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src/ ./upload-api/src/ + rsync -av --delete ../upload-api/migration-sitecore/ ./upload-api/migration-sitecore/ git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} @@ -137,8 +137,8 @@ jobs: rsync -av --delete ${{ env.RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete ${{ env.RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src ./upload-api/src - rsync -av --delete ../upload-api/migration-contentful ./upload-api/migration-contentful + rsync -av --delete ${{ env.RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src/ ./upload-api/src/ + rsync -av --delete ../upload-api/migration-contentful/ ./upload-api/migration-contentful/ git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} @@ -182,8 +182,8 @@ jobs: rsync -av --delete ${{ env.RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/ rsync -av --delete ../cli/ ./cli/ rsync -av --delete ../ui/ ./ui/ - rsync -av --delete ${{ env.RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src ./upload-api/src - rsync -av --delete ../upload-api/migration-wordpress ./upload-api/migration-wordpress + rsync -av --delete ${{ env.RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES }} ../upload-api/src/ ./upload-api/src/ + rsync -av --delete ../upload-api/migration-wordpress/ ./upload-api/migration-wordpress/ git add . git commit -m "Sync changes from migration-v2 PR #${{ github.event.pull_request.number }}" git push origin sync-from-migration-v2-${{ github.event.pull_request.number }} From 62847721d475623117bc128a532b7fd580da09c8 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 19:01:55 +0530 Subject: [PATCH 16/19] workflow test --- api/src/services/contentful.service.ts | 1 - api/src/services/contentful/jsonRTE.ts | 1 - api/src/services/sitecore.service.ts | 1 - api/src/services/wordpress.service.ts | 1 - index.js | 1 + upload-api/migration-contentful/config/index.json | 1 + upload-api/migration-sitecore/constants/index.js | 1 - upload-api/migration-wordpress/config/index.json | 1 + upload-api/src/config/index.ts | 1 - upload-api/src/models/contentful.json | 1 + upload-api/src/models/wordpress.json | 1 + upload-api/src/services/contentful/index.ts | 1 + upload-api/src/validators/contentful/index.ts | 1 + upload-api/src/validators/sitecore/index.ts | 1 + upload-api/src/validators/wordpress/index.ts | 1 + 15 files changed, 9 insertions(+), 6 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 4a866ca4..8d533e64 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -333,7 +333,6 @@ const processField = ( } }; - // Helper function to clean up brackets in non-numeric lang_value const cleanBrackets = (lang_value: any) => { const myJSON = JSON.stringify(lang_value); diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 34d054c3..90d168b9 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -314,7 +314,6 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack }; } - function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index 6d403a3d..f64bcad6 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -10,7 +10,6 @@ import { getLogMessage } from '../utils/index.js'; import customLogger from '../utils/custom-logger.utils.js'; const append = "a"; - const baseDirName = MIGRATION_DATA_CONFIG.DATA const { ENTRIES_DIR_NAME, diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index 668c2a68..f48641a9 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -11,7 +11,6 @@ import { getLogMessage } from "../utils/index.js"; import { Advanced } from "../models/FieldMapper.js"; const { JSDOM } = jsdom; - const virtualConsole = new jsdom.VirtualConsole(); // Get the current file's path const __filename = fileURLToPath(import.meta.url); diff --git a/index.js b/index.js index 92e49eb6..8993572b 100644 --- a/index.js +++ b/index.js @@ -33,6 +33,7 @@ const envContents = { 'Upload-API': uploadAPIEnvContent, }; + // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index 3c2a1243..975eb765 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -4,4 +4,5 @@ "contentful": "contentfulSchema" }, "fileName": "en-us.json" + } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index 3496c92d..e7268fb1 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -42,7 +42,6 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } - module.exports = { MIGRATION_DATA_CONFIG }; \ No newline at end of file diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index d5ef93c8..cf50910c 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -48,4 +48,5 @@ "masterfile": "posts.json" } } + } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index 3d9d5784..1682e698 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -13,7 +13,6 @@ export default { buketKey: 'project/package 45.zip' }, - localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' }; diff --git a/upload-api/src/models/contentful.json b/upload-api/src/models/contentful.json index 5ff2ce71..441a70ba 100644 --- a/upload-api/src/models/contentful.json +++ b/upload-api/src/models/contentful.json @@ -31,4 +31,5 @@ "name": "roles", "required": "false" } + } diff --git a/upload-api/src/models/wordpress.json b/upload-api/src/models/wordpress.json index b0f90bfd..52e3e0f3 100644 --- a/upload-api/src/models/wordpress.json +++ b/upload-api/src/models/wordpress.json @@ -23,4 +23,5 @@ "name": "wp\\:attachment_url", "required": "false" } + } diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index 9ca2c637..cbd5d6c7 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -7,6 +7,7 @@ import { Config } from '../../models/types'; const { extractContentTypes, createInitialMapper } = require('migration-contentful'); + const createContentfulMapper = async ( projectId: string | string[], app_token: string | string[], diff --git a/upload-api/src/validators/contentful/index.ts b/upload-api/src/validators/contentful/index.ts index 16a7e681..a0711634 100644 --- a/upload-api/src/validators/contentful/index.ts +++ b/upload-api/src/validators/contentful/index.ts @@ -5,6 +5,7 @@ interface JsonData { [key: string]: any; } + //function to validate json file data function contentfulValidator(data: string): boolean { let jsonData: JsonData; diff --git a/upload-api/src/validators/sitecore/index.ts b/upload-api/src/validators/sitecore/index.ts index 551c50e6..69ecb291 100644 --- a/upload-api/src/validators/sitecore/index.ts +++ b/upload-api/src/validators/sitecore/index.ts @@ -9,6 +9,7 @@ interface props { data: items; } + async function sitecoreValidator({ data }: props) { try { let templates: any[] = []; diff --git a/upload-api/src/validators/wordpress/index.ts b/upload-api/src/validators/wordpress/index.ts index 86570598..2a758e99 100644 --- a/upload-api/src/validators/wordpress/index.ts +++ b/upload-api/src/validators/wordpress/index.ts @@ -1,6 +1,7 @@ import Config from '../../models/wordpress.json'; import * as Cheerio from 'cheerio'; + //function to validate xml file data function wordpressValidator(data: string): boolean { try { From afa1a81e5c12ac7ad6bf8d323278e003690c0db1 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 19:11:14 +0530 Subject: [PATCH 17/19] workflow test --- .github/workflows/repo-sync.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index e9c8c441..0e5aeddc 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -14,9 +14,9 @@ env: RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts' --exclude 'src/services/wordpress.service.ts'" # Include only sitecore services RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/wordpress.service.ts' --exclude 'src/services/sitecore.service.ts'" # Include only contentful services RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES: "--exclude 'src/services/contentful/' --exclude 'src/services/contentful.service.ts' --exclude 'src/services/sitecore.service.ts'" # Include only wordpress services - RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/wordpress/' --exclude 'models/contentful.json' --exclude 'models/wordpress.json' --exclude 'services/contentful/' --exclude 'validators/aem/' --exclude 'validators/contentful/' --exclude 'validators/wordpress/'" # Include only sitecore upload-api - RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/sitecore/' --exclude 'models/wordpress.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/wordpress/'" # Include only contentful upload-api - RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/sitecore/' --exclude 'models/contentful.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/contentful/'" # Include only wordpress upload-api + RSYNC_SITECORE_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/wordpress/' --exclude 'controllers/contentful/' --exclude 'models/contentful.json' --exclude 'models/wordpress.json' --exclude 'services/contentful/' --exclude 'validators/aem/' --exclude 'validators/contentful/' --exclude 'validators/wordpress/'" # Include only sitecore upload-api + RSYNC_CONTENTFUL_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/sitecore/' --exclude 'controllers/wordpress/' --exclude 'models/wordpress.json' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/wordpress/'" # Include only contentful upload-api + RSYNC_WORDPRESS_UPLOAD_API_SRC_EXCLUDES: "--exclude 'controllers/sitecore/' --exclude 'controllers/contentful/' --exclude 'models/contentful.json' --exclude 'services/contentful/' --exclude 'validators/aem/' --exclude 'validators/sitecore/' --exclude 'validators/contentful/'" # Include only wordpress upload-api jobs: sync-on-merge: runs-on: ubuntu-latest From daa26fb1ccd2f6a6fbdb24bc7cb00bb846c175f0 Mon Sep 17 00:00:00 2001 From: RohitKini <57343193+RohitKini@users.noreply.github.com> Date: Wed, 5 Mar 2025 19:14:42 +0530 Subject: [PATCH 18/19] test --- api/src/services/contentful.service.ts | 1 + api/src/services/contentful/jsonRTE.ts | 1 + api/src/services/sitecore.service.ts | 1 + api/src/services/wordpress.service.ts | 1 + index.js | 1 - upload-api/migration-contentful/config/index.json | 1 - upload-api/migration-sitecore/constants/index.js | 1 + upload-api/migration-wordpress/config/index.json | 1 - upload-api/src/config/index.ts | 1 + upload-api/src/models/contentful.json | 1 - upload-api/src/models/wordpress.json | 1 - upload-api/src/services/contentful/index.ts | 1 - upload-api/src/validators/contentful/index.ts | 1 - upload-api/src/validators/sitecore/index.ts | 1 - upload-api/src/validators/wordpress/index.ts | 1 - 15 files changed, 6 insertions(+), 9 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 8d533e64..4a866ca4 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -333,6 +333,7 @@ const processField = ( } }; + // Helper function to clean up brackets in non-numeric lang_value const cleanBrackets = (lang_value: any) => { const myJSON = JSON.stringify(lang_value); diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 90d168b9..34d054c3 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -314,6 +314,7 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack }; } + function parseBlockquote(obj: any): any { const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); return { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index f64bcad6..6d403a3d 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -10,6 +10,7 @@ import { getLogMessage } from '../utils/index.js'; import customLogger from '../utils/custom-logger.utils.js'; const append = "a"; + const baseDirName = MIGRATION_DATA_CONFIG.DATA const { ENTRIES_DIR_NAME, diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index f48641a9..668c2a68 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -11,6 +11,7 @@ import { getLogMessage } from "../utils/index.js"; import { Advanced } from "../models/FieldMapper.js"; const { JSDOM } = jsdom; + const virtualConsole = new jsdom.VirtualConsole(); // Get the current file's path const __filename = fileURLToPath(import.meta.url); diff --git a/index.js b/index.js index 8993572b..92e49eb6 100644 --- a/index.js +++ b/index.js @@ -33,7 +33,6 @@ const envContents = { 'Upload-API': uploadAPIEnvContent, }; - // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index 975eb765..3c2a1243 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -4,5 +4,4 @@ "contentful": "contentfulSchema" }, "fileName": "en-us.json" - } \ No newline at end of file diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index e7268fb1..d355f817 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -42,6 +42,7 @@ const MIGRATION_DATA_CONFIG = { EXPORT_INFO_FILE: "export-info.json" } + module.exports = { MIGRATION_DATA_CONFIG }; \ No newline at end of file diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index cf50910c..d5ef93c8 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -48,5 +48,4 @@ "masterfile": "posts.json" } } - } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index 1682e698..3d9d5784 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -13,6 +13,7 @@ export default { buketKey: 'project/package 45.zip' }, + localPath: '/home/gaurishn/Documents/contentstack/sitetitle.xml' //package 45.zip' // localPath: '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45.zip' }; diff --git a/upload-api/src/models/contentful.json b/upload-api/src/models/contentful.json index 441a70ba..5ff2ce71 100644 --- a/upload-api/src/models/contentful.json +++ b/upload-api/src/models/contentful.json @@ -31,5 +31,4 @@ "name": "roles", "required": "false" } - } diff --git a/upload-api/src/models/wordpress.json b/upload-api/src/models/wordpress.json index 52e3e0f3..b0f90bfd 100644 --- a/upload-api/src/models/wordpress.json +++ b/upload-api/src/models/wordpress.json @@ -23,5 +23,4 @@ "name": "wp\\:attachment_url", "required": "false" } - } diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index cbd5d6c7..9ca2c637 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -7,7 +7,6 @@ import { Config } from '../../models/types'; const { extractContentTypes, createInitialMapper } = require('migration-contentful'); - const createContentfulMapper = async ( projectId: string | string[], app_token: string | string[], diff --git a/upload-api/src/validators/contentful/index.ts b/upload-api/src/validators/contentful/index.ts index a0711634..16a7e681 100644 --- a/upload-api/src/validators/contentful/index.ts +++ b/upload-api/src/validators/contentful/index.ts @@ -5,7 +5,6 @@ interface JsonData { [key: string]: any; } - //function to validate json file data function contentfulValidator(data: string): boolean { let jsonData: JsonData; diff --git a/upload-api/src/validators/sitecore/index.ts b/upload-api/src/validators/sitecore/index.ts index 69ecb291..551c50e6 100644 --- a/upload-api/src/validators/sitecore/index.ts +++ b/upload-api/src/validators/sitecore/index.ts @@ -9,7 +9,6 @@ interface props { data: items; } - async function sitecoreValidator({ data }: props) { try { let templates: any[] = []; diff --git a/upload-api/src/validators/wordpress/index.ts b/upload-api/src/validators/wordpress/index.ts index 2a758e99..86570598 100644 --- a/upload-api/src/validators/wordpress/index.ts +++ b/upload-api/src/validators/wordpress/index.ts @@ -1,7 +1,6 @@ import Config from '../../models/wordpress.json'; import * as Cheerio from 'cheerio'; - //function to validate xml file data function wordpressValidator(data: string): boolean { try { From 3e114e7fbfb86604065c66c5fa0f129531b77501 Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Thu, 6 Mar 2025 02:28:37 +0530 Subject: [PATCH 19/19] fixed test migartion --- api/src/services/contentful.service.ts | 51 +++++++++++++-------- api/src/services/contentful/jsonRTE.ts | 2 +- api/src/services/migration.service.ts | 9 ++-- api/src/utils/content-type-creator.utils.ts | 2 +- api/src/utils/test-folder-creator.utils.ts | 39 ++++++++-------- 5 files changed, 58 insertions(+), 45 deletions(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 8d533e64..db542422 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -10,7 +10,7 @@ import { JSDOM } from "jsdom"; import { jsonToHtml, jsonToMarkdown, htmlToJson } from '@contentstack/json-rte-serializer'; -import { CHUNK_SIZE, MIGRATION_DATA_CONFIG } from "../constants/index.js"; +import { CHUNK_SIZE, MIGRATION_DATA_CONFIG, LOCALE_MAPPER } from "../constants/index.js"; import { Locale } from "../models/types.js"; import jsonRTE from "./contentful/jsonRTE.js"; import { getAllLocales, getLogMessage } from "../utils/index.js"; @@ -234,7 +234,7 @@ const processField = ( return assetsData; } else { const id = lang_value?.sys?.id; - if (id in assetId) { + if (assetId && id in assetId) { return assetId?.[id] } return null; @@ -253,7 +253,7 @@ const processField = ( return refs; } const id = lang_value?.sys?.id; - if (id in entryId) return [[entryId?.[id]]]; + if (entryId && id in entryId) return [[entryId?.[id]]]; return null; } @@ -436,11 +436,11 @@ const saveAsset = async ( const publishDetails: { environment: any; version: number; locale: any }[] = []; const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); - const environmentsId = await readFile(path.join(DATA, destination_stack_id, ENVIRONMENTS_DIR_NAME), ENVIRONMENTS_FILE_NAME); + const environmentsId = await readFile(path.join(DATA, destination_stack_id, ENVIRONMENTS_DIR_NAME), ENVIRONMENTS_FILE_NAME) ?? {}; const localeId = await readFile(path.join(DATA, destination_stack_id, LOCALE_DIR_NAME), LOCALE_CF_LANGUAGE); if (assets.fields.file && assets.fields.title) { - Object.values(environmentsId).forEach((env: any) => { + Object?.values?.(environmentsId)?.length && Object.values(environmentsId).forEach((env: any) => { if (env?.name === assets?.sys?.environment?.sys?.id) { Object.values(localeId).forEach((locale: any) => { publishDetails.push({ @@ -696,6 +696,18 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string, } }; +const mapLocales = ({ masterLocale, locale, locales }: any) => { + if (locales?.masterLocale?.[masterLocale ?? ''] === locale) { + return Object?.keys(locales?.masterLocale)?.[0] + } + for (const [key, value] of Object?.entries?.(locales) ?? {}) { + if (typeof value !== 'object' && value === locale) { + return key; + } + } + return locale.toLowerCase(); +} + /** * Creates and processes entries from a given package file and saves them to the destination stack directory. * @@ -722,7 +734,8 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string, * * @throws Will log errors encountered during file reading, processing, or writing of entries. */ -const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any): Promise => { +const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string): Promise => { + console.info("🚀 ~ createEntry ~ master_locale:", master_locale) const srcFunc = 'createEntry'; try { const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME); @@ -735,7 +748,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje if (entries && entries.length > 0) { const assetId = await readFile(assetsSave, ASSETS_SCHEMA_FILE) ?? []; const entryId = await readFile(path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME), REFERENCES_FILE_NAME); - const environmentsId = await readFile(environmentSave, ENVIRONMENTS_FILE_NAME); + const environmentsId = await readFile(environmentSave, ENVIRONMENTS_FILE_NAME) ?? {}; const displayField: { [key: string]: any } = {} content.map((item: any) => { displayField[item.name.toLowerCase().replace(/[^a-z0-9]+/g, "_")] = @@ -781,13 +794,15 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje }); const pathName = getDisplayName(name, displayField); locales.forEach((locale) => { - const publishDetails = Object?.values?.(environmentsId) + const localeCode = mapLocales({ masterLocale: master_locale, locale, locales: LOCALE_MAPPER }); + console.info("🚀 ~ locales.forEach ~ localeCode:", localeCode); + const publishDetails = Object?.values?.(environmentsId)?.length ? Object?.values?.(environmentsId) .filter((env: any) => env?.name === environment_id) ?.map((env: any) => ({ environment: env?.uid, version: 1, - locale: locale.toLowerCase(), - })); + locale: locale?.toLowerCase(), + })) : []; const title = entryData[name][locale][id][pathName] || ""; const urlTitle = title @@ -814,21 +829,18 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje ); }); }); - return entryData; }, {} ); - const writePromises = []; - - for (const [newKey, values] of Object.entries(result)) { + for await (const [newKey, values] of Object.entries(result)) { const currentCT = contentTypes?.find((ct: any) => ct?.otherCmsUid === newKey); const ctName = currentCT?.contentstackUid in mapperKeys ? mapperKeys?.[currentCT?.contentstackUid] : (currentCT?.contentstackUid ?? newKey.replace(/([A-Z])/g, "_$1").toLowerCase()); - for (const [localeKey, localeValues] of Object.entries( + for await (const [localeKey, localeValues] of Object.entries( values as { [key: string]: any } )) { - const chunks = await makeChunks(localeValues); + const chunks = makeChunks(localeValues); for (const [entryKey, entryValue] of Object.entries(localeValues)) { const message = getLogMessage( srcFunc, @@ -843,14 +855,13 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje entriesSave, ctName, localeKey.toLowerCase() ); - for (const [chunkId, chunkData] of Object.entries(chunks)) { + for await (const [chunkId, chunkData] of Object.entries(chunks)) { refs[chunkIndex++] = `${chunkId}-entries.json`; - writePromises.push(writeFile(filePath, `${chunkId}-entries.json`, chunkData)) + await writeFile(filePath, `${chunkId}-entries.json`, chunkData) } - writePromises.push(writeFile(filePath, ENTRIES_MASTER_FILE, refs)); + await writeFile(filePath, ENTRIES_MASTER_FILE, refs); } } - await Promise.all(writePromises); } else { const message = getLogMessage( srcFunc, diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 90d168b9..0d7da75c 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -250,7 +250,7 @@ function parseBlockReference(obj: any, lang?: LangType, destination_stack_id?: S function parseInlineReference(obj: any, lang?: LangType, destination_stack_id?: StackId): any { const entryId: { [key: string]: any } = destination_stack_id && readFile(path.join(process.cwd(), DATA, destination_stack_id, RTE_REFERENCES_DIR_NAME, RTE_REFERENCES_FILE_NAME)); - const entry = Object.entries(entryId).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue[obj.data.target.sys.id]); + const entry = entryId && Object.entries(entryId).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue[obj.data.target.sys.id]); if (entry) { const [arrayKey, arrayValue] = entry; diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 5b7a6c78..c7d4fad2 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -25,6 +25,7 @@ import { extensionService } from "./extension.service.js"; + /** * Creates a test stack. * @@ -259,7 +260,7 @@ const startTestMigration = async (req: Request): Promise => { await contentfulService?.createWebhooks(file_path, project?.current_test_stack_id, projectId); await contentfulService?.createEnvironment(file_path, project?.current_test_stack_id, projectId); await contentfulService?.createAssets(file_path, project?.current_test_stack_id, projectId, true); - await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys); + await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale); await contentfulService?.createVersionFile(project?.current_test_stack_id, projectId); break; } @@ -299,8 +300,8 @@ const startMigration = async (req: Request): Promise => { await customLogger(projectId, project?.destination_stack_id, 'info', message); await setLogFilePath(loggerPath); const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.destination_stack_id, region, user_id }); - await marketPlaceAppService?.createAppManifest({ orgId, destinationStackId: project?.current_test_stack_id, region, userId: user_id }); - await extensionService?.createExtension({ destinationStackId: project?.current_test_stack_id }); + await marketPlaceAppService?.createAppManifest({ orgId, destinationStackId: project?.destination_stack_id, region, userId: user_id }); + await extensionService?.createExtension({ destinationStackId: project?.destination_stack_id }); switch (cms) { case CMS.SITECORE_V8: case CMS.SITECORE_V9: @@ -337,7 +338,7 @@ const startMigration = async (req: Request): Promise => { await contentfulService?.createWebhooks(file_path, project?.destination_stack_id, projectId); await contentfulService?.createEnvironment(file_path, project?.destination_stack_id, projectId); await contentfulService?.createAssets(file_path, project?.destination_stack_id, projectId); - await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys); + await contentfulService?.createEntry(file_path, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale); await contentfulService?.createVersionFile(project?.destination_stack_id, projectId); break; } diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index 290130ca..66424018 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -748,7 +748,7 @@ export const contenTypeMaker = async ({ contentType, destinationStackId, project if (currentCt?.uid) { ct = await mergeTwoCts(ct, currentCt); } - if (ct?.uid) { + if (ct?.uid && ct?.schema?.length) { if (contentType?.type === 'global_field') { const globalSave = path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, GLOBAL_FIELDS_DIR_NAME); const message = getLogMessage(srcFunc, `Global Field ${ct?.uid} has been successfully Transformed.`, {}); diff --git a/api/src/utils/test-folder-creator.utils.ts b/api/src/utils/test-folder-creator.utils.ts index cdbafae4..f3d95b32 100644 --- a/api/src/utils/test-folder-creator.utils.ts +++ b/api/src/utils/test-folder-creator.utils.ts @@ -8,7 +8,6 @@ const { ENTRIES_DIR_NAME, ASSETS_DIR_NAME, ASSETS_SCHEMA_FILE, - CONTENT_TYPES_DIR_NAME, CONTENT_TYPES_SCHEMA_FILE, ENTRIES_MASTER_FILE, GLOBAL_FIELDS_DIR_NAME, @@ -240,22 +239,24 @@ const sortGlobalField = async (baseDir: string, finalData: any) => { } } -const sortContentType = async (baseDir: string, finalData: any) => { - const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME); - const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME); - const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8')); - await sortGlobalField(baseDir, finalData); - const contentTypes: any = []; - for await (const ct of finalData) { - const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType); - await lookForReference(findCtData, finalData); - contentTypes?.push(findCtData); - } - await deleteFolderAsync(contentTypePath); - for await (const ctItem of contentTypes) { - await saveContent(ctItem, contentSave); - } -} +//this code can be used in feature + +// const sortContentType = async (baseDir: string, finalData: any) => { +// const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME); +// const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME); +// const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8')); +// await sortGlobalField(baseDir, finalData); +// const contentTypes: any = []; +// for await (const ct of finalData) { +// const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType); +// await lookForReference(findCtData, finalData); +// contentTypes?.push(findCtData); +// } +// await deleteFolderAsync(contentTypePath); +// for await (const ctItem of contentTypes) { +// await saveContent(ctItem, contentSave); +// } +// } @@ -275,7 +276,7 @@ export const testFolderCreator = async ({ destinationStackId }: any) => { const pathParts = normalizedPath.split(path.sep); const ct = pathParts?.[0]; // First directory const locale = pathParts?.[1]; // Second directory - allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length, entryData, filePath, locale }) + allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length ?? 0, entryData, filePath, locale }) } } } @@ -292,7 +293,7 @@ export const testFolderCreator = async ({ destinationStackId }: any) => { finalData?.push({ contentType: et?.contentType, entryObj, locale: et?.locale }); }); await sortAssets(baseDir); - await sortContentType(baseDir, finalData); + // await sortContentType(baseDir, finalData); await deleteFolderAsync(entryDelete); for await (const entry of finalData) { const fileMeta = { "1": `${entry?.locale}.json` };