diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b6192f200..e27d32582 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,5 @@ +* @contentstack/tso-migration-pr-reviewers .github/workflows/sca-scan.yml @contentstack/security-admin .github/workflows/sast-scan.yml @contentstack/security-admin .github/workflows/jira.yml @contentstack/security-admin -**/.snyk @contentstack/security-admin +**/.snyk @contentstack/security-admin \ No newline at end of file diff --git a/README.md b/README.md index 8620f2f54..3f079c37e 100644 --- a/README.md +++ b/README.md @@ -8,29 +8,135 @@ Clone the repository: git clone https://github.com/contentstack/migration-v2.git cd migration-v2 ``` + +Check for readme.md files and install dependencies for folders + +## Migration-v2 API + +This is the migration V2's node server. + +### Environment Variables +The following environment variables are used in this project: + +- `APP_TOKEN_KEY`: The token key for the application. Default is `MIGRATION_V2`. +- `PORT`: The port number on which the application runs. Default is `5001`. + +Make sure to set these variables in a `.env` file at the root of your project. + +1. To run the development server, create a `./development.env` file and add environment variables as per `./example.env` +2. To run the production server, create a `./production.env` file and add environment variables as per `./example.env` + +### To start the server +Run `npm run dev` + + +## Migration UI + +Migration UI is a web application designed to facilitate the migration of content. It provides a user-friendly interface for managing and migrating content efficiently. + +### Features +- User-friendly interface for content migration +- Integration with Contentstack +- State management with Redux Toolkit +- Comprehensive testing with Testing Library + +### Installation + +1. Navigate to the project directory: + ```sh + cd ui + ``` + +2. Install the dependencies: + ```sh + npm install + ``` + +### Environment Variables + +The following environment variables are used in this project: + +- `REACT_APP_WEBSITE_BASE_URL`: The base URL for the website. Default is `http://localhost:3000/`. +- `REACT_APP_BASE_API_URL`: The base URL for the API. Default is `http://localhost:5001/`. +- `REACT_APP_API_VERSION`: The version of the API. Default is `v2`. +- `REACT_APP_HOST`: The host URL for the application. Default is `http://localhost:3000`. +- `REACT_APP_UPLOAD_SERVER`: The URL for the upload server. Default is `http://localhost:4002/`. +- `REACT_APP_OFFLINE_CMS`: A flag to indicate if the CMS is offline. Default is `true`. + +Make sure to set these variables in a `.env` file at the root of your ui project. + +### Usage +Start the development server: +```sh +npm start +``` + +Open your browser and navigate to http://localhost:3000. + +### Dependencies +- @contentstack/json-rte-serializer: ^2.0.5 +- @contentstack/venus-components: ^2.2.4 +- @reduxjs/toolkit: ^2.2.5 +- @testing-library/jest-dom: ^5.17.0 +- @testing-library/react: ^13.4.0 +- @testing-library/user-event: ^13.5.0 +- @types/react: ^18.2.28 +- @types/react-dom: ^18.2.13 +- @types/react-redux: ^7.1.33 + +## Migration v2 upload-api +### Overview +The migration-v2 upload-api project is designed to facilitate the migration of different CMS to Contentstack functionalities. + +### Installation +Navigate to the project directory: +``` +cd migration-v2/upload-api +``` Install dependencies: ``` npm install ``` -Check for readme.md files and install dependencies for folders +### Environment Variables -1. go to api folder - ``` - cd api - npm install - ``` +The following environment variables are used in this project: + +- `PORT`: The port number on which the application runs. Default is `4002`. +- `NODE_BACKEND_API`: The backend API endpoint. Default is `http://localhost:5001`. + +Make sure to set these variables in a `.env` file at the root of your project. + +### Configuration +Please refer sample.config.json and provide this config in index.ts of config folder of upload-api/src +The following configuration is used in this project: -2. go to ui folder +- `plan.dropdown.optionLimit`: The limit for dropdown options. Default is `100`. +- `cmsType`: The type of CMS used. Default is `sitecore`. +- `isLocalPath`: A flag to indicate if the path is local. Default is `true`. + +#### AWS Data +- `awsRegion`: The AWS region. Default is `us-east-2`. +- `awsAccessKeyId`: The AWS access key ID. +- `awsSecretAccessKey`: The AWS secret access key. +- `awsSessionToken`: The AWS session token. +- `bucketName`: The name of the AWS S3 bucket. Default is `migartion-test`. +- `buketKey`: The key for the AWS S3 bucket. Default is `project/package 45.zip`. + +#### Local Path +- `localPath`: The local path to the extracted files. for example `/upload-api/extracted_files/package 45.zip`. + +## Cli +Navigate to the project directory + 1. Install pnpm + Since pnpm is required, you need to install it globally if it's not already installed: ``` - cd ui - npm install + npm install -g pnpm ``` - -3. go to upload-api folder + 2. Run the Setup Script + Now, run the main setup script specified in the package.json: ``` - cd upload-api - npm install + npm run setup-repo ``` ## Scripts diff --git a/api/README.md b/api/README.md index e92712735..e69de29bb 100644 --- a/api/README.md +++ b/api/README.md @@ -1,18 +0,0 @@ -# migration-v2 API - -This is the migration V2's node server. - -## To start the server - -1. Run `npm i` -2. Install and start MongoDB - -### To start MongoDB - -1. Install MongoDB on your system. -2. Start the MongoDB service. -3. Verify that MongoDB is running by opening a command prompt and running the command `mongo --version`. - -3. To run the development server, create a `./development.env` file and add environment variables as per `./example.env` -4. To run the production server, create a `./production.env` file and add environment variables as per `./example.env` - diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index d02d1ba63..2c3fa357e 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -192,7 +192,8 @@ const convertToSchemaFormate = ({ field, advanced = true }: any) => { "unique": field?.advanced?.unique ?? false, "non_localizable": field.advanced?.nonLocalizable ?? false }; - data.field_metadata.default_value = field?.advanced?.default_value ?? null; + const default_value = field?.advanced?.options?.length ? (field?.advanced?.options?.find((item: any) => (item?.key === field?.advanced?.default_value) || (item?.key === field?.advanced?.default_value))) : { value: field?.advanced?.default_value }; + data.field_metadata.default_value = default_value?.value ?? null; return data; } case 'radio': { diff --git a/api/src/utils/test-folder-creator.utils.ts b/api/src/utils/test-folder-creator.utils.ts index 8e83f670e..79e993cb9 100644 --- a/api/src/utils/test-folder-creator.utils.ts +++ b/api/src/utils/test-folder-creator.utils.ts @@ -10,7 +10,9 @@ const { ASSETS_SCHEMA_FILE, CONTENT_TYPES_DIR_NAME, CONTENT_TYPES_SCHEMA_FILE, - ENTRIES_MASTER_FILE + ENTRIES_MASTER_FILE, + GLOBAL_FIELDS_DIR_NAME, + GLOBAL_FIELDS_FILE_NAME } = MIGRATION_DATA_CONFIG; @@ -201,10 +203,41 @@ const sortAssets = async (baseDir: string) => { await fs.promises.writeFile(path.join(assetsPath, ASSETS_SCHEMA_FILE), JSON?.stringify?.(assetsMeta)); } +const writeGlobalField = async (schema: any, globalSave: string, filePath: string) => { + try { + await fs.promises.access(globalSave); + } catch (err) { + try { + await fs.promises.mkdir(globalSave, { recursive: true }); + } catch (mkdirErr) { + console.error("🚀 ~ fs.mkdir ~ err:", mkdirErr); + return; + } + } + try { + await fs.promises.writeFile(filePath, JSON.stringify(schema, null, 2)); + } catch (writeErr) { + console.error("🚀 ~ fs.writeFile ~ err:", writeErr); + } +}; + +const sortGlobalField = async (baseDir: string, finalData: any) => { + const globalSave = path.join(process.cwd(), baseDir, GLOBAL_FIELDS_DIR_NAME); + const globalPath = path.join(globalSave, GLOBAL_FIELDS_FILE_NAME); + const globalData = await JSON.parse(await fs.promises.readFile(globalPath, 'utf8')); + const globalResult = []; + for await (const ct of globalData) { + await lookForReference(ct, finalData); + globalResult?.push(ct); + } + await writeGlobalField(globalResult, globalPath, globalPath); +} + const sortContentType = async (baseDir: string, finalData: any) => { const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME); const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME); const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8')); + await sortGlobalField(baseDir, finalData); const contentTypes: any = []; for await (const ct of finalData) { const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType); @@ -218,6 +251,7 @@ const sortContentType = async (baseDir: string, finalData: any) => { } + export const testFolderCreator = async ({ destinationStackId }: any) => { const sanitizedStackId = path.basename(destinationStackId); const baseDir = path.join(MIGRATION_DATA_CONFIG.DATA, sanitizedStackId); diff --git a/cli/packages/contentstack-dev-dependencies/.snyk b/cli/packages/contentstack-dev-dependencies/.snyk deleted file mode 100644 index 69b495d0e..000000000 --- a/cli/packages/contentstack-dev-dependencies/.snyk +++ /dev/null @@ -1,10 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.25.0 -# ignores vulnerabilities until expiry date; change duration by modifying expiry date -ignore: - SNYK-JS-ASYNC-7414156: - - '*': - reason: 'https://contentstack.atlassian.net/browse/CLOUD-6762' - expires: 2024-08-02T13:54:17.307Z - created: 2024-07-03T13:54:17.314Z -patch: {} diff --git a/ui/README.md b/ui/README.md index 5fe4fea2f..8b1378917 100644 --- a/ui/README.md +++ b/ui/README.md @@ -1,51 +1 @@ -# Migration UI -Migration UI is a web application designed to facilitate the migration of content. It provides a user-friendly interface for managing and migrating content efficiently. - -## Features -- User-friendly interface for content migration -- Integration with Contentstack -- State management with Redux Toolkit -- Comprehensive testing with Testing Library - -## Installation - -1. Clone the repository: - ```sh - git clone https://github.com/contentstack/migration-v2 - ``` - -2. Navigate to the project directory: - ```sh - cd ui - ``` - -3. Install the dependencies: - ```sh - npm install - ``` - -## Usage -Start the development server: -```sh -npm start -``` - -Open your browser and navigate to http://localhost:3000. - -## Dependencies -- @contentstack/json-rte-serializer: ^2.0.5 -- @contentstack/venus-components: ^2.2.4 -- @reduxjs/toolkit: ^2.2.5 -- @testing-library/jest-dom: ^5.17.0 -- @testing-library/react: ^13.4.0 -- @testing-library/user-event: ^13.5.0 -- @types/react: ^18.2.28 -- @types/react-dom: ^18.2.13 -- @types/react-redux: ^7.1.33 - -## Contributing -Contributions are welcome! Please open an issue or submit a pull request for any changes. - -## License -This project is licensed under the MIT License. diff --git a/upload-api/.env b/upload-api/.env index 1c5d9649a..ea5f15669 100644 --- a/upload-api/.env +++ b/upload-api/.env @@ -1,2 +1,2 @@ PORT=4002 -NODE_BACKEND_API=http://localhost:5000 +NODE_BACKEND_API=http://localhost:5001 diff --git a/upload-api/README.md b/upload-api/README.md index 1b18a9ddc..e69de29bb 100644 --- a/upload-api/README.md +++ b/upload-api/README.md @@ -1,36 +0,0 @@ -# Migration v2 upload-api -## Overview -The migration-v2 upload-api project is designed to facilitate the migration of different CMS to Contentstack functionalities. - -## Installation -Clone the repository: -``` -git clone https://github.com/contentstack/migration-v2.git -cd migration-v2/upload-api -``` -Install dependencies: -``` -npm install -``` - -## Environment Variables - -The following environment variables are used in this project: - -- `PORT`: The port number on which the application runs. Default is `4002`. -- `NODE_BACKEND_API`: The backend API endpoint. Default is `http://localhost:5000`. - -Make sure to set these variables in a `.env` file at the root of your project. - -## Repository -- Type: git -- URL: https://github.com/contentstack/migration-v2.git - -## License -This project is licensed under the ISC License. - -## Author -The author information is not specified. - -## Contact -For further assistance, please contact the project maintainer through the issues page on GitHub. diff --git a/upload-api/migration-sitecore/utils/index.js b/upload-api/migration-sitecore/utils/index.js index cae85bd5f..519e6ac8d 100644 --- a/upload-api/migration-sitecore/utils/index.js +++ b/upload-api/migration-sitecore/utils/index.js @@ -1,48 +1,73 @@ const restrictedUid = [ - "uid", "api_key", - "created_at", - "deleted_at", - "updated_at", - "tags_array", - "klass_id", - "applikation_id", - "*_ids", - "id", - "_id", - "ACL", - "SYS_ACL", - "DEFAULT_ACL", - "app_user_object_uid", + "built_io_application_user", + "built_io_application_user_role", + "built_io_audit_log", + "built_io_environment", + "built_io_extensions", + "built_io_installation_data", + "built_io_label", + "built_io_language", + "built_io_publish_queue", + "built_io_release", "built_io_upload", + "cs_branches", + "org_uid", + "sys_asset", + "sys_metadata", + "_exists", + "_regex", + "*_ids", // Fields cannot have UID ending with "_ids." + "__indexes", "__loc", - "tags", + "__meta", + "__v", + "_id", "_owner", + "_publish_locales", + "_shouldFilter", + "_shouldLean", "_version", - "toJSON", - "save", - "update", + "ACL", + "api_key", + "app_user_object_uid", + "applikation_id", + "built_io_upload", + "contentstackFilters", + "created_at", + "created_by", + "DEFAULT_ACL", + "deleted_at", + "dimension", "domain", - "shard_account", - "shard_app", - "shard_random", + "embedded_items", "hook", - "__indexes", - "__meta", - "created_by", - "updated_by", + "id", "inbuilt_class", - "tenant_id", - "isSystemUser", "isApplicationUser", "isNew", - "_shouldLean", - "_shouldFilter", - "options", - "_version", - "__v", + "isSystemUser", + "klass_id", "locale", - "publish_details" + "options", + "org_uid", + "publish_details", + "save", + "shard_account", + "shard_app", + "shard_random", + "SYS_ACL", + "sys_assets", + "sys_metadata", + "tags", + "tags_array", + "taxonomies", + "tenant_id", + "toJSON", + "uid", + "update", + "updated_at", + "updated_by" ]; module.exports = restrictedUid; \ No newline at end of file