Feature/rohit logger

.gitignore

@@ -355,4 +355,3 @@ package-lock.json
 ui/.env
 upload-api/sitecoreMigrationData
 upload-api/extracted_files
-locale-cli

api/.gitignore

@@ -361,4 +361,5 @@ package-lock.json
 !example.env
 
 database/
-/sitecoreMigrationData
+/sitecoreMigrationData
+/migration-data

api/package.json

@@ -35,6 +35,7 @@
     "express": "^4.18.2",
     "express-validator": "^7.0.1",
     "express-winston": "^4.2.0",
+    "fs-extra": "^11.2.0",
     "fs-readdir-recursive": "^1.1.0",
     "helmet": "^7.1.0",
     "html-to-json-parser": "^2.0.1",
@@ -50,6 +51,7 @@
   "devDependencies": {
     "@types/cors": "^2.8.17",
     "@types/express": "^4.17.21",
+    "@types/fs-extra": "^11.0.4",
     "@types/fs-readdir-recursive": "^1.1.3",
     "@types/jsdom": "^21.1.7",
     "@types/jsonwebtoken": "^9.0.5",

api/src/config/index.ts

@@ -10,6 +10,7 @@ dotenv.config({
   path: path.resolve(process.cwd(), `${process.env.NODE_ENV}.env`),
 });
 
+
 /**
  * Configuration type for the application.
  */
@@ -34,6 +35,7 @@ export type ConfigType = {
     AZURE_EU?: string;
     GCP_NA?: string;
   };
+  LOG_FILE_PATH: string;
 };
 
 /**

api/src/controllers/migration.controller.ts

@@ -21,7 +21,7 @@ const createTestStack = async (req: Request, res: Response): Promise<void> => {
  * @returns {Promise<void>} - A Promise that resolves when the stack is deleted.
  */
 const fieldMapping = async (req: Request, res: Response): Promise<void> => {
-  const resp = await migrationService.fieldMapping(req);
+  const resp = migrationService.fieldMapping(req);
   res.status(200).json(resp);
 };
 

api/src/server.ts

@@ -21,86 +21,109 @@ import { Server } from "socket.io";
 import http from "http";
 import fs from "fs";
 
+// Initialize file watcher for the log file
+const watcher = chokidar.watch(config.LOG_FILE_PATH,{
+  usePolling: true,     // Enables polling to detect changes in all environments
+  interval: 100,        // Poll every 100ms (you can adjust this if needed)
+  awaitWriteFinish: {   // Wait for file to finish being written before triggering
+    stabilityThreshold: 500,  // Time to wait before considering the file stable
+    pollInterval: 100,        // Interval at which to poll for file stability
+  },
+  persistent: true,     // Keeps watching the file even after initial change
+}); // Initialize with initial log path
+
+let io: Server; // Socket.IO server instance
+
+// Dynamically change the log file path and update the watcher
+export async function setLogFilePath(path:string) {
+  console.info(`Setting new log file path: ${path}`);
+
+  // Stop watching the old log file
+  watcher.unwatch(config.LOG_FILE_PATH);
+
+  // Update the config and start watching the new log file
+  config.LOG_FILE_PATH = path;
+  watcher.add(path);
+}
+
 try {
   const app = express();
+
+  // Set security-related HTTP headers
   app.use(
     helmet({
-      crossOriginOpenerPolicy: false,
+      crossOriginOpenerPolicy: false, // Disable to allow cross-origin resource sharing
     })
   );
 
+  // Enable CORS for all origins
   app.use(cors({ origin: "*" }));
+
+  // Parsing request bodies
   app.use(express.urlencoded({ extended: false, limit: "10mb" }));
   app.use(express.json({ limit: "10mb" }));
+
+  // Custom middleware for logging and request headers
   app.use(loggerMiddleware);
   app.use(requestHeadersMiddleware);
 
-  // Routes
+  // Define routes
   app.use("/v2/auth", authRoutes);
   app.use("/v2/user", authenticateUser, userRoutes);
   app.use("/v2/org/:orgId", authenticateUser, orgRoutes);
   app.use("/v2/org/:orgId/project", authenticateUser, projectRoutes);
   app.use("/v2/mapper", authenticateUser, contentMapperRoutes);
   app.use("/v2/migration", authenticateUser, migrationRoutes);
 
-  //For unmatched route patterns
+  // Handle unmatched routes
   app.use(unmatchedRoutesMiddleware);
 
-  // Error Middleware
+  // Handle errors
   app.use(errorMiddleware);
 
-  // starting the server & DB connection.
+  // Start the server and establish DB connection
   (async () => {
-    await connectToDatabase();
+    await connectToDatabase(); // Establish DB connection
+
     const server = app.listen(config.PORT, () =>
       logger.info(`Server listening at port ${config.PORT}`)
     );
-    // Chokidar - Watch for log file changes
-    const logFilePath = config.LOG_FILE_PATH;
-    const watcher = chokidar.watch(logFilePath);
-    // Socket.IO - Send logs to client
-    /**
-     * The Socket.IO server instance.
-     *
-     * @remarks
-     * This server instance is responsible for handling real-time communication between the client and the server using the Socket.IO library.
-     *
-     * @type {Server}
-     */
-    const io = new Server(
-      server,
-      (http,
-      {
-        cors: {
-          origin: "*", // This allows all origins. For production, specify exact origins for security.
-          methods: ["GET", "POST"], // Specify which HTTP methods are allowed.
-          allowedHeaders: ["my-custom-header"], // Specify which headers are allowed.
-          credentials: true, // If your client needs to send cookies or credentials with the requests.
-        },
-      })
-    );
 
+    // Initialize Socket.IO for real-time log updates
+    io = new Server(server, {
+      cors: {
+        origin: "*", // Allow all origins; adjust for production
+        methods: ["GET", "POST"],
+        allowedHeaders: ["my-custom-header"],
+        credentials: true,
+      },
+    });
+
+    // Emit initial log file content to connected clients
+
+    // File watcher for log file changes
     watcher.on("change", (path) => {
-      // Read the updated log file
+      console.info(`File changed: ${path}`);
+
+      // Read the updated file content
       fs.readFile(path, "utf8", (err, data) => {
         if (err) {
           logger.error(`Error reading log file: ${err}`);
           return;
         }
-        // Get just the updated data
-        // const updatedData = data.slice(data.lastIndexOf("\n") + 1);
-        console.info("updates", data);
-        // Emit the updated data to all connected clients
+
         try {
-          const parsedData = data;
-          io.emit("logUpdate", parsedData);
+          // Emit the updated log content to connected clients
+          io.emit("logUpdate", data);
         } catch (error) {
-          logger.error(`Error parsing data: ${error}`);
+          logger.error(`Error emitting log data: ${error}`);
         }
       });
     });
+
   })();
 } catch (e) {
   logger.error("Error while starting the server!");
   logger.error(e);
 }
+

api/src/services/auth.service.ts

@@ -23,7 +23,6 @@ import logger from "../utils/logger.js";
  */
 const login = async (req: Request): Promise<LoginServiceType> => {
   const srcFun = "Login";
-  const cliUtilities: any = await import('@contentstack/cli-utilities');
   /*
   handles user authentication by making a request to an API, 
   performing various checks and validations, 
@@ -110,10 +109,6 @@ const login = async (req: Request): Promise<LoginServiceType> => {
         data.users[userIndex].authtoken = res?.data.user?.authtoken;
         data.users[userIndex].updated_at = new Date().toISOString();
       }
-
-      cliUtilities?.configHandler?.set('region', userData?.region);
-      cliUtilities?.configHandler?.set('authtoken', res?.data.user?.authtoken);
-
     });
 
     // JWT token generation

api/src/services/migration.service.ts

@@ -1,5 +1,5 @@
 import { Request } from "express";
-import cliUtilities from '@contentstack/cli-utilities';
+// import cliUtilities from '@contentstack/cli-utilities';
 import { config } from "../config/index.js";
 import { safePromise, getLogMessage } from "../utils/index.js";
 import https from "../utils/https.utils.js";
@@ -10,12 +10,16 @@ import { HTTP_TEXTS, HTTP_CODES, CS_REGIONS } from "../constants/index.js";
 import { ExceptionFunction } from "../utils/custom-errors.utils.js";
 import { fieldAttacher } from "../utils/field-attacher.utils.js";
 import ProjectModelLowdb from "../models/project-lowdb.js";
-// import shell from 'shelljs'
-// import path from "path";
+import shell from 'shelljs'
+import path from "path";
 import AuthenticationModel from "../models/authentication.js";
 import { siteCoreService } from "./sitecore.service.js";
+import { fileURLToPath } from 'url';
+import { copyDirectory } from '../utils/index.js'
+import { setLogFilePath } from "../server.js";
+
+
 
-// const importCmd: any = await import('@contentstack/cli-cm-import');
 
 /**
  * Creates a test stack.
@@ -198,65 +202,92 @@ const cliLogger = (child: any) => {
     console.info(`Error: Failed to install @contentstack/cli. Exit code: ${child.code}`);
     console.info(`stderr: ${child.stderr}`);
   } else {
-    console.info('Installation successful', child?.stdout);
+    console.info('Installation successful', child, child?.stdout);
   }
 };
 
-const runCli = async (rg: string, user_id: string) => {
+const runCli = async (rg: string, user_id: string, project: any) => {
   try {
     const regionPresent = CS_REGIONS?.find((item: string) => item === rg) ?? 'NA';
-    // const email = 'umesh.more+10@contentstack.com'
     await AuthenticationModel.read();
     const userData = AuthenticationModel.chain
       .get("users")
       .find({ region: regionPresent, user_id })
       .value();
-    if (userData?.authtoken) {
-
-      cliUtilities?.configHandler?.set('region', regionPresent);
-      cliUtilities?.configHandler?.set('authtoken', userData?.authtoken);
-      // shell.cd(path.resolve(process.cwd(), `../cli/packages/contentstack`));
-      // const pwd = shell.exec('pwd');
-      // cliLogger(pwd);
-      // const region = shell.exec(`node bin/run config:set:region ${regionPresent}`);
-      // cliLogger(region);
-      // const login = shell.exec(`node bin/run login -a ${userData?.authtoken}  -e ${email}`)
-      // cliLogger(login);
-      // const exportData = shell.exec(`node bin/run cm:stacks:import  -k blt3e7d2a4135d8bfab -d "/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/data" --backup-dir="/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/migrations/blt3e7d2a4135d8bfab"`);
-      // cliLogger(exportData);
-      // const cmd = [`-k ${userData?.authtoken}`, "-d /Users/umesh.more/Documents/ui-migration/migration-v2-node-server/api/sitecoreMigrationData", "--backup-dir=/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/migrations/blt3e7d2a4135d8bfab", "--yes"]
-
-      // await importCmd.default.run(cmd);  // This will bypass the type issue
-      // shell.cd(path.resolve(process.cwd(), '..', 'locale-cli', 'packages', 'contentstack'));
-      // const pwd = shell.exec('pwd');
-      // cliLogger(pwd);
-      // const region = shell.exec(`node bin/run config:set:region ${regionPresent}`);
-      // cliLogger(region);
-      // const login = shell.exec(`node bin/run login -a ${userData?.authtoken}  -e ${email}`)
-      // cliLogger(login);
-      // const exportData = shell.exec(`node bin/run cm:stacks:import  -k blt69235b992c3d99c6 -d "/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/api/sitecoreMigrationData" --backup-dir="/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/test"`);
-      // cliLogger(exportData);
+    if (userData?.authtoken && project?.destination_stack_id) {
+      // Manually define __filename and __dirname
+      const __filename = fileURLToPath(import.meta.url);
+      const dirPath = path.join(path.dirname(__filename), '..', '..');
+      const sourcePath = path.join(dirPath, 'sitecoreMigrationData', project?.destination_stack_id);
+      const backupPath = path.join(process.cwd(), 'migration-data', project?.destination_stack_id);
+      await copyDirectory(sourcePath, backupPath);
+
+      const loggerPath = path.join(backupPath, 'logs', 'import','success.log');
+      console.info('loggerPath', loggerPath);
+      await setLogFilePath(loggerPath);
+      shell.cd(path.join(process.cwd(), '..', 'cli', 'packages', 'contentstack'));
+      const pwd = shell.exec('pwd');
+      cliLogger(pwd);
+      const region = shell.exec(`node bin/run config:set:region ${regionPresent}`);
+      cliLogger(region);
+      const login = shell.exec(`node bin/run login -a ${userData?.authtoken}  -e ${userData?.email}`);
+      cliLogger(login);
+      const exportData = shell.exec(`node bin/run cm:stacks:import  -k ${project?.destination_stack_id} -d ${sourcePath} --backup-dir=${backupPath}  --yes`, { async: true });
+      cliLogger(exportData);
     } else {
       console.info('user not found.')
     }
   } catch (er) {
-    console.info("🚀 ~ runCli ~ er:", er)
+    console.error("🚀 ~ runCli ~ er:", er)
   }
 }
 
 const fieldMapping = async (req: Request): Promise<any> => {
   const { orgId, projectId } = req?.params ?? {};
-  const contentTypes = await fieldAttacher({ orgId, projectId });
-  const packagePath = '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45';
-  await siteCoreService?.createEntry({ packagePath, contentTypes });
-  await siteCoreService?.createLocale(req);
-  await siteCoreService?.createVersionFile();
   const { region, user_id } = req?.body?.token_payload ?? {};
-  await runCli(region, user_id);
+  const project = ProjectModelLowdb.chain
+    .get("projects")
+    .find({ id: projectId })
+    .value();
+  if (project?.extract_path && project?.destination_stack_id) {
+    const packagePath = project?.extract_path;
+    const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.destination_stack_id });
+    await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.destination_stack_id });
+    await siteCoreService?.createLocale(req, project?.destination_stack_id);
+    await siteCoreService?.createVersionFile(project?.destination_stack_id);
+    await runCli(region, user_id, project);
+  }
 }
 
 export const migrationService = {
   createTestStack,
   deleteTestStack,
   fieldMapping
 };
+
+
+
+
+
+// cliUtilities?.configHandler?.set('region', regionPresent);
+// cliUtilities?.configHandler?.set('authtoken', userData?.authtoken);
+// shell.cd(path.resolve(process.cwd(), `../cli/packages/contentstack`));
+// const pwd = shell.exec('pwd');
+// cliLogger(pwd);
+// const region = shell.exec(`node bin/run config:set:region ${regionPresent}`);
+// cliLogger(region);
+// const login = shell.exec(`node bin/run login -a ${userData?.authtoken}  -e ${email}`)
+// cliLogger(login);
+// const exportData = shell.exec(`node bin/run cm:stacks:import  -k blt3e7d2a4135d8bfab -d "/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/data" --backup-dir="/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/migrations/blt3e7d2a4135d8bfab"`);
+// cliLogger(exportData);
+// const cmd = [`-k ${userData?.authtoken}`, "-d /Users/umesh.more/Documents/ui-migration/migration-v2-node-server/api/sitecoreMigrationData", "--backup-dir=/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/migrations/blt3e7d2a4135d8bfab", "--yes"]
+// await importCmd.default.run(cmd);  // This will bypass the type issue
+// shell.cd(path.resolve(`${path?.dirname}`, '..', 'cli', 'packages', 'contentstack'));
+// const importCmd: any = await import('@contentstack/cli-cm-import');
+
+
+
+
+
+
+