improvement for readCSV file and code refactoring

.nvmrc

@@ -0,0 +1 @@
+20.8.0

index.js

@@ -1,74 +1,95 @@
 import fs from "node:fs/promises"
-import { existsSync } from "node:fs"
-import { createIfNot } from "./utils/fileUtils.mjs"
-import path from "node:path"
+
 async function writeSQL(statement, saveFileAs = "") {
   try {
     const destinationFile = process.argv[2] || saveFileAs
     if (!destinationFile) {
       throw new Error("Missing saveFileAs parameter")
     }
-    createIfNot(path.resolve(`./sql/${destinationFile}`))
     await fs.writeFile(`sql/${process.argv[2]}.sql`, statement)
   } catch (err) {
     console.log(err)
   }
 }
-async function readCSV(csvFileName = "") {
+
+async function readCSV(csvFileName = '', batchSize = 500) {
   try {
-    const fileAndTableName = process.argv[2] || csvFileName
+    const fileAndTableName = process.argv[2] || csvFileName;
+    const isAppend = false; // Initialize isAppend directly
+
     if (!fileAndTableName) {
-      throw new Error("Missing csvFileName parameter")
+      throw new Error('Missing csvFileName parameter');
     }
-    if (!existsSync(path.resolve(`./csv/${fileAndTableName}.csv`))) {
-      console.log("file not found")
-      return
+
+    const filePath = `csv/${fileAndTableName}.csv`;
+
+    if (!(await fileExists(filePath))) {
+      console.log('File not found:', filePath);
+      return;
     }
-    const data = await fs.readFile(`csv/${fileAndTableName}.csv`, {
-      encoding: "utf8",
-    })
-    const linesArray = data.split(/\r|\n/).filter(line => line)
-    const columnNames = linesArray.shift().split(",")
-    let beginSQLInsert = `INSERT INTO ${fileAndTableName} (`
-    columnNames.forEach(name => (beginSQLInsert += `${name}, `))
-    beginSQLInsert = beginSQLInsert.slice(0, -2) + ")\nVALUES\n"
-    let values = ""
-    linesArray.forEach(line => {
-      // Parses each line of CSV into field values array
-      const arr = line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/)
-      if (arr.length > columnNames.length) {
-        console.log(arr)
-        throw new Error("Too Many Values in row")
-      } else if (arr.length < columnNames.length) {
-        console.log(arr)
-        throw new Error("Too Few Values in row")
+
+    const data = await fs.readFile(filePath, { encoding: 'utf8' });
+    const linesArray = data.split(/\r|\n/).filter((line) => line);
+    const columnNames = linesArray.shift()?.split(',') || [];
+    let beginSQLInsert = `INSERT INTO ${fileAndTableName} (${columnNames.join(', ')})\nVALUES\n`;
+    let values = '';
+
+    linesArray.forEach((line, index) => {
+      const arr = line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/);
+
+      // Validate the number of values in each row
+      validateRow(arr, columnNames);
+
+      if (index > 0 && index % batchSize === 0) {
+        values = `${values.slice(0, -2)};\n\n`;
+        const sqlStatement = `${beginSQLInsert}${values}`;
+
+        writeSQL(sqlStatement, fileAndTableName, isAppend);
+        values = '';
+        isAppend = true;
       }
-      let valueLine = "\t("
-      arr.forEach(value => {
-        // Matches NULL values, Numbers,
-        // Strings accepted as numbers, and Booleans (0 or 1)
-        if (value === "NULL" || !isNaN(+value)) {
-          valueLine += `${value}, `
-        } else {
-          // If a string is wrapped in quotes, it doesn't need more
-          if (value.at(0) === '"') valueLine += `${value}, `
-          else {
-            // This wraps strings in quotes
-            // also wraps timestamps
-            valueLine += `"${value}", `
-          }
-        }
-      })
-      valueLine = valueLine.slice(0, -2) + "),\n"
-      values += valueLine
-    })
-    values = values.slice(0, -2) + ";"
-    const sqlStatement = beginSQLInsert + values
-    // Write File
-    writeSQL(sqlStatement)
+
+      let valueLine = '\t(';
+      arr.forEach((value) => {
+        // Handle NULL values, Numbers, Strings, and Booleans
+        valueLine += value === 'NULL' || !isNaN(+value)
+          ? `${value}, `
+          : value.at(0) === '"'
+            ? `${value}, `
+            : `"${value}", `;
+      });
+
+      valueLine = `${valueLine.slice(0, -2)}),\n`;
+      values += valueLine;
+    });
+
+    values = `${values.slice(0, -2)};`;
+    const sqlStatement = `${beginSQLInsert}${values}`;
+
+    writeSQL(sqlStatement, fileAndTableName, isAppend);
   } catch (err) {
-    console.log(err)
+    console.error(err);
+  }
+}
+
+//file existence check 
+async function fileExists(filePath) {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+//Extracted the validation logic
+function validateRow(arr, columnNames) {
+  if (arr.length !== columnNames.length) {
+    console.error(arr);
+    throw new Error(`Mismatched number of values in row. Expected: ${columnNames.length}, Actual: ${arr.length}`);
   }
 }
-readCSV()
+
+readCSV();
+
 console.log("Finished!")