#RI-3478 - automatic update dependencies versions

.circleci/config.yml

@@ -555,6 +555,10 @@ jobs:
             yarn --cwd redisinsight/api/ install --ignore-optional
             yarn --cwd redisinsight/ install --ignore-optional
             yarn install
+          no_output_timeout: 15m
+      - run:
+          name: Install plugins dependencies and build plugins
+          command: |
             yarn build:statics
           no_output_timeout: 15m
       - run:
@@ -599,6 +603,7 @@ jobs:
             - release/RedisInsight*.snap
             - release/*-linux.yml
             - release/redisstack
+
   macosx:
     macos:
       xcode: 14.2.0
@@ -793,6 +798,32 @@ jobs:
       - store_artifacts:
           path: docker-release
           destination: docker-release
+  licenses-check:
+    executor: linux-executor
+    steps:
+      - checkout
+      - restore_cache:
+          <<: *uiDepsCacheKey
+          <<: *apiDepsCacheKey
+      - run:
+          name: Run install all dependencies
+          command: |
+            yarn install
+            yarn --cwd redisinsight/api install
+            yarn --cwd tests/e2e install
+            # Install plugins dependencies
+            export pluginsOnlyInstall=1
+            yarn build:statics
+      - run:
+          name: Generate licenses csv files and send csv data to google sheet
+          command: |
+            npm i -g license-checker
+
+            echo "$GOOGLE_ACCOUNT_SERVICE_KEY_BASE64" | base64 -id > gasKey.json
+            SPREADSHEET_ID=$GOOGLE_SPREADSHEET_DEPENDENCIES_ID node .circleci/deps-licenses-report.js
+      - store_artifacts:
+          path: licenses
+          destination: licenses
 
   # Release jobs
   store-build-artifacts:
@@ -1139,6 +1170,7 @@ workflows:
             - Build app - Linux (dev)
             - Build app - MacOS (dev)
             - Build app - Windows (dev)
+
   # Main workflow for release/* and latest branches only
   release:
     jobs:
@@ -1214,6 +1246,7 @@ workflows:
             - Build app - Linux (stage)
             - Build app - MacOS (stage)
             - Build app - Windows (stage)
+
       # Needs approval from QA team that build was tested before merging to latest
       - qa-approve:
           name: Approved by QA team
@@ -1414,3 +1447,16 @@ workflows:
       #     report: true
       #     requires:
       #       - Build app - Windows (stage)
+
+weekly:
+  triggers:
+    - schedule:
+        cron: '0 0 * * 1'
+        filters:
+          branches:
+            only:
+              - main
+  jobs:
+    # Process all licenses
+    - licenses-check:
+        name: Process licenses of packages

.circleci/deps-licenses-report.js

@@ -0,0 +1,252 @@
+const fs = require('fs');
+const { join } = require('path');
+const { last, set } = require('lodash');
+const { google } = require('googleapis');
+const { exec } = require('child_process');
+const csvParser = require('csv-parser');
+const { stringify } = require('csv-stringify');
+
+const licenseFolderName = 'licenses';
+const spreadsheetId = process.env.SPREADSHEET_ID;
+const outputFilePath = `./${licenseFolderName}/licenses.csv`;
+const summaryFilePath = `./${licenseFolderName}/summary.csv`;
+const allData = [];
+let csvFiles = [];
+
+
+// Main function
+async function main() {
+  const folderPath = './';
+  const packageJsons = findPackageJsonFiles(folderPath); // Find all package.json files in the given folder
+
+  console.log('All package.jsons was found:', packageJsons);
+
+  // Create the folder if it doesn't exist
+  if (!fs.existsSync(licenseFolderName)) {
+    fs.mkdirSync(licenseFolderName);
+  }
+
+  try {
+    await Promise.all(packageJsons.map(runLicenseCheck));
+    console.log('All csv files was generated');
+    await generateSummary()
+    await sendLicensesToGoogleSheet()
+  } catch (error) {
+    console.error('An error occurred:', error);
+    process.exit(1);
+  }
+}
+
+main();
+
+// Function to find all package.json files in a given folder
+function findPackageJsonFiles(folderPath) {
+  const packageJsonPaths = [];
+  const packageJsonName = 'package.json';
+  const excludeFolders = ['dist', 'node_modules', 'static', 'electron', 'redisgraph'];
+
+  // Recursive function to search for package.json files
+  function searchForPackageJson(currentPath) {
+    const files = fs.readdirSync(currentPath);
+
+    for (const file of files) {
+      const filePath = join(currentPath, file);
+      const stats = fs.statSync(filePath);
+
+      if (stats.isDirectory() && !excludeFolders.includes(file)) {
+        searchForPackageJson(filePath);
+      } else if (file === packageJsonName) {
+        packageJsonPaths.push(`./${filePath.slice(0, -packageJsonName.length - 1)}`);
+      }
+    }
+  }
+
+  searchForPackageJson(folderPath);
+  return packageJsonPaths;
+}
+
+// Function to run license check for a given package.json file
+async function runLicenseCheck(path) {
+  const name = last(path.split('/')) || 'electron';
+
+  const COMMANDS = [
+    `license-checker --start ${path} --csv --out ./${licenseFolderName}/${name}_prod.csv --production`,
+    `license-checker --start ${path} --csv --out ./${licenseFolderName}/${name}_dev.csv --development`,
+  ]
+
+  return await Promise.all(COMMANDS.map((command) =>
+    new Promise((resolve, reject) => {
+      exec(command, (error, stdout, stderr) => {
+        if (error) {
+          console.error(`Failed command: ${commandProd}, error:`, stderr);
+          reject(error);
+        }
+        resolve();
+      });
+    })
+  ));
+}
+
+async function sendLicensesToGoogleSheet() {
+  try {
+    const serviceAccountKey = JSON.parse(fs.readFileSync('./gasKey.json', 'utf-8'));
+
+    // Set up JWT client
+    const jwtClient = new google.auth.JWT(
+      serviceAccountKey.client_email,
+      null,
+      serviceAccountKey.private_key,
+      ['https://www.googleapis.com/auth/spreadsheets']
+    );
+
+    const sheets = google.sheets('v4');
+
+    // Read all .csv files in the 'licenses' folder
+    csvFiles.forEach((csvFile) => {
+      // Extract sheet name from file name
+      const sheetName = csvFile.replace('.csv', '').replaceAll('_', ' ');
+
+      const data = [];
+      fs.createReadStream(`./${licenseFolderName}/${csvFile}`)
+        .pipe(csvParser({ headers: false }))
+        .on('data', (row) => {
+          data.push(Object.values(row));
+        })
+        .on('end', async () => {
+          const resource = { values: data };
+
+          try {
+            const response = await sheets.spreadsheets.get({
+              auth: jwtClient,
+              spreadsheetId,
+            });
+
+            const sheet = response.data.sheets.find((sheet) => sheet.properties.title === sheetName);
+            if (sheet) {
+              // Clear contents of the sheet starting from cell A2
+              await sheets.spreadsheets.values.clear({
+                auth: jwtClient,
+                spreadsheetId,
+                range: `${sheetName}!A1:Z`, // Assuming Z is the last column
+              });
+            } else {
+              // Create the sheet if it doesn't exist
+              await sheets.spreadsheets.batchUpdate({
+                auth: jwtClient,
+                spreadsheetId,
+                resource: set({}, 'requests[0].addSheet.properties.title', sheetName),
+              });
+            }
+          } catch (error) {
+            console.error(`Error checking/creating sheet for ${sheetName}:`, error);
+          }
+
+          try {
+            await sheets.spreadsheets.values.batchUpdate({
+              auth: jwtClient,
+              spreadsheetId,
+              resource: {
+                valueInputOption: 'RAW',
+                data: [
+                  {
+                    range: `${sheetName}!A1`, // Use the sheet name as the range and start from A2
+                    majorDimension: 'ROWS',
+                    values: data,
+                  },
+                ],
+              },
+            });
+
+            console.log(`CSV data has been inserted into ${sheetName} sheet.`);
+          } catch (err) {
+            console.error(`Error inserting data for ${sheetName}:`, err);
+          }
+        });
+      });
+  } catch (error) {
+    console.error('Error loading service account key:', error);
+  }
+}
+
+// Function to read and process each CSV file
+const processCSVFile = (file) => {
+  return new Promise((resolve, reject) => {
+    const parser = csvParser({ columns: true, trim: true });
+    const input = fs.createReadStream(`./${licenseFolderName}/${file}`);
+
+    parser.on('data', (record) => {
+      allData.push(record);
+    });
+
+    parser.on('end', () => {
+      resolve();
+    });
+
+    parser.on('error', (err) => {
+      reject(err);
+    });
+
+    input.pipe(parser);
+  });
+};
+
+// Process and aggregate license data
+const processLicenseData = () => {
+  const licenseCountMap = {};
+  for (const record of allData) {
+    const license = record.license;
+    licenseCountMap[license] = (licenseCountMap[license] || 0) + 1;
+  }
+  return licenseCountMap;
+};
+
+// Create summary CSV data
+const createSummaryData = (licenseCountMap) => {
+  const summaryData = [['License', 'Count']];
+  for (const license in licenseCountMap) {
+    summaryData.push([license, licenseCountMap[license]]);
+  }
+  return summaryData;
+};
+
+// Write summary CSV file
+const writeSummaryCSV = async (summaryData) => {
+  try {
+    const summaryCsvString = await stringifyPromise(summaryData);
+    fs.writeFileSync(summaryFilePath, summaryCsvString);
+    csvFiles.push(last(summaryFilePath.split('/')));
+    console.log(`Summary CSV saved as ${summaryFilePath}`);
+  } catch (err) {
+    console.error(`Error: ${err}`);
+  }
+};
+
+// Stringify as a promise
+const stringifyPromise = (data) => {
+  return new Promise((resolve, reject) => {
+    stringify(data, (err, csvString) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(csvString);
+      }
+    });
+  });
+};
+
+async function generateSummary() {
+  csvFiles = fs.readdirSync(licenseFolderName).filter(file => file.endsWith('.csv')).sort();
+
+  for (const file of csvFiles) {
+    try {
+      await processCSVFile(file);
+    } catch (err) {
+      console.error(`Error processing ${file}: ${err}`);
+    }
+  }
+
+  const licenseCountMap = processLicenseData();
+  const summaryData = createSummaryData(licenseCountMap);
+
+  await writeSummaryCSV(summaryData);
+}

.gitignore

@@ -58,6 +58,7 @@ main.js.map
 vendor
 redisinsight/main.js.LICENSE.txt
 redisinsight/main.prod.js.LICENSE.txt
+licenses
 
 
 # E2E tests report

package.json

@@ -159,6 +159,8 @@
     "cross-env": "^7.0.2",
     "css-loader": "^5.0.1",
     "css-minimizer-webpack-plugin": "^2.0.0",
+    "csv-parser": "^3.0.0",
+    "csv-stringify": "^6.4.0",
     "electron": "25.1.1",
     "electron-builder": "^23.6.0",
     "electron-builder-notarize": "^1.5.1",
@@ -178,6 +180,8 @@
     "eslint-plugin-react-hooks": "^4.0.8",
     "eslint-plugin-sonarjs": "^0.10.0",
     "file-loader": "^6.0.0",
+    "google-auth-library": "^9.0.0",
+    "googleapis": "^125.0.0",
     "html-webpack-plugin": "^5.5.0",
     "husky": "^4.2.5",
     "identity-obj-proxy": "^3.0.0",
@@ -186,6 +190,7 @@
     "jest": "^27.5.1",
     "jest-runner-groups": "^2.2.0",
     "jest-when": "^3.2.1",
+    "license-checker": "^25.0.1",
     "lint-staged": "^10.2.11",
     "mini-css-extract-plugin": "2.7.2",
     "moment": "^2.29.3",