feat(server): send to InfluxDB

#9

package.json

@@ -33,6 +33,7 @@
     "expand-home-dir": "0.0.3",
     "express": "^4.17.1",
     "googleapis": "^59.0.0",
+    "influx": "^5.6.3",
     "lighthouse": "^6.2.0",
     "socket.io": "^2.3.0",
     "xlsx": "^0.15.6"

src/actions/sendToInfluxDB.js

@@ -0,0 +1,81 @@
+const fs = require('fs');
+const os = require('os');
+const url = require('url');
+const Influx = require('influx');
+
+function init(options) {
+  const influx = new Influx.InfluxDB({
+   host: options.influxdb.host,
+   port: options.influxdb.port,
+   database: options.influxdb.database,
+   schema: options.influxdb.schema,
+   username: options.influxdb.username,
+   password: options.influxdb.password,
+  });
+
+  return influx;
+}
+
+async function getPoint(item, schema) {
+  const point = {
+    measurement: schema.measurement,
+    tags: {
+      host: os.hostname(),
+      domain: url.parse(item.url).hostname,
+      url: item.url
+    },
+    fields: {},
+  }
+
+  for (let fieldName in schema.fields) {
+    const val = item[fieldName];
+    if (val !== undefined && val !== null) {
+      point.fields[fieldName] = val;
+    }
+  }
+
+  // console.log('point: ', point);
+  return point;
+}
+
+function buildSchemaByFields(fields, measurement) {
+  const schema = { fields: {}, measurement, tags: ['host', 'domain', 'url'] };
+  for (let field of fields) {
+    /* if (field.name === 'url') {
+      schema.fields[field.name] = Influx.FieldType.STRING;
+    } */
+    if (field.type === 'integer') {
+      schema.fields[field.name] = Influx.FieldType.INTEGER;
+    }
+    if (field.type === 'boolean') {
+      schema.fields[field.name] = Influx.FieldType.INTEGER;
+    }
+  }
+  // console.log('schema: ', schema);
+  return schema;
+}
+
+module.exports = async (jsonPath, options) => {
+  // console.log('options.influxdb: ', options.influxdb);
+  const jsonRaw = fs.readFileSync(jsonPath);
+  const data = JSON.parse(jsonRaw);
+
+  const measurement = options.influxdb.measurement || 'site_audit_seo';
+  const schema = buildSchemaByFields(data.fields, measurement);
+  options.influxdb.schema = [schema];
+  const influx = init(options);
+
+  // points list for influx
+  const points = [];
+  for (let item of data.items) {
+    // console.log('item: ', item);
+    const point = await getPoint(item, schema);
+    points.push(point);
+  }
+
+  // console.log('writePoints');
+  // console.log('points: ', points);
+  // console.log('options.influxdb: ', options.influxdb);
+  await influx.writePoints(points);
+  return points;
+};

src/program.js

@@ -183,6 +183,9 @@ program.postParse = async () => {
     // program.defaultFilter = 'depth>1';
   }
 
+  // influxdb config fron ~/.site-audit-seo.conf.js
+  program.influxdb = getConfigVal('influxdb', false);
+
   program.outDir = expandHomedir(program.outDir);
   createDirIfNotExists(program.outDir);
 }
@@ -217,6 +220,7 @@ program.getOptions = () => {
     upload: program.upload,                     // выгружать json на сервер
     consoleValidate: program.consoleValidate,   // выводить данные валидации в консоль
     obeyRobotsTxt: !program.ignoreRobotsTxt,    // не учитывать блокировки в robots.txt
+    influxdb: program.influxdb,                 // конфиг influxdb
   };
   return opts;
 }
@@ -301,7 +305,7 @@ program.outBrief = (options) => {
 
   console.log('');
   for (let line of brief) {
-    const nameCol = line.name.padEnd(22, ' ');
+    const nameCol = line.name.padEnd(20, ' ');
     const valueCol = `${line.value}`.padEnd(10, ' ');
     const comment = line.comment ? ` ${line.comment}` : '';
     console.log(color.white + nameCol + valueCol + color.reset + comment);

src/scrap-site.js

@@ -1,7 +1,7 @@
 // see API - https://github.com/yujiosaka/headless-chrome-crawler/blob/master/docs/API.md#event-requeststarted
 const fs = require('fs');
 const path = require('path');
-const {saveAsXlsx, saveAsJson, uploadJson, publishGoogleDrive, startViewer} = require(
+const {saveAsXlsx, saveAsJson, uploadJson, publishGoogleDrive, startViewer, sendToInfluxDB} = require(
   './actions');
 const axios = require('axios');
 const HCCrawler = require('@popstas/headless-chrome-crawler');
@@ -547,24 +547,29 @@ module.exports = async (baseUrl, options = {}) => {
   };
 
   if (options.webService) {
-    await saveAsJson(csvPath, jsonPath, options.lang, options.preset, options.defaultFilter);
-    const jsonName = getJsonName(jsonPath);
-
-    // TODO: remove
-    if (options.upload) {
-      webPath = await uploadJson(jsonPath, options);
+    try {
+      await saveAsJson(csvPath, jsonPath, options.lang, options.preset, options.defaultFilter);
+      const jsonName = getJsonName(jsonPath);
 
-      if (options.socket) {
-        // const webViewer = `https://popstas-server:5302/?url=${webPath}`;
-        options.socket.emit('result', {json: webPath});
-        // options.socket.emit('status', `<a target="_blank" href="${webViewer}">${webViewer}</a>`);
+      if (options.influxdb) {
+        log('send to InfluxDB...');
+        const points = await sendToInfluxDB(jsonPath, options);
+        log(`sent ${points.length} points`);
       }
-    }
-    else {
+
       // copy to local reports
       const localPath = 'data/reports/' + jsonName;
       fs.copyFileSync(jsonPath, localPath);
-      options.socket.emit('result', {name: jsonName});
+      if (options.socket) options.socket.emit('result', {name: jsonName});
+
+      // TODO: error upload 8MB+
+      if (options.upload) {
+        webPath = await uploadJson(jsonPath, options);
+        if (options.socket) options.socket.emit('result', {json: webPath});
+      }
+    }
+    catch (e) {
+      log('error after scan: ' + e.message);
     }
   }
   else {

src/server.js

@@ -51,10 +51,19 @@ app.post("/scan", async (req, res) => {
   const url = req.body.url;
   const args = req.body.args.split(" ");
   if (!url) {
-    userSocket.emit('status', 'URL not defined!')
+    log('URL not defined!');
     return;
   }
-  program.parse([...['', ''], ...args]);
+
+  program.exitOverride();
+  try {
+    program.parse([...['', ''], ...args]);
+  } catch (e) {
+    log('failed to parse arguments: ' + e.message);
+    res.send('ERROR');
+    return;
+  }
+
   await program.postParse();
 
   const opts = program.getOptions();
@@ -63,7 +72,13 @@ app.post("/scan", async (req, res) => {
 
   program.outBrief(opts);
 
-  scan(url, opts);
+  try {
+    scan(url, opts);
+  }
+  catch(e) {
+    log('error while scan:', e.message);
+  }
+
   res.send('OK');
 });