feat: plugin types: afterRequest, afterScan

CONTRIBUTING.md

@@ -20,29 +20,84 @@ docker-compose up -d
 ```
 
 ## Plugins
-Example package.json:
+
+### AfterScan plugin
+
+#### AfterScan package.json:
 
 ``` json
 {
   "name": "site-audit-seo-export-influxdb",
   "site-audit-seo": {
     "plugins": {
-      "export-influxdb": "sendToInfluxDB.js"
+      "export-influxdb": {
+        "main": "sendToInfluxDB.js",
+        "type": "afterScan",
+      }
     }
   }
 }
 ```
 
+#### Minimal `afterScan` plugin code:
+
+``` js
+function afterScan(jsonPath, options) {
+  const jsonRaw = fs.readFileSync(jsonPath);
+  const data = JSON.parse(jsonRaw);
+}
+
+module.exports = afterScan;
+```
+
+
+### AfterRequest plugin
+
+### AfterRequest package.json:
+
+``` json
+{
+  "name": "site-audit-seo-export-influxdb",
+  "site-audit-seo": {
+    "plugins": {
+      "readability": {
+        "main": "readability.js",
+        "type": "afterRequest",
+        "fields": [
+          {
+            "name": "readability_time",
+            "comment": "Читать, секунд?",
+            "comment_en": "Reading, time",
+            "groups": ["readability"],
+            "type": "integer"
+          }
+        ]
+      }
+    }
+  }
+}
+```
+
+#### Minimal `afterRequest` plugin code:
+
+``` js
+function afterRequest(result, options) {
+  result.newField = 123;
+}
+
+module.exports = afterRequest;
+```
+
 See core plugins at [src/plugins](src/plugins).
 
 ### Plugin types:
-- `plugins` - launch after scan, for export data, send notifications and so on
+- `plugins` - launch depends of plugin type: `afterScan`, `afterRequest`, for export data, send notifications and so on
 
 ### Install plugins:
 You can install plugins with `npm` to `data` directory.
 
 ### Places for plugin:
-- [ ] Extractors data from page
-- [ ] Analyze html of page
+- [x] Extractors data from page
+- [x] Analyze html of page
 - [x] Actions after scan (implemented)
 - [ ] Command line arguments

README.md

@@ -350,3 +350,4 @@ site-audit-seo -u https://example.com --lighthouse
 - Breadcrumbs - https://github.com/glitchdigital/structured-data-testing-tool
 - joeyguerra/schema.js - https://gist.github.com/joeyguerra/7740007
 - smhg/microdata-js - https://github.com/smhg/microdata-js
+- indicate page scan error

src/plugins/site-audit-seo-export-influxdb/package.json

@@ -2,7 +2,10 @@
   "name": "site-audit-seo-export-influxdb",
   "site-audit-seo": {
     "plugins": {
-      "export-influxdb": "sendToInfluxDB.js"
+      "export-influxdb": {
+        "main": "sendToInfluxDB.js",
+        "type": "afterScan"
+      }
     }
   }
 }

src/presets/fields.js

@@ -1,3 +1,4 @@
+const registry = require('../registry');
 const fieldsLighthouse = require('./fields-lighthouse');
 const fieldsLighthouseEn = require('./fields-lighthouse-en');
 const fields = [
@@ -383,4 +384,21 @@ for (let lhf of fieldsLighthouse) {
   fields.push(lhf);
 }
 
+// plugins fields
+const plugins = registry.getPlugins();
+for (let plugin of plugins) {
+  if (plugin.fields) for(let field of plugin.fields) {
+    if (typeof field === 'string') {
+      fields.push({
+        name: field,
+        groups: [plugin.name]
+      });
+    } else {
+      fields.push(field);
+    }
+    // console.log(`push ${field}`);
+  }
+}
+
+
 module.exports = {fields, getFieldByName};

src/registry.js

@@ -32,17 +32,25 @@ function loadPluginsFromDir(dir) {
     const info = getModuleInfo(dirPath);
 
     if (info.plugins) for (let pluginName in info.plugins) {
-      const pluginFile = info.plugins[pluginName];
+      const pluginInfo = info.plugins[pluginName];
+      const pluginFile = typeof pluginInfo === 'string' ? pluginInfo : pluginInfo.main;
       const pluginPath = path.join(dirPath, pluginFile);
 
       if (!fs.existsSync(pluginPath)) {
         console.log(`plugin ${pluginName} not found: ${pluginPath}`);
         continue;
       }
-      const plugin = {
+      let plugin = {
         name: pluginName,
-        path: pluginPath
+        path: pluginPath,
+        type: pluginInfo.type
       };
+      if (typeof pluginInfo === 'object') {
+        plugin = {...plugin, ...{
+          type: pluginInfo.type,
+          fields: pluginInfo.fields,
+        }}
+      }
       plugins.push(plugin);
       dirPlugins.push(plugin);
     }
@@ -64,16 +72,19 @@ function getPlugins() {
   return plugins;
 }
 
-async function execPlugins(jsonPath, options) {
+async function execPlugins(jsonPath, options, type = 'any') {
   load();
   if (!plugins) return;
-  console.log(`\n${color.white}exec plugins:${color.reset}`);
+  // console.log(`\n${color.white}exec plugins (${type}):${color.reset}`);
   for (let plugin of plugins) {
-    console.log(`exec plugin ${plugin.name}:`);
+    if (type !== 'any' && plugin.type != type) continue;
+
+    // console.log(`exec plugin ${plugin.name} (type ${type}):`);
+    // console.log('plugin: ', plugin);
     const relPath = path.join('..', plugin.path);
     const pluginObj = require(relPath);
     await pluginObj(jsonPath, options);
-    console.log('');
+    // console.log('');
   }
 }
 

src/scrap-site.js

@@ -113,6 +113,19 @@ module.exports = async (baseUrl, options = {}) => {
     }
   }
 
+  // plugins fields
+  const plugins = registry.getPlugins();
+  for (let plugin of plugins) {
+    if (plugin.fields) for(let field of plugin.fields) {
+      if (typeof field === 'string') {
+        fields.push(field);
+      } else {
+        fields.push(field.name);
+      }
+      // console.log(`push ${field}`);
+    }
+  }
+
   // skip static
   if (options.skipStatic !== undefined) {
     SKIP_IMAGES = SKIP_CSS = SKIP_JS = options.skipStatic;
@@ -434,8 +447,9 @@ module.exports = async (baseUrl, options = {}) => {
       }
 
       result.result.mixed_content_url = mixedContentUrl;
-      if (result.response.url) result.response.url = decodeURI(
-        result.response.url);
+      if (result.response.url) {
+        result.response.url = decodeURI(result.response.url);
+      }
 
       // console validate output
       // was in onSuccess(), but causes exception on docs
@@ -452,6 +466,18 @@ module.exports = async (baseUrl, options = {}) => {
 
       // You can access the page object after requests
       result.content = await page.content();
+
+      // plugins afterRequest
+      try {
+        await registry.execPlugins(result, options, 'afterRequest');
+      } catch (e) {
+        console.log('Error while plugins afterRequest');
+        console.log(e);
+      }
+      // console.log('result.readability_reading: ', result.readability_reading);
+      // console.log('result.readability_readingMinutes: ', result.readability_readingMinutes);
+      // console.log('result.readability_readingTime: ', result.readability_readingTime);
+
       // You need to extend and return the crawled result
       return result;
     },
@@ -543,7 +569,7 @@ module.exports = async (baseUrl, options = {}) => {
       if (!options.removeJson) console.log('Saved to ' + jsonPath);
 
       // user plugins
-      await registry.execPlugins(jsonPath, options);
+      await registry.execPlugins(jsonPath, options, 'afterScan');
 
       if (!options.webService) {
         if (options.upload) webPath = await uploadJson(jsonPath);