feat: now default output dir is ~/site-audit-seo/, not current dir

.site-audit-seo.conf.js

@@ -8,7 +8,7 @@ module.exports = {
   // headless: true,
   // removeCsv: true,
   // removeJson: true,
-  // outDir: '~/site-audit-seo-reports/.',
+  // outDir: '~/site-audit-seo/.',
   // gdrive: false,
   // json: false,
   // xlsx: false,

README.md

@@ -72,10 +72,10 @@ Options:
 - Validation summary after scan
 - Documents with the extensions `doc`,` docx`, `xls`,` xlsx`, `ppt`,` pptx`, `pdf`,` rar`, `zip` are added to the list with a depth == 0
 - Search pages with SSL mixed content
-- Each site is saved to a file with a domain name
+- Each site is saved to a file with a domain name in `~/site-audit-seo/`
 - Does not follow links outside the scanned domain (configurable)
 - Does not load images, css, js (configurable)
-- Some URLs are ignored ([`preRequest` in `src/scrap-site.js`](src/scrap-site.js#L112))
+- Some URLs are ignored ([`preRequest` in `src/scrap-site.js`](src/scrap-site.js#L98))
 - Analyse each page with Lighthouse (see below)
 
 ### XLSX features
@@ -139,6 +139,7 @@ Options:
 - lighthouse.interactive
 - lighthouse.total-blocking-time
 - lighthouse.cumulative-layout-shift
+- and 150 more lighthouse tests!
 
 
 ## Custom fields
@@ -174,9 +175,9 @@ site-audit-seo -u https://example.com --lighthouse
 You can copy [.site-audit-seo.conf.js](.site-audit-seo.conf.js) to your home directory and tune options.
 
 ## Bugs
-1. Sometimes it writes identical pages to csv. This happens in 2 cases:  
-1.1. Redirect from another page to this (solved by setting `skipRequestedRedirect: true`, hardcoded).  
-1.2. Simultaneous request of the same page in parallel threads.  
+1. Sometimes it writes identical pages to csv. This happens in 2 cases:
+1.1. Redirect from another page to this (solved by setting `skipRequestedRedirect: true`, hardcoded).
+1.2. Simultaneous request of the same page in parallel threads.
 2. Sometimes a number appears instead of the URL, it occurs at the stage of converting csv to xlsx, don't know why.
 
 
@@ -269,9 +270,9 @@ site-audit-seo -u https://example.com --lighthouse
 
 
 ## Баги
-1. Иногда пишет в csv одинаковые страницы. Это бывает в 2 случаях:  
-1.1. Редирект с другой страницы на эту (решается установкой `skipRequestedRedirect: true`, сделано).  
-1.2. Одновременный запрос одной и той же страницы в параллельных потоках.  
+1. Иногда пишет в csv одинаковые страницы. Это бывает в 2 случаях:
+1.1. Редирект с другой страницы на эту (решается установкой `skipRequestedRedirect: true`, сделано).
+1.2. Одновременный запрос одной и той же страницы в параллельных потоках.
 2. Иногда вместо URL появляется цифра, происходит на этапе конвертации csv в xlsx, не знаю почему.
 
 

package.json

@@ -28,6 +28,7 @@
     "chrome-launcher": "^0.13.4",
     "commander": "^5.0.0",
     "csvtojson": "^2.0.10",
+    "expand-home-dir": "0.0.3",
     "express": "^4.17.1",
     "googleapis": "^59.0.0",
     "lighthouse": "^6.2.0",

src/index.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 const fs = require('fs');
+const path = require('path');
 const process = require('process');
 const {program} = require('commander');
 const packageJson = require('../package.json');
@@ -9,6 +10,7 @@ const {saveAsXlsx, saveAsJson, uploadJson, publishGoogleSheets, startViewer} = r
   './actions');
 const {exec} = require('child_process');
 const os = require('os');
+const expandHomedir = require('expand-home-dir');
 const color = require('./color');
 
 const defaultDocs = [
@@ -93,7 +95,7 @@ program.option('-u --urls <urls>', 'Comma separated url list for scan', list).
     getConfigVal('removeJson', true)).
   option('--no-remove-csv', `No delete csv after xlsx generate`).
   option('--no-remove-json', `No delete json after serve`).
-  option('--out-dir <dir>', `Output directory`, getConfigVal('outDir', '.')).
+  option('--out-dir <dir>', `Output directory`, getConfigVal('outDir', '~/site-audit-seo/')).
   option('--csv <path>', `Skip scan, only convert csv to xlsx`).
   option('--xlsx', `Save as XLSX`, getConfigVal('xlsx', false)).
   option('--gdrive', `Publish sheet to google docs`,
@@ -146,9 +148,9 @@ async function start() {
 
   if (program.csv) {
     program.removeCsv = false;
-    const csvPath = program.csv;
-    const xlsxPath = csvPath.replace(/\.csv$/, '.xlsx');
-    let jsonPath = csvPath.replace(/\.csv$/, '.json');
+    const csvPath = expandHomedir(program.csv);
+    const xlsxPath = path.normalize(csvPath.replace(/\.csv$/, '.xlsx'));
+    let jsonPath = path.normalize(csvPath.replace(/\.csv$/, '.json'));
     let webPath;
     try {
       if (program.xlsx) {
@@ -199,6 +201,8 @@ async function start() {
   // c = 2, when lighthouse c = 1
   if (!program.concurrency) program.concurrency = program.lighthouse ? 1 : 2;
 
+  program.outDir = expandHomedir(program.outDir);
+
   outBrief(program);
 
   for (let site of sites) {

src/scrap-site.js

@@ -1,5 +1,6 @@
 // see API - https://github.com/yujiosaka/headless-chrome-crawler/blob/master/docs/API.md#event-requeststarted
 const fs = require('fs');
+const path = require('path');
 const {saveAsXlsx, saveAsJson, uploadJson, publishGoogleSheets, startViewer} = require(
   './actions');
 const HCCrawler = require('@popstas/headless-chrome-crawler');
@@ -9,26 +10,29 @@ const {validateResults, getValidationSum} = require('./validate');
 const {exec} = require('child_process');
 const lighthouse = require('lighthouse');
 const chromeLauncher = require('chrome-launcher');
+// поля описаны в API по ссылке выше
+const fieldsPresets = require('./presets/scraperFields');
+const color = require('./color');
 
 const DEBUG = true; // выключить, если не нужны console.log на каждый запрос (не будет видно прогресс)
 
-const color = require('./color');
 
 // запреты браузеру на подгрузку статики, ускоряет
 let SKIP_IMAGES = true;
 let SKIP_CSS = true;
 let SKIP_JS = true;
-const finishTries = 5;
 
-// поля описаны в API по ссылке выше
-const fieldsPresets = require('./presets/scraperFields');
+// кол-во попыток выполнить actions
+const finishTries = 5;
 
 module.exports = async (baseUrl, options = {}) => {
+  createDirIfNotExists(options.outDir);
   const domain = url.parse(baseUrl).hostname;
   const protocol = url.parse(baseUrl).protocol;
-  const csvPath = `${options.outDir}/${domain}.csv`;
-  const xlsxPath = `${options.outDir}/${domain}.xlsx`;
-  const jsonPath = `${options.outDir}/${domain}.json`;
+
+  const csvPath = path.normalize(`${options.outDir}/${domain}.csv`);
+  const xlsxPath = path.normalize(`${options.outDir}/${domain}.xlsx`);
+  const jsonPath = path.normalize(`${options.outDir}/${domain}.json`);
   let webPath;
 
   if (!options.color) color.white = color.red = color.reset = color.yellow = '';
@@ -502,3 +506,15 @@ module.exports = async (baseUrl, options = {}) => {
 
   await tryFinish(finishTries);
 };
+
+function createDirIfNotExists(path) {
+  const exists = fs.existsSync(path);
+  if(exists && fs.statSync(path).isFile()) {
+    throw new Error(`File exists, cannot save here: ${path}`);
+    return false;
+  }
+
+  if(!exists) fs.mkdirSync(path, { recursive: true })
+
+  return path;
+}