feat: working docker-compose

.dockerignore

@@ -10,3 +10,4 @@ node_modules
 node_modules/
 node_modules/*
 node_modules/**
+data/

.env

@@ -1,2 +1,6 @@
 # General
 NAMESPACE=sas
+SERVER_PORT=5301
+FRONT_PORT=5302
+JSON_URL=http://localhost:5301/reports/last.json
+SERVER_URL=http://localhost:5301

Dockerfile

@@ -1,22 +1,29 @@
-FROM mhart/alpine-node:12
+FROM buildkite/puppeteer:latest
 
 # ARG PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=${PUPPETEER_SKIP_CHROMIUM_DOWNLOAD:-"false"}
 
-RUN apk update && \ 
-    apk upgrade && \
-    apk add --no-cache git
+# RUN apk update && \
+#     apk upgrade && \
+#     apk add --no-cache git
 
 WORKDIR /app
+COPY . .
+# COPY package.json ./
 
-RUN adduser -D node -h /app -s /bin/sh && \
-    su node -c 'cd /app; mkdir -p config outputs'
+# RUN adduser -D node -h /app -s /bin/sh && \
+    # su node -c 'cd /app; mkdir -p config data/reports'
 
+RUN chown -R node:node /app
 USER node
 
-RUN npm install -g site-audit-seo --unsafe-perm=true && \
-    chown -R "$(id -u):$(id -g)" "$(npm prefix -g)/lib/node_modules/site-audit-seo/node_modules/puppeteer/.local-chromium/"
+RUN npm install
+# RUN npm install -g site-audit-seo --unsafe-perm=true && \
+    # chown -R "$(id -u):$(id -g)" "$(npm prefix -g)/lib/node_modules/site-audit-seo/node_modules/puppeteer/.local-chromium/"
+# RUN chown -R "$(id -u):$(id -g)" "/app/node_modules/puppeteer/.local-chromium/"
 
-VOLUME ["/app/outputs"]
-EXPOSE 3000
+VOLUME ["/app/data/reports"]
+EXPOSE 5301
+ENV PORT=5301
 
 CMD  ["npm", "run", "server"]
+# CMD  ["ls", "-l", "/app"]

README.md

@@ -36,6 +36,12 @@ sudo chown -R $USER:$USER "$(npm prefix -g)/lib/node_modules/site-audit-seo/node
 
 Error details [Invalid file descriptor to ICU data received](https://github.com/puppeteer/puppeteer/issues/2519).
 
+#### docker-compose
+`docker-compose up -d`
+
+##### Default ports:
+- Backend: `5301`
+- Frontend: `5302`
 
 ## Usage:
 ```

docker-compose.yml

@@ -1,62 +1,45 @@
 ---
+version: '3'
+
 services:
 
   backend:
-    image: viasite/site-audit-seo:latest-alpine
+    image: viasite/site-audit-seo:latest
     container_name: ${NAMESPACE}-backend
     build:
       context: .
       dockerfile: Dockerfile
-    expose:
-    - 3000
+    environment:
+      PORT: ${SERVER_PORT}
     ports:
-    - 3000:3000
-    networks:
-    - internal
-    - web
+    - 5301:${SERVER_PORT}
+    # networks:
+    # - internal
+    # - web
     volumes:
-    - reports:/app/reports
+    - ./data/reports:/app/data/reports
 
   frontend:
     image: viasite/site-audit-seo-viewer:latest
     container_name: ${NAMESPACE}-frontend
-    build:
-      context: .
-      dockerfile: Dockerfile.viewer
-    networks:
-    - web
-    expose:
-    - 3001
-    ports:
-    - 3001:3001
-    networks:
-    - web
-    volumes:
-    - reports:/app/reports
-
-  redis:
-    image: redis:6-alpine
-    container_name: redis
-    networks:
-    - internal
-    volumes:
-    - redis-data:/data
-    command:
-    - "redis-server"
-    - "--maxmemory 3G"
-    - "--maxmemory-policy allkeys-lru"
-    restart: unless-stopped
+    build: data/front
+    # env_file:
+    # - data/popstas-server.env
+    environment:
+      HOST: 0.0.0.0 # for nuxt
+      PORT: ${FRONT_PORT}
+      SERVER_URL: ${SERVER_URL}
+      JSON_URL: ${JSON_URL}
 
-networks:
-  internal:
-    driver: bridge
-  web:
-    external: true
+    ports:
+    - 5302:${FRONT_PORT}
+    # networks:
+    # - web
+    # volumes:
+    # - ./data/reports:/app/static/reports
 
-volumes:
-  reports:
-    driver_opts:
-      type: none
-      o: bind
-      device: ${PWD}/outputs
-  redis-data:
+# networks:
+#   internal:
+#     driver: bridge
+#   web:
+#     external: true

src/actions/uploadJson.js

@@ -6,15 +6,7 @@ module.exports = async (jsonPath, options) => {
   const data = fs.readFileSync(jsonPath, 'utf8');
   // const raw = JSON.stringify(data);
 
-  const offset = new Date().getTimezoneOffset() * 60000;
-  const dateLocal = new Date(Date.now() - offset)
-  const date = dateLocal.toISOString().
-    replace(/:/g, '-').
-    replace('T', '_').
-    replace('Z', '');
-  // const dateStr = date.slice(0,10);
-  const name = path.basename(jsonPath).replace(/[^0-9a-zа-я_.-]/ig, '');
-  const uploadName = date + '_' + name;
+  const uploadName = getJsonName(jsonPath);
 
   console.log('\nUploading to https://site-audit.viasite.ru...');
   const res = await axios.post('https://site-audit.viasite.ru/upload/', {
@@ -28,3 +20,16 @@ module.exports = async (jsonPath, options) => {
   }
   return res.data.url;
 };
+
+function getJsonName(jsonPath) {
+  const offset = new Date().getTimezoneOffset() * 60000;
+  const dateLocal = new Date(Date.now() - offset)
+  const date = dateLocal.toISOString().
+    replace(/:/g, '-').
+    replace('T', '_').
+    replace('Z', '');
+  // const dateStr = date.slice(0,10);
+  const name = path.basename(jsonPath).replace(/[^0-9a-zа-я_.-]/ig, '');
+  const uploadName = date + '_' + name;
+  return uploadName;
+}

src/scrap-site.js

@@ -120,7 +120,7 @@ module.exports = async (baseUrl, options = {}) => {
     allowedDomains: options.limitDomain ? [domain] : undefined,
     skipRequestedRedirect: true, // all redirects marks as visited
     depthPriority: false, // without it find not all pages
-    args: ['--no-sandbox'], // puppeteer freezes without it
+    args: ['--no-sandbox', '--disable-dev-shm-usage'], // puppeteer freezes without it
     exporter,
 
     // url ignore rules
@@ -548,22 +548,39 @@ module.exports = async (baseUrl, options = {}) => {
 
   if (options.webService) {
     await saveAsJson(csvPath, jsonPath, options.lang, options.preset, options.defaultFilter);
+    const jsonName = getJsonName(jsonPath);
+
+    // TODO: remove
     if (options.upload) {
       webPath = await uploadJson(jsonPath, options);
+
       if (options.socket) {
-        const webViewer = `https://viasite.github.io/site-audit-seo-viewer/?url=${webPath}`;
-        options.socket.emit('status', `<a target="_blank" href="${webViewer}">${webViewer}</a>`);
-      }
-      return {
-        webPath
+        // const webViewer = `https://popstas-server:5302/?url=${webPath}`;
+        options.socket.emit('result', {json: webPath});
+        // options.socket.emit('status', `<a target="_blank" href="${webViewer}">${webViewer}</a>`);
       }
     }
     else {
-      const jsonData = fs.readFileSync(jsonPath, 'utf-8');
-      return JSON.parse(jsonData);
+      // copy to local reports
+      const localPath = 'data/reports/' + jsonName;
+      fs.copyFileSync(jsonPath, localPath);
+      options.socket.emit('result', {name: jsonName});
     }
   }
   else {
     await tryFinish(finishTries);
   }
 };
+
+function getJsonName(jsonPath) {
+  const offset = new Date().getTimezoneOffset() * 60000;
+  const dateLocal = new Date(Date.now() - offset)
+  const date = dateLocal.toISOString().
+    replace(/:/g, '-').
+    replace('T', '_').
+    replace('Z', '');
+  // const dateStr = date.slice(0,10);
+  const name = path.basename(jsonPath).replace(/[^0-9a-zа-я_.-]/ig, '');
+  const uploadName = date + '_' + name;
+  return uploadName;
+}

src/server.js

@@ -1,7 +1,8 @@
 const program = require("./program");
 const scrapSite = require("./scrap-site");
 
-const app = require("express")();
+const express = require("express");
+const app = express();
 const bodyParser = require('body-parser')
 const http = require('http').createServer(app);
 const io = require('socket.io')(http, {
@@ -12,27 +13,27 @@ const io = require('socket.io')(http, {
   }
 });
 
-const port = process.env.PORT || 3001;
+const port = process.env.PORT || 5301;
 
 let userSocket;
 
-http.listen(port, () => {
-  console.log(`Listening at http://localhost:${port}`);
-});
+function log(msg) {
+  console.log(msg);
+  if (userSocket) userSocket.emit('status', msg);
+}
 
 io.on('connection', (socket) => {
-  console.log('a user connected');
-  socket.emit('status', 'connected to server');
   userSocket = socket;
+  log('user connected to server');
   socket.on('disconnect', () => {
+    userSocket = null;
     console.log('user disconnected');
   });
 });
 
-app.use( bodyParser.json() );       // to support JSON-encoded bodies
-app.use(bodyParser.urlencoded({     // to support URL-encoded bodies
-  extended: true
-})); 
+http.listen(port, () => {
+  console.log(`Listening at http://localhost:${port}`);
+});
 
 // CORS
 app.use(function(req, res, next) {
@@ -41,6 +42,11 @@ app.use(function(req, res, next) {
   next();
 });
 
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: true }));
+
+app.use('/reports', express.static('data/reports'));
+
 app.post("/scan", async (req, res) => {
   const url = req.body.url;
   const args = req.body.args.split(" ");
@@ -62,16 +68,19 @@ app.get("/", async (req, res) => {
 
   const url = program.urls[0]; // TODO: support multiple urls queue
 
-  const data = await scan(url, program);
-
-
-  if (data.webPath) {
-    const webViewer = `https://viasite.github.io/site-audit-seo-viewer/?url=${data.webPath}`;
-    res.send(`<a target="_blank" href="${webViewer}">${webViewer}</a>`);
-  }
-  else {
-    res.json(data);
-  }
+  // try {
+    const data = await scan(url, program);
+    if (data.webPath) {
+      const webViewer = `https://viasite.github.io/site-audit-seo-viewer/?url=${data.webPath}`;
+      res.send(`<a target="_blank" href="${webViewer}">${webViewer}</a>`);
+    }
+    else {
+      res.json(data);
+    }
+  // }
+  /* catch(e) {
+    console.warn(e.message)
+  }; */
 });
 
 async function scan(url, program) {
@@ -109,9 +118,9 @@ async function scan(url, program) {
   opts.webService = true;
   opts.socket = userSocket;
 
-  userSocket.emit('status', `start audit: ${url}`);
+  if (userSocket) userSocket.emit('status', `start audit: ${url}`);
   const data = await scrapSite(url, opts);
-  socket.emit('status', `finish audit: ${url}`);
+  if (userSocket) userSocket.emit('status', `finish audit: ${url}`);
   return data;
 }