Merge pull request #27 from crawlab-team/develop

Develop

.dockerignore

@@ -5,3 +5,4 @@ data/
 .ipynb_checkpoints/
 .idea/
 .models/
+dev/

.env.example

@@ -0,0 +1 @@
+WEBSPOT_DATABASE_URL=

.github/workflows/dockerpublish.yml

@@ -2,7 +2,9 @@ name: "Docker Image CI: websplot"
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - main
+      - develop
 
 env:
   IMAGE_NAME: crawlabteam/webspot

.github/workflows/main_webspot.yml

@@ -7,6 +7,7 @@ on:
   push:
     branches:
       - main
+      - develop
   workflow_dispatch:
 
 jobs:

.gitignore

@@ -139,3 +139,7 @@ node_modules/
 
 *.db
 *.db-journal
+
+dev/data/
+
+*.lock

Dockerfile

@@ -1,3 +1,13 @@
+FROM golang:1.19 AS build
+
+WORKDIR /go/src/app
+COPY ./webspot_rod .
+
+ENV GO111MODULE on
+
+RUN go mod tidy \
+  && go install -v ./...
+
 FROM python:3.10.9
 
 # Working directory
@@ -7,6 +17,36 @@ WORKDIR /app
 RUN echo `uname -a`
 RUN echo `python --version`
 
+# Install supervisor
+RUN apt-get update && apt-get install -y supervisor
+
+# Install dependencies
+RUN apt-get -qq install --no-install-recommends -y \
+    # chromium dependencies
+    libnss3 \
+    libxss1 \
+    libasound2 \
+    libxtst6 \
+    libgtk-3-0 \
+    libgbm1 \
+    ca-certificates \
+    # fonts
+    fonts-liberation fonts-noto-color-emoji fonts-noto-cjk
+
+# Start and enable SSH
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends dialog \
+    && apt-get install -y --no-install-recommends openssh-server \
+    && echo "root:Docker!" | chpasswd
+COPY webspot_rod/conf/sshd_config /etc/ssh/
+
+# Expose SSH port
+EXPOSE 8000 2222
+
+# Copy webspot_rod
+COPY --from=build /go/bin/webspot_rod /go/bin/webspot_rod
+COPY webspot_rod/conf/supervisord.conf /etc/supervisor/supervisord.conf
+
 # Install requirements
 COPY ./requirements.txt /app
 RUN pip install --upgrade pip
@@ -22,5 +62,7 @@ ADD . /app
 ENV PORT 80
 EXPOSE 80
 
-ENTRYPOINT ["python", "main.py"]
-CMD ["web"]
+# Change mode of entrypoint.sh
+RUN chmod u+x ./entrypoint.sh
+
+CMD ["sh", "entrypoint.sh"]

README.md

@@ -32,6 +32,22 @@ Then you can access the web UI at http://localhost:80.
 webspot web --help
 ```
 
+## Architecture
+
+### Overview
+
+The overall process of how Webspot detects meaningful elements from HTML or web pages is shown in the following figure.
+
+```mermaid
+graph LR
+    hr[HtmlRequester]
+    gl[GraphLoader]
+    d[Detector]
+    r[Results]
+
+    hr --"html + json"--> gl --"graph"--> d --"output"--> r
+```
+
 ## Development
 
 Development with Webspot is easy. You can follow the following guidance to get started.

dev/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3.3'
+services:
+  mysql:
+    image: mysql:8
+    container_name: mysql
+    ports:
+      - "3306:3306"
+    environment:
+      MYSQL_DATABASE: "webspot"
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+    volumes:
+      - ./data/mysql:/var/lib/mysql

docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3.3'
+services:
+  webspot:
+    image: crawlabteam/webspot:develop
+    container_name: webspot
+    environment:
+      WEBSPOT_DATABASE_URL: mongodb://mongo:27017/webspot
+    ports:
+      - '9999:80'
+
+  mongo:
+    image: mongo:4.2
+    container_name: webspot_mongo

entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+# Start ssh
+service ssh start
+
+# Start supervisor
+supervisord -c /etc/supervisor/supervisord.conf

main.py

@@ -1,8 +1,13 @@
 from argparse import ArgumentParser
 
+from dotenv import load_dotenv
+
 from webspot.cmd.crawl import cmd_crawl
 from webspot.cmd.request import cmd_request
 from webspot.cmd.web import cmd_web
+from webspot.constants.html_request_method import HTML_REQUEST_METHOD_REQUEST
+
+load_dotenv()
 
 parser = ArgumentParser()
 
@@ -27,7 +32,7 @@
 
 request_parser = subparsers.add_parser('request')
 request_parser.add_argument('--url', '-U', help='url to request', required=True)
-request_parser.add_argument('--method', '-M', help='request method', default='rod')
+request_parser.add_argument('--method', '-M', help='request method', default=HTML_REQUEST_METHOD_REQUEST)
 request_parser.set_defaults(func=cmd_request)
 
 if __name__ == '__main__':