Add Docker files and additional Visitors and Mappers

Dockerfile

@@ -0,0 +1,42 @@
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM --platform=linux/amd64 python:3.9
+
+WORKDIR /app
+
+# Python settings: Force unbuffered stdout and stderr (i.e. they are flushed to terminal immediately)
+ENV PYTHONUNBUFFERED 1
+# Python settings: do not write pyc files
+ENV PYTHONDONTWRITEBYTECODE 1
+
+# OS requirements as per
+# https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+       bzip2 \
+       xz-utils \
+       zlib1g \
+       libxml2-dev \
+       libxslt1-dev \
+       libpopt0 \
+       bzip2 \
+       tar \
+       unzip \
+       zip \
+       libsasl2-dev \
+       libldap-dev \
+       openssl \
+       wait-for-it \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+COPY setup.cfg setup.py /app/
+RUN mkdir -p /app/matchcode-toolkit/src/
+COPY matchcode-toolkit/setup.cfg matchcode-toolkit/setup.py /app/matchcode-toolkit/
+RUN pip install https://github.com/nexB/commoncode/archive/refs/heads/main.zip
+RUN pip install https://github.com/nexB/scancode-toolkit/archive/refs/heads/maven-pom-parse-dep-backport.zip
+RUN pip install -e matchcode-toolkit
+RUN pip install -e .
+
+COPY . /app

configure

@@ -27,7 +27,7 @@ CLI_ARGS=$1
 # Defaults. Change these variables to customize this script
 ################################
 
-CUSTOM_PACKAGES="https://github.com/nexB/commoncode/archive/refs/heads/48-correctly-assign-codebase-attributes.zip https://github.com/nexB/scancode-toolkit/archive/refs/heads/maven-pom-parse-dep-backport.zip"
+CUSTOM_PACKAGES="https://github.com/nexB/scancode-toolkit/archive/refs/heads/maven-pom-parse-dep-backport.zip"
 
 # Requirement arguments passed to pip and used by default or with --dev.
 REQUIREMENTS="$CUSTOM_PACKAGES --editable matchcode-toolkit --editable . --constraint requirements.txt"

docker-compose.yml

@@ -0,0 +1,90 @@
+version: "3"
+
+services:
+  db:
+    image: postgres:13
+    env_file:
+      - docker.env
+    volumes:
+      - db_data:/var/lib/postgresql/data/
+
+  web:
+    build: .
+    command: sh -c "
+        python manage.py migrate &&
+        python manage.py collectstatic --no-input --verbosity 0 --clear &&
+        gunicorn purldb.wsgi:application --bind :8000 --timeout 600 --workers 8"
+    env_file:
+      - docker.env
+    expose:
+      - 8000
+    volumes:
+      - /etc/purldb/:/etc/purldb/
+      - static:/var/purldb/static/
+    depends_on:
+      - db
+
+  visitor:
+    build: .
+    command: sh -c "
+        wait-for-it web:8000 -- python manage.py seed &&
+        python manage.py run_visit --ignore-robots --ignore-throttle"
+    env_file:
+      - docker.env
+    volumes:
+      - /etc/purldb/:/etc/purldb/
+    profiles:
+      - visit_and_map
+    depends_on:
+      - db
+      - web # Ensure that potential db migrations run first
+
+  mapper:
+    build: .
+    command: wait-for-it web:8000 -- python manage.py run_map
+    env_file:
+      - docker.env
+    volumes:
+      - /etc/purldb/:/etc/purldb/
+    profiles:
+      - visit_and_map
+    depends_on:
+      - db
+      - web # Ensure that potential db migrations run first
+
+  clearsync:
+    build: .
+    command: wait-for-it web:8000 -- clearsync --save-to-db  --verbose -n 3
+    env_file:
+      - docker.env
+    volumes:
+      - /etc/purldb/:/etc/purldb/
+    profiles:
+      - clearsync
+    depends_on:
+      - db
+      - web # Ensure that potential db migrations run first
+
+  clearindex:
+    build: .
+    command: wait-for-it web:8000 -- python manage.py run_clearindex
+    profiles:
+      - clearsync
+    depends_on:
+      - db
+      - web # Ensure that potential db migrations run first
+
+  nginx:
+    image: nginx
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - ./etc/nginx/conf.d/:/etc/nginx/conf.d/
+      - static:/var/purldb/static/
+    depends_on:
+      - web
+
+volumes:
+  db_data:
+  static:

docker.env

@@ -0,0 +1,6 @@
+POSTGRES_DB=packagedb
+POSTGRES_USER=packagedb
+POSTGRES_PASSWORD=packagedb
+POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8
+
+PACKAGEDB_DB_HOST=db

etc/nginx/conf.d/default.conf

@@ -0,0 +1,20 @@
+upstream gunicorn_app {
+    server web:8000;
+}
+
+server {
+    listen 80;
+
+    location / {
+        proxy_pass http://gunicorn_app;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+        client_max_body_size 10G;
+        proxy_read_timeout 600s;
+    }
+
+    location /static/ {
+        alias /var/purldb/static/;
+    }
+}

etc/nginx/examples/ssl.conf

@@ -0,0 +1,28 @@
+upstream gunicorn_app {
+    server web:8000;
+}
+
+server {
+    listen 80;
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen 443 ssl;
+    ssl_certificate /etc/nginx/conf.d/fullchain.pem;
+    ssl_certificate_key /etc/nginx/conf.d/privkey.pem;
+
+    location / {
+        proxy_pass http://gunicorn_app;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+        client_max_body_size 10G;
+        proxy_read_timeout 600s;
+    }
+
+    location /static/ {
+        alias /var/purldb/static/;
+    }
+}