dockerize, add ci/cd

.dockerignore

@@ -0,0 +1,4 @@
+**/*.sqlite3
+venv
+.git
+**/.gitignore

.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+name: CI
+on:
+  push:
+jobs:
+  ci:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - env:
+        DOCKER_APP_IMAGE_NAME: "ghcr.io/hasadna/openlaw-archive/openlaw-archive-app"
+        DOCKER_NGINX_IMAGE_NAME: "ghcr.io/hasadna/openlaw-archive/openlaw-archive-nginx"
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        HASADNA_K8S_DEPLOY_KEY: ${{ secrets.HASADNA_K8S_DEPLOY_KEY }}
+        MAIN_BRANCH: main
+      run: |
+        echo "${GITHUB_TOKEN}" | docker login ghcr.io -u hasadna --password-stdin &&\
+        if docker pull "${DOCKER_APP_IMAGE_NAME}:latest"; then
+          CACHE_FROM_ARG="--cache-from ${DOCKER_APP_IMAGE_NAME}:latest"
+        else
+          CACHE_FROM_ARG=""
+        fi &&\
+        docker build $CACHE_FROM_ARG --build-arg VERSION=${GITHUB_SHA} -t openlaw-archive-app . &&\
+        docker tag openlaw-archive-app "${DOCKER_APP_IMAGE_NAME}:${GITHUB_SHA}" &&\
+        docker push "${DOCKER_APP_IMAGE_NAME}:${GITHUB_SHA}" &&\
+        if docker pull "${DOCKER_NGINX_IMAGE_NAME}:latest"; then
+          CACHE_FROM_ARG="--cache-from ${DOCKER_NGINX_IMAGE_NAME}:latest"
+        else
+          CACHE_FROM_ARG=""
+        fi &&\
+        docker build $CACHE_FROM_ARG --build-arg VERSION=${GITHUB_SHA} -t openlaw-archive-nginx -f nginx.Dockerfile . &&\
+        docker tag openlaw-archive-nginx "${DOCKER_NGINX_IMAGE_NAME}:${GITHUB_SHA}" &&\
+        docker push "${DOCKER_NGINX_IMAGE_NAME}:${GITHUB_SHA}" &&\
+        if [ "${GITHUB_REF}" == "refs/heads/${MAIN_BRANCH}" ]; then
+          docker tag openlaw-archive-app "${DOCKER_APP_IMAGE_NAME}:latest" &&\
+          docker push "${DOCKER_APP_IMAGE_NAME}:latest" &&\
+          docker tag openlaw-archive-nginx "${DOCKER_NGINX_IMAGE_NAME}:latest" &&\
+          docker push "${DOCKER_NGINX_IMAGE_NAME}:latest" &&\
+          if ! git log -1 --pretty=format:"%s" | grep -- --no-deploy; then
+            cd `mktemp -d` &&\
+            echo "${HASADNA_K8S_DEPLOY_KEY}" > hasadna_k8s_deploy_key &&\
+            chmod 400 hasadna_k8s_deploy_key &&\
+            export GIT_SSH_COMMAND="ssh -i `pwd`/hasadna_k8s_deploy_key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" &&\
+            git clone git@github.com:hasadna/hasadna-k8s.git &&\
+            cd hasadna-k8s &&\
+            python update_yaml.py '{"archiveAppImage":"'"${DOCKER_APP_IMAGE_NAME}:${GITHUB_SHA}"'","archiveNginxImage":"'"${DOCKER_NGINX_IMAGE_NAME}:${GITHUB_SHA}"'"}' apps/openlaw/values-hasadna-auto-updated.yaml &&\
+            git config --global user.name "openlaw-archive CI" &&\
+            git config --global user.email "openlaw-archive-ci@localhost" &&\
+            git add apps/openlaw/values-hasadna-auto-updated.yaml && git commit -m "automatic update of openlaw-archive app" &&\
+            git push origin master
+          fi
+        fi

Dockerfile

@@ -0,0 +1,8 @@
+# Pulled Apr 24, 2023
+FROM python:3.8@sha256:6aea47c16a4fe8a30f184060a2347caa24ea156b224041543fdc6b901dbec699
+WORKDIR /srv
+COPY requirements.txt ./
+RUN pip install --upgrade pip && pip install -r requirements.txt && rm requirements.txt
+COPY djang ./djang
+WORKDIR /srv/djang
+ENTRYPOINT ["/srv/djang/entrypoint.sh"]

README.md

@@ -0,0 +1,39 @@
+# OpenLaw Archive
+
+## Setup
+
+```
+make init
+```
+
+## Running
+
+```
+make serve
+```
+
+
+## Docker Compose development
+
+This environment resembles the production environment as closely as possible.
+
+Run migrations:
+
+```bash
+docker-compose run --build --rm migrate
+```
+
+Start the web app:
+
+```bash
+docker-compose up -d --build ingress
+```
+
+Access at http://localhost:8000
+
+Start a shell to run management commands:
+
+```bash
+docker-compose exec web bash
+pytyhon manage.py
+```

compose.yaml

@@ -0,0 +1,71 @@
+services:
+  db:
+    # Pulled Apr 24, 2023
+    image: postgres:15@sha256:6cc97262444f1c45171081bc5a1d4c28b883ea46a6e0d1a45a8eac4a7f4767ab
+    environment:
+      POSTGRES_PASSWORD: "123456"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready --username=postgres"]
+      interval: 5s
+      timeout: 10s
+      retries: 5
+      start_period: 80s
+
+  migrate:
+    image: openlaw-archive-app
+    build: .
+    restart: on-failure
+    environment: &commonenv
+      DJANGO_SECRET_KEY: "#-@^btdzoe9n-swq=fx5(db5^ibkqsytrt0ie2n55efz7wtpk#"
+      DEBUG: "True"
+      DJANGO_DATABASE_ENGINE: "postgres"
+      DJANGO_DATABASE_NAME: "postgres"
+      DJANGO_DATABASE_USER: "postgres"
+      DJANGO_DATABASE_PASSWORD: "123456"
+      DJANGO_DATABASE_HOST: "db"
+      DJANGO_DATABASE_PORT: "5432"
+      DJANGO_ALLOWED_HOSTS: "*"
+      DJANGO_CSRF_TRUSTED_ORIGINS: "http://localhost:8000"
+    depends_on: &dependsondb
+      db:
+        condition: service_healthy
+    command: ["migrate"]
+
+  nginx:
+    image: openlaw-archive-nginx
+    build:
+      context: .
+      dockerfile: nginx.Dockerfile
+
+  web:
+    image: openlaw-archive-app
+    build: .
+    environment:
+      <<: *commonenv
+    depends_on: *dependsondb
+    command: ["web"]
+
+  ingress:
+    # Pulled Apr 24, 2023
+    image: nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94
+    volumes:
+      - ./ingress-nginx-default.conf:/etc/nginx/conf.d/default.conf
+    ports:
+      - "8000:80"
+    depends_on:
+      - nginx
+      - web
+
+#  qcluster:
+#    image: open-pension-ng
+#    build: .
+#    environment:
+#      <<: *commonenv
+#      DJANGO_Q_CLUSTER_SETTINGS_JSON: '{"workers": 2}'
+#    depends_on: *dependsondb
+#    command: ["qcluster"]
+
+volumes:
+  pgdata:

djang/djang/settings.py

@@ -9,8 +9,10 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/4.2/ref/settings/
 """
-
+import os
+import json
 from pathlib import Path
+from django.core.management.utils import get_random_secret_key
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -20,12 +22,27 @@
 # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = "django-insecure-muy(&9-si^dpl(2%g)ckdjp1y+!0!0_khm7qx*gx(c6zy9ude!"
+SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY")
+if not SECRET_KEY:
+    # for developement it's OK, for production we should set the DJANGO_SECRET_KEY env variable
+    SECRET_KEY = get_random_secret_key()
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = os.environ.get("DJANGO_DEBUG") != "False"
+
+ALLOWED_HOSTS = [
+    h.strip()
+    for h in os.environ.get("DJANGO_ALLOWED_HOSTS", "").split(",")
+    if h.strip()
+]
+if not ALLOWED_HOSTS:
+    ALLOWED_HOSTS = ["*"]
 
-ALLOWED_HOSTS = []
+CSRF_TRUSTED_ORIGINS = [
+    h.strip()
+    for h in os.environ.get("DJANGO_CSRF_TRUSTED_ORIGINS", "").split(",")
+    if h.strip()
+]
 
 
 # Application definition
@@ -75,12 +92,25 @@
 # Database
 # https://docs.djangoproject.com/en/4.2/ref/settings/#databases
 
-DATABASES = {
-    "default": {
+database_engine = os.environ.get("DJANGO_DATABASE_ENGINE")
+if database_engine == "postgres":
+    default_database = {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": os.environ["DJANGO_DATABASE_NAME"],
+        "USER": os.environ["DJANGO_DATABASE_USER"],
+        "PASSWORD": os.environ["DJANGO_DATABASE_PASSWORD"],
+        "HOST": os.environ["DJANGO_DATABASE_HOST"],
+        "PORT": os.environ["DJANGO_DATABASE_PORT"],
+    }
+elif not database_engine:
+    default_database = {
         "ENGINE": "django.db.backends.sqlite3",
         "NAME": BASE_DIR / "db.sqlite3",
     }
-}
+else:
+    raise Exception(f"Unsupported database engine: {database_engine}")
+
+DATABASES = {"default": default_database}
 
 
 # Password validation
@@ -118,6 +148,7 @@
 # https://docs.djangoproject.com/en/4.2/howto/static-files/
 
 STATIC_URL = "static/"
+STATIC_ROOT = os.environ.get("DJANGO_STATIC_ROOT")
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field

djang/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+if [ "${1}" == "web" ]; then
+  exec gunicorn -k uvicorn.workers.UvicornWorker -c gunicorn_conf.py djang.asgi:application
+else
+  exec python manage.py "$@"
+fi

djang/gunicorn_conf.py

@@ -0,0 +1,68 @@
+# copied from https://github.com/tiangolo/uvicorn-gunicorn-docker/blob/master/docker-images/gunicorn_conf.py August 24, 2021
+import json
+import multiprocessing
+import os
+
+workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1")
+max_workers_str = os.getenv("MAX_WORKERS")
+use_max_workers = None
+if max_workers_str:
+    use_max_workers = int(max_workers_str)
+web_concurrency_str = os.getenv("WEB_CONCURRENCY", None)
+
+host = os.getenv("HOST", "0.0.0.0")
+port = os.getenv("PORT", "80")
+bind_env = os.getenv("BIND", None)
+use_loglevel = os.getenv("LOG_LEVEL", "info")
+if bind_env:
+    use_bind = bind_env
+else:
+    use_bind = f"{host}:{port}"
+
+cores = multiprocessing.cpu_count()
+workers_per_core = float(workers_per_core_str)
+default_web_concurrency = workers_per_core * cores
+if web_concurrency_str:
+    web_concurrency = int(web_concurrency_str)
+    assert web_concurrency > 0
+else:
+    web_concurrency = max(int(default_web_concurrency), 2)
+    if use_max_workers:
+        web_concurrency = min(web_concurrency, use_max_workers)
+accesslog_var = os.getenv("ACCESS_LOG", "-")
+use_accesslog = accesslog_var or None
+errorlog_var = os.getenv("ERROR_LOG", "-")
+use_errorlog = errorlog_var or None
+graceful_timeout_str = os.getenv("GRACEFUL_TIMEOUT", "120")
+timeout_str = os.getenv("TIMEOUT", "120")
+keepalive_str = os.getenv("KEEP_ALIVE", "5")
+
+# Gunicorn config variables
+loglevel = use_loglevel
+workers = web_concurrency
+bind = use_bind
+errorlog = use_errorlog
+worker_tmp_dir = "/dev/shm"
+accesslog = use_accesslog
+graceful_timeout = int(graceful_timeout_str)
+timeout = int(timeout_str)
+keepalive = int(keepalive_str)
+
+
+# For debugging and testing
+log_data = {
+    "loglevel": loglevel,
+    "workers": workers,
+    "bind": bind,
+    "graceful_timeout": graceful_timeout,
+    "timeout": timeout,
+    "keepalive": keepalive,
+    "errorlog": errorlog,
+    "accesslog": accesslog,
+    # Additional, non-gunicorn variables
+    "workers_per_core": workers_per_core,
+    "use_max_workers": use_max_workers,
+    "host": host,
+    "port": port,
+}
+print(json.dumps(log_data))

ingress-nginx-default.conf

@@ -0,0 +1,12 @@
+server {
+    listen       80;
+    server_name  localhost;
+
+    location /static {
+        proxy_pass http://nginx/static;
+    }
+
+    location / {
+        proxy_pass http://web;
+    }
+}

nginx.Dockerfile

@@ -0,0 +1,7 @@
+FROM openlaw-archive-app
+RUN DJANGO_STATIC_ROOT=/srv/static python manage.py collectstatic --noinput -c
+
+# Pulled Apr 24, 2023
+FROM nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94
+COPY --from=0 /srv/static /usr/share/nginx/html/static
+RUN rm /usr/share/nginx/html/*.html

requirements.txt

@@ -7,3 +7,6 @@ idna==3.4
 requests==2.29.0
 sqlparse==0.4.4
 urllib3==1.26.15
+gunicorn==20.1.0
+uvicorn==0.20.0
+psycopg2-binary==2.9.5