feat: native Dockerfile

Dockerfile

@@ -1,33 +1,135 @@
-FROM ubuntu:xenial as openedx
-
-RUN apt update && \
-  apt-get install -y software-properties-common && \
-  apt-add-repository -y ppa:deadsnakes/ppa && apt-get update && \
-  apt-get install -y curl && \
-  apt-get upgrade -qy && \
-  apt install -y git-core language-pack-en build-essential python3.8-dev python3.8-distutils libmysqlclient-dev && \
-  curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
-  python3.8 get-pip.py && python3.8 -m pip install --upgrade pip setuptools && \
+FROM ubuntu:focal as app
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt update && apt-get install -qy \
+  curl \
+  vim \
+  git-core \
+  language-pack-en \
+  build-essential \
+  python3.8-dev \
+  python3-virtualenv \
+  python3.8-distutils \
+  libmysqlclient-dev \
+  libssl-dev \
+  gettext && \
   rm -rf /var/lib/apt/lists/*
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
-ENV ANALYTICS_DASHBOARD_CFG /edx/etc/insights.yml
 
-WORKDIR /edx/app/analytics_dashboard
-COPY requirements /edx/app/analytics_dashboard/requirements
-RUN python3.8 -m pip install -r requirements/production.txt
+
+ARG COMMON_CFG_DIR "/edx/etc"
+ARG INSIGHTS_CFG_DIR "${COMMON_CFG_DIR}/insights"
+ARG ANALYTICS_DASHBOARD_CFG "${COMMON_CFG_DIR}/insights.yml"
+ENV ANALYTICS_DASHBOARD_CFG "${ANALYTICS_DASHBOARD_CFG}"
+
+ARG COMMON_APP_DIR="/edx/app"
+ARG INSIGHTS_APP_DIR="${COMMON_APP_DIR}/insights"
+ENV INSIGHTS_APP_DIR ${INSIGHTS_APP_DIR}
+ARG SUPERVISOR_APP_DIR="${COMMON_APP_DIR}/supervisor"
+ARG INSIGHTS_VENV_DIR="${COMMON_APP_DIR}/insights/venvs/insights"
+ARG SUPERVISOR_VENVS_DIR="${SUPERVISOR_APP_DIR}/venvs"
+ARG SUPERVISOR_VENV_DIR="${SUPERVISOR_VENVS_DIR}/supervisor"
+ARG INSIGHTS_CODE_DIR="${INSIGHTS_APP_DIR}/edx_analytics_dashboard"
+ARG INSIGHTS_NODEENV_DIR="${COMMON_APP_DIR}/insights/nodeenvs/insights"
+ARG SUPERVISOR_AVAILABLE_DIR="${COMMON_APP_DIR}/supervisor/conf.available.d"
+ARG SUPERVISOR_VENV_BIN="${SUPERVISOR_VENV_DIR}/bin"
+ARG SUPEVISOR_CTL="${SUPERVISOR_VENV_BIN}/supervisorctl"
+ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d"
+ARG SUPERVISOR_VERSION="4.2.1"
+ARG INSIGHTS_NODE_VERSION="16.14.0"
+ARG INSIGHTS_NPM_VERSION="8.5.x"
+
+ENV PATH "${INSIGHTS_VENV_DIR}/bin:${INSIGHTS_NODEENV_DIR}/bin:$PATH"
+
+RUN addgroup insights
+RUN adduser --disabled-login --disabled-password insights --ingroup insights
+
+# No need to activate insights virtualenv as it is already activated by putting in the path
+RUN virtualenv -p python3.8 --always-copy ${INSIGHTS_VENV_DIR}
+RUN virtualenv -p python3.8 --always-copy ${SUPERVISOR_VENV_DIR}
+COPY requirements ${INSIGHTS_CODE_DIR}/requirements
+
+ENV PATH="${INSIGHTS_CODE_DIR}/node_modules/.bin:$PATH"
+
+WORKDIR ${INSIGHTS_CODE_DIR}/
+
+#Configurations from edx_service task
+RUN mkdir ${INSIGHTS_APP_DIR}/data/
+RUN mkdir ${INSIGHTS_APP_DIR}/staticfiles/
+RUN mkdir /edx/var/
+RUN mkdir /edx/var/insights/
+# Log dir
+RUN mkdir /edx/var/log/
+
+
+#install supervisor and deps in its virtualenv
+RUN . ${SUPERVISOR_VENV_BIN}/activate && \
+  pip install supervisor==${SUPERVISOR_VERSION} backoff==1.4.3 boto==2.48.0 && \
+  deactivate
+
+# create supervisor job
+COPY /configuration_files/supervisor.conf /etc/systemd/system/supervisor.service
+# These files are already present in the current insights image but do we have a reasonable use case of these files or should they be deleted...?
+# COPY /configuration_files/lms.conf ${SUPERVISOR_CFG_DIR}/lms.conf
+# COPY /configuration_files/cms.conf ${SUPERVISOR_CFG_DIR}/cms.conf
+COPY /configuration_files/supervisorctl ${SUPERVISOR_VENV_BIN}/supervisorctl
+
+# insights service config commands below
+RUN pip install -r ${INSIGHTS_CODE_DIR}/requirements/production.txt 
+
+RUN nodeenv ${INSIGHTS_NODEENV_DIR} --node=${INSIGHTS_NODE_VERSION} --prebuilt
+RUN npm install -g npm@${INSIGHTS_NPM_VERSION}
+
+# Tried to cache the dependencies by copying related files after the npm install step but npm post install fails in that case.
+# COPY package.json ${INSIGHTS_CODE_DIR}/package.json
+# COPY package-lock.json ${INSIGHTS_CODE_DIR}/package-lock.json
+# COPY npm-post-install.sh ${INSIGHTS_CODE_DIR}/npm-post-install.sh
+
+COPY . ${INSIGHTS_CODE_DIR}/
+RUN npm set progress=false && npm ci
+
+COPY configuration_files/insights.yml /edx/etc/insights.yml
+
+# Enable supervisor script
+COPY /scripts/insights.sh ${INSIGHTS_APP_DIR}/insights.sh
+COPY /configuration_files/insights.conf ${SUPERVISOR_AVAILABLE_DIR}/insights.conf
+COPY /configuration_files/insights.conf ${SUPERVISOR_CFG_DIR}/insights.conf
+# Manage.py symlink
+COPY /manage.py /edx/bin/manage.insights
 
 EXPOSE 8110
-CMD gunicorn -b 127.0.0.1:8110 --workers 2 --timeout=300 analytics_dashboard.wsgi:application
+EXPOSE 18110
+
+
+FROM app as production
+
+# DJANGO_SETTINGS_MODULE was defined previously in `insights-env` file and sourced from there while running the devstack. To me it makes more sense to define
+# this variable within the Dockerfile and it will also eliminate the use of two different files(devstack-env, production-env)
+ENV DJANGO_SETTINGS_MODULE "analytics_dashboard.settings.production"
+
+# For devstack these are run from the provisioning steps and hence no need for them in the dev target.
+RUN webpack --config webpack.prod.config.js
+RUN python3 manage.py collectstatic --noinput
+
+ENTRYPOINT ["${INSIGHTS_APP_DIR}/insights.sh"]
+
+FROM app as dev
+
+RUN pip install -r requirements/local.txt
+
+ENV DJANGO_SETTINGS_MODULE "analytics_dashboard.settings.devstack"
+
+COPY /scripts/devstack.sh ${INSIGHTS_APP_DIR}/devstack.sh
+COPY /configuration_files/insights-env ${INSIGHTS_APP_DIR}/insights_env
+
+RUN chown insights:insights ${INSIGHTS_APP_DIR}/devstack.sh && chmod a+x ${INSIGHTS_APP_DIR}/devstack.sh
 
-RUN useradd -m --shell /bin/false app
-USER app
-COPY . /edx/app/analytics_dashboard
 
-FROM openedx as edx.org
-RUN python3.8 -m pip install newrelic
-CMD newrelic-admin run-program gunicorn -b 127.0.0.1:8110 --workers 2 --timeout=300 analytics_dashboard.wsgi:application
+ENTRYPOINT ["/edx/app/insights/devstack.sh"]
 
+CMD ["start"]

configuration_files/insights-env

@@ -0,0 +1,3 @@
+export ANALYTICS_DASHBOARD_CFG="/edx/etc/insights.yml"
+export PATH="/edx/app/insights/nodeenvs/insights/bin:/edx/app/insights/venvs/insights/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+export THEME_SCSS="sass/themes/open-edx.scss"

configuration_files/insights.conf

@@ -0,0 +1,9 @@
+[program:insights]
+
+command=/edx/app/insights/insights.sh
+user=www-data
+directory=/edx/app/insights/edx_analytics_dashboard
+stdout_logfile=/edx/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/edx/var/log/supervisor/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

configuration_files/insights.yml

@@ -0,0 +1,58 @@
+---
+
+APPLICATION_NAME: Insights
+BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL: http://edx.devstack.lms:18000/oauth2
+CACHES:
+    default:
+        BACKEND: django.core.cache.backends.memcached.MemcachedCache
+        KEY_PREFIX: default_env-default_deployment-insights
+        LOCATION:
+        - edx.devstack.memcached:11211
+CDN_DOMAIN: null
+CMS_COURSE_SHORTCUT_BASE_URL: http://edx.devstack.lms:18000/course
+COURSE_API_URL: http://edx.devstack.lms:18000/api/courses/v1/
+CSRF_COOKIE_NAME: insights_csrftoken
+CSRF_COOKIE_SECURE: false
+DATABASES:
+    default:
+        ENGINE: django.db.backends.mysql
+        HOST: edx.devstack.mysql
+        NAME: dashboard
+        PASSWORD: secret
+        PORT: '3306'
+        USER: rosencrantz
+DATA_API_AUTH_TOKEN: edx
+DATA_API_URL: http://edx.devstack.analyticsapi:18100/api/v0
+DOCUMENTATION_LOAD_ERROR_URL: http://127.0.0.1/en/latest/Reference.html#error-conditions
+EMAIL_HOST: smtp.example.com
+EMAIL_HOST_PASSWORD: mail_password
+EMAIL_HOST_USER: mail_user
+EMAIL_PORT: 587
+ENABLE_AUTO_AUTH: true
+GRADING_POLICY_API_URL: http://edx.devstack.lms:18000/api/grades/v1/
+HELP_URL: http://127.0.0.1/en/latest
+LANGUAGE_CODE: en-us
+LANGUAGE_COOKIE_NAME: insights_language
+LEARNER_API_LIST_DOWNLOAD_FIELDS: null
+LMS_COURSE_SHORTCUT_BASE_URL: URL_FOR_LMS_COURSE_LIST_PAGE
+MODULE_PREVIEW_URL: http://edx.devstack.lms:18000/xblock
+OPEN_SOURCE_URL: http://set-me-please
+PLATFORM_NAME: edX
+PRIVACY_POLICY_URL: http://example.com/privacy-policy
+RESEARCH_URL: https://www.edx.org/research-pedagogy
+SECRET_KEY: YOUR_SECRET_KEY_HERE
+SEGMENT_IGNORE_EMAIL_REGEX: null
+SEGMENT_IO_KEY: YOUR_KEY
+SESSION_COOKIE_NAME: insights_sessionid
+SESSION_EXPIRE_AT_BROWSER_CLOSE: false
+SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
+SOCIAL_AUTH_EDX_OAUTH2_ISSUER: http://localhost:18000
+SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT: http://edx.devstack.lms:18000
+SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT: http://localhost:18000
+SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL: http://localhost:18000/logout
+STATICFILES_DIRS:
+- /edx/app/insights/edx_analytics_dashboard/analytics_dashboard/static
+STATIC_ROOT: /edx/var/insights/staticfiles
+SUPPORT_EMAIL: ''
+TERMS_OF_SERVICE_URL: http://example.com/terms-service
+TIME_ZONE: UTC

configuration_files/supervisor.conf

@@ -0,0 +1,29 @@
+[Unit]
+Description=supervisord - Supervisor process control system
+Documentation=http://supervisord.org
+After=network.target
+
+
+[Service]
+
+# User will be applied only to ExecStart, not other commands (i.e. ExecStartPre)
+# This is needed because pre_supervisor needs to write to supervisor/conf.d, which
+# supervisor_service_user does not have permission to do.
+PermissionsStartOnly=true
+User=www-data
+
+Type=forking
+TimeoutSec=432000
+
+ExecStart=/edx/app/supervisor/venvs/supervisor/bin/supervisord --configuration /edx/app/supervisor/supervisord.conf
+ExecReload=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl reload
+ExecStop=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl shutdown
+
+# Trust supervisor to kill all its children
+# Otherwise systemd will see that ExecStop ^ comes back synchronously and say "Oh, I can kill everyone in this cgroup"
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStop=
+# https://www.freedesktop.org/software/systemd/man/systemd.kill.html
+KillMode=none
+
+[Install]
+WantedBy=multi-user.target

configuration_files/supervisorctl

@@ -0,0 +1,8 @@
+#!/edx/app/supervisor/venvs/supervisor/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from supervisor.supervisorctl import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

scripts/devstack.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+COMMAND=$1
+
+case $COMMAND in
+    start)
+        /edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/supervisord.conf
+        ;;
+    open)
+        . /edx/app/insights/venvs/insights/bin/activate
+        cd /edx/app/insights/insights
+
+        /bin/bash
+        ;;
+    exec)
+        shift
+
+        . /edx/app/insights/venvs/insights/bin/activate
+        cd /edx/app/insights/insights
+
+        "$@"
+        ;;
+    *)
+        "$@"
+        ;;
+esac

scripts/insights.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+
+source /edx/app/insights/insights_env
+
+# We exec so that gunicorn is the child of supervisor and can be managed properly
+exec /edx/app/insights/venvs/insights/bin/gunicorn --pythonpath=/edx/app/insights/edx_analytics_dashboard/analytics_dashboard -b 127.0.0.1:8110 -w 2 --timeout=300  analytics_dashboard.wsgi:application