deliverable as mainly created by Niels de Water

.env_example

@@ -0,0 +1,26 @@
+DJANGO_DEBUG=False
+DJANGO_SECRET_KEY=django-insecure-=!uw69v8!e=bk16^==ls)r!xum5znr3boawo0k#shm4ukg34g&
+DMPONLINE_AUTH_URL=https://dmponline.tudelft.nl/api/v1/authenticate
+DMPONLINE_TOKEN=foo
+DMPONLINE_USER_EMAIL=foo
+DMPONLINE_API_V0_URL=https://dmponline.tudelft.nl/api/v0/
+DMPONLINE_API_V1_URL=https://dmponline.tudelft.nl/api/v1/
+DMPONLINE_VERIFY=True
+AVG_REGISTRY_URL=https://here-goes-the-avg-registry-url.nl/api/
+AVG_REGISTRY_TOKEN=foo
+PARSE_TEST_PLANS=True
+ESB_TOKEN=foo
+ESB_URL=https://here-goes-the-esb-url.nl/
+ESB_VERIFY=True
+EMAIL_HOST=mail.ahost.com
+#EMAIL_PORT=465  #SSL
+EMAIL_PORT=587  #TLS
+EMAIL_USE_TLS=True
+#EMAIL_USE_SSL=True
+EMAIL_HOST_USER=user@email.com
+EMAIL_HOST_PASSWORD=apassword
+DEFAULT_RECIPIENT=user@email.com
+SHAREPOINT_URL=https://here-goes-the-sharepoint-url.nl/
+SHAREPOINT_USERNAME=domain\user
+# only use 1 '\' (don't escape) to separate domain and user
+SHAREPOINT_PASSWORD=foo

.gitignore

@@ -0,0 +1,5 @@
+__pycache__
+.idea
+db.sqlite3
+.env
+venv

Dockerfile

@@ -0,0 +1,8 @@
+# syntax=docker/dockerfile:1
+FROM python:3
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code
+COPY . /code/
+RUN pip install -r requirements.txt
+RUN python ./manage.py makemigrations
+RUN python ./manage.py migrate

README.md

@@ -0,0 +1,14 @@
+DMP to AVG registry and TOPdesk
+
+- after cloning, in Django root (same as manage.py)
+- create .env file with necessary authentication information (tokens, email) (copy .env_example)
+- `python3 -m venv venv`
+- `.venv/bin/activate`
+- `pip install -r requirements.txt`
+- `python manage.py migrate`
+- `python manage.py runserver 0.0.0.0:8000`
+- Running the script for cron: `python manage.py fetch -b [first page] -e [last_page]` where pages refer to API pages of DMPonline
+- Testing is done with pytest: `pytest`
+- If caching problems occur: `pytest -o cache_dir=/tmp`
+- Test coverage is calculated with: `coverage run -m pytest && coverage html`
+- Docker image can be build with: `docker-compose up`

dmps/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for dmps project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dmps.settings")
+
+application = get_asgi_application()

dmps/settings.py

@@ -0,0 +1,213 @@
+"""
+Django settings for dmps project.
+
+Generated by 'django-admin startproject' using Django 3.2.7.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.2/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/3.2/ref/settings/
+"""
+
+import sys
+from pathlib import Path
+from environs import Env
+import colorlog
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+
+env = Env()
+env.read_env()
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+DMPONLINE_AUTH_URL = env.str("DMPONLINE_AUTH_URL")
+DMPONLINE_API_V0_URL = env.str("DMPONLINE_API_V0_URL")
+DMPONLINE_API_V1_URL = env("DMPONLINE_API_V1_URL")
+DMPONLINE_TOKEN = env.str("DMPONLINE_TOKEN")
+DMPONLINE_USER_EMAIL = env.str("DMPONLINE_USER_EMAIL")
+DMPONLINE_VERIFY = env.str(
+    "DMPONLINE_VERIFY"
+)  # on purpose not as boolean, because it also can be a file path
+AVG_REGISTRY_URL = env.str("AVG_REGISTRY_URL")
+AVG_REGISTRY_TOKEN = env.str("AVG_REGISTRY_TOKEN")
+PARSE_TEST_PLANS = env.bool("PARSE_TEST_PLANS")
+ESB_TOKEN = env.str("ESB_TOKEN")
+ESB_URL = env.str("ESB_URL")
+ESB_VERIFY = env.str(
+    "ESB_VERIFY"
+)  # on purpose not as boolean, because it also can be a file path
+
+SHAREPOINT_URL = env.str("SHAREPOINT_URL")
+SHAREPOINT_USERNAME = env.str("SHAREPOINT_USERNAME")
+SHAREPOINT_PASSWORD = env.str("SHAREPOINT_PASSWORD")
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = env.str("DJANGO_SECRET_KEY")
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = env.bool("DJANGO_DEBUG")
+
+ALLOWED_HOSTS = ["*"]
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "stats.apps.StatsConfig",
+    "colorlog",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+]
+
+ROOT_URLCONF = "dmps.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [BASE_DIR / "templates"],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ],
+        },
+    },
+]
+
+
+""""""
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "verbose": {
+            "format": "{levelname} {asctime} [{module}] [{name}]: {message}",  # {process:d} {thread:d}
+            "style": "{",
+        },
+        "simple": {
+            "format": "{levelname} {message}",
+            "style": "{",
+        },
+        "colored": {
+            "()": "colorlog.ColoredFormatter",
+            "format": "%(log_color)s %(levelname)-8s %(asctime)s "
+            "%(module)s %(reset)s %(white)s%(message)s",
+        },
+    },
+    "handlers": {
+        "log_to_stdout": {
+            "level": "DEBUG",
+            "class": "logging.StreamHandler",
+            "stream": sys.stdout,
+            "formatter": "colored",
+        },
+    },
+    "loggers": {
+        "main": {
+            "handlers": ["log_to_stdout"],
+            "level": "DEBUG",
+            "propagate": True,
+        },
+        "mappings": {
+            "handlers": ["log_to_stdout"],
+            "level": "DEBUG",
+            "propagate": True,
+        },
+    },
+}
+
+
+WSGI_APPLICATION = "dmps.wsgi.application"
+
+
+# Database
+# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": BASE_DIR / "db.sqlite3",
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/3.2/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+
+TIME_ZONE = "Europe/Amsterdam"
+
+USE_I18N = True
+
+USE_L10N = True
+
+EMAIL_HOST = env.str("EMAIL_HOST")
+# EMAIL_PORT = 465  # SSL
+EMAIL_PORT = env.int("EMAIL_PORT")  # TLS
+EMAIL_USE_TLS = env.bool("EMAIL_USE_TLS")
+# EMAIL_USE_SSL = True
+EMAIL_HOST_USER = env.str("EMAIL_HOST_USER")
+EMAIL_HOST_PASSWORD = env.str("EMAIL_HOST_PASSWORD")
+DEFAULT_FROM_EMAIL = env.str("EMAIL_HOST_USER")
+SERVER_EMAIL = env.str("EMAIL_HOST_USER")
+DEFAULT_RECIPIENT = env.str("DEFAULT_RECIPIENT")
+
+USE_TZ = True
+
+DATA_UPLOAD_MAX_NUMBER_FIELDS = 5000
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.2/howto/static-files/
+
+STATIC_URL = "/static/"
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"

dmps/urls.py

@@ -0,0 +1,22 @@
+"""dmps URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/3.2/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path, include
+
+urlpatterns = [
+    path("admin/", admin.site.urls),
+    path("", include("stats.urls")),
+]

dmps/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for dmps project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dmps.settings")
+
+application = get_wsgi_application()

docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3.3"
+
+services:
+#  db:
+#    image: postgres
+#    volumes:
+#      - ./data/db:/var/lib/postgresql/data
+#    environment:
+#      - POSTGRES_DB=postgres
+#      - POSTGRES_USER=postgres
+#      - POSTGRES_PASSWORD=postgres
+  web:
+    build: .
+    command: >
+      sh -c "python manage.py makemigrations &&
+             python manage.py migrate &&
+             python manage.py runserver 0.0.0.0:8080"
+    volumes:
+      - .:/code
+    ports:
+      - "8080:8080"
+#    network_mode: host
+#    depends_on:
+#      - db

manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dmps.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+DJANGO_SETTINGS_MODULE = dmps.settings
+python_files = stats/tests.py
+filterwarnings = ignore::DeprecationWarning
+                 ignore::urllib3.exceptions.InsecureRequestWarning
+