Add Django backend to store and compare issues

.dockerignore

@@ -0,0 +1,2 @@
+*/node_modules
+*.sqlite*

.gitignore

@@ -1,3 +1,5 @@
 .vscode/
 *.pyc
 *.egg-info
+*.sqlite*
+backend/hgmo

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
-known_first_party = code_review_bot,code_review_tools,code_review_events,conftest
-known_third_party = influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,setuptools,structlog,taskcluster,toml
+known_first_party = code_review_backend,code_review_bot,code_review_tools,code_review_events,conftest
+known_third_party = dj_database_url,django,influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,rest_framework,setuptools,structlog,taskcluster,toml
 force_single_line = True
 default_section=FIRSTPARTY
 line_length=159

.taskcluster.yml

@@ -98,6 +98,26 @@ tasks:
         owner: bastien@mozilla.com
         source: https://github.com/mozilla/code-review
 
+    - taskId: {$eval: as_slugid("backend_check_tests")}
+      provisionerId: aws-provisioner-v1
+      workerType: github-worker
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        maxRunTime: 3600
+        image: python:3
+        command:
+          - sh
+          - -lxce
+          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
+            cd /src/backend && pip install -q . && pip install -q -r requirements-dev.txt &&
+            ./manage.py test"
+      metadata:
+        name: "Code Review Backend checks: unit tests"
+        description: Check python code with Django tests
+        owner: bastien@mozilla.com
+        source: https://github.com/mozilla/code-review
+
     - taskId: {$eval: as_slugid("frontend_build")}
       provisionerId: aws-provisioner-v1
       workerType: github-worker
@@ -205,6 +225,47 @@ tasks:
         owner: bastien@mozilla.com
         source: https://github.com/mozilla/code-review
 
+    - taskId: {$eval: as_slugid("backend_build")}
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      provisionerId: aws-provisioner-v1
+      workerType: releng-svc
+      dependencies:
+        - {$eval: as_slugid("check_lint")}
+        - {$eval: as_slugid("backend_check_tests")}
+      payload:
+        capabilities:
+          privileged: true
+        maxRunTime: 3600
+        image: "${taskboot_image}"
+        env:
+          GIT_REPOSITORY: ${repository}
+          GIT_REVISION: ${head_rev}
+        command:
+          - taskboot
+          - build
+          - --image
+          - mozilla/code-review
+          - --tag
+          - "${channel}"
+          - --tag
+          - "${head_rev}"
+          - --write
+          - /backend.tar
+          - backend/Dockerfile
+        artifacts:
+          public/code-review-backend.tar:
+            expires: {$fromNow: '2 weeks'}
+            path: /backend.tar
+            type: file
+      scopes:
+        - docker-worker:capability:privileged
+      metadata:
+        name: Code Review Backend docker build
+        description: Build docker image of code review backend
+        owner: bastien@mozilla.com
+        source: https://github.com/mozilla/code-review
+
     - $if: 'channel in ["testing", "production"]'
       then:
         taskId: {$eval: as_slugid("frontend_deploy")}
@@ -326,3 +387,34 @@ tasks:
           description: Deploy docker image on Heroku
           owner: bastien@mozilla.com
           source: https://github.com/mozilla/code-review
+
+    # TODO: remove backend branch after tests are OK
+    - $if: 'channel in ["testing", "production"] || head_branch == "backend"'
+      then:
+        taskId: {$eval: as_slugid("backend_deploy")}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '1 hour'}
+        provisionerId: aws-provisioner-v1
+        workerType: github-worker
+        dependencies:
+          - {$eval: as_slugid("backend_build")}
+        payload:
+          features:
+            taskclusterProxy: true
+          maxRunTime: 3600
+          image: "${taskboot_image}"
+          command:
+            - taskboot
+            - deploy-heroku
+            - --heroku-app
+            - "code-review-backend-${channel}"
+            - web:public/code-review-backend.tar
+          env:
+            TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}"
+        scopes:
+          - "secrets:get:project/relman/code-review/deploy-${channel}"
+        metadata:
+          name: "Code Review Backend deployment (${channel})"
+          description: Deploy docker image on Heroku
+          owner: bastien@mozilla.com
+          source: https://github.com/mozilla/code-review

backend/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.7-slim
+
+ADD backend /src/backend
+
+WORKDIR /src/backend
+
+# Activate Django settings for in docker image
+ENV DJANGO_DOCKER=true
+
+RUN pip install --no-cache-dir .
+
+# Collect all static files
+RUN ./manage.py collectstatic --no-input
+
+CMD gunicorn code_review_backend.app.wsgi

backend/README.md

@@ -0,0 +1,35 @@
+# Code Review Backend
+
+## Developer setup
+
+```
+mkvirtualenv -p /usr/bin/python3 code-review-backend
+cd backend
+pip install -r requirements.txt
+./manage migrate
+./manage createsuperuser
+./manage runserver
+./manage loaddata fixtures/repositories.json
+```
+
+At this point, you can log into http://127.0.0.1:8000/admin/ with the credentials you mentioned during the `createsuperuser` step.
+
+## Load existing issues
+
+To load remote issues from production (default configuration):
+
+```
+./manage load_issues
+```
+
+To load already retrieved issues
+
+```
+./manage load_issues --offline
+```
+
+To load from testing
+
+```
+./manage load_issues --environment=testing
+```

backend/VERSION

@@ -0,0 +1 @@
+1.0.4

backend/code_review_backend/app/settings.py

@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Django settings for backend project.
+
+Generated by 'django-admin startproject' using Django 2.2.6.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.2/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/2.2/ref/settings/
+"""
+
+import logging
+import os
+
+import dj_database_url
+
+logger = logging.getLogger(__name__)
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ROOT_DIR = os.path.dirname(BASE_DIR)
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = "t!+s!@x5p!85x19q83jufr#95_z0fv7$!u5z*c&gi!%hr3^w+r"
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "rest_framework",
+    "code_review_backend.issues",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+]
+
+ROOT_URLCONF = "code_review_backend.app.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ]
+        },
+    }
+]
+
+WSGI_APPLICATION = "code_review_backend.app.wsgi.application"
+
+
+# Database
+# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": os.path.join(ROOT_DIR, "db.sqlite3"),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"
+    },
+    {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
+    {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
+    {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/2.2/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+
+TIME_ZONE = "UTC"
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+# API configuration
+REST_FRAMEWORK = {
+    # Use Django's standard `django.contrib.auth` permissions,
+    # or allow read-only access for unauthenticated users.
+    "DEFAULT_PERMISSION_CLASSES": [
+        "rest_framework.permissions.IsAuthenticatedOrReadOnly"
+    ],
+    # Setup pagination
+    "PAGE_SIZE": 50,
+    "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",
+}
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/2.2/howto/static-files/
+
+STATIC_URL = "/static/"
+
+# Static files are set in a dedicated path in Docker image
+if "DJANGO_DOCKER" in os.environ:
+    STATIC_ROOT = "/static"
+
+    # Enable GZip and cache, and build a manifest during collectstatic
+    STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
+
+
+# Internal logging setup
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {"console": {"class": "logging.StreamHandler"}},
+    "loggers": {
+        "django": {"handlers": ["console"], "level": "INFO"},
+        "code_review_backend": {"handlers": ["console"], "level": "INFO"},
+    },
+}
+
+# Heroku settings override to run the web app in production mode
+if "DYNO" in os.environ:
+    logger.info("Setting up Heroku environment")
+    ALLOWED_HOSTS = ["*"]
+    DEBUG = os.environ.get("DEBUG", "false").lower() == "true"
+
+    # Database setup
+    if "DATABASE_URL" in os.environ:
+        logger.info("Using remote database from $DATABASE_URL")
+        DATABASES["default"] = dj_database_url.parse(
+            os.environ["DATABASE_URL"], ssl_require=True
+        )
+    else:
+        logger.info("DATABASE_URL not found, will use sqlite. Data may be lost.")
+
+    # Insert Whitenoise Middleware after the security one
+    MIDDLEWARE.insert(1, "whitenoise.middleware.WhiteNoiseMiddleware")

backend/code_review_backend/app/urls.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from django.contrib import admin
+from django.shortcuts import redirect
+from django.urls import include
+from django.urls import path
+
+from code_review_backend.issues import api
+
+urlpatterns = [
+    path("", lambda request: redirect("v1/", permanent=False)),
+    path("v1/", include(api.router.urls)),
+    path("admin/", admin.site.urls),
+]

backend/code_review_backend/app/wsgi.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+WSGI config for backend project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "code_review_backend.app.settings")
+
+application = get_wsgi_application()