diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..57d9cd579 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +*/node_modules +*.sqlite* diff --git a/.gitignore b/.gitignore index 2e7fa2ee3..fa99d4b36 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .vscode/ *.pyc *.egg-info +*.sqlite* +backend/hgmo diff --git a/.isort.cfg b/.isort.cfg index 41aa15cec..ef244a0fe 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] -known_first_party = code_review_bot,code_review_tools,code_review_events,conftest -known_third_party = influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,setuptools,structlog,taskcluster,toml +known_first_party = code_review_backend,code_review_bot,code_review_tools,code_review_events,conftest +known_third_party = dj_database_url,django,influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,rest_framework,setuptools,structlog,taskcluster,toml force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/.taskcluster.yml b/.taskcluster.yml index a4eb5529f..62472ec82 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -98,6 +98,26 @@ tasks: owner: bastien@mozilla.com source: https://github.com/mozilla/code-review + - taskId: {$eval: as_slugid("backend_check_tests")} + provisionerId: aws-provisioner-v1 + workerType: github-worker + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + payload: + maxRunTime: 3600 + image: python:3 + command: + - sh + - -lxce + - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && + cd /src/backend && pip install -q . && pip install -q -r requirements-dev.txt && + ./manage.py test" + metadata: + name: "Code Review Backend checks: unit tests" + description: Check python code with Django tests + owner: bastien@mozilla.com + source: https://github.com/mozilla/code-review + - taskId: {$eval: as_slugid("frontend_build")} provisionerId: aws-provisioner-v1 workerType: github-worker @@ -205,6 +225,47 @@ tasks: owner: bastien@mozilla.com source: https://github.com/mozilla/code-review + - taskId: {$eval: as_slugid("backend_build")} + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + provisionerId: aws-provisioner-v1 + workerType: releng-svc + dependencies: + - {$eval: as_slugid("check_lint")} + - {$eval: as_slugid("backend_check_tests")} + payload: + capabilities: + privileged: true + maxRunTime: 3600 + image: "${taskboot_image}" + env: + GIT_REPOSITORY: ${repository} + GIT_REVISION: ${head_rev} + command: + - taskboot + - build + - --image + - mozilla/code-review + - --tag + - "${channel}" + - --tag + - "${head_rev}" + - --write + - /backend.tar + - backend/Dockerfile + artifacts: + public/code-review-backend.tar: + expires: {$fromNow: '2 weeks'} + path: /backend.tar + type: file + scopes: + - docker-worker:capability:privileged + metadata: + name: Code Review Backend docker build + description: Build docker image of code review backend + owner: bastien@mozilla.com + source: https://github.com/mozilla/code-review + - $if: 'channel in ["testing", "production"]' then: taskId: {$eval: as_slugid("frontend_deploy")} @@ -326,3 +387,34 @@ tasks: description: Deploy docker image on Heroku owner: bastien@mozilla.com source: https://github.com/mozilla/code-review + + # TODO: remove backend branch after tests are OK + - $if: 'channel in ["testing", "production"] || head_branch == "backend"' + then: + taskId: {$eval: as_slugid("backend_deploy")} + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + provisionerId: aws-provisioner-v1 + workerType: github-worker + dependencies: + - {$eval: as_slugid("backend_build")} + payload: + features: + taskclusterProxy: true + maxRunTime: 3600 + image: "${taskboot_image}" + command: + - taskboot + - deploy-heroku + - --heroku-app + - "code-review-backend-${channel}" + - web:public/code-review-backend.tar + env: + TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}" + scopes: + - "secrets:get:project/relman/code-review/deploy-${channel}" + metadata: + name: "Code Review Backend deployment (${channel})" + description: Deploy docker image on Heroku + owner: bastien@mozilla.com + source: https://github.com/mozilla/code-review diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 000000000..963b48045 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +ADD backend /src/backend + +WORKDIR /src/backend + +# Activate Django settings for in docker image +ENV DJANGO_DOCKER=true + +RUN pip install --no-cache-dir . + +# Collect all static files +RUN ./manage.py collectstatic --no-input + +CMD gunicorn code_review_backend.app.wsgi diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 000000000..df9ede4dd --- /dev/null +++ b/backend/README.md @@ -0,0 +1,35 @@ +# Code Review Backend + +## Developer setup + +``` +mkvirtualenv -p /usr/bin/python3 code-review-backend +cd backend +pip install -r requirements.txt +./manage migrate +./manage createsuperuser +./manage runserver +./manage loaddata fixtures/repositories.json +``` + +At this point, you can log into http://127.0.0.1:8000/admin/ with the credentials you mentioned during the `createsuperuser` step. + +## Load existing issues + +To load remote issues from production (default configuration): + +``` +./manage load_issues +``` + +To load already retrieved issues + +``` +./manage load_issues --offline +``` + +To load from testing + +``` +./manage load_issues --environment=testing +``` diff --git a/backend/VERSION b/backend/VERSION new file mode 100644 index 000000000..ee90284c2 --- /dev/null +++ b/backend/VERSION @@ -0,0 +1 @@ +1.0.4 diff --git a/backend/code_review_backend/__init__.py b/backend/code_review_backend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/app/__init__.py b/backend/code_review_backend/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/app/settings.py b/backend/code_review_backend/app/settings.py new file mode 100644 index 000000000..460f2f199 --- /dev/null +++ b/backend/code_review_backend/app/settings.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Django settings for backend project. + +Generated by 'django-admin startproject' using Django 2.2.6. + +For more information on this file, see +https://docs.djangoproject.com/en/2.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.2/ref/settings/ +""" + +import logging +import os + +import dj_database_url + +logger = logging.getLogger(__name__) + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ROOT_DIR = os.path.dirname(BASE_DIR) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "t!+s!@x5p!85x19q83jufr#95_z0fv7$!u5z*c&gi!%hr3^w+r" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "rest_framework", + "code_review_backend.issues", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "code_review_backend.app.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ] + }, + } +] + +WSGI_APPLICATION = "code_review_backend.app.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/2.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(ROOT_DIR, "db.sqlite3"), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" + }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, +] + + +# Internationalization +# https://docs.djangoproject.com/en/2.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# API configuration +REST_FRAMEWORK = { + # Use Django's standard `django.contrib.auth` permissions, + # or allow read-only access for unauthenticated users. + "DEFAULT_PERMISSION_CLASSES": [ + "rest_framework.permissions.IsAuthenticatedOrReadOnly" + ], + # Setup pagination + "PAGE_SIZE": 50, + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", +} + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.2/howto/static-files/ + +STATIC_URL = "/static/" + +# Static files are set in a dedicated path in Docker image +if "DJANGO_DOCKER" in os.environ: + STATIC_ROOT = "/static" + + # Enable GZip and cache, and build a manifest during collectstatic + STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" + + +# Internal logging setup +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": {"console": {"class": "logging.StreamHandler"}}, + "loggers": { + "django": {"handlers": ["console"], "level": "INFO"}, + "code_review_backend": {"handlers": ["console"], "level": "INFO"}, + }, +} + +# Heroku settings override to run the web app in production mode +if "DYNO" in os.environ: + logger.info("Setting up Heroku environment") + ALLOWED_HOSTS = ["*"] + DEBUG = os.environ.get("DEBUG", "false").lower() == "true" + + # Database setup + if "DATABASE_URL" in os.environ: + logger.info("Using remote database from $DATABASE_URL") + DATABASES["default"] = dj_database_url.parse( + os.environ["DATABASE_URL"], ssl_require=True + ) + else: + logger.info("DATABASE_URL not found, will use sqlite. Data may be lost.") + + # Insert Whitenoise Middleware after the security one + MIDDLEWARE.insert(1, "whitenoise.middleware.WhiteNoiseMiddleware") diff --git a/backend/code_review_backend/app/urls.py b/backend/code_review_backend/app/urls.py new file mode 100644 index 000000000..0e18f219f --- /dev/null +++ b/backend/code_review_backend/app/urls.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from django.contrib import admin +from django.shortcuts import redirect +from django.urls import include +from django.urls import path + +from code_review_backend.issues import api + +urlpatterns = [ + path("", lambda request: redirect("v1/", permanent=False)), + path("v1/", include(api.router.urls)), + path("admin/", admin.site.urls), +] diff --git a/backend/code_review_backend/app/wsgi.py b/backend/code_review_backend/app/wsgi.py new file mode 100644 index 000000000..ca3a145ea --- /dev/null +++ b/backend/code_review_backend/app/wsgi.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +WSGI config for backend project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "code_review_backend.app.settings") + +application = get_wsgi_application() diff --git a/backend/code_review_backend/issues/__init__.py b/backend/code_review_backend/issues/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/issues/admin.py b/backend/code_review_backend/issues/admin.py new file mode 100644 index 000000000..7ab73d94a --- /dev/null +++ b/backend/code_review_backend/issues/admin.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from django.contrib import admin + +from code_review_backend.issues.models import Diff +from code_review_backend.issues.models import Issue +from code_review_backend.issues.models import Repository +from code_review_backend.issues.models import Revision + + +class RepositoryAdmin(admin.ModelAdmin): + list_display = ("slug", "url") + + +class DiffInline(admin.TabularInline): + # Read only inline + model = Diff + readonly_fields = ("id", "mercurial", "phid", "review_task_id") + + +class RevisionAdmin(admin.ModelAdmin): + list_display = ("id", "title", "bugzilla_id", "repository") + list_filter = ("repository",) + inlines = (DiffInline,) + + +class IssueAdmin(admin.ModelAdmin): + list_filter = ("analyzer",) + list_display = ("id", "path", "line", "level", "analyzer", "check", "diff") + + +admin.site.register(Repository, RepositoryAdmin) +admin.site.register(Revision, RevisionAdmin) +admin.site.register(Issue, IssueAdmin) + +# Naming +admin.site.site_header = "Mozilla Code Review Backend" diff --git a/backend/code_review_backend/issues/api.py b/backend/code_review_backend/issues/api.py new file mode 100644 index 000000000..c514e40a6 --- /dev/null +++ b/backend/code_review_backend/issues/api.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from rest_framework import mixins +from rest_framework import routers +from rest_framework import viewsets + +from code_review_backend.issues.models import Diff +from code_review_backend.issues.models import Issue +from code_review_backend.issues.models import Revision +from code_review_backend.issues.serializers import DiffSerializer +from code_review_backend.issues.serializers import IssueSerializer +from code_review_backend.issues.serializers import RevisionSerializer + + +class CreateListRetrieveViewSet( + mixins.CreateModelMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """ + A viewset that allows creation, listing and retrieval of Model instances + From https://www.django-rest-framework.org/api-guide/viewsets/#custom-viewset-base-classes + """ + + +class RevisionViewSet(CreateListRetrieveViewSet): + queryset = Revision.objects.all() + serializer_class = RevisionSerializer + + +class DiffViewSet(CreateListRetrieveViewSet): + queryset = Diff.objects.all() + serializer_class = DiffSerializer + + +class IssueViewSet(CreateListRetrieveViewSet): + serializer_class = IssueSerializer + + def get_queryset(self): + return Issue.objects.filter(diff_id=self.kwargs["diff_id"]) + + def perform_create(self, serializer): + # Attach diff to issue created + serializer.save(diff=Diff.objects.get(pk=self.kwargs["diff_id"])) + + +router = routers.DefaultRouter() +router.register(r"revision", RevisionViewSet) +router.register(r"diff", DiffViewSet) +router.register(r"diff/(?P\d+)/issues", IssueViewSet, basename="issues") diff --git a/backend/code_review_backend/issues/apps.py b/backend/code_review_backend/issues/apps.py new file mode 100644 index 000000000..fe6b5d320 --- /dev/null +++ b/backend/code_review_backend/issues/apps.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from django.apps import AppConfig + + +class IssuesConfig(AppConfig): + name = "issues" diff --git a/backend/code_review_backend/issues/management/__init__.py b/backend/code_review_backend/issues/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/issues/management/commands/__init__.py b/backend/code_review_backend/issues/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/issues/management/commands/load_issues.py b/backend/code_review_backend/issues/management/commands/load_issues.py new file mode 100644 index 000000000..aae1943fb --- /dev/null +++ b/backend/code_review_backend/issues/management/commands/load_issues.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os +import tempfile + +import taskcluster +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError +from django.db import transaction + +from code_review_backend.issues.models import Issue +from code_review_backend.issues.models import Repository + +logger = logging.getLogger(__name__) + +INDEX_PATH = "project.relman.{environment}.code-review.phabricator.diff" + + +def positive_int(x): + """ + Some analyzer may give line & char positions below 0 to indicate a full file + We store it as null values + """ + return x if isinstance(x, int) and x >= 0 else None + + +class Command(BaseCommand): + help = "Load issues from remote taskcluster reports" + + def add_arguments(self, parser): + parser.add_argument( + "--offline", + action="store_true", + default=False, + help="Use only previously downloaded reports", + ) + parser.add_argument( + "-e", + "--environment", + default="production", + choices=("production", "testing"), + help="Specify the environment to load issues from", + ) + + def handle(self, *args, **options): + # Check repositories + for repo in ("mozilla-central", "nss"): + try: + Repository.objects.get(slug=repo) + except Repository.DoesNotExist: + raise CommandError(f"Missing repository {repo}") + + # Setup cache dir + self.cache_dir = os.path.join( + tempfile.gettempdir(), "code-review-reports", options["environment"] + ) + os.makedirs(self.cache_dir, exist_ok=True) + + # Load available tasks from Taskcluster or already downloaded + tasks = ( + self.load_local_reports() + if options["offline"] + else self.load_tasks(options["environment"]) + ) + + for task_id, report in tasks: + + # Build revision & diff + revision, diff = self.build_hierarchy(report["revision"], task_id) + + with transaction.atomic(): + # Remove all issues from diff + diff.issues.all().delete() + + # Build all issues for that diff, in a single DB call + issues = Issue.objects.bulk_create( + Issue( + diff=diff, + path=i["path"], + line=positive_int(i["line"]), + nb_lines=i.get("nb_lines", 1), + char=positive_int(i.get("char")), + level=i.get("level", "warning"), + check=i.get("kind") or i.get("check"), + message=i.get("message"), + analyzer=i["analyzer"], + hash=i["hash"], + ) + for i in report["issues"] + ) + logger.info(f"Imported task {task_id} - {len(issues)}") + + def load_tasks(self, environment, chunk=200): + # Direct unauthenticated usage + index = taskcluster.Index({"rootUrl": "https://taskcluster.net"}) + queue = taskcluster.Queue({"rootUrl": "https://taskcluster.net"}) + + token = None + while True: + + query = {"limit": chunk} + if token is not None: + query["continuationToken"] = token + data = index.listTasks( + INDEX_PATH.format(environment=environment), query=query + ) + + for task in data["tasks"]: + + if not task["data"].get("issues"): + continue + + # Lookup artifact in cache + path = os.path.join(self.cache_dir, task["taskId"]) + if os.path.exists(path): + artifact = json.load(open(path)) + + else: + + # Download the task report + logging.info(f"Download task {task['taskId']}") + try: + artifact = queue.getLatestArtifact( + task["taskId"], "public/results/report.json" + ) + except taskcluster.exceptions.TaskclusterRestFailure as e: + if e.status_code == 404: + logging.info(f"Missing artifact") + continue + raise + + # Check the artifact has repositories & revision + revision = artifact["revision"] + assert "repository" in revision, "Missing repository" + assert "target_repository" in revision, "Missing target_repository" + assert ( + "mercurial_revision" in revision + ), "Missing mercurial_revision" + + # Store artifact in cache + with open(path, "w") as f: + json.dump(artifact, f, sort_keys=True, indent=4) + + yield task["taskId"], artifact + + token = data.get("continuationToken") + if token is None: + break + + def load_local_reports(self): + for task_id in os.listdir(self.cache_dir): + report = json.load(open(os.path.join(self.cache_dir, task_id))) + yield task_id, report + + def build_hierarchy(self, data, task_id): + """Build or retrieve a revision and diff in current repo from report's data""" + repository = Repository.objects.get(slug=data["target_repository"]) + revision, _ = repository.revisions.get_or_create( + id=data["id"], + defaults={ + "phid": data["phid"], + "title": data["title"], + "bugzilla_id": int(data["bugzilla_id"]) + if data["bugzilla_id"] + else None, + }, + ) + diff, _ = revision.diffs.get_or_create( + id=data["diff_id"], + defaults={ + "phid": data["diff_phid"], + "review_task_id": task_id, + "mercurial": data["mercurial_revision"], + }, + ) + return revision, diff diff --git a/backend/code_review_backend/issues/migrations/0001_initial.py b/backend/code_review_backend/issues/migrations/0001_initial.py new file mode 100644 index 000000000..985c76cb4 --- /dev/null +++ b/backend/code_review_backend/issues/migrations/0001_initial.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Generated by Django 2.2.6 on 2019-10-17 15:23 + +import uuid + +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Diff", + fields=[ + ("id", models.PositiveIntegerField(primary_key=True, serialize=False)), + ("phid", models.CharField(max_length=40, unique=True)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ("review_task_id", models.CharField(max_length=30, unique=True)), + ("mercurial", models.CharField(max_length=40)), + ], + options={"ordering": ("id",), "abstract": False}, + ), + migrations.CreateModel( + name="Repository", + fields=[ + ("id", models.PositiveIntegerField(primary_key=True, serialize=False)), + ("phid", models.CharField(max_length=40, unique=True)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ("slug", models.SlugField(unique=True)), + ("url", models.URLField(unique=True)), + ("try_url", models.URLField(blank=True, null=True)), + ], + options={"verbose_name_plural": "repositories"}, + ), + migrations.CreateModel( + name="Revision", + fields=[ + ("id", models.PositiveIntegerField(primary_key=True, serialize=False)), + ("phid", models.CharField(max_length=40, unique=True)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ("title", models.CharField(max_length=250)), + ("bugzilla_id", models.PositiveIntegerField(null=True)), + ( + "repository", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="revisions", + to="issues.Repository", + ), + ), + ], + options={"ordering": ("id",), "abstract": False}, + ), + migrations.CreateModel( + name="Issue", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, primary_key=True, serialize=False + ), + ), + ("path", models.CharField(max_length=250)), + ("line", models.PositiveIntegerField(null=True)), + ("nb_lines", models.PositiveIntegerField(null=True)), + ("char", models.PositiveIntegerField(null=True)), + ( + "level", + models.CharField( + choices=[("warning", "Warning"), ("error", "Error")], + max_length=20, + ), + ), + ("check", models.CharField(max_length=250, null=True)), + ("message", models.TextField(null=True)), + ("analyzer", models.CharField(max_length=50)), + ("hash", models.CharField(max_length=32)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ( + "diff", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="issues", + to="issues.Diff", + ), + ), + ], + options={"ordering": ("diff", "path", "line", "analyzer")}, + ), + migrations.AddField( + model_name="diff", + name="revision", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="diffs", + to="issues.Revision", + ), + ), + ] diff --git a/backend/code_review_backend/issues/migrations/__init__.py b/backend/code_review_backend/issues/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/issues/models.py b/backend/code_review_backend/issues/models.py new file mode 100644 index 000000000..c88d7482c --- /dev/null +++ b/backend/code_review_backend/issues/models.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import uuid + +from django.db import models + +LEVEL_WARNING = "warning" +LEVEL_ERROR = "error" +ISSUE_LEVELS = ((LEVEL_WARNING, "Warning"), (LEVEL_ERROR, "Error")) + + +class PhabricatorModel(models.Model): + id = models.PositiveIntegerField(primary_key=True) + phid = models.CharField(max_length=40, unique=True) + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Meta: + abstract = True + ordering = ("id",) + + +class Repository(PhabricatorModel): + + slug = models.SlugField(unique=True) + url = models.URLField(unique=True) + try_url = models.URLField(null=True, blank=True) + + class Meta: + verbose_name_plural = "repositories" + + def __str__(self): + return self.slug + + +class Revision(PhabricatorModel): + repository = models.ForeignKey( + Repository, related_name="revisions", on_delete=models.CASCADE + ) + + title = models.CharField(max_length=250) + bugzilla_id = models.PositiveIntegerField(null=True) + + def __str__(self): + return f"D{self.id} - {self.title}" + + +class Diff(PhabricatorModel): + revision = models.ForeignKey( + Revision, related_name="diffs", on_delete=models.CASCADE + ) + + review_task_id = models.CharField(max_length=30, unique=True) + mercurial = models.CharField(max_length=40) + + def __str__(self): + return f"Diff {self.id}" + + +class Issue(models.Model): + """An issue detected on a Phabricator patch""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4) + diff = models.ForeignKey(Diff, related_name="issues", on_delete=models.CASCADE) + + # Raw issue data + path = models.CharField(max_length=250) + line = models.PositiveIntegerField(null=True) + nb_lines = models.PositiveIntegerField(null=True) + char = models.PositiveIntegerField(null=True) + level = models.CharField(max_length=20, choices=ISSUE_LEVELS) + check = models.CharField(max_length=250, null=True) + message = models.TextField(null=True) + analyzer = models.CharField(max_length=50) + + # Calculated hash identifying issue + hash = models.CharField(max_length=32) + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ("diff", "path", "line", "analyzer") diff --git a/backend/code_review_backend/issues/serializers.py b/backend/code_review_backend/issues/serializers.py new file mode 100644 index 000000000..b2c3a5554 --- /dev/null +++ b/backend/code_review_backend/issues/serializers.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from rest_framework import serializers + +from code_review_backend.issues.models import Diff +from code_review_backend.issues.models import Issue +from code_review_backend.issues.models import Repository +from code_review_backend.issues.models import Revision + + +class RevisionSerializer(serializers.HyperlinkedModelSerializer): + """ + Serialize a Revision in a Repository + """ + + repository = serializers.SlugRelatedField( + queryset=Repository.objects.all(), slug_field="slug" + ) + + class Meta: + model = Revision + fields = ("id", "repository", "phid", "title", "bugzilla_id") + + +class DiffSerializer(serializers.HyperlinkedModelSerializer): + """ + Serialize a Diff in a Revision + """ + + revision = serializers.PrimaryKeyRelatedField(queryset=Revision.objects.all()) + issues_url = serializers.HyperlinkedIdentityField( + view_name="issues-list", lookup_url_kwarg="diff_id" + ) + + class Meta: + model = Diff + fields = ("id", "revision", "phid", "review_task_id", "mercurial", "issues_url") + + +class IssueSerializer(serializers.HyperlinkedModelSerializer): + """ + Serialize an Issue in a Diff + """ + + class Meta: + model = Issue + fields = ( + "id", + "hash", + "analyzer", + "path", + "line", + "nb_lines", + "char", + "level", + "check", + "message", + ) diff --git a/backend/code_review_backend/issues/tests/__init__.py b/backend/code_review_backend/issues/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/issues/tests/test_api.py b/backend/code_review_backend/issues/tests/test_api.py new file mode 100644 index 000000000..776a3df49 --- /dev/null +++ b/backend/code_review_backend/issues/tests/test_api.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from django.contrib.auth.models import User +from rest_framework import status +from rest_framework.test import APITestCase + +from code_review_backend.issues.models import Diff +from code_review_backend.issues.models import Issue +from code_review_backend.issues.models import Repository +from code_review_backend.issues.models import Revision + + +class CreationAPITestCase(APITestCase): + def setUp(self): + # Create a user + self.user = User.objects.create(username="crash_user") + + # Create a repo + self.repo = Repository.objects.create( + id=1, phid="PHID-REPO-xxx", slug="myrepo", url="http://repo.test/myrepo" + ) + + def test_create_revision(self): + """ + Check we can create a revision through the API + """ + data = { + "id": 123, + "phid": "PHID-REV-xxx", + "title": "Bug XXX - Some bug", + "bugzilla_id": 123456, + "repository": "myrepo", + } + + # No auth will give a permission denied + response = self.client.post("/v1/revision/", data, format="json") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + # Once authenticated, creation will work + self.assertEqual(Revision.objects.count(), 0) + self.client.force_authenticate(user=self.user) + response = self.client.post("/v1/revision/", data, format="json") + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + # Check a revision has been created + self.assertEqual(Revision.objects.count(), 1) + revision = Revision.objects.get(pk=123) + self.assertEqual(revision.title, "Bug XXX - Some bug") + self.assertEqual(revision.bugzilla_id, 123456) + + def test_create_diff(self): + """ + Check we can create a diff through the API + """ + data = { + "id": 1234, + "revision": 123, + "phid": "PHID-DIFF-xxx", + "review_task_id": "deadbeef123", + "mercurial": "coffee12345", + } + + # No auth will give a permission denied + response = self.client.post("/v1/revision/", data, format="json") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + # Once authenticated, creation will require the revision to exist + self.assertEqual(Diff.objects.count(), 0) + self.client.force_authenticate(user=self.user) + response = self.client.post("/v1/diff/", data, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertDictEqual( + response.json(), {"revision": ['Invalid pk "123" - object does not exist.']} + ) + + # Create the requested revision + revision = Revision.objects.create( + id=123, + phid="PHID-REV-XXX", + repository=self.repo, + title="Bug XXX - Another bug", + bugzilla_id=123456, + ) + + # Now creation will work + self.assertEqual(Diff.objects.count(), 0) + self.client.force_authenticate(user=self.user) + response = self.client.post("/v1/diff/", data, format="json") + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + # Response should have url to create issues + self.assertEqual( + response.json()["issues_url"], "http://testserver/v1/diff/1234/issues/" + ) + + # Check a diff has been created + self.assertEqual(Diff.objects.count(), 1) + diff = Diff.objects.get(pk=1234) + self.assertEqual(diff.mercurial, "coffee12345") + self.assertEqual(diff.revision, revision) + + def test_create_issue(self): + """ + Check we can create a issue through the API + """ + # Create revision and diff + revision = self.repo.revisions.create( + id=456, + phid="PHID-REV-XXX", + title="Bug XXX - Yet Another bug", + bugzilla_id=78901, + ) + diff = revision.diffs.create( + id=1234, + phid="PHID-DIFF-xxx", + review_task_id="deadbeef123", + mercurial="coffee12345", + ) + + data = { + "hash": "somemd5hash", + "line": 1, + "analyzer": "remote-flake8", + "level": "error", + "path": "path/to/file.py", + } + + # No auth will give a permission denied + response = self.client.post("/v1/diff/1234/issues/", data, format="json") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + # Once authenticated, creation will work + self.assertEqual(Issue.objects.count(), 0) + self.client.force_authenticate(user=self.user) + response = self.client.post("/v1/diff/1234/issues/", data, format="json") + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + # Check a revision has been created + self.assertEqual(Diff.objects.count(), 1) + issue = Issue.objects.first() + self.assertEqual(issue.path, "path/to/file.py") + self.assertEqual(issue.line, 1) + self.assertEqual(issue.diff, diff) + self.assertEqual(issue.diff.revision, revision) diff --git a/backend/fixtures/repositories.json b/backend/fixtures/repositories.json new file mode 100644 index 000000000..5525b9a6a --- /dev/null +++ b/backend/fixtures/repositories.json @@ -0,0 +1,26 @@ +[ +{ + "model": "issues.repository", + "pk": 1, + "fields": { + "phid": "PHID-REPO-saax4qdxlbbhahhp2kg5", + "created": "2019-10-17T07:17:06.396Z", + "updated": "2019-10-17T07:17:06.396Z", + "url": "https://hg.mozilla.org/mozilla-central", + "try_url": "https://hg.mozilla.org/try", + "slug": "mozilla-central" + } +}, +{ + "model": "issues.repository", + "pk": 8, + "fields": { + "phid": "PHID-REPO-3lrloqw4qf6fluy2a5ni", + "created": "2019-10-17T07:17:32.970Z", + "updated": "2019-10-17T07:17:32.970Z", + "url": "https://hg.mozilla.org/projects/nss", + "try_url": "https://hg.mozilla.org/projects/nss-try", + "slug": "nss" + } +} +] diff --git a/backend/manage.py b/backend/manage.py new file mode 100755 index 000000000..fac678717 --- /dev/null +++ b/backend/manage.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "code_review_backend.app.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt new file mode 100644 index 000000000..f555665e4 --- /dev/null +++ b/backend/requirements-dev.txt @@ -0,0 +1 @@ +pre-commit==1.18.3 diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 000000000..0222ff9f1 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,10 @@ +Django==2.2.6 +dj-database-url==0.5.0 +djangorestframework==3.10.3 +gunicorn==19.9.0 +psycopg2-binary==2.8.3 +pytz==2019.3 +sqlparse==0.3.0 +taskcluster==19.0.0 +taskcluster-urls==11.0.0 +whitenoise==4.1.4 diff --git a/backend/setup.py b/backend/setup.py new file mode 100644 index 000000000..1dc11cae2 --- /dev/null +++ b/backend/setup.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import setuptools + + +def read_requirements(file_): + lines = [] + with open(file_) as f: + for line in f.readlines(): + line = line.strip() + if ( + line.startswith("-e ") + or line.startswith("http://") + or line.startswith("https://") + ): + extras = "" + if "[" in line: + extras = "[" + line.split("[")[1].split("]")[0] + "]" + line = line.split("#")[1].split("egg=")[1] + extras + elif line == "" or line.startswith("#") or line.startswith("-"): + continue + line = line.split("#")[0].strip() + lines.append(line) + return sorted(list(set(lines))) + + +with open("VERSION") as f: + VERSION = f.read().strip() + + +setuptools.setup( + name="code_review_backend", + version=VERSION, + description="Store and compare issues found in Mozilla code review tasks", + author="Mozilla Release Management", + author_email="release-mgmt-analysis@mozilla.com", + url="https://github.com/mozilla/code-review", + tests_require=read_requirements("requirements-dev.txt"), + install_requires=read_requirements("requirements.txt"), + packages=setuptools.find_packages(), + include_package_data=True, + zip_safe=False, + license="MPL2", +)