Store Revision, Diff and related Issues in the backend

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
 known_first_party = code_review_backend,code_review_bot,code_review_tools,code_review_events,conftest
-known_third_party = dj_database_url,django,influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,setuptools,structlog,taskcluster,toml
+known_third_party = dj_database_url,django,influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,rest_framework,setuptools,structlog,taskcluster,toml
 force_single_line = True
 default_section=FIRSTPARTY
 line_length=159

backend/README.md

@@ -8,7 +8,28 @@ cd backend
 pip install -r requirements.txt
 ./manage.py migrate
 ./manage.py createsuperuser
+./manage.py loaddata fixtures/repositories.json
 ./manage.py runserver
 ```
 
 At this point, you can log into http://127.0.0.1:8000/admin/ with the credentials you mentioned during the `createsuperuser` step.
+
+## Load existing issues
+
+To load remote issues from production (default configuration):
+
+```
+./manage.py load_issues
+```
+
+To load already retrieved issues
+
+```
+./manage.py load_issues --offline
+```
+
+To load from testing
+
+```
+./manage.py load_issues --environment=testing
+```

backend/code_review_backend/app/settings.py

@@ -50,6 +50,7 @@
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "rest_framework",
+    "code_review_backend.issues",
 ]
 
 MIDDLEWARE = [

backend/code_review_backend/app/urls.py

@@ -5,9 +5,13 @@
 
 from django.contrib import admin
 from django.shortcuts import redirect
+from django.urls import include
 from django.urls import path
 
+from code_review_backend.issues import api
+
 urlpatterns = [
-    path("", lambda request: redirect("admin/", permanent=False)),
+    path("", lambda request: redirect("v1/", permanent=False)),
+    path("v1/", include(api.router.urls)),
     path("admin/", admin.site.urls),
 ]

backend/code_review_backend/issues/admin.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from django.contrib import admin
+
+from code_review_backend.issues.models import Diff
+from code_review_backend.issues.models import Issue
+from code_review_backend.issues.models import Repository
+from code_review_backend.issues.models import Revision
+
+
+class RepositoryAdmin(admin.ModelAdmin):
+    list_display = ("slug", "url")
+
+
+class DiffInline(admin.TabularInline):
+    # Read only inline
+    model = Diff
+    readonly_fields = ("id", "mercurial_hash", "phid", "review_task_id")
+
+
+class RevisionAdmin(admin.ModelAdmin):
+    list_display = ("id", "title", "bugzilla_id", "repository")
+    list_filter = ("repository",)
+    inlines = (DiffInline,)
+
+
+class IssueAdmin(admin.ModelAdmin):
+    list_filter = ("analyzer",)
+    list_display = ("id", "path", "line", "level", "analyzer", "check", "diff")
+
+
+admin.site.register(Repository, RepositoryAdmin)
+admin.site.register(Revision, RevisionAdmin)
+admin.site.register(Issue, IssueAdmin)
+
+# Naming
+admin.site.site_header = "Mozilla Code Review Backend"

backend/code_review_backend/issues/api.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from rest_framework import mixins
+from rest_framework import routers
+from rest_framework import viewsets
+
+from code_review_backend.issues.models import Diff
+from code_review_backend.issues.models import Issue
+from code_review_backend.issues.models import Revision
+from code_review_backend.issues.serializers import DiffSerializer
+from code_review_backend.issues.serializers import IssueSerializer
+from code_review_backend.issues.serializers import RevisionSerializer
+
+
+class CreateListRetrieveViewSet(
+    mixins.CreateModelMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    viewsets.GenericViewSet,
+):
+    """
+    A viewset that allows creation, listing and retrieval of Model instances
+    From https://www.django-rest-framework.org/api-guide/viewsets/#custom-viewset-base-classes
+    """
+
+
+class RevisionViewSet(CreateListRetrieveViewSet):
+    queryset = Revision.objects.all()
+    serializer_class = RevisionSerializer
+
+
+class DiffViewSet(CreateListRetrieveViewSet):
+    queryset = Diff.objects.all()
+    serializer_class = DiffSerializer
+
+
+class IssueViewSet(CreateListRetrieveViewSet):
+    serializer_class = IssueSerializer
+
+    def get_queryset(self):
+        return Issue.objects.filter(diff_id=self.kwargs["diff_id"])
+
+    def perform_create(self, serializer):
+        # Attach diff to issue created
+        serializer.save(diff=Diff.objects.get(pk=self.kwargs["diff_id"]))
+
+
+router = routers.DefaultRouter()
+router.register(r"revision", RevisionViewSet)
+router.register(r"diff", DiffViewSet)
+router.register(r"diff/(?P<diff_id>\d+)/issues", IssueViewSet, basename="issues")

backend/code_review_backend/issues/apps.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from django.apps import AppConfig
+
+
+class IssuesConfig(AppConfig):
+    name = "issues"

backend/code_review_backend/issues/management/commands/load_issues.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import logging
+import os
+import tempfile
+
+import taskcluster
+from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
+from django.db import transaction
+
+from code_review_backend.issues.models import Issue
+from code_review_backend.issues.models import Repository
+
+logger = logging.getLogger(__name__)
+
+INDEX_PATH = "project.relman.{environment}.code-review.phabricator.diff"
+
+
+class Command(BaseCommand):
+    help = "Load issues from remote taskcluster reports"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--offline",
+            action="store_true",
+            default=False,
+            help="Use only previously downloaded reports",
+        )
+        parser.add_argument(
+            "-e",
+            "--environment",
+            default="production",
+            choices=("production", "testing"),
+            help="Specify the environment to load issues from",
+        )
+
+    def handle(self, *args, **options):
+        # Check repositories
+        for repo in ("mozilla-central", "nss"):
+            try:
+                Repository.objects.get(slug=repo)
+            except Repository.DoesNotExist:
+                raise CommandError(f"Missing repository {repo}")
+
+        # Setup cache dir
+        self.cache_dir = os.path.join(
+            tempfile.gettempdir(), "code-review-reports", options["environment"]
+        )
+        os.makedirs(self.cache_dir, exist_ok=True)
+
+        # Load available tasks from Taskcluster or already downloaded
+        tasks = (
+            self.load_local_reports()
+            if options["offline"]
+            else self.load_tasks(options["environment"])
+        )
+
+        for task_id, report in tasks:
+
+            # Build revision & diff
+            revision, diff = self.build_revision_and_diff(report["revision"], task_id)
+
+            # Save all issues in a single db transaction
+            try:
+                issues = self.save_issues(diff, report["issues"])
+                logger.info(f"Imported task {task_id} - {len(issues)}")
+            except Exception as e:
+                logger.error(f"Failed to save issues for {task_id}: {e}")
+
+    @transaction.atomic
+    def save_issues(self, diff, issues):
+        # Remove all issues from diff
+        diff.issues.all().delete()
+
+        # Build all issues for that diff, in a single DB call
+        return Issue.objects.bulk_create(
+            Issue(
+                diff=diff,
+                path=i["path"],
+                line=i["line"],
+                nb_lines=i.get("nb_lines", 1),
+                char=i.get("char"),
+                level=i.get("level", "warning"),
+                check=i.get("kind") or i.get("check"),
+                message=i.get("message"),
+                analyzer=i["analyzer"],
+                hash=i["hash"],
+            )
+            for i in issues
+        )
+
+    def load_tasks(self, environment, chunk=200):
+        # Direct unauthenticated usage
+        index = taskcluster.Index({"rootUrl": "https://taskcluster.net"})
+        queue = taskcluster.Queue({"rootUrl": "https://taskcluster.net"})
+
+        token = None
+        while True:
+
+            query = {"limit": chunk}
+            if token is not None:
+                query["continuationToken"] = token
+            data = index.listTasks(
+                INDEX_PATH.format(environment=environment), query=query
+            )
+
+            for task in data["tasks"]:
+
+                if not task["data"].get("issues"):
+                    continue
+
+                # Lookup artifact in cache
+                path = os.path.join(self.cache_dir, task["taskId"])
+                if os.path.exists(path):
+                    artifact = json.load(open(path))
+
+                else:
+
+                    # Download the task report
+                    logging.info(f"Download task {task['taskId']}")
+                    try:
+                        artifact = queue.getLatestArtifact(
+                            task["taskId"], "public/results/report.json"
+                        )
+                    except taskcluster.exceptions.TaskclusterRestFailure as e:
+                        if e.status_code == 404:
+                            logging.info(f"Missing artifact")
+                            continue
+                        raise
+
+                    # Check the artifact has repositories & revision
+                    revision = artifact["revision"]
+                    assert "repository" in revision, "Missing repository"
+                    assert "target_repository" in revision, "Missing target_repository"
+                    assert (
+                        "mercurial_revision" in revision
+                    ), "Missing mercurial_revision"
+
+                    # Store artifact in cache
+                    with open(path, "w") as f:
+                        json.dump(artifact, f, sort_keys=True, indent=4)
+
+                yield task["taskId"], artifact
+
+            token = data.get("continuationToken")
+            if token is None:
+                break
+
+    def load_local_reports(self):
+        for task_id in os.listdir(self.cache_dir):
+            report = json.load(open(os.path.join(self.cache_dir, task_id)))
+            yield task_id, report
+
+    def build_revision_and_diff(self, data, task_id):
+        """Build or retrieve a revision and diff in current repo from report's data"""
+        repository = Repository.objects.get(slug=data["target_repository"])
+        revision, _ = repository.revisions.get_or_create(
+            id=data["id"],
+            defaults={
+                "phid": data["phid"],
+                "title": data["title"],
+                "bugzilla_id": int(data["bugzilla_id"])
+                if data["bugzilla_id"]
+                else None,
+            },
+        )
+        diff, _ = revision.diffs.get_or_create(
+            id=data["diff_id"],
+            defaults={
+                "phid": data["diff_phid"],
+                "review_task_id": task_id,
+                "mercurial_hash": data["mercurial_revision"],
+            },
+        )
+        return revision, diff