adding example upload with ddash results and log files

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.gitignore

@@ -16,3 +16,4 @@ db.sqlite3
 test-db.sqlite
 dyninst_dashboard.egg-info/
 migrations
+./data

ddash/apps/api/permissions.py

@@ -0,0 +1,57 @@
+from rest_framework.permissions import BasePermission, SAFE_METHODS
+from rest_framework.authtoken.models import Token
+
+
+class AllowAnyGet(BasePermission):
+    """Allows an anonymous user access for GET requests only."""
+
+    def has_permission(self, request, view):
+
+        if request.user.is_anonymous and request.method == "GET":
+            return (True,)
+
+        if request.user.is_staff or request.user.is_superuser:
+            return True
+
+        return request.method in SAFE_METHODS
+
+
+def check_user_authentication(request):
+    """Given a request, check that the user is authenticated via a token in
+    the header.
+    """
+    token = get_token(request)
+
+    # Case 1: no token and auth is required, prompt the user for it
+    if not token:
+        return None, 401
+
+    # Case 2: the token is not associated with a user
+    try:
+        token = Token.objects.get(key=token)
+    except:
+        return None, 403
+
+    return token.user, 200
+
+
+def get_token(request):
+    """The same as validate_token, but return the token object to check the
+    associated user.
+    """
+    # Coming from HTTP, look for authorization as bearer token
+    token = request.META.get("HTTP_AUTHORIZATION")
+
+    if token:
+        token = token.split(" ")[-1].strip()
+        try:
+            return Token.objects.get(key=token)
+        except Token.DoesNotExist:
+            pass
+
+    # Next attempt - try to get token via user session
+    elif request.user.is_authenticated and not request.user.is_anonymous:
+        try:
+            return Token.objects.get(user=request.user)
+        except Token.DoesNotExist:
+            pass

ddash/apps/api/urls.py

@@ -38,6 +38,7 @@
         api_views.ServiceInfo.as_view(),
         name="service_info",
     ),
+    path("results/log/", api_views.upload_log),
     path(
         "results/new/",
         api_views.NewTestResult.as_view(),

ddash/apps/api/views/__init__.py

@@ -1,4 +1,4 @@
 from .auth import GetAuthToken
 from .base import ServiceInfo
-from .results import NewTestResult
+from .results import NewTestResult, upload_log
 from .tables import ResultsTable

ddash/apps/api/views/results.py

@@ -1,14 +1,20 @@
 from django.conf import settings
 from django.db.models import F
-from django.shortcuts import get_object_or_404
+from django.shortcuts import get_object_or_404, render
 from django.urls import reverse
 
 from ratelimit.decorators import ratelimit
 from django.views.decorators.cache import never_cache
 from django.utils.decorators import method_decorator
+from django.views.decorators.csrf import csrf_exempt
+from django.http import JsonResponse
 
 from ddash.settings import cfg
-from ddash.apps.main.tasks import import_result
+from ddash.apps.main.tasks import (
+    import_result,
+    import_test_log,
+    import_build_log,
+)
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
@@ -17,6 +23,41 @@
 import json
 
 
+@csrf_exempt
+def upload_log(request):
+
+    # Acceptable log types
+    log_types = ["test_log", "dyninst_build_log", "test_build_log"]
+
+    # Both logs will be post with a log file
+    if request.method == "POST" and request.FILES.get("logfile"):
+        log_type = request.POST.get("log_type")
+        run_id = request.POST.get("run_id")
+        log_file = request.FILES["logfile"]
+
+        # Both file_type and run_id are required
+        if not log_type or not run_id:
+            return JsonResponse(
+                status=400, data={"message": "log_type and run_id are required"}
+            )
+
+        # We must recognize the log type!
+        if log_type not in log_types:
+            return JsonResponse(
+                status=400,
+                data={"message": "log_type must be one of %s" % ",".join(log_types)},
+            )
+
+        if log_type == "test_log":
+            data = import_test_log(run_id, log_file)
+        if log_type == "dyninst_build_log":
+            data = import_build_log(run_id, log_file, "dyninst")
+        if log_type == "test_build_log":
+            data = import_build_log(run_id, log_file, "testsuite")
+        return JsonResponse(status=data["code"], data=data)
+    return JsonResponse(status=400, data={"message": "Malformed request."})
+
+
 class NewTestResult(APIView):
     permission_classes = []
     allowed_methods = ("POST",)
@@ -39,7 +80,6 @@ def post(self, request, *args, **kwargs):
 
         # Generate the config
         data = json.loads(request.body)
-        print(data)
 
         # We require the spack version and the spec
         result = import_result(data.get("result"))

ddash/apps/main/management/commands/import_db.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand, CommandError
 from django.contrib.auth import authenticate
 from ddash.apps.main.models import *
-from datetime import datetime
+from datetime import datetime, timedelta
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import make_aware
 import sqlite3
@@ -111,8 +111,8 @@ def handle(self, *args, **options):
         compiler_id_lookup = add_compilers(conn)
         test_run_lookup, test_result_lookup = add_runs(conn)
 
-        # Finally, add regressions
-        add_regressions(conn, test_run_lookup)
+        # Finally, add regressions (skipping we will calculate again)
+        # add_regressions(conn, test_run_lookup)
 
 
 def add_regressions(conn, test_run_lookup):
@@ -133,9 +133,11 @@ def add_regressions(conn, test_run_lookup):
             next_run = TestRun.objects.get(id=test_run_lookup[reg[idx["next_run"]]])
         except:
             next_run = None
-        regression, _ = Regressions.objects.get_or_create(
-            previous_run=previous_run, next_run=next_run, count=reg[idx["count"]]
-        )
+
+        if previous_run and next_run:
+            regression, _ = Regressions.objects.get_or_create(
+                previous_run=previous_run, next_run=next_run, count=reg[idx["count"]]
+            )
 
 
 def add_runs(conn):
@@ -158,20 +160,39 @@ def add_runs(conn):
     test_result_lookup = {}
     total_runs = len(runs)
 
+    # Keep track of repeated run count
+    repeated_run_count = 0
+    too_old_count = 0
+    imported_runs = 0
+    bad_test_mode_count = 0
+
+    # Don't add if older than 6 months ago
+    six_months_ago = make_aware(datetime.today() - timedelta(days=6 * 365 / 12))
+
     for i, run in enumerate(runs):
 
+        # Datetime converted from string
+        date_run = make_aware(parse_datetime(run[idx["run_date"]]))
+
+        # If it's older than 6 months, do not continue
+        if date_run < six_months_ago:
+            too_old_count += 1
+            continue
+
         print("Adding run %s of %s" % (i + 1, total_runs))
 
         # Derive the PR id either from dyninst or testsuite (I checked, it's either OR)
         if "PR" in run[idx["dyninst_branch"]]:
-            pr_id = re.sub("(PR|pr|dev_)", "", run[idx["dyninst_branch"]])
+            pr_id = re.sub("(PR|pr|dev_|prdev_|PRdev_)", "", run[idx["dyninst_branch"]])
             pr, _ = PullRequest.objects.get_or_create(
                 url="https://github.com/dyninst/dyninst/pull/%s" % pr_id,
                 user="unknown",
                 pr_id=int(pr_id),
             )
         elif "PR" in run[idx["testsuite_branch"]]:
-            pr_id = re.sub("(PR|pr|_dev)", "", run[idx["testsuite_branch"]])
+            pr_id = re.sub(
+                "(PR|pr|dev_|prdev_|PRdev_)", "", run[idx["testsuite_branch"]]
+            )
             pr, _ = PullRequest.objects.get_or_create(
                 url="https://github.com/dyninst/testsuite/pull/%s" % pr_id,
                 user="unknown",
@@ -247,31 +268,39 @@ def add_runs(conn):
             name=run[idx["compiler_name"]], version="Unknown"
         )
 
-        # Datetime converted from string
-        date_run = make_aware(parse_datetime(run[idx["run_date"]]))
-
         # Now create the testrun and keep track of the lookup
         # NOTE: we don't have any commands here!
-        test_run, _ = TestRun.objects.get_or_create(
-            date_run=date_run,
-            dyninst=dyninst_repo,
-            testsuite=testsuite_repo,
-            environment=env,
-            pull_request=pr,
-            cirun_url="unknown",
-            compiler=compiler,
-            result=test_run_result,
-        )
-        test_run_lookup[run[idx["id"]]] = test_run.id
+        # 656
+        try:
+            test_run, _ = TestRun.objects.get_or_create(
+                date_run=date_run,
+                dyninst=dyninst_repo,
+                testsuite=testsuite_repo,
+                environment=env,
+                pull_request=pr,
+                cirun_url="unknown",
+                compiler=compiler,
+                result=test_run_result,
+            )
+            test_run_lookup[run[idx["id"]]] = test_run.id
+        except:
+            repeated_run_count += 1
+            continue
+
+        imported_runs += 1
 
         # Query for the test results
         test_results = conn.run_query(
             "SELECT * FROM test_result WHERE runid=%s;" % run[idx["id"]]
         )
         for result in test_results:
 
-            # Get the test mode
-            test_mode = TestMode.objects.get(name=result[ridx["mode"]])
+            # Get the test mode (found multiple empty string '')
+            try:
+                test_mode = TestMode.objects.get(name=result[ridx["mode"]])
+            except:
+                bad_test_mode_count += 1
+                test_mode = None
 
             # prepare booleans
             pic = False if result[ridx["pic"]] == "nonPIC" else True
@@ -297,4 +326,10 @@ def add_runs(conn):
             )
             test_result_lookup[result[ridx["resultid"]]] = test_result.id
 
+    print("RESULTS =========")
+    print("         total runs: %s" % total_runs)
+    print("         imported runs: %s" % imported_runs)
+    print("   import run error: %s" % repeated_run_count)
+    print("older than 6 months: %s" % too_old_count)
+    print("      bad test mode: %s" % bad_test_mode_count)
     return test_run_lookup, test_result_lookup

ddash/apps/main/models.py

@@ -263,7 +263,7 @@ class TestResult(models.Model):
         "main.Compiler", null=False, blank=False, on_delete=models.CASCADE
     )
     test_mode = models.ForeignKey(
-        "main.TestMode", null=False, blank=False, on_delete=models.CASCADE
+        "main.TestMode", null=True, blank=True, on_delete=models.CASCADE
     )
 
     isPIC = models.BooleanField(default=False)

ddash/apps/main/storage.py

@@ -0,0 +1,11 @@
+from django.core.files.storage import FileSystemStorage
+from django.conf import settings
+import os
+
+
+class OverwriteStorage(FileSystemStorage):
+    def get_available_name(self, name, max_length=None):
+        # If the filename already exists, remove it as if it was a true file system
+        if self.exists(name):
+            os.remove(os.path.join(settings.MEDIA_ROOT, name))
+        return name

ddash/apps/main/tasks.py

@@ -1,3 +1,6 @@
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import make_aware
+
 from ddash.apps.main.models import (
     TestRun,
     TestResult,
@@ -202,14 +205,15 @@ def import_result(data):
 
     # Create a new test run
     test_run = TestRun(
-        date_run=data["date_run"],
+        date_run=make_aware(parse_datetime(data["date_run"])),
         dyninst=repos.get("dyninst"),
         testsuite=repos.get("testsuite"),
         environment=environ,
         pull_request=pr,
         cirun_url=data.get("cirun_url"),
         compiler=compiler,
         result=result,
+        command=data.get("command"),
     )
     test_run.save()
 
@@ -218,3 +222,50 @@ def import_result(data):
 
     data = {"test_run": test_run.id}
     return {"message": "success", "data": data, "code": 201}
+
+
+def import_build_log(run_id, request_file, result_type):
+    if result_type not in ["dyninst", "testsuite"]:
+        return {
+            "code": 400,
+            "message": "Test build log must be of type dyninst or testsuite",
+        }
+    try:
+        test_run = TestRun.objects.get(id=run_id)
+    except TestRun.DoesNotExist:
+        return {"code": 404, "message": "TestRun with ID %s does not exist." % run_id}
+
+    if result_type == "dyninst":
+        test_run.result.dyninst_build.log.save(
+            "dyninst-build-" + request_file.name, request_file
+        )
+        test_run.result.dyninst_build.save()
+        test_run.result.save()
+        test_run.save()
+        return {"code": 201, "message": "TestBuildResult log for dyninst was saved."}
+
+    test_run.result.testsuite_build.log.save(
+        "testsuite-build-" + request_file.name, request_file
+    )
+    test_run.result.testsuite_build.save()
+    test_run.result.save()
+    test_run.save()
+    return {"code": 201, "message": "TestBuildResult log for the testsuite was saved."}
+
+
+def import_test_log(run_id, request_file):
+    try:
+        test_run = TestRun.objects.get(id=run_id)
+    except TestRun.DoesNotExist:
+        return {"code": 404, "message": "TestRun with ID %s does not exist." % run_id}
+
+    # This should not happen
+    if not test_run.result:
+        return {
+            "code": 404,
+            "message": "TestRun with ID %s does not have a TestRunResult." % run_id,
+        }
+    test_run.result.test_log.save("test-log-" + request_file.name, request_file)
+    test_run.result.save()
+    test_run.save()
+    return {"code": 201, "message": "Test log was added to TestRun result."}