From 029f285fdee5f7921e7d11ca0af26a68a218a083 Mon Sep 17 00:00:00 2001 From: Miquel Torres Date: Mon, 5 Apr 2010 00:13:00 +0200 Subject: [PATCH] BIG DB Changes. New Project and Commitlog Models. Revision now has a commitid charfield. --- speedcenter/codespeed/admin.py | 14 +- speedcenter/codespeed/models.py | 35 +++- speedcenter/codespeed/tests.py | 37 ++-- speedcenter/codespeed/views.py | 179 +++++++++++------- speedcenter/templates/codespeed/overview.html | 13 +- speedcenter/templates/codespeed/timeline.html | 3 +- tools/import_from_json.py | 77 ++------ tools/save_single_result.py | 16 +- tools/saveresults.py | 16 +- tools/test_saveresults.py | 8 +- 10 files changed, 211 insertions(+), 187 deletions(-) diff --git a/speedcenter/codespeed/admin.py b/speedcenter/codespeed/admin.py index bc72a66e..b92bfcd6 100644 --- a/speedcenter/codespeed/admin.py +++ b/speedcenter/codespeed/admin.py @@ -1,9 +1,19 @@ # -*- coding: utf-8 -*- -from codespeed.models import Revision, Interpreter, Benchmark, Result, Environment +from codespeed.models import Project, Revision, Commitlog, Interpreter, Benchmark, Result, Environment from django.contrib import admin +class ProjectAdmin(admin.ModelAdmin): + list_display = ('name', 'rcType', 'rcURL', 'isdefault') + +admin.site.register(Project, ProjectAdmin) + +class CommitlogAdmin(admin.ModelAdmin): + list_display = ('revision', 'commitid', 'author', 'date', 'message') + +admin.site.register(Commitlog, CommitlogAdmin) + class RevisionAdmin(admin.ModelAdmin): - list_display = ('number', 'project', 'branch', 'tag', 'date') + list_display = ('commitid', 'project', 'branch', 'tag', 'date') list_filter = ('project', 'tag') admin.site.register(Revision, RevisionAdmin) diff --git a/speedcenter/codespeed/models.py b/speedcenter/codespeed/models.py index 96553738..7aa3667f 100644 --- a/speedcenter/codespeed/models.py +++ b/speedcenter/codespeed/models.py @@ -1,17 +1,38 @@ # -*- coding: utf-8 -*- from django.db import models +class Project(models.Model): + RC_TYPES = ( + ('N', 'none'), + ('G', 'git'), + ('S', 'svn'), + ) + def __unicode__(self): + return str(self.name) + name = models.CharField(unique=True, max_length=20) + rcType = models.CharField(max_length=1, choices=RC_TYPES, default='N') + rcURL = models.URLField(blank=True, null=True) + isdefault = models.BooleanField(default=False) + + class Revision(models.Model): def __unicode__(self): - return str(self.number) - number = models.IntegerField() - project = models.CharField(max_length=20) + return str(self.commitid) + commitid = models.CharField(max_length=40)#git's SHA-1 length is 40 + project = models.ForeignKey(Project) branch = models.CharField(max_length=20, default='trunk') tag = models.CharField(max_length=20, blank=True) - message = models.CharField(max_length=100, blank=True) - date = models.DateTimeField(blank=True, null=True) + date = models.DateTimeField(null=True) class Meta: - unique_together = ("number", "project", "branch") + unique_together = ("commitid", "branch", "project") + + +class Commitlog(models.Model): + commitid = models.CharField(max_length=40)#git's SHA-1 length is 40 + author = models.CharField(max_length=20) + date = models.DateTimeField() + message = models.CharField(max_length=200, blank=True) + revision = models.ForeignKey(Revision) class Interpreter(models.Model): @@ -19,6 +40,7 @@ def __unicode__(self): return str(self.name + " " + str(self.coptions)) name = models.CharField(max_length=50) coptions = models.CharField("compile options", max_length=100) + project = models.ForeignKey(Project) class Meta: unique_together = ("name", "coptions") @@ -61,4 +83,3 @@ class Result(models.Model): class Meta: unique_together = ("revision", "interpreter", "benchmark", "environment") - \ No newline at end of file diff --git a/speedcenter/codespeed/tests.py b/speedcenter/codespeed/tests.py index b4704236..d386d2be 100644 --- a/speedcenter/codespeed/tests.py +++ b/speedcenter/codespeed/tests.py @@ -2,7 +2,7 @@ from django.test import TestCase from datetime import datetime from django.test.client import Client -from codespeed.models import Benchmark, Revision, Interpreter, Environment, Result +from codespeed.models import Project, Benchmark, Revision, Interpreter, Environment, Result from django.core.urlresolvers import reverse class AddResultTest(TestCase): @@ -13,12 +13,12 @@ def setUp(self): self.e.save() self.cdate = datetime.today() self.data = { - 'revision_number': '23232', - 'revision_project': 'pypy', - 'revision_branch': 'trunk', + 'commitid': '23232', + 'project': 'pypy', + 'branch': 'trunk', 'interpreter_name': 'pypy-c', 'interpreter_coptions': 'gc=Böhm', - 'benchmark_name': 'Richards', + 'benchmark': 'Richards', 'environment': 'bigdog', 'result_value': 456, 'result_date': self.cdate, @@ -35,23 +35,25 @@ def test_add_default_result(self): self.assertEquals(b.benchmark_type, "C") self.assertEquals(b.units, "seconds") self.assertEquals(b.lessisbetter, True) - r = Revision.objects.get(number='23232', project='pypy', branch="trunk") + p = Project.objects.get(name='pypy') + r = Revision.objects.get(commitid='23232', project=p, branch="trunk") + self.assertEquals(r.date, self.cdate) i = Interpreter.objects.get(name='pypy-c', coptions='gc=Böhm') - self.assertTrue(Result.objects.get( - value=456, - date=self.cdate, + res = Result.objects.get( revision=r, interpreter=i, benchmark=b, environment=e - )) + ) + self.assertTrue(res.value, 456) + self.assertTrue(res.date, self.cdate) def test_add_non_default_result(self): """ Add result data with non-default options """ modified_data = self.data - modified_data['revision_branch'] = "experimental" + modified_data['branch'] = "experimental" modified_data['benchmark_type'] = "O" modified_data['units'] = "fps" modified_data['lessisbetter'] = False @@ -61,18 +63,13 @@ def test_add_non_default_result(self): response = self.client.post(self.path, modified_data) self.assertEquals(response.status_code, 200) self.assertEquals(response.content, "Result data saved succesfully") - e = Environment.objects.get(name='bigdog') - b = Benchmark.objects.get(name='Richards') - self.assertEquals(b.benchmark_type, "O") - self.assertEquals(b.units, "fps") - self.assertEquals(b.lessisbetter, False) - r = Revision.objects.get(number='23232', project='pypy', branch='experimental') + e = Environment.objects.get(name='bigdog') + p = Project.objects.get(name='pypy') + r = Revision.objects.get(commitid='23232', project=p, branch='experimental') self.assertEquals(r.branch, "experimental") i = Interpreter.objects.get(name='pypy-c', coptions='gc=Böhm') - + b = Benchmark.objects.get(name='Richards') res = Result.objects.get( - value=456, - date=self.cdate, revision=r, interpreter=i, benchmark=b, diff --git a/speedcenter/codespeed/views.py b/speedcenter/codespeed/views.py index 7e213672..7148fe40 100644 --- a/speedcenter/codespeed/views.py +++ b/speedcenter/codespeed/views.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- from django.shortcuts import get_object_or_404, render_to_response -from codespeed.models import Revision, Result, Interpreter, Benchmark, Environment +from codespeed.models import Project, Revision, Result, Interpreter, Benchmark, Environment from django.http import HttpResponse, Http404, HttpResponseNotAllowed, HttpResponseBadRequest, HttpResponseNotFound from codespeed import settings +from datetime import datetime +from time import sleep import json def getbaselineinterpreters(): @@ -11,12 +13,10 @@ def getbaselineinterpreters(): try: for entry in settings.baselinelist: interpreter = Interpreter.objects.get(id=entry['interpreter']) - rev = Revision.objects.filter(number=entry['revision']) + rev = Revision.objects.filter( + commitid=entry['revision'], project=interpreter.project + ) if len(rev) > 1: - for r in rev: - if(r.project in interpreter.name): - rev = r - elif len(rev): rev = rev[0] else: raise Revision.DoesNotExist @@ -25,12 +25,12 @@ def getbaselineinterpreters(): #shortname += " " + interpreter.coptions name = interpreter.name + " " + interpreter.coptions if rev.tag: name += " " + rev.tag - else: name += " " + str(rev.number) + else: name += " " + str(rev.commitid) baseline.append({ 'interpreter': interpreter.id, 'name': name, 'shortname': shortname, - 'revision': rev.number, + 'revision': rev.commitid, 'project': rev.project, }) except (Interpreter.DoesNotExist, Revision.DoesNotExist): @@ -40,20 +40,20 @@ def getbaselineinterpreters(): revs = Revision.objects.exclude(tag="") interpreters = Interpreter.objects.all() for rev in revs: - #add interpreters that correspond to each tagged revission. + #add interpreters that correspond to each tagged revision. for interpreter in interpreters: - if interpreter.name in rev.project: + if interpreter.project == rev.project: shortname = interpreter.name #if interpreter.coptions != "default": #shortname += " " + interpreter.coptions name = interpreter.name + " " + interpreter.coptions if rev.tag: name += " " + rev.tag - else: name += " " + str(rev.number) + else: name += " " + str(rev.commitid) baseline.append({ 'interpreter': interpreter.id, 'name': name, 'shortname': shortname, - 'revision': rev.number, + 'revision': rev.commitid, 'project': rev.project, }) # move default to first place @@ -83,33 +83,36 @@ def getdefaultenvironment(): def getdefaultinterpreters(): default = [] + defaultproject = Project.objects.filter(isdefault=True)[0] if hasattr(settings, 'defaultinterpreters'): try: for interpreter in settings.defaultinterpreters: i = Interpreter.objects.get(id=interpreter) default.append(interpreter) except Interpreter.DoesNotExist: - i_list = Interpreter.objects.filter(name__startswith=settings.PROJECT_NAME) + i_list = Interpreter.objects.filter(project=defaultproject) for i in i_list: default.append(i.id) else: - i_list = Interpreter.objects.filter(name__startswith=settings.PROJECT_NAME) + i_list = Interpreter.objects.filter(project=defaultproject) for i in i_list: default.append(i.id) - + return default def gettimelinedata(request): if request.method != 'GET': return HttpResponseNotAllowed('GET') data = request.GET - + + defaultproject = Project.objects.filter(isdefault=True)[0] + timeline_list = {'error': 'None', 'timelines': []} interpreters = data['interpreters'].split(",") if interpreters[0] == "": timeline_list['error'] = "No interpreters selected" return HttpResponse(json.dumps( timeline_list )) - + benchmarks = [] number_of_rev = data['revisions'] if data['benchmark'] == 'grid': @@ -121,10 +124,14 @@ def gettimelinedata(request): baseline = getbaselineinterpreters() baselinerev = None if data['baseline'] == "true" and len(baseline): + p = Project.objects.get(name=baseline['project']) baseline = baseline[0] - baselinerev = Revision.objects.get(number=baseline['revision'], project=baseline['project']) + baselinerev = Revision.objects.get( + commitid=baseline['revision'], project=p + ) for bench in benchmarks: + append = False timeline = {} timeline['benchmark'] = bench.name timeline['benchmark_id'] = bench.id @@ -135,17 +142,22 @@ def gettimelinedata(request): ).value for interpreter in interpreters: resultquery = Result.objects.filter( - revision__project=settings.PROJECT_NAME + revision__project=defaultproject ).filter( benchmark=bench ).filter( interpreter=interpreter - ).order_by('-revision__number')[:number_of_rev] + ).order_by('-revision__commitid')[:number_of_rev] results = [] for res in resultquery: - results.append([res.revision.number, res.value]) + results.append([res.revision.commitid, res.value]) timeline['interpreters'][interpreter] = results - timeline_list['timelines'].append(timeline) + if len(results): append = True + if append: timeline_list['timelines'].append(timeline) + + if not len(timeline_list['timelines']): + response = 'No data found for project "' + defaultproject.name + '"' + timeline_list['error'] = response return HttpResponse(json.dumps( timeline_list )) def timeline(request): @@ -154,18 +166,25 @@ def timeline(request): data = request.GET # Configuration of default parameters - baseline = getbaselineinterpreters() - if len(baseline): baseline = baseline[0] - defaultbaseline = True - if data.has_key("baseline"): - if data["baseline"] == "false": - defaultbaseline = False + defaultproject = Project.objects.filter(isdefault=True) + if not len(defaultproject): + return HttpResponse("You need to configure at least one Project as default") + else: defaultproject = defaultproject[0] defaultenvironment = getdefaultenvironment() if not defaultenvironment: return HttpResponse("You need to configure at least one Environment") defaultenvironment = defaultenvironment.id + baseline = getbaselineinterpreters() + + defaultbaseline = True + if data.has_key("baseline"): + if data["baseline"] == "false": + defaultbaseline = False + if len(baseline): baseline = baseline[0] + else: defaultbaseline = False + defaultbenchmark = "grid" if data.has_key("benchmark"): try: @@ -187,7 +206,7 @@ def timeline(request): defaultlast = data["revisions"] # Information for template - interpreters = Interpreter.objects.filter(name__startswith=settings.PROJECT_NAME) + interpreters = Interpreter.objects.filter(project=defaultproject) benchmarks = Benchmark.objects.all() hostlist = Environment.objects.all() return render_to_response('codespeed/timeline.html', { @@ -205,29 +224,31 @@ def timeline(request): def getoverviewtable(request): data = request.GET + + defaultproject = Project.objects.filter(isdefault=True)[0] interpreter = int(data["interpreter"]) trendconfig = int(data["trend"]) revision = int(data["revision"]) lastrevisions = Revision.objects.filter( - project=settings.PROJECT_NAME - ).filter(number__lte=revision).order_by('-number')[:trendconfig+1] - lastrevision = lastrevisions[0].number + project=defaultproject + ).filter(commitid__lte=revision).order_by('-commitid')[:trendconfig+1] + lastrevision = lastrevisions[0].commitid change_list = None pastrevisions = None if len(lastrevisions) > 1: - changerevision = lastrevisions[1].number + changerevision = lastrevisions[1].commitid change_list = Result.objects.filter( - revision__number=changerevision + revision__commitid=changerevision ).filter( - revision__project=settings.PROJECT_NAME + revision__project=defaultproject ).filter(interpreter=interpreter) pastrevisions = lastrevisions[trendconfig-2:trendconfig+1] result_list = Result.objects.filter( - revision__number=lastrevision + revision__commitid=lastrevision ).filter( - revision__project=settings.PROJECT_NAME + revision__project=defaultproject ).filter(interpreter=interpreter) # TODO: remove baselineflag @@ -240,10 +261,11 @@ def getoverviewtable(request): base = int(data['baseline']) - 1 baseline = getbaselineinterpreters() baseinterpreter = baseline[base] + p = Project.objects.get(name=baseline[base]['project']) base_list = Result.objects.filter( - revision__number=baseline[base]['revision'] + revision__commitid=baseline[base]['revision'] ).filter( - revision__project=baseline[base]['project'] + revision__project=p ).filter(interpreter=baseline[base]['interpreter']) table_list = [] @@ -266,9 +288,9 @@ def getoverviewtable(request): if pastrevisions != None: for rev in pastrevisions: past_rev = Result.objects.filter( - revision__number=rev.number + revision__commitid=rev.commitid ).filter( - revision__project=settings.PROJECT_NAME + revision__project=defaultproject ).filter( interpreter=interpreter ).filter(benchmark=bench) @@ -317,6 +339,10 @@ def overview(request): data = request.GET # Configuration of default parameters + defaultproject = Project.objects.filter(isdefault=True) + if not len(defaultproject): + return HttpResponse("You need to configure at least one Project as default") + else: defaultproject = defaultproject[0] defaultenvironment = getdefaultenvironment() if not defaultenvironment: return HttpResponse("You need to configure at least one Environment") @@ -344,34 +370,41 @@ def overview(request): if len(baseline) < defaultbaseline: defaultbaseline = 1 # Information for template - interpreters = Interpreter.objects.filter(name__startswith=settings.PROJECT_NAME) + interpreters = Interpreter.objects.filter(project=defaultproject) lastrevisions = Revision.objects.filter( - project=settings.PROJECT_NAME - ).order_by('-number')[:20] + project=defaultproject + ).order_by('-commitid')[:20] if not len(lastrevisions): - response = 'No data found for project "' + settings.PROJECT_NAME + '"' + response = 'No data found for project "' + defaultproject.name + '"' return HttpResponse(response) - selectedrevision = lastrevisions[0].number + selectedrevision = lastrevisions[0].commitid if data.has_key("revision"): if data["revision"] > 0: # TODO: Create 404 html embeded in the overview - selectedrevision = get_object_or_404(Revision, number=data["revision"]) + selectedrevision = get_object_or_404(Revision, commitid=data["revision"]) hostlist = Environment.objects.all() return render_to_response('codespeed/overview.html', locals()) +def getcommitdate(project, branch, commitid): + # FIXME: if project has defined a repository, read real commitdate + if project.rcType == 'N': + return None + else: + return None + def addresult(request): if request.method != 'POST': return HttpResponseNotAllowed('POST') data = request.POST mandatory_data = [ - 'revision_number', - 'revision_project', - 'revision_branch', + 'commitid', + 'project', + 'branch', 'interpreter_name', 'interpreter_coptions', - 'benchmark_name', + 'benchmark', 'environment', 'result_value', 'result_date', @@ -383,36 +416,40 @@ def addresult(request): return HttpResponseBadRequest('Key "' + key + '" empty in request') else: return HttpResponseBadRequest('Key "' + key + '" missing from request') - b, created = Benchmark.objects.get_or_create(name=data["benchmark_name"]) - if data.has_key('benchmark_type'): - b.benchmark_type = data['benchmark_type'] - if data.has_key('units'): - b.units = data['units'] - if data.has_key('lessisbetter'): - l = 0 - if data['lessisbetter'] == True: l = 1 - b.lessisbetter = l - b.save() - rev, created = Revision.objects.get_or_create( - number=data['revision_number'], - project=data['revision_project'], - branch=data['revision_branch'], - ) - if data.has_key('revision_date'): - rev.date = data['revision_date'] - rev.save() - inter, created = Interpreter.objects.get_or_create(name=data['interpreter_name'], coptions=data['interpreter_coptions']) + # Check that Environment exists try: e = get_object_or_404(Environment, name=data['environment']) except Http404: return HttpResponseNotFound("Environment " + data["environment"] + " not found") + + p, created = Project.objects.get_or_create(name=data["project"]) + b, created = Benchmark.objects.get_or_create(name=data["benchmark"]) + rev, created = Revision.objects.get_or_create( + commitid=data['commitid'], + project=p, + branch=data['branch'], + ) + + temp_date = getcommitdate(p, data['branch'], data['commitid']) + if temp_date: + rev.date = temp_date + else: + rev.date = data["result_date"] + rev.save() + + inter, created = Interpreter.objects.get_or_create( + name=data['interpreter_name'], + coptions=data['interpreter_coptions'], + project=p + ) r, created = Result.objects.get_or_create( - value = data["result_value"], + value=data["result_value"], revision=rev, interpreter=inter, benchmark=b, environment=e ) + r.date = data["result_date"] if data.has_key('std_dev'): r.std_dev = data['std_dev'] if data.has_key('min'): r.val_min = data['min'] diff --git a/speedcenter/templates/codespeed/overview.html b/speedcenter/templates/codespeed/overview.html index 263d89e8..ef1c325c 100644 --- a/speedcenter/templates/codespeed/overview.html +++ b/speedcenter/templates/codespeed/overview.html @@ -62,16 +62,21 @@ var tdwidth = parseInt($("#results thead tr").find("th:eq(5)").css("width")); $("#results > tbody > tr").each(function() { //Size plot bars - var bar = transToLogBars(58, tdwidth, parseFloat($(this).children("td:eq(4)").text())); - $(this).children("td:eq(5)").find("span").css("width", bar["width"]).css("margin-left", bar["margin"]); + var found = $(this).children("td:eq(2)").length; + if ($(this).children("td:eq(4)").length) { + var bar = transToLogBars(58, tdwidth, parseFloat($(this).children("td:eq(4)").text())); + $(this).children("td:eq(5)").find("span").css("width", bar["width"]).css("margin-left", bar["margin"]); + } //Link rows to timelines $(this).click(function () { permalinkToTimeline($(this).children("td:eq(0)").text()); }); }); //Configure table as tablesorter - $.tablesorter.defaults.sortList = [[4,1]];//sort by default by the 5th column - $.tablesorter.defaults.headers = { 5: { sorter: false } };//disable sorting for bar column + if ($(this).children("td:eq(4)").length) { + $.tablesorter.defaults.sortList = [[4,1]];//sort by default by the 5th column + $.tablesorter.defaults.headers = { 5: { sorter: false } };//disable sorting for bar column + } $("#results").tablesorter({widgets: ['zebra']}); $("#results tbody td").hover(function() { $(this).parents('tr').addClass('highlight'); diff --git a/speedcenter/templates/codespeed/timeline.html b/speedcenter/templates/codespeed/timeline.html index da6acdd0..c9c0626d 100644 --- a/speedcenter/templates/codespeed/timeline.html +++ b/speedcenter/templates/codespeed/timeline.html @@ -203,11 +203,12 @@
{% endfor %} + {% if defaultbaseline %} + {% endif %}