Permalink
Browse files

Store parents elsewhere

  • Loading branch information...
1 parent 102c8ae commit c68bbae5381f6ab4733256ab0af0ca21af34abcf @dcramer dcramer committed Nov 2, 2012
@@ -166,12 +166,21 @@ def get_changes(previous_build, objects):
def get_git_changes(build, previous_build):
+ # TODO: reenable this when it doesnt hit github on each request
+ return None
+
project = build.project
- results = github.compare_commits(project.github_user, project.github_repo,
- previous_build.revision.label, build.revision.label)
+ try:
+ results = github.compare_commits(project.github_user, project.github_repo,
+ previous_build.revision.label, build.revision.label)
+ except Exception:
+ # XXX: likely we hit rate limit
+ return None
+
commits = []
for commit in results['commits']:
revision = Revision.get_or_create(build.project, commit['sha'])[0]
commits.append(revision)
- return commits
+
+ return reversed(commits)
@@ -1,20 +1,33 @@
from django.db import transaction
from django.core.management.base import BaseCommand
+from optparse import make_option
from zumanji.github import github
from zumanji.models import Project, Revision
class Command(BaseCommand):
help = 'Updates all revisions by hammering GitHub'
+ option_list = BaseCommand.option_list + (
+ make_option('--project', '-p', dest='projects', help='Limit to project(s)', action='append'),
+ make_option('--refetch', '-r', dest='refetch', help='Refetch all data from GitHub', action='store_true'),
+ )
+
@transaction.commit_on_success
- def handle(self, **options):
- for project in Project.objects.filter(label__contains='/'):
+ def handle(self, projects=None, refetch=False, **options):
+ if projects:
+ project_list = list(Project.objects.filter(label__in=projects))
+ assert len(projects) == len(projects), 'one or more project label was not found'
+ else:
+ project_list = Project.objects.filter(label__contains='/')
+
+ for project in project_list:
print "Project %r" % project.label
for data in github.iter_commits(project.github_user, project.github_repo):
print " Revision %r (%s; %s)" % (data['sha'], data['commit']['author']['name'],
data['commit']['committer']['date'])
- rev, created = Revision.get_or_create(project, data['sha'], data=data, recurse=False)
- if not created and not rev.data:
- rev.update_from_github(data, recurse=False)
- rev.save()
+ rev, created = Revision.get_or_create(project, data['sha'], data=data)
+ if not created:
+ if not rev.data or refetch:
+ rev.update_from_github(data)
+ rev.save()
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding model 'RevisionParent'
+ db.create_table('zumanji_revisionparent', (
+ ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('project', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['zumanji.Project'])),
+ ('revision_label', self.gf('django.db.models.fields.CharField')(max_length=64, null=True)),
+ ('parent_label', self.gf('django.db.models.fields.CharField')(max_length=64, null=True)),
+ ))
+ db.send_create_signal('zumanji', ['RevisionParent'])
+
+ # Adding unique constraint on 'RevisionParent', fields ['project', 'revision_label', 'parent_label']
+ db.create_unique('zumanji_revisionparent', ['project_id', 'revision_label', 'parent_label'])
+
+ # Deleting field 'Revision.parent_label'
+ db.delete_column('zumanji_revision', 'parent_label')
+
+ # Deleting field 'Revision.parent'
+ db.delete_column('zumanji_revision', 'parent_id')
+
+
+ def backwards(self, orm):
+ # Removing unique constraint on 'RevisionParent', fields ['project', 'revision_label', 'parent_label']
+ db.delete_unique('zumanji_revisionparent', ['project_id', 'revision_label', 'parent_label'])
+
+ # Deleting model 'RevisionParent'
+ db.delete_table('zumanji_revisionparent')
+
+ # Adding field 'Revision.parent_label'
+ db.add_column('zumanji_revision', 'parent_label',
+ self.gf('django.db.models.fields.CharField')(max_length=64, null=True),
+ keep_default=False)
+
+ # Adding field 'Revision.parent'
+ db.add_column('zumanji_revision', 'parent',
+ self.gf('django.db.models.fields.related.ForeignKey')(to=orm['zumanji.Revision'], null=True),
+ keep_default=False)
+
+
+ models = {
+ 'zumanji.build': {
+ 'Meta': {'unique_together': "(('revision', 'datetime'),)", 'object_name': 'Build'},
+ 'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
+ 'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'num_tests': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
+ 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"}),
+ 'result': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True'}),
+ 'revision': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Revision']"}),
+ 'total_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'})
+ },
+ 'zumanji.buildtag': {
+ 'Meta': {'object_name': 'BuildTag'},
+ 'builds': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'tags'", 'symmetrical': 'False', 'to': "orm['zumanji.Build']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+ },
+ 'zumanji.project': {
+ 'Meta': {'object_name': 'Project'},
+ 'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
+ },
+ 'zumanji.revision': {
+ 'Meta': {'unique_together': "(('project', 'label'),)", 'object_name': 'Revision'},
+ 'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
+ 'datetime': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+ 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"})
+ },
+ 'zumanji.revisionparent': {
+ 'Meta': {'unique_together': "(('project', 'revision_label', 'parent_label'),)", 'object_name': 'RevisionParent'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'parent_label': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}),
+ 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"}),
+ 'revision_label': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'})
+ },
+ 'zumanji.test': {
+ 'Meta': {'unique_together': "(('build', 'label'),)", 'object_name': 'Test'},
+ 'build': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Build']"}),
+ 'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'null': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'lower_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'}),
+ 'mean_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'}),
+ 'num_tests': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
+ 'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Test']", 'null': 'True'}),
+ 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"}),
+ 'result': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True'}),
+ 'revision': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Revision']"}),
+ 'upper90_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'}),
+ 'upper_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'})
+ },
+ 'zumanji.testdata': {
+ 'Meta': {'unique_together': "(('test', 'key'),)", 'object_name': 'TestData'},
+ 'build': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Build']"}),
+ 'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+ 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"}),
+ 'revision': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Revision']"}),
+ 'test': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Test']"})
+ }
+ }
+
+ complete_apps = ['zumanji']
View
@@ -73,8 +73,6 @@ class Revision(models.Model):
project = models.ForeignKey(Project)
label = models.CharField(max_length=64)
datetime = models.DateTimeField(null=True)
- parent = models.ForeignKey('self', null=True)
- parent_label = models.CharField(max_length=64, null=True)
data = GzippedJSONField(default={}, blank=True)
class Meta:
@@ -95,7 +93,7 @@ def sanitize_github_data(cls, data):
return result
@classmethod
- def get_or_create(cls, project, label, data=None, recurse=True):
+ def get_or_create(cls, project, label, data=None):
"""
Get a revision, and if it doesnt exist, attempt to pull it from Git.
"""
@@ -104,7 +102,7 @@ def get_or_create(cls, project, label, data=None, recurse=True):
created = False
except Revision.DoesNotExist:
rev = cls(project=project, label=label)
- rev.update_from_github(data=data, recurse=recurse)
+ rev.update_from_github(data=data)
rev.save()
created = True
@@ -132,32 +130,37 @@ def author(self):
return {}
return self.data['commit']['author']
- def update_from_github(self, data=None, recurse=False):
+ def get_parents(self):
+ return list(RevisionParent.objects.filter(
+ project=self.project, revision_label=self.label
+ ).values_list('parent_label', flat=True))
+
+ def update_from_github(self, data=None):
if data is None:
data = github.get_commit(self.project.github_user, self.project.github_repo, self.label)
datetime = dateutil.parser.parse(data['commit']['committer']['date'])
- # LOL MULTIPLE PARENTS HOW DOES GIT WORK
- # (dont care about the merge commits parent for our system)
- if data.get('parents'):
- parent_sha = data['parents'][0]['sha']
- if recurse:
- parent = type(self).get_or_create(self.project, parent_sha, recurse=recurse)[0]
- else:
- try:
- parent = type(self).objects.get(project=self.project, label=parent_sha)
- except:
- parent = None
- else:
- parent = None
- parent_sha = None
- self.parent = parent
- self.parent_label = parent_sha
+ for parent in data.get('parents', []):
+ RevisionParent.objects.get_or_create(
+ project=self.project,
+ revision_label=self.label,
+ parent_label=parent['sha'],
+ )
+
self.datetime = datetime
self.data = type(self).sanitize_github_data(data)
+class RevisionParent(models.Model):
+ project = models.ForeignKey(Project)
+ revision_label = models.CharField(max_length=64, null=True)
+ parent_label = models.CharField(max_length=64, null=True)
+
+ class Meta:
+ unique_together = (('project', 'revision_label', 'parent_label'),)
+
+
class Build(models.Model):
project = models.ForeignKey(Project)
revision = models.ForeignKey(Revision)
@@ -187,14 +190,20 @@ def _get_temporal_sibling(self, tag=None, previous=False):
qs = type(self).objects.filter(**filter_args)
- if self.revision.datetime:
+ # if datetime is present, it means we've parsed the commit
+ parents = self.revision.get_parents()
+ if parents:
if previous:
qs = qs.filter(
- Q(revision=self.revision.parent) | Q(revision__isnull=True)
+ revision__label__in=parents,
)
order_field = ('-revision__datetime', '-datetime')
else:
- qs = qs.filter(revision__parent=self.revision)
+ qs = qs.extra(
+ tables=['zumanji_revisionparent'],
+ where=['zumanji_revisionparent.revision_label = %s'],
+ params=[self.revision.label],
+ )
order_field = ('revision__datetime', 'datetime')
else:
if previous:
@@ -271,11 +280,11 @@ def get_test_in_previous_build(self):
).exclude(
build=self.build,
)
+ parents = self.revision.get_parents()
if self.revision.datetime:
qs = qs.filter(
build__revision__datetime__lt=self.revision.datetime,
- ).filter(
- Q(build__revision=self.revision.parent) | Q(build__revision__isnull=True)
+ build__revision__label__in=parents,
).order_by('-build__revision__datetime', '-build__datetime')
else:
qs = qs.filter(
@@ -295,9 +304,14 @@ def get_test_in_next_build(self):
build=self.build,
)
if self.revision.datetime:
- qs = qs.filter(
+ qs = qs.extra(
+ tables=['zumanji_revisionparent'],
+ where=[
+ 'zumanji_revisionparent.parent_label = %s',
+ ],
+ params=[self.revision.label],
+ ).filter(
build__revision__datetime__gt=self.revision.datetime,
- build__revision__parent=self.revision,
).order_by('build__revision__datetime', 'build__datetime')
else:
qs = qs.filter(
@@ -28,7 +28,15 @@
<div class="span6">
<dl class="dl-horizontal">
<dt>Revision</dt>
- <dd>{{ build.revision.label }}{% if build.revision.data %} [<a href="{{ build.revision.details_url }}">git</a>]{% endif %}</dd>
+ <dd>
+ {{ build.revision.label }}{% if build.revision.data %} [<a href="{{ build.revision.details_url }}">git</a>]{% endif %}
+ {% with build.revision.get_parents as parents %}
+ {% if parents %}
+ <br><small>{{ parents|length }} parent{% if parents|length != 1 %}s{% endif %}:
+ {% for parent in parents %}{{ parent }}{% if not forloop.last %}, {% endif %}{% endfor %}</small>
+ {% endif %}
+ {% endwith %}
+ </dd>
<dt>Author</dt>
<dd>{{ build.revision.data.commit.author.name }} ({{ build.revision.data.commit.author.email }})</dd>
<dt>Datetime</dt>

0 comments on commit c68bbae

Please sign in to comment.