Skip to content

Commit

Permalink
Store parents elsewhere
Browse files Browse the repository at this point in the history
  • Loading branch information
@dcramer
dcramer committed Nov 2, 2012
1 parent 102c8ae commit c68bbae
Show file tree
Hide file tree
Showing 5 changed files with 198 additions and 38 deletions.
15 changes: 12 additions & 3 deletions src/zumanji/helpers.py
View file
Expand Up @@ -166,12 +166,21 @@ def get_changes(previous_build, objects):


def get_git_changes(build, previous_build):
# TODO: reenable this when it doesnt hit github on each request
return None

project = build.project

results = github.compare_commits(project.github_user, project.github_repo,
previous_build.revision.label, build.revision.label)
try:
results = github.compare_commits(project.github_user, project.github_repo,
previous_build.revision.label, build.revision.label)
except Exception:
# XXX: likely we hit rate limit
return None

commits = []
for commit in results['commits']:
revision = Revision.get_or_create(build.project, commit['sha'])[0]
commits.append(revision)
return commits

return reversed(commits)
25 changes: 19 additions & 6 deletions src/zumanji/management/commands/poll_github.py
View file
@@ -1,20 +1,33 @@
from django.db import transaction
from django.core.management.base import BaseCommand
from optparse import make_option
from zumanji.github import github
from zumanji.models import Project, Revision


class Command(BaseCommand):
help = 'Updates all revisions by hammering GitHub'

option_list = BaseCommand.option_list + (
make_option('--project', '-p', dest='projects', help='Limit to project(s)', action='append'),
make_option('--refetch', '-r', dest='refetch', help='Refetch all data from GitHub', action='store_true'),
)

@transaction.commit_on_success
def handle(self, **options):
for project in Project.objects.filter(label__contains='/'):
def handle(self, projects=None, refetch=False, **options):
if projects:
project_list = list(Project.objects.filter(label__in=projects))
assert len(projects) == len(projects), 'one or more project label was not found'
else:
project_list = Project.objects.filter(label__contains='/')

for project in project_list:
print "Project %r" % project.label
for data in github.iter_commits(project.github_user, project.github_repo):
print " Revision %r (%s; %s)" % (data['sha'], data['commit']['author']['name'],
data['commit']['committer']['date'])
rev, created = Revision.get_or_create(project, data['sha'], data=data, recurse=False)
if not created and not rev.data:
rev.update_from_github(data, recurse=False)
rev.save()
rev, created = Revision.get_or_create(project, data['sha'], data=data)
if not created:
if not rev.data or refetch:
rev.update_from_github(data)
rev.save()
116 changes: 116 additions & 0 deletions ...migrations/0005_auto__add_revisionparent__add_unique_revisionparent_project_revision_l.py
View file
@@ -0,0 +1,116 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

def forwards(self, orm):
# Adding model 'RevisionParent'
db.create_table('zumanji_revisionparent', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('project', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['zumanji.Project'])),
('revision_label', self.gf('django.db.models.fields.CharField')(max_length=64, null=True)),
('parent_label', self.gf('django.db.models.fields.CharField')(max_length=64, null=True)),
))
db.send_create_signal('zumanji', ['RevisionParent'])

# Adding unique constraint on 'RevisionParent', fields ['project', 'revision_label', 'parent_label']
db.create_unique('zumanji_revisionparent', ['project_id', 'revision_label', 'parent_label'])

# Deleting field 'Revision.parent_label'
db.delete_column('zumanji_revision', 'parent_label')

# Deleting field 'Revision.parent'
db.delete_column('zumanji_revision', 'parent_id')


def backwards(self, orm):
# Removing unique constraint on 'RevisionParent', fields ['project', 'revision_label', 'parent_label']
db.delete_unique('zumanji_revisionparent', ['project_id', 'revision_label', 'parent_label'])

# Deleting model 'RevisionParent'
db.delete_table('zumanji_revisionparent')

# Adding field 'Revision.parent_label'
db.add_column('zumanji_revision', 'parent_label',
self.gf('django.db.models.fields.CharField')(max_length=64, null=True),
keep_default=False)

# Adding field 'Revision.parent'
db.add_column('zumanji_revision', 'parent',
self.gf('django.db.models.fields.related.ForeignKey')(to=orm['zumanji.Revision'], null=True),
keep_default=False)


models = {
'zumanji.build': {
'Meta': {'unique_together': "(('revision', 'datetime'),)", 'object_name': 'Build'},
'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'num_tests': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"}),
'result': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True'}),
'revision': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Revision']"}),
'total_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'})
},
'zumanji.buildtag': {
'Meta': {'object_name': 'BuildTag'},
'builds': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'tags'", 'symmetrical': 'False', 'to': "orm['zumanji.Build']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'label': ('django.db.models.fields.CharField', [], {'max_length': '255'})
},
'zumanji.project': {
'Meta': {'object_name': 'Project'},
'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
},
'zumanji.revision': {
'Meta': {'unique_together': "(('project', 'label'),)", 'object_name': 'Revision'},
'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'label': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"})
},
'zumanji.revisionparent': {
'Meta': {'unique_together': "(('project', 'revision_label', 'parent_label'),)", 'object_name': 'RevisionParent'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'parent_label': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}),
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"}),
'revision_label': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'})
},
'zumanji.test': {
'Meta': {'unique_together': "(('build', 'label'),)", 'object_name': 'Test'},
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Build']"}),
'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'label': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'lower_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'}),
'mean_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'}),
'num_tests': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Test']", 'null': 'True'}),
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"}),
'result': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True'}),
'revision': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Revision']"}),
'upper90_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'}),
'upper_duration': ('django.db.models.fields.FloatField', [], {'default': '0.0'})
},
'zumanji.testdata': {
'Meta': {'unique_together': "(('test', 'key'),)", 'object_name': 'TestData'},
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Build']"}),
'data': ('django.db.models.fields.TextField', [], {'default': '{}', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Project']"}),
'revision': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Revision']"}),
'test': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['zumanji.Test']"})
}
}

complete_apps = ['zumanji']
70 changes: 42 additions & 28 deletions src/zumanji/models.py
View file
Expand Up @@ -73,8 +73,6 @@ class Revision(models.Model):
project = models.ForeignKey(Project)
label = models.CharField(max_length=64)
datetime = models.DateTimeField(null=True)
parent = models.ForeignKey('self', null=True)
parent_label = models.CharField(max_length=64, null=True)
data = GzippedJSONField(default={}, blank=True)

class Meta:
Expand All @@ -95,7 +93,7 @@ def sanitize_github_data(cls, data):
return result

@classmethod
def get_or_create(cls, project, label, data=None, recurse=True):
def get_or_create(cls, project, label, data=None):
"""
Get a revision, and if it doesnt exist, attempt to pull it from Git.
"""
Expand All @@ -104,7 +102,7 @@ def get_or_create(cls, project, label, data=None, recurse=True):
created = False
except Revision.DoesNotExist:
rev = cls(project=project, label=label)
rev.update_from_github(data=data, recurse=recurse)
rev.update_from_github(data=data)
rev.save()
created = True

Expand Down Expand Up @@ -132,32 +130,37 @@ def author(self):
return {}
return self.data['commit']['author']

def update_from_github(self, data=None, recurse=False):
def get_parents(self):
return list(RevisionParent.objects.filter(
project=self.project, revision_label=self.label
).values_list('parent_label', flat=True))

def update_from_github(self, data=None):
if data is None:
data = github.get_commit(self.project.github_user, self.project.github_repo, self.label)

datetime = dateutil.parser.parse(data['commit']['committer']['date'])
# LOL MULTIPLE PARENTS HOW DOES GIT WORK
# (dont care about the merge commits parent for our system)
if data.get('parents'):
parent_sha = data['parents'][0]['sha']
if recurse:
parent = type(self).get_or_create(self.project, parent_sha, recurse=recurse)[0]
else:
try:
parent = type(self).objects.get(project=self.project, label=parent_sha)
except:
parent = None
else:
parent = None
parent_sha = None

self.parent = parent
self.parent_label = parent_sha
for parent in data.get('parents', []):
RevisionParent.objects.get_or_create(
project=self.project,
revision_label=self.label,
parent_label=parent['sha'],
)

self.datetime = datetime
self.data = type(self).sanitize_github_data(data)


class RevisionParent(models.Model):
project = models.ForeignKey(Project)
revision_label = models.CharField(max_length=64, null=True)
parent_label = models.CharField(max_length=64, null=True)

class Meta:
unique_together = (('project', 'revision_label', 'parent_label'),)


class Build(models.Model):
project = models.ForeignKey(Project)
revision = models.ForeignKey(Revision)
Expand Down Expand Up @@ -187,14 +190,20 @@ def _get_temporal_sibling(self, tag=None, previous=False):

qs = type(self).objects.filter(**filter_args)

if self.revision.datetime:
# if datetime is present, it means we've parsed the commit
parents = self.revision.get_parents()
if parents:
if previous:
qs = qs.filter(
Q(revision=self.revision.parent) | Q(revision__isnull=True)
revision__label__in=parents,
)
order_field = ('-revision__datetime', '-datetime')
else:
qs = qs.filter(revision__parent=self.revision)
qs = qs.extra(
tables=['zumanji_revisionparent'],
where=['zumanji_revisionparent.revision_label = %s'],
params=[self.revision.label],
)
order_field = ('revision__datetime', 'datetime')
else:
if previous:
Expand Down Expand Up @@ -271,11 +280,11 @@ def get_test_in_previous_build(self):
).exclude(
build=self.build,
)
parents = self.revision.get_parents()
if self.revision.datetime:
qs = qs.filter(
build__revision__datetime__lt=self.revision.datetime,
).filter(
Q(build__revision=self.revision.parent) | Q(build__revision__isnull=True)
build__revision__label__in=parents,
).order_by('-build__revision__datetime', '-build__datetime')
else:
qs = qs.filter(
Expand All @@ -295,9 +304,14 @@ def get_test_in_next_build(self):
build=self.build,
)
if self.revision.datetime:
qs = qs.filter(
qs = qs.extra(
tables=['zumanji_revisionparent'],
where=[
'zumanji_revisionparent.parent_label = %s',
],
params=[self.revision.label],
).filter(
build__revision__datetime__gt=self.revision.datetime,
build__revision__parent=self.revision,
).order_by('build__revision__datetime', 'build__datetime')
else:
qs = qs.filter(
Expand Down
10 changes: 9 additions & 1 deletion src/zumanji/templates/zumanji/includes/build_details.html
View file
Expand Up @@ -28,7 +28,15 @@
<div class="span6">
<dl class="dl-horizontal">
<dt>Revision</dt>
<dd>{{ build.revision.label }}{% if build.revision.data %} [<a href="{{ build.revision.details_url }}">git</a>]{% endif %}</dd>
<dd>
{{ build.revision.label }}{% if build.revision.data %} [<a href="{{ build.revision.details_url }}">git</a>]{% endif %}
{% with build.revision.get_parents as parents %}
{% if parents %}
<br><small>{{ parents|length }} parent{% if parents|length != 1 %}s{% endif %}:
{% for parent in parents %}{{ parent }}{% if not forloop.last %}, {% endif %}{% endfor %}</small>
{% endif %}
{% endwith %}
</dd>
<dt>Author</dt>
<dd>{{ build.revision.data.commit.author.name }} ({{ build.revision.data.commit.author.email }})</dd>
<dt>Datetime</dt>
Expand Down

0 comments on commit c68bbae

Please sign in to comment.