Make bug refresh async (hack).

* Add crude script to refresh dev db from prod. * Add fields to BugzillaURL to enable queue like functions.
bensternthal · Aug 22, 2012 · 9d10381 · 9d10381
1 parent 112f58d
commit 9d10381
Show file tree

Hide file tree

Showing 10 changed files with 238 additions and 48 deletions.
diff --git a/copy_prod_db_to_dev.sh b/copy_prod_db_to_dev.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+test -z "$1" && echo "usage: $0 <DB_ALIAS_NAME>" && exit 1
+
+heroku pgbackups:capture -r prod
+heroku pgbackups:restore "$1" `heroku pgbackups:url -r prod` -r dev
diff --git a/scrum/cron.py b/scrum/cron.py
@@ -1,19 +1,29 @@
 from __future__ import absolute_import
 
+import sys
+from datetime import datetime, timedelta
+
+from django.conf import settings
+
 from cronjobs import register
 
 from scrum.email import get_bugmails
 from scrum.models import BugzillaURL, Project
 from scrum.utils import get_bz_url_for_bug_ids
 
 
+CACHE_BUGS_FOR = timedelta(hours=getattr(settings, 'CACHE_BUGS_FOR', 4))
+
+
 @register
-def sync_bugs():
+def sync_bugmail():
     """
     Check bugmail for updated bugs, and get their data from Bugzilla.
     """
+    counter = 0
     bugids = get_bugmails()
     for slug, ids in bugids.items():
+        counter += len(ids)
         url = BugzillaURL(url=get_bz_url_for_bug_ids(ids))
         proj = None
         if slug:
@@ -24,3 +34,35 @@ def sync_bugs():
         if proj:
             url.project = proj
         url.get_bugs()
+        sys.stdout.write('.')
+        sys.stdout.flush()
+    if counter:
+        print "\nSynced {0} bugs".format(counter)
+
+
+@register
+def sync_backlogs():
+    """
+    Get the bugs data for all urls in the system updated more than
+    CACHE_BUGS_FOR hours ago.
+    """
+    counter = 0
+    synced_urls = []
+    sync_time = datetime.utcnow() - CACHE_BUGS_FOR
+    for url in BugzillaURL.objects.filter(date_synced__lt=sync_time):
+        # avoid dupes
+        # need to do this here instead of setting the DB column unique b/c
+        # it is possible for 2 projects to use the same search url.
+        if url.url in synced_urls:
+            if url.one_time:
+                url.delete()
+            continue
+        synced_urls.append(url.url)
+        url.get_bugs()
+        if url.one_time:
+            url.delete()
+        sys.stdout.write('.')
+        sys.stdout.flush()
+        counter += 1
+    if counter:
+        print "\nSynced {0} urls".format(counter)
diff --git a/...migrations/0007_auto__add_field_bugzillaurl_date_synced__add_field_bugzillaurl_one_tim.py b/...migrations/0007_auto__add_field_bugzillaurl_date_synced__add_field_bugzillaurl_one_tim.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'BugzillaURL.date_synced'
+        db.add_column('scrum_bugzillaurl', 'date_synced',
+                      self.gf('django.db.models.fields.DateTimeField')(default='2000-01-01'),
+                      keep_default=False)
+
+        # Adding field 'BugzillaURL.one_time'
+        db.add_column('scrum_bugzillaurl', 'one_time',
+                      self.gf('django.db.models.fields.BooleanField')(default=False),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'BugzillaURL.date_synced'
+        db.delete_column('scrum_bugzillaurl', 'date_synced')
+
+        # Deleting field 'BugzillaURL.one_time'
+        db.delete_column('scrum_bugzillaurl', 'one_time')
+
+
+    models = {
+        'scrum.bug': {
+            'Meta': {'ordering': "('id',)", 'object_name': 'Bug'},
+            'assigned_to': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'backlog': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backlog_bugs'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['scrum.Project']"}),
+            'blocks': ('jsonfield.fields.JSONField', [], {'blank': 'True'}),
+            'comments': ('scrum.models.CompressedJSONField', [], {'blank': 'True'}),
+            'comments_count': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '0'}),
+            'component': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'creation_time': ('django.db.models.fields.DateTimeField', [], {}),
+            'depends_on': ('jsonfield.fields.JSONField', [], {'blank': 'True'}),
+            'history': ('scrum.models.CompressedJSONField', [], {}),
+            'id': ('django.db.models.fields.PositiveIntegerField', [], {'primary_key': 'True'}),
+            'last_change_time': ('django.db.models.fields.DateTimeField', [], {}),
+            'last_synced_time': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}),
+            'priority': ('django.db.models.fields.CharField', [], {'max_length': '2', 'blank': 'True'}),
+            'product': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bugs'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['scrum.Project']"}),
+            'resolution': ('django.db.models.fields.CharField', [], {'max_length': '20', 'blank': 'True'}),
+            'sprint': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bugs'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['scrum.Sprint']"}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
+            'story_component': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'story_points': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '0'}),
+            'story_user': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'summary': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'whiteboard': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'scrum.bugsprintlog': {
+            'Meta': {'ordering': "('-timestamp',)", 'object_name': 'BugSprintLog'},
+            'action': ('django.db.models.fields.PositiveSmallIntegerField', [], {}),
+            'bug': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'sprint_actions'", 'to': "orm['scrum.Bug']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sprint': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bug_actions'", 'to': "orm['scrum.Sprint']"}),
+            'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'})
+        },
+        'scrum.bugzillaurl': {
+            'Meta': {'ordering': "('id',)", 'object_name': 'BugzillaURL'},
+            'date_synced': ('django.db.models.fields.DateTimeField', [], {'default': "'2000-01-01'"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'one_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'urls'", 'null': 'True', 'to': "orm['scrum.Project']"}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '2048'})
+        },
+        'scrum.project': {
+            'Meta': {'object_name': 'Project'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'}),
+            'team': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'projects'", 'null': 'True', 'to': "orm['scrum.Team']"})
+        },
+        'scrum.sprint': {
+            'Meta': {'ordering': "['-start_date']", 'unique_together': "(('team', 'slug'),)", 'object_name': 'Sprint'},
+            'bugs_data_cache': ('jsonfield.fields.JSONField', [], {'null': 'True'}),
+            'bz_url': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'created_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'end_date': ('django.db.models.fields.DateField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'notes_html': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'slug': ('django.db.models.fields.CharField', [], {'max_length': '200', 'db_index': 'True'}),
+            'start_date': ('django.db.models.fields.DateField', [], {}),
+            'team': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'sprints'", 'to': "orm['scrum.Team']"})
+        },
+        'scrum.team': {
+            'Meta': {'object_name': 'Team'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'})
+        }
+    }
+
+    complete_apps = ['scrum']
diff --git a/scrum/models.py b/scrum/models.py
@@ -14,7 +14,7 @@
 from django.db import models, transaction
 from django.db.models.query import QuerySet
 from django.db.models.query_utils import Q
-from django.db.models.signals import pre_save, post_save
+from django.db.models.signals import pre_save
 from django.dispatch import receiver
 from django.utils.encoding import force_unicode
 
@@ -180,7 +180,8 @@ def get_bz_search_url(self, bugs=None):
 
     def refresh_bugs_data(self, bugs=None):
         bzurl = self.get_bz_search_url(bugs)
-        bzurl.get_bugs()
+        bzurl.one_time = True
+        bzurl.save()
 
     def get_bugs(self, **kwargs):
         kwargs['bug_filters'] = {'sprint__isnull': True}
@@ -233,8 +234,9 @@ def date_cached(self):
 
     def refresh_backlog(self):
         self._clear_bugs_data_cache()
-        bugs = self._get_url_items('bugs')
-        self.update_backlog_bugs(bugs)
+        for url in self.urls.all():
+            url.date_synced = '2000-01-01'
+            url.save()
 
     def get_backlog(self, **kwargs):
         """Get a unique set of bugs from all bz urls"""
@@ -371,7 +373,8 @@ def get_edit_url(self):
     def refresh_bugs_data(self, bugs=None):
         self._clear_bugs_data_cache()
         bzurl = self.get_bz_search_url(bugs)
-        bzurl.get_bugs()
+        bzurl.one_time = True
+        bzurl.save()
 
     def update_bugs(self, bugs):
         """
@@ -419,14 +422,22 @@ def get_cached_bugs_data(self):
 
 
 class EmptyBugzillaURL(object):
+    one_time = False
+
     def get_bugs(self):
         return set()
 
+    def save(self):
+        pass
+
 
 class BugzillaURL(models.Model):
     url = models.URLField(verbose_name='Bugzilla URL', max_length=2048)
     project = models.ForeignKey(Project, null=True, blank=True,
                                 related_name='urls')
+    # default in the past
+    date_synced = models.DateTimeField(default='2000-01-01')
+    one_time = models.BooleanField(default=False)
 
     date_cached = None
     num_no_data_bugs = 0
@@ -465,9 +476,12 @@ def get_bugs(self):
             args = dict((k.encode('utf-8'), v) for k, v in
                         args.iterlists())
             data = BZAPI.bug.get(**args)
-            data['date_received'] = datetime.now()
+            data['date_received'] = datetime.utcnow()
         except Exception:
             raise BZError("Couldn't retrieve bugs from Bugzilla")
+        if not self.one_time:
+            self.date_synced = datetime.utcnow()
+            self.save()
         return set(store_bugs(data['bugs'], self.project))
 
     def get_products(self):
@@ -490,7 +504,8 @@ def sync_bugs(self):
         bids = self.only('id')
         if bids:
             url = BugzillaURL(url=get_bz_url_for_buglist(bids))
-            url.get_bugs()
+            url.one_time = True
+            url.save()
 
     def scrum_only(self):
         return self.filter(~Q(story_component='') |
@@ -782,12 +797,3 @@ def process_notes(sender, instance, **kwargs):
             output_format='html5',
             safe_mode=True,
         )
-
-
-@receiver(post_save, sender=BugzillaURL)
-def get_bugzilla_bugs_data(sender, instance, **kwargs):
-    """
-    After a `BugzillaURL` is saved, get the bugs data and associate it with
-    the url's project's backlog if available.
-    """
-    instance.get_bugs()
diff --git a/scrum/templates/scrum/project.html b/scrum/templates/scrum/project.html
@@ -2,30 +2,33 @@
 {% from "scrum/includes/macros.html" import bug_table with context %}
 
 {% block content %}
-  {% if bzerror %}
-    {% include "scrum/bzerror.html" %}
-  {% else %}
-    {% if perms.scrum.change_project %}
-      <a class="btn btn-primary pull-right"
-         href="{{ url('scrum_project_edit', project.slug) }}">
-        <i class="icon-edit icon-white"></i> Edit
-      </a>
-    {% endif %}
-    <h1>{{ project.name }} Project</h1>
-    <p>Part of the <a href="{{ project.team.get_absolute_url() }}">{{ project.team.name }}</a> team.</p>
-    <hr>
-    <h2>Currently Sprinting</h2>
-    {{ bug_table(currently_sprinting, show_sprint=True, show_project=False) }}
-    <hr>
-    {% if perms.scrum.change_project %}
-      <a class="btn btn-small btn-primary pull-right"
-         href="{{ url('scrum_project_bugs', project.slug) }}">
-        <i class="icon-list icon-white"></i> Manage Ready Stories
-      </a>
-    {% endif %}
-    <h2>Ready Backlog</h2>
-    {{ bug_table(project.get_bugs(), 'backlog_table', show_project=False) }}
-  {% endif %}
+  <div class="row">
+    <div class="span12">
+      {% if perms.scrum.change_project %}
+        <a class="btn btn-primary pull-right"
+           href="{{ url('scrum_project_edit', project.slug) }}">
+          <i class="icon-edit icon-white"></i> Edit
+        </a>
+      {% endif %}
+      <h1>{{ project.name }} Project</h1>
+      <p>Part of the <a href="{{ project.team.get_absolute_url() }}">{{ project.team.name }}</a> team.</p>
+    </div>
+  </div>
+  <div class="row">
+    <div class="span12">
+      <h2>Currently Sprinting</h2>
+      {{ bug_table(sprinting, show_sprint=True, show_project=False, blocked_bugs=sprinting_blocked) }}
+      <hr>
+      {% if perms.scrum.change_project %}
+        <a class="btn btn-small btn-primary pull-right"
+           href="{{ url('scrum_project_bugs', project.slug) }}">
+          <i class="icon-list icon-white"></i> Manage Ready Stories
+        </a>
+      {% endif %}
+      <h2>Sprint Ready</h2>
+      {{ bug_table(bugs, 'backlog_table', show_project=False, blocked_bugs=blocked_bugs) }}
+    </div>
+  </div>
 {% endblock %}
 
 {% block js %}

diff --git a/scrum/templates/scrum/project_bugs.html b/scrum/templates/scrum/project_bugs.html
@@ -5,7 +5,6 @@
   <div class="row">
     <div class="span12">
       <button class="btn btn-primary pull-right stats-toggle" data-toggle="button">Stats</button>
-
       <h1>
         <a href="{{ project.get_absolute_url() }}">{{ project.name }}</a> Stories
       </h1>