From 9d10381712c5435ad6c5bed8bb293254b1cf16d3 Mon Sep 17 00:00:00 2001 From: Paul McLanahan Date: Wed, 22 Aug 2012 16:31:38 -0400 Subject: [PATCH] Make bug refresh async (hack). * Add crude script to refresh dev db from prod. * Add fields to BugzillaURL to enable queue like functions. --- copy_prod_db_to_dev.sh | 6 ++ scrum/cron.py | 44 +++++++- ...e_synced__add_field_bugzillaurl_one_tim.py | 102 ++++++++++++++++++ scrum/models.py | 38 ++++--- scrum/templates/scrum/project.html | 51 ++++----- scrum/templates/scrum/project_bugs.html | 1 - scrum/tests.py | 33 +++++- scrum/utils.py | 2 +- scrum/views.py | 7 +- settings/base.py | 2 +- 10 files changed, 238 insertions(+), 48 deletions(-) create mode 100755 copy_prod_db_to_dev.sh create mode 100644 scrum/migrations/0007_auto__add_field_bugzillaurl_date_synced__add_field_bugzillaurl_one_tim.py diff --git a/copy_prod_db_to_dev.sh b/copy_prod_db_to_dev.sh new file mode 100755 index 0000000..7de9f8c --- /dev/null +++ b/copy_prod_db_to_dev.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +test -z "$1" && echo "usage: $0 " && exit 1 + +heroku pgbackups:capture -r prod +heroku pgbackups:restore "$1" `heroku pgbackups:url -r prod` -r dev diff --git a/scrum/cron.py b/scrum/cron.py index 83de6c9..f1873e0 100644 --- a/scrum/cron.py +++ b/scrum/cron.py @@ -1,5 +1,10 @@ from __future__ import absolute_import +import sys +from datetime import datetime, timedelta + +from django.conf import settings + from cronjobs import register from scrum.email import get_bugmails @@ -7,13 +12,18 @@ from scrum.utils import get_bz_url_for_bug_ids +CACHE_BUGS_FOR = timedelta(hours=getattr(settings, 'CACHE_BUGS_FOR', 4)) + + @register -def sync_bugs(): +def sync_bugmail(): """ Check bugmail for updated bugs, and get their data from Bugzilla. """ + counter = 0 bugids = get_bugmails() for slug, ids in bugids.items(): + counter += len(ids) url = BugzillaURL(url=get_bz_url_for_bug_ids(ids)) proj = None if slug: @@ -24,3 +34,35 @@ def sync_bugs(): if proj: url.project = proj url.get_bugs() + sys.stdout.write('.') + sys.stdout.flush() + if counter: + print "\nSynced {0} bugs".format(counter) + + +@register +def sync_backlogs(): + """ + Get the bugs data for all urls in the system updated more than + CACHE_BUGS_FOR hours ago. + """ + counter = 0 + synced_urls = [] + sync_time = datetime.utcnow() - CACHE_BUGS_FOR + for url in BugzillaURL.objects.filter(date_synced__lt=sync_time): + # avoid dupes + # need to do this here instead of setting the DB column unique b/c + # it is possible for 2 projects to use the same search url. + if url.url in synced_urls: + if url.one_time: + url.delete() + continue + synced_urls.append(url.url) + url.get_bugs() + if url.one_time: + url.delete() + sys.stdout.write('.') + sys.stdout.flush() + counter += 1 + if counter: + print "\nSynced {0} urls".format(counter) diff --git a/scrum/migrations/0007_auto__add_field_bugzillaurl_date_synced__add_field_bugzillaurl_one_tim.py b/scrum/migrations/0007_auto__add_field_bugzillaurl_date_synced__add_field_bugzillaurl_one_tim.py new file mode 100644 index 0000000..a2e33b3 --- /dev/null +++ b/scrum/migrations/0007_auto__add_field_bugzillaurl_date_synced__add_field_bugzillaurl_one_tim.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'BugzillaURL.date_synced' + db.add_column('scrum_bugzillaurl', 'date_synced', + self.gf('django.db.models.fields.DateTimeField')(default='2000-01-01'), + keep_default=False) + + # Adding field 'BugzillaURL.one_time' + db.add_column('scrum_bugzillaurl', 'one_time', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'BugzillaURL.date_synced' + db.delete_column('scrum_bugzillaurl', 'date_synced') + + # Deleting field 'BugzillaURL.one_time' + db.delete_column('scrum_bugzillaurl', 'one_time') + + + models = { + 'scrum.bug': { + 'Meta': {'ordering': "('id',)", 'object_name': 'Bug'}, + 'assigned_to': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'backlog': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backlog_bugs'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['scrum.Project']"}), + 'blocks': ('jsonfield.fields.JSONField', [], {'blank': 'True'}), + 'comments': ('scrum.models.CompressedJSONField', [], {'blank': 'True'}), + 'comments_count': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '0'}), + 'component': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'creation_time': ('django.db.models.fields.DateTimeField', [], {}), + 'depends_on': ('jsonfield.fields.JSONField', [], {'blank': 'True'}), + 'history': ('scrum.models.CompressedJSONField', [], {}), + 'id': ('django.db.models.fields.PositiveIntegerField', [], {'primary_key': 'True'}), + 'last_change_time': ('django.db.models.fields.DateTimeField', [], {}), + 'last_synced_time': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}), + 'priority': ('django.db.models.fields.CharField', [], {'max_length': '2', 'blank': 'True'}), + 'product': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bugs'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['scrum.Project']"}), + 'resolution': ('django.db.models.fields.CharField', [], {'max_length': '20', 'blank': 'True'}), + 'sprint': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bugs'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['scrum.Sprint']"}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '20'}), + 'story_component': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}), + 'story_points': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '0'}), + 'story_user': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}), + 'summary': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'whiteboard': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}) + }, + 'scrum.bugsprintlog': { + 'Meta': {'ordering': "('-timestamp',)", 'object_name': 'BugSprintLog'}, + 'action': ('django.db.models.fields.PositiveSmallIntegerField', [], {}), + 'bug': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'sprint_actions'", 'to': "orm['scrum.Bug']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sprint': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bug_actions'", 'to': "orm['scrum.Sprint']"}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}) + }, + 'scrum.bugzillaurl': { + 'Meta': {'ordering': "('id',)", 'object_name': 'BugzillaURL'}, + 'date_synced': ('django.db.models.fields.DateTimeField', [], {'default': "'2000-01-01'"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'one_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'urls'", 'null': 'True', 'to': "orm['scrum.Project']"}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '2048'}) + }, + 'scrum.project': { + 'Meta': {'object_name': 'Project'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'}), + 'team': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'projects'", 'null': 'True', 'to': "orm['scrum.Team']"}) + }, + 'scrum.sprint': { + 'Meta': {'ordering': "['-start_date']", 'unique_together': "(('team', 'slug'),)", 'object_name': 'Sprint'}, + 'bugs_data_cache': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'bz_url': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'created_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'end_date': ('django.db.models.fields.DateField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'notes_html': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'slug': ('django.db.models.fields.CharField', [], {'max_length': '200', 'db_index': 'True'}), + 'start_date': ('django.db.models.fields.DateField', [], {}), + 'team': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'sprints'", 'to': "orm['scrum.Team']"}) + }, + 'scrum.team': { + 'Meta': {'object_name': 'Team'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'}) + } + } + + complete_apps = ['scrum'] \ No newline at end of file diff --git a/scrum/models.py b/scrum/models.py index 2de2ba6..3ff2258 100644 --- a/scrum/models.py +++ b/scrum/models.py @@ -14,7 +14,7 @@ from django.db import models, transaction from django.db.models.query import QuerySet from django.db.models.query_utils import Q -from django.db.models.signals import pre_save, post_save +from django.db.models.signals import pre_save from django.dispatch import receiver from django.utils.encoding import force_unicode @@ -180,7 +180,8 @@ def get_bz_search_url(self, bugs=None): def refresh_bugs_data(self, bugs=None): bzurl = self.get_bz_search_url(bugs) - bzurl.get_bugs() + bzurl.one_time = True + bzurl.save() def get_bugs(self, **kwargs): kwargs['bug_filters'] = {'sprint__isnull': True} @@ -233,8 +234,9 @@ def date_cached(self): def refresh_backlog(self): self._clear_bugs_data_cache() - bugs = self._get_url_items('bugs') - self.update_backlog_bugs(bugs) + for url in self.urls.all(): + url.date_synced = '2000-01-01' + url.save() def get_backlog(self, **kwargs): """Get a unique set of bugs from all bz urls""" @@ -371,7 +373,8 @@ def get_edit_url(self): def refresh_bugs_data(self, bugs=None): self._clear_bugs_data_cache() bzurl = self.get_bz_search_url(bugs) - bzurl.get_bugs() + bzurl.one_time = True + bzurl.save() def update_bugs(self, bugs): """ @@ -419,14 +422,22 @@ def get_cached_bugs_data(self): class EmptyBugzillaURL(object): + one_time = False + def get_bugs(self): return set() + def save(self): + pass + class BugzillaURL(models.Model): url = models.URLField(verbose_name='Bugzilla URL', max_length=2048) project = models.ForeignKey(Project, null=True, blank=True, related_name='urls') + # default in the past + date_synced = models.DateTimeField(default='2000-01-01') + one_time = models.BooleanField(default=False) date_cached = None num_no_data_bugs = 0 @@ -465,9 +476,12 @@ def get_bugs(self): args = dict((k.encode('utf-8'), v) for k, v in args.iterlists()) data = BZAPI.bug.get(**args) - data['date_received'] = datetime.now() + data['date_received'] = datetime.utcnow() except Exception: raise BZError("Couldn't retrieve bugs from Bugzilla") + if not self.one_time: + self.date_synced = datetime.utcnow() + self.save() return set(store_bugs(data['bugs'], self.project)) def get_products(self): @@ -490,7 +504,8 @@ def sync_bugs(self): bids = self.only('id') if bids: url = BugzillaURL(url=get_bz_url_for_buglist(bids)) - url.get_bugs() + url.one_time = True + url.save() def scrum_only(self): return self.filter(~Q(story_component='') | @@ -782,12 +797,3 @@ def process_notes(sender, instance, **kwargs): output_format='html5', safe_mode=True, ) - - -@receiver(post_save, sender=BugzillaURL) -def get_bugzilla_bugs_data(sender, instance, **kwargs): - """ - After a `BugzillaURL` is saved, get the bugs data and associate it with - the url's project's backlog if available. - """ - instance.get_bugs() diff --git a/scrum/templates/scrum/project.html b/scrum/templates/scrum/project.html index 50c45ec..4d7f0a8 100644 --- a/scrum/templates/scrum/project.html +++ b/scrum/templates/scrum/project.html @@ -2,30 +2,33 @@ {% from "scrum/includes/macros.html" import bug_table with context %} {% block content %} - {% if bzerror %} - {% include "scrum/bzerror.html" %} - {% else %} - {% if perms.scrum.change_project %} - - Edit - - {% endif %} -

{{ project.name }} Project

-

Part of the {{ project.team.name }} team.

-
-

Currently Sprinting

- {{ bug_table(currently_sprinting, show_sprint=True, show_project=False) }} -
- {% if perms.scrum.change_project %} - - Manage Ready Stories - - {% endif %} -

Ready Backlog

- {{ bug_table(project.get_bugs(), 'backlog_table', show_project=False) }} - {% endif %} +
+
+ {% if perms.scrum.change_project %} + + Edit + + {% endif %} +

{{ project.name }} Project

+

Part of the {{ project.team.name }} team.

+
+
+
+
+

Currently Sprinting

+ {{ bug_table(sprinting, show_sprint=True, show_project=False, blocked_bugs=sprinting_blocked) }} +
+ {% if perms.scrum.change_project %} + + Manage Ready Stories + + {% endif %} +

Sprint Ready

+ {{ bug_table(bugs, 'backlog_table', show_project=False, blocked_bugs=blocked_bugs) }} +
+
{% endblock %} {% block js %} diff --git a/scrum/templates/scrum/project_bugs.html b/scrum/templates/scrum/project_bugs.html index 431d509..e8fb5ab 100644 --- a/scrum/templates/scrum/project_bugs.html +++ b/scrum/templates/scrum/project_bugs.html @@ -5,7 +5,6 @@
-

{{ project.name }} Stories

diff --git a/scrum/tests.py b/scrum/tests.py index ff22fd1..3b3513f 100644 --- a/scrum/tests.py +++ b/scrum/tests.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from copy import deepcopy -from datetime import date, timedelta +from datetime import date, timedelta, datetime from email.parser import Parser from mock import Mock, patch @@ -14,6 +14,7 @@ from django.test import TestCase from django.utils import simplejson as json +from scrum import cron as scrum_cron from scrum import email as scrum_email from scrum import models as scrum_models from scrum.forms import BZURLForm, CreateProjectForm, SprintBugsForm @@ -49,6 +50,32 @@ def get_messages_mock(delete=True): scrum_email.get_messages.side_effect = get_messages_mock +class TestCron(TestCase): + fixtures = ['test_data.json'] + + def setUp(self): + self.p = Project.objects.get(pk=1) + + def test_default_sync_date_urls_synced(self): + """Test that BugzillaURL objects with NULL sync dates are synced.""" + BugzillaURL.objects.create(url=GOOD_BZ_URL, project=self.p) + scrum_cron.sync_backlogs() + eq_(self.p.backlog_bugs.count(), 20) + + def test_recently_synced_urls_not_synced(self): + hour_ago = datetime.utcnow() - timedelta(hours=1) + url = BugzillaURL.objects.create(url=GOOD_BZ_URL, date_synced=hour_ago) + scrum_cron.sync_backlogs() + url = BugzillaURL.objects.get(id=url.id) + eq_(url.date_synced, hour_ago) + + def test_one_time_urls_deleted(self): + url = BugzillaURL.objects.create(url=GOOD_BZ_URL, one_time=True) + scrum_cron.sync_backlogs() + with self.assertRaises(BugzillaURL.DoesNotExist): + BugzillaURL.objects.get(id=url.id) + + class TestEmail(TestCase): def test_is_bugmail(self): m = scrum_email.get_messages()[0] @@ -78,6 +105,7 @@ def setUp(self): self.s = Sprint.objects.get(slug='2.2') self.p = Project.objects.get(pk=1) self.url = Project.objects.get(pk=1) + scrum_cron.sync_backlogs() @patch.object(Bug, 'points_history') def test_points_for_date_default(self, mock_bug): @@ -126,11 +154,11 @@ def test_refreshing_bugs_not_remove_from_sprint(self): def test_adding_bzurl_adds_backlog_bugs(self): """Adding a url to a project should populate the backlog.""" # should have created and loaded bugs when fixture loaded - eq_(self.p.backlog_bugs.count(), 20) self.p.backlog_bugs.clear() eq_(self.p.backlog_bugs.count(), 0) # now this should update the existing bugs w/ the backlog BugzillaURL.objects.create(url=GOOD_BZ_URL, project=self.p) + scrum_cron.sync_backlogs() eq_(self.p.backlog_bugs.count(), 20) @@ -141,6 +169,7 @@ def setUp(self): cache.clear() self.s = Sprint.objects.get(slug='2.2') self.p = Project.objects.get(pk=1) + scrum_cron.sync_backlogs() def test_sprint_creation(self): User.objects.create_superuser('admin', 'admin@admin.com', 'admin') diff --git a/scrum/utils.py b/scrum/utils.py index f914b02..04b5f30 100644 --- a/scrum/utils.py +++ b/scrum/utils.py @@ -19,7 +19,7 @@ # 'list_id', 'columnlist', ) -CLOSED_STATUSES = ['RESOLVED', 'VERIFIED'] +CLOSED_STATUSES = ['RESOLVED', 'VERIFIED', 'CLOSED'] def parse_whiteboard(wb): diff --git a/scrum/views.py b/scrum/views.py index 2a46e06..51b75ff 100644 --- a/scrum/views.py +++ b/scrum/views.py @@ -55,6 +55,8 @@ def get_context_data(self, **kwargs): # clear cache if requested if self.request.META.get('HTTP_CACHE_CONTROL') == 'no-cache': self.bugs_kwargs['refresh'] = True + messages.info(self.request, "The bugs will be refreshed from " + "Bugzilla in 10 minutes or so.") if 'all' in self.request.GET: self.bugs_kwargs['scrum_only'] = False try: @@ -91,7 +93,7 @@ class HomeView(TemplateView): home = HomeView.as_view() -class ProjectView(ProjectsMixin, DetailView): +class ProjectView(BugsDataMixin, ProjectsMixin, DetailView): template_name = 'scrum/project.html' def get_context_data(self, **kwargs): @@ -100,7 +102,8 @@ def get_context_data(self, **kwargs): bugs = Bug.objects.filter(sprint__start_date__lte=today, sprint__end_date__gte=today, project=self.object) - context['currently_sprinting'] = bugs + context['sprinting'] = bugs + context['sprinting_blocked'] = get_blocked_bugs(bugs) return context diff --git a/settings/base.py b/settings/base.py index 1ac2fb0..1915fa2 100644 --- a/settings/base.py +++ b/settings/base.py @@ -16,7 +16,7 @@ BZ_SHOW_URL = 'https://bugzilla.mozilla.org/show_bug.cgi?' BZ_FILE_URL = 'https://bugzilla.mozilla.org/enter_bug.cgi?' BZ_SEARCH_URL = 'https://bugzilla.mozilla.org/buglist.cgi?' -CACHE_BUGS_FOR = 2 # hours +CACHE_BUGS_FOR = 4 # hours SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db' MESSAGE_STORAGE = 'django.contrib.messages.storage.fallback.FallbackStorage'