Permalink
Browse files

Merge pull request #1903 from lonnen/957301-bugzilla-cron

bug 957301 - refactor bugzilla cron
  • Loading branch information...
lonnen committed Feb 22, 2014
2 parents 059de23 + 81ff5b2 commit 68ebf1d0febe2184c3b77cc44c416ab928f163fe
Showing with 82 additions and 66 deletions.
  1. +4 −2 requirements.txt
  2. +74 −63 socorro/cron/jobs/bugzilla.py
  3. +4 −1 socorro/unittest/cron/jobs/test_bugzilla.py
View
@@ -62,8 +62,8 @@ requests==1.2.3
simplejson==2.5.0
# sha256: 8EWv1t_7dVzAQRrLfOmsxN4OcSYdS1-R3i5o2apfg2c
six==1.4.1
-# sha256: 1coRGB3yjavWHUKc4SBs0C2xubGAV_WiSsLowvY6nZo
-statsd==0.5.1
+# sha256: RdsWgxKmgg_XhpfHtv0BlvOvkiyOvIhUDt0Nb6_tVU0
+statsd==2.1.2
# sha256: 1VJK5SO7ngnFe829HvriwofSBgNojqMfYCDtGApImvA
suds==0.4
# sha256: gLPAyGSc5ZC9JYcVUPZnDqvi6YpdHvXkD5K4hZfYCB0
@@ -74,3 +74,5 @@ web.py==0.36
wsgiref==0.1.2
# sha256: aM-CXyJ8guGsYeQjz8rZI_9zTCe1vdcXRJXRYsQsYCs
ujson==1.33
+# sha256: TETsP5_wV7jHtMeL7KX92HEGAOqaHfQvMb_Lri8Fne4
+python-dateutil==2.1
@@ -5,24 +5,27 @@
import datetime
import urllib2
import csv
+
+from dateutil import tz
from configman import Namespace
-from socorro.lib.datetimeutil import utc_now
from socorro.database.database import singleRowSql, SQLDidNotReturnSingleRow
from socorro.cron.base import PostgresTransactionManagedCronApp
-_URL = ('https://bugzilla.mozilla.org/buglist.cgi?query_format=advanced&short_d'
- 'esc_type=allwordssubstr&short_desc=&long_desc_type=allwordssubstr&long'
- '_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&status_whiteboar'
- 'd_type=allwordssubstr&status_whiteboard=&keywords_type=allwords&keywor'
- 'ds=&deadlinefrom=&deadlineto=&emailassigned_to1=1&emailtype1=substring'
- '&email1=&emailassigned_to2=1&emailreporter2=1&emailqa_contact2=1&email'
- 'cc2=1&emailtype2=substring&email2=&bugidtype=include&bug_id=&votes=&ch'
- 'fieldfrom=%s&chfieldto=Now&chfield=[Bug+creation]&chfield=resolution&c'
- 'hfield=bug_status&chfield=short_desc&chfield=cf_crash_signature&chfiel'
- 'dvalue=&cmdtype=doit&order=Importance&field0-0-0=noop&type0-0-0=noop&v'
- 'alue0-0-0=&columnlist=bug_id,bug_status,resolution,short_desc,cf_crash'
- '_signature&ctype=csv')
+_URL = (
+ 'https://bugzilla.mozilla.org/buglist.cgi?query_format=advanced&short_'
+ 'desc_type=allwordssubstr&short_desc=&long_desc_type=allwordssubstr&lo'
+ 'ng_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&status_whiteb'
+ 'oard_type=allwordssubstr&status_whiteboard=&keywords_type=allwords&ke'
+ 'ywords=&deadlinefrom=&deadlineto=&emailassigned_to1=1&emailtype1=subs'
+ 'tring&email1=&emailassigned_to2=1&emailreporter2=1&emailqa_contact2=1'
+ '&emailcc2=1&emailtype2=substring&email2=&bugidtype=include&bug_id=&vo'
+ 'tes=&chfieldfrom=%s&chfieldto=Now&chfield=[Bug+creation]&chfield=reso'
+ 'lution&chfield=bug_status&chfield=short_desc&chfield=cf_crash_signatu'
+ 're&chfieldvalue=&cmdtype=doit&order=Importance&field0-0-0=noop&type0-'
+ '0-0=noop&value0-0-0=&columnlist=bug_id,bug_status,resolution,short_de'
+ 'sc,cf_crash_signature&ctype=csv'
+)
class BugzillaCronApp(PostgresTransactionManagedCronApp):
@@ -51,14 +54,16 @@ def run(self, connection):
# TypeError if self.job_information is None
last_run = self.job_information['last_success']
except (KeyError, TypeError):
- last_run = utc_now() - datetime.timedelta(
- days=self.config.days_into_past and
- self.config.days_into_past or
- 30)
- last_run_formatted = last_run.strftime('%Y-%m-%d')
+ last_run = (datetime.datetime.now(tz.gettz('UTC')) -
+ datetime.timedelta(days=self.config.days_into_past))
+
+ # bugzilla runs on PST, so we need to communicate in its time zone
+ PST = tz.gettz('PST8PDT')
+ last_run_formatted = last_run.astimezone(PST).strftime('%Y-%m-%d')
query = self.config.query % last_run_formatted
cursor = connection.cursor()
- for bug_id, status, resolution, short_desc, signature_set in self._iterator(query):
+ for i in self._iterator(query):
+ bug_id, status, resolution, short_desc, signature_set = i
logger.debug(
"bug %s (%s, %s) %s: %s",
bug_id, status, resolution, short_desc, signature_set)
@@ -71,66 +76,74 @@ def run(self, connection):
insert_made = False
try:
status_db, resolution_db, short_desc_db = singleRowSql(
- cursor, """
+ cursor,
+ """
SELECT status, resolution, short_desc
FROM bugs
WHERE id = %s
""",
- (bug_id,)
- )
- if status_db != status or resolution_db != resolution or short_desc_db != short_desc:
- cursor.execute("""
- UPDATE bugs SET
- status = %s, resolution = %s, short_desc = %s
- WHERE id = %s""",
- (status, resolution, short_desc, bug_id)
- )
+ (bug_id,))
+ if (status_db != status or resolution_db != resolution
+ or short_desc_db != short_desc):
+ cursor.execute(
+ """
+ UPDATE bugs SET
+ status = %s, resolution = %s, short_desc = %s
+ WHERE id = %s
+ """,
+ (status, resolution, short_desc, bug_id))
logger.info("bug status updated: %s - %s, %s",
bug_id, status, resolution)
useful = True
- cursor.execute("""
- SELECT signature FROM bug_associations WHERE bug_id = %s""",
+ cursor.execute(
+ """
+ SELECT signature FROM bug_associations WHERE bug_id = %s
+ """,
(bug_id,)
)
signatures_db = [x[0] for x in cursor.fetchall()]
for signature in signatures_db:
if signature not in signature_set:
- cursor.execute("""
- DELETE FROM bug_associations
- WHERE signature = %s and bug_id = %s""",
- (signature, bug_id)
- )
- logger.info('association removed: %s - "%s"', bug_id, signature)
+ cursor.execute(
+ """
+ DELETE FROM bug_associations
+ WHERE signature = %s and bug_id = %s
+ """,
+ (signature, bug_id))
+ logger.info('association removed: %s - "%s"',
+ bug_id, signature)
useful = True
except SQLDidNotReturnSingleRow:
- cursor.execute("""
- INSERT INTO bugs
- (id, status, resolution, short_desc)
- VALUES (%s, %s, %s, %s)""",
- (bug_id, status, resolution, short_desc)
- )
+ cursor.execute(
+ """
+ INSERT INTO bugs
+ (id, status, resolution, short_desc)
+ VALUES (%s, %s, %s, %s)
+ """,
+ (bug_id, status, resolution, short_desc))
insert_made = True
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
if self._has_signature_report(signature, cursor):
- cursor.execute("""
- INSERT INTO bug_associations (signature, bug_id)
- VALUES (%s, %s)""",
- (signature, bug_id)
- )
+ cursor.execute(
+ """
+ INSERT INTO bug_associations (signature, bug_id)
+ VALUES (%s, %s)
+ """,
+ (signature, bug_id))
logger.info('new association: %s - "%s"',
bug_id, signature)
useful = True
else:
logger.info(
- 'rejecting association (no reports with this '
- 'signature): %s - "%s"',
- bug_id,
- signature)
+ 'rejecting association (no reports with this '
+ 'signature): %s - "%s"',
+ bug_id,
+ signature)
if useful:
connection.commit()
@@ -149,15 +162,13 @@ def run(self, connection):
bug_id, status, resolution, short_desc)
def _iterator(self, query):
- ##assert query.startswith('file://'), query## DEBUGGGINGG
- opener = urllib2.urlopen
- for report in csv.DictReader(opener(query)):
+ for report in csv.DictReader(urllib2.urlopen(query)):
yield (
- int(report['bug_id']),
- report['bug_status'],
- report['resolution'],
- report['short_desc'],
- self._signatures_found(report['cf_crash_signature'])
+ int(report['bug_id']),
+ report['bug_status'],
+ report['resolution'],
+ report['short_desc'],
+ self._signatures_found(report['cf_crash_signature'])
)
def _signatures_found(self, signature):
@@ -178,9 +189,9 @@ def _signatures_found(self, signature):
def _has_signature_report(self, signature, cursor):
try:
- singleRowSql(cursor, """
- SELECT 1 FROM reports
- WHERE signature = %s LIMIT 1""",
+ singleRowSql(
+ cursor,
+ """SELECT 1 FROM reports WHERE signature = %s LIMIT 1""",
(signature,)
)
return True
@@ -5,6 +5,7 @@
import datetime
import os
from nose.plugins.attrib import attr
+from dateutil import tz
from socorro.cron import crontabber
from ..base import IntegrationTestCaseBase
@@ -35,8 +36,10 @@ def tearDown(self):
super(IntegrationTestBugzilla, self).tearDown()
def _setup_config_manager(self, days_into_past):
- datestring = ((datetime.datetime.utcnow() -
+ PST = tz.gettz('PST8PDT')
+ datestring = ((datetime.datetime.now(PST) -
datetime.timedelta(days=days_into_past))
+ .astimezone(PST)
.strftime('%Y-%m-%d'))
filename = os.path.join(self.tempdir, 'sample-%s.csv' % datestring)
with open(filename, 'w') as f:

0 comments on commit 68ebf1d

Please sign in to comment.