From a941d4f2b33a841b1374a7c1ce8d78e4715d64d2 Mon Sep 17 00:00:00 2001 From: calixteman Date: Tue, 21 May 2019 14:54:15 +0200 Subject: [PATCH] Migrate to python3 + cleanup (#602) --- .gitignore | 6 +- .travis.yml | 6 +- MANIFEST.in | 2 - auto_nag/bugzilla/__init__.py | 19 - auto_nag/bugzilla/agents.py | 63 --- auto_nag/bugzilla/fields.py | 23 - auto_nag/bugzilla/models.py | 275 ---------- auto_nag/bugzilla/settings.py | 2 - auto_nag/bugzilla/utils.py | 196 ------- auto_nag/common.py | 34 -- auto_nag/escalation.py | 2 - auto_nag/next_release.py | 2 +- auto_nag/scripts/attach.py | 145 ------ auto_nag/scripts/email_nag.py | 489 ------------------ .../mismatch_priority_tracking_beta.py | 4 +- .../scripts/mismatch_priority_tracking_esr.py | 3 +- .../mismatch_priority_tracking_nightly.py | 4 +- .../mismatch_priority_tracking_release.py | 4 +- auto_nag/scripts/missing_beta_status.py | 11 +- auto_nag/scripts/no_crashes.py | 2 +- auto_nag/scripts/phonebook.py | 94 ---- auto_nag/scripts/tracked_needinfo.py | 5 +- auto_nag/scripts/tracking.py | 2 +- auto_nag/scripts/unlanded.py | 9 + auto_nag/tests/_get_credential.py | 11 - auto_nag/tests/people.json | 18 - auto_nag/tests/templates/channel_meeting_wiki | 27 - auto_nag/tests/templates/daily_email | 18 - auto_nag/tests/test_agents.py | 78 --- auto_nag/tests/test_bzcleaner.py | 8 +- auto_nag/tests/test_email_nag.py | 62 --- auto_nag/tests/test_mail.py | 6 +- auto_nag/tests/test_models.py | 168 ------ auto_nag/tests/test_phonebook.py | 40 -- auto_nag/tests/test_round_robin.py | 2 +- auto_nag/tests/test_utils.py | 82 --- auto_nag/utils.py | 57 +- requirements.txt | 6 - runauto_nag_daily.sh | 113 +++- runauto_nag_daily_py2.sh | 104 ---- runauto_nag_daily_py3.sh | 25 - runauto_nag_hourly.sh | 91 +++- runauto_nag_hourly_py2.sh | 84 --- runauto_nag_hourly_py3.sh | 20 - setup.py | 36 -- templates/channel_meeting_wiki | 28 - templates/common.html | 9 - templates/daily_email | 21 - templates/tracked_affected_email | 16 - templates/unlanded.html | 2 +- tox.ini | 2 +- 51 files changed, 296 insertions(+), 2240 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 auto_nag/bugzilla/__init__.py delete mode 100644 auto_nag/bugzilla/agents.py delete mode 100644 auto_nag/bugzilla/fields.py delete mode 100644 auto_nag/bugzilla/models.py delete mode 100644 auto_nag/bugzilla/settings.py delete mode 100644 auto_nag/bugzilla/utils.py delete mode 100644 auto_nag/common.py delete mode 100644 auto_nag/scripts/attach.py delete mode 100755 auto_nag/scripts/email_nag.py delete mode 100644 auto_nag/scripts/phonebook.py delete mode 100644 auto_nag/tests/_get_credential.py delete mode 100644 auto_nag/tests/people.json delete mode 100644 auto_nag/tests/templates/channel_meeting_wiki delete mode 100644 auto_nag/tests/templates/daily_email delete mode 100644 auto_nag/tests/test_agents.py delete mode 100644 auto_nag/tests/test_email_nag.py delete mode 100644 auto_nag/tests/test_models.py delete mode 100644 auto_nag/tests/test_phonebook.py delete mode 100644 auto_nag/tests/test_utils.py delete mode 100755 runauto_nag_daily_py2.sh delete mode 100755 runauto_nag_daily_py3.sh delete mode 100755 runauto_nag_hourly_py2.sh delete mode 100755 runauto_nag_hourly_py3.sh delete mode 100644 setup.py delete mode 100644 templates/channel_meeting_wiki delete mode 100644 templates/daily_email delete mode 100644 templates/tracked_affected_email diff --git a/.gitignore b/.gitignore index d2f838111..b48e976a3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,10 +2,6 @@ *.pyc *~ .DS_Store -build/ -bztools.egg-info/ -dist/ -queries/ scripts/configs/config.json auto_nag/scripts/configs/* !auto_nag/scripts/configs/tools.json @@ -18,3 +14,5 @@ venv3/ db/* !db/lock cache/ +.coverage +.tox/ diff --git a/.travis.yml b/.travis.yml index 040d8d81a..9f7a4978e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ -sudo: required +dist: xenial language: python env: - - TOX_ENV=py27 + - TOX_ENV=py37 python: - - "2.7" + - "3.7" install: - travis_retry pip install -r requirements-test.txt diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9d5d250d0..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include LICENSE -include README.rst diff --git a/auto_nag/bugzilla/__init__.py b/auto_nag/bugzilla/__init__.py deleted file mode 100644 index 35e03662e..000000000 --- a/auto_nag/bugzilla/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -try: - import httplib - from remoteobjects import http - - # Printing throws an error if we are printing using ascii - import sys - __all__ = ['models', 'utils', 'agents'] - reload(sys) - sys.setdefaultencoding('utf-8') - - # Monkey patch remoteobjects to accept 202 status codes. - http.HttpObject.response_has_content[httplib.ACCEPTED] = False -except ImportError: - # Python 3 - pass - - -VERSION = (0, 0, 1) -__version__ = '.'.join(map(str, VERSION)) diff --git a/auto_nag/bugzilla/agents.py b/auto_nag/bugzilla/agents.py deleted file mode 100644 index 24f71a7f6..000000000 --- a/auto_nag/bugzilla/agents.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import httplib2 -import urlparse -from auto_nag.bugzilla.models import BugSearch -from auto_nag.bugzilla.utils import urljoin, qs, hide_personal_info - - -class Http(httplib2.Http): - def __init__(self, api_key=None): - super(Http, self).__init__() - self.api_key = api_key - self.proxy_info = None - - def request(self, uri, method='GET', body=None, headers=None, redirections=5, connection_type=None): - if headers is None: - headers = {} - if 'user-agent' not in headers: - headers['user-agent'] = 'relman-auto-nag' - if self.api_key is not None: - scheme, netloc, path, query, frag = urlparse.urlsplit(uri) - if 'api_key' not in urlparse.parse_qs(query): - query += '&api_key=%s' % self.api_key - uri = urlparse.urlunsplit((scheme, netloc, path, query, frag)) - return super(Http, self).request(uri, method=method, body=body, headers=headers, redirections=redirections, connection_type=connection_type) - - -class InvalidAPI_ROOT(Exception): - def __str__(self): - return "Invalid API url specified. " + \ - "Please set BZ_API_ROOT in your environment " + \ - "or pass it to the agent constructor" - - -class BugzillaAgent(object): - def __init__(self, api_root=None, api_key=None): - if not api_root: - api_root = os.environ.get('BZ_API_ROOT') - if not api_root: - raise InvalidAPI_ROOT - self.API_ROOT = api_root - self.http = Http(api_key) - - def get_bug(self, bug, include_fields='_default', exclude_fields=None, params={}): - params['include_fields'] = [include_fields] - params['exclude_fields'] = [exclude_fields] - - url = urljoin(self.API_ROOT, 'bug/%s?%s' % (bug, qs(**params))) - try: - return BugSearch.get(url, http=self.http).bugs[0] - except Exception as e: - raise Exception(hide_personal_info(str(e))) - - def get_bug_list(self, params={}): - url = urljoin(self.API_ROOT, 'bug?%s' % (qs(**params))) - try: - return BugSearch.get(url, http=self.http).bugs - except Exception as e: - raise Exception(hide_personal_info(str(e))) - - -class BMOAgent(BugzillaAgent): - def __init__(self, api_key=None): - super(BMOAgent, self).__init__('https://bugzilla.mozilla.org/rest/', api_key) diff --git a/auto_nag/bugzilla/fields.py b/auto_nag/bugzilla/fields.py deleted file mode 100644 index 389879b82..000000000 --- a/auto_nag/bugzilla/fields.py +++ /dev/null @@ -1,23 +0,0 @@ -from datetime import datetime - -from remoteobjects import fields -import dateutil.parser - - -class StringBoolean(fields.Field): - """Decodes a boolean hidden in a string.""" - - def decode(self, value): - return bool(int(value)) - - -class Datetime(fields.Datetime): - """Uses python-dateutil for working with datetimes.""" - - def decode(self, value): - return dateutil.parser.parse(value) - - def encode(self, value): - if not isinstance(value, datetime): - raise TypeError('Value to encode %r is not a datetime' % (value,)) - return value.replace(microsecond=0).strftime(self.dateformat) diff --git a/auto_nag/bugzilla/models.py b/auto_nag/bugzilla/models.py deleted file mode 100644 index abe9f05bc..000000000 --- a/auto_nag/bugzilla/models.py +++ /dev/null @@ -1,275 +0,0 @@ -import urlparse - -from remoteobjects import RemoteObject as RemoteObject_, fields - -from .fields import StringBoolean, Datetime -from utils import getVersions - -# The datetime format is inconsistent. -DATETIME_FORMAT_WITH_SECONDS = '%Y-%m-%d %H:%M:%S %z' -DATETIME_FORMAT = '%Y-%m-%d %H:%M %Z' - - -class RemoteObject(RemoteObject_): - - def post_to(self, url): - self._location = url - self.post(self) - return self.api_data['ref'] - - def put_to(self, url): - self._location = url - self.put() - - def _get_location(self): - if self.__location is not None: - return self.__location - else: - return self.api_data.get('ref', None) - - def _set_location(self, url): - self.__location = url - - _location = property(_get_location, _set_location) - - -class Comments(fields.List): - """This is a bit of a mix between Link and List, retrieving the comments - from an API call and then turning the returned data into a list of Comment - objets""" - def __get__(self, instance, owner): - if instance is None: - return self - if self.attrname not in instance.__dict__: - if instance._location is None: - raise AttributeError('Cannot find URL of %s relative to URL-less %s' - % (self.fld.cls.__name__, owner.__name__)) - newurl = urlparse.urljoin(instance._location + '/', self.api_name) - value = RemoteObject.get(newurl, http=instance._http) - value = value.api_data['bugs'][str(instance.id)]['comments'] - value = self.decode(value) - instance.__dict__[self.attrname] = value - return instance.__dict__[self.attrname] - - -class Bug(RemoteObject): - id = fields.Field() - summary = fields.Field() - assigned_to = fields.Object('User', api_name='assigned_to_detail') - creator = fields.Object('User', api_name='creator_detail') - target_milestone = fields.Field() - attachments = fields.List(fields.Object('Attachment')) - comments = Comments(fields.Object('Comment'), api_name='comment') - history = fields.List(fields.Object('Changeset')) - keywords = fields.List(fields.Object('Keyword')) - status = fields.Field() - resolution = fields.Field() - - # TODO: These are Mozilla specific and should be generalized - cf_blocking_20 = fields.Field() - cf_blocking_fennec = fields.Field() - cf_crash_signature = fields.Field() - - creation_time = Datetime(DATETIME_FORMAT_WITH_SECONDS) - flags = fields.List(fields.Object('Flag')) - blocks = fields.List(fields.Field()) - depends_on = fields.List(fields.Field()) - url = fields.Field() - cc = fields.List(fields.Object('User'), api_name='cc_detail') - keywords = fields.List(fields.Field()) - whiteboard = fields.Field() - - op_sys = fields.Field() - platform = fields.Field() - priority = fields.Field() - product = fields.Field() - qa_contact = fields.Object('User', api_name='qa_contact_detail') - severity = fields.Field() - see_also = fields.List(fields.Field()) - version = fields.Field() - - alias = fields.Field() - classification = fields.Field() - component = fields.Field() - is_cc_accessible = StringBoolean() - is_everconfirmed = StringBoolean() - is_creator_accessible = StringBoolean() - last_change_time = Datetime(DATETIME_FORMAT_WITH_SECONDS) - ref = fields.Field() - - # Needed for submitting changes. - token = fields.Field(api_name='update_token') - - # Time tracking. - actual_time = fields.Field() - deadline = Datetime(DATETIME_FORMAT_WITH_SECONDS) - estimated_time = fields.Field() - groups = fields.List(fields.Field()) - percentage_complete = fields.Field() - remaining_time = fields.Field() - work_time = fields.Field() - - def __repr__(self): - return '' % (self.id, self.summary) - - def __str__(self): - return "[Bug %s] - %s" % (self.id, self.summary) - - def __hash__(self): - return self.id - - def get_fx_affected_versions(self): - affected = [] - for version in getVersions(): - if "esr" in version: - version = "_" + version - if ('cf_status_firefox' + version in self.api_data and - self.api_data['cf_status_firefox' + version] == 'affected'): - affected.append(version) - - return affected - - -class User(RemoteObject): - - name = fields.Field() - real_name = fields.Field() - ref = fields.Field() - - def __repr__(self): - return '' % self.real_name - - def __str__(self): - return self.real_name or self.name - - def __hash__(self): - if not self or not self.name: - return 0 - return self.name.__hash__() - - -class Attachment(RemoteObject): - - # Attachment data. - id = fields.Field() - attacher = fields.Object('User') - creation_time = Datetime(DATETIME_FORMAT_WITH_SECONDS) - last_change_time = Datetime(DATETIME_FORMAT_WITH_SECONDS) - description = fields.Field() - bug_id = fields.Field() - bug_ref = fields.Field() - - # File data. - file_name = fields.Field() - size = fields.Field() - content_type = fields.Field() - - # Attachment metadata. - flags = fields.List(fields.Object('Flag')) - is_obsolete = StringBoolean() - is_private = StringBoolean() - is_patch = StringBoolean() - - # Used for submitting changes. - token = fields.Field() - ref = fields.Field() - - # Only with attachmentdata=1 - data = fields.Field() - encoding = fields.Field() - - def __repr__(self): - return '' % (self.id, self.description) - - def __hash__(self): - return self.id - - -class Comment(RemoteObject): - - id = fields.Field() - creator = fields.Field() - creation_time = Datetime(DATETIME_FORMAT_WITH_SECONDS) - text = fields.Field() - is_private = StringBoolean() - - def __repr__(self): - return '' % ( - self.creator, self.creation_time.strftime(DATETIME_FORMAT)) - - def __str__(self): - return self.text - - def __hash__(self): - return self.id - - -class Change(RemoteObject): - - field_name = fields.Field() - added = fields.Field() - removed = fields.Field() - - def __repr__(self): - return ' "%s">' % (self.field_name, self.removed, - self.added) - - -class Changeset(RemoteObject): - - changer = fields.Object('User') - changes = fields.List(fields.Object('Change')) - change_time = Datetime(DATETIME_FORMAT_WITH_SECONDS) - - def __repr__(self): - return '' % ( - self.changer, self.change_time.strftime(DATETIME_FORMAT)) - - -class Flag(RemoteObject): - - id = fields.Field() - name = fields.Field() - setter = fields.Field() - status = fields.Field() - requestee = fields.Field() - type_id = fields.Field() - - def __repr__(self): - return '' % self.name - - def __str__(self): - return self.name - - def __hash__(self): - return self.id - - -class Keyword(RemoteObject): - - name = fields.Field() - - def __repr__(self): - return '' % self.name - - def __str__(self): - return self.name - - def __hash__(self): - if not self or not self.name: - return 0 - return self.name.__hash__() - - -class BugList(fields.List): - def __get__(self, obj, cls): - ret = super(BugList, self).__get__(obj, cls) - for bug in ret: - bug._location = urlparse.urljoin(obj._location, str(bug.id)) - bug._http = obj._http - return ret - - -class BugSearch(RemoteObject): - - bugs = BugList(fields.Object('Bug')) diff --git a/auto_nag/bugzilla/settings.py b/auto_nag/bugzilla/settings.py deleted file mode 100644 index ff4dfae72..000000000 --- a/auto_nag/bugzilla/settings.py +++ /dev/null @@ -1,2 +0,0 @@ -API_ROOT = "https://bugzilla.mozilla.org/" -BZ_API_ROOT = API_ROOT + "rest/" diff --git a/auto_nag/bugzilla/utils.py b/auto_nag/bugzilla/utils.py deleted file mode 100644 index 4d9fa3dd1..000000000 --- a/auto_nag/bugzilla/utils.py +++ /dev/null @@ -1,196 +0,0 @@ -import base64 -try: - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser -import getpass -import os -import re -import posixpath -try: - from urllib.parse import quote -except ImportError: - from urllib import quote -import datetime -import requests -from auto_nag.common import get_current_versions - - -def get_project_root_path(): - """ - Get project root path - return:string: Project ROOT folder - """ - pwd = os.getcwd() - root_dir = pwd.split('auto_nag')[0] - if root_dir[-1] != '/': - # if the current dir is the ROOT, add / - root_dir += '/' - return root_dir - - -def get_config_path(): - """ - Get config path location - return:string: config file location - """ - return get_project_root_path() + 'auto_nag/scripts/configs/config.json' - - -def urljoin(base, *args): - """Remove any leading slashes so no subpaths look absolute.""" - return posixpath.join(base, *[str(s).lstrip('/') for s in args]) - - -def hide_personal_info(error): - """ Hides bugzilla user information from remoteobject error""" - pattern = re.compile( - r"https://bugzilla.mozilla.org*.+&api_key=(.*?)&") - try: - api_key = pattern.findall(error)[0] - error_msg = error.replace(api_key, '*' * len(api_key)) - except IndexError: - error_msg = error - return error_msg - - -def qs(**kwargs): - """Build a URL query string.""" - url = '' - for k, v in kwargs.iteritems(): - if k == 'username' or k == 'password': - pass - for value in v: - url += '&%s=%s' % (quote(k), value) - return url - - -def get_credentials(username=None): - - # Try to get it from the environment first - if not username: - username = os.environ.get('BZ_USERNAME', None) - password = os.environ.get('BZ_PASSWORD', None) - - # Try to get it from the system keychain next - if not username and not password: - try: - import keyring - if not username: - # Grab the default username as we weren't passed in a specific one - username = keyring.get_password("bugzilla", 'default_username') - if username: - # Get the password for the username - password = keyring.get_password("bugzilla", username) - except ImportError: - # If they don't have the keyring lib, fall back to next method - pass - - # Then try a config file in their home directory - if not (username and password): - rcfile = os.path.expanduser('~/.bztoolsrc') - config = ConfigParser() - config.add_section('bugzilla') - if os.path.exists(rcfile): - try: - config.read(rcfile) - username = config.get('bugzilla', 'username') - _password = config.get('bugzilla', 'password') - if _password: - password = base64.b64decode(_password) - except Exception: - pass - - # Finally, prompt the user for the info if we didn't get it above - if not (username and password): - username = raw_input('Bugzilla username: ') - password = getpass.getpass('Bugzilla password: ') - try: - # Save the data to the keyring if possible - import keyring - keyring.set_password("bugzilla", 'default_username', username) - keyring.set_password("bugzilla", username, password) - except ImportError: - # Otherwise save it to a config file - config.set('bugzilla', 'username', username) - config.set('bugzilla', 'password', base64.b64encode(password)) - with open(rcfile, 'wb') as configfile: - config.write(configfile) - - return username, password - - -FILE_TYPES = { - 'text': 'text/plain', - 'html': 'text/html', - 'xml': 'application/xml', - 'gif': 'image/gif', - 'jpg': 'image/jpeg', - 'png': 'image/png', - 'svg': 'image/svg+xml', - 'binary': 'application/octet-stream', - 'xul': 'application/vnd.mozilla.xul+xml', -} - - -def createQuery(queries_dir, title, short_title, url): - file_name = queries_dir + str(datetime.date.today()) + '_' + short_title - if not os.path.exists(queries_dir): - os.makedirs(queries_dir) - qf = open(file_name, 'w') - qf.write("query_name = \'" + title + "\'\n") - qf.write("query_url = \'" + url + "\'\n") - return file_name - - -def createQueriesList(queries_dir, weekday, urls, print_all): - queries = [] - for url in urls: - if weekday >= 0 and weekday < 5 and url[0] == 5: - queries.append(createQuery(queries_dir, title=url[1][0], short_title=url[1][1], url=url[1][2])) - if weekday == 0 and url[0] == 0: - queries.append(createQuery(queries_dir, title=url[1][0], short_title=url[1][1], url=url[1][2])) - if weekday == 3 and url[0] == 3: - queries.append(createQuery(queries_dir, title=url[1][0], short_title=url[1][1], url=url[1][2])) - print(queries) - return queries - - -def cleanUp(queries_dir): - try: - for file in os.listdir(queries_dir): - if file.startswith(str(datetime.date.today())): - os.remove(os.path.join(queries_dir, file)) - return True - except Exception as error: - print("Error: ", str(error)) - return False - - -def __getTemplateValue(url): - version_regex = re.compile(".*

(.*)

.*") - template_page = str(requests.get(url).text.encode('utf-8')).replace('\n', '') - parsed_template = version_regex.match(template_page) - return parsed_template.groups()[0] - - -versions = get_current_versions() -release_version = versions['release'] -beta_version = versions['beta'] -central_version = versions['central'] -esr_version = versions['esr'] - - -def getVersions(channel=None): - if channel and isinstance(channel, basestring): - channel = channel.lower() - if channel == 'release': - return release_version - elif channel == 'beta': - return beta_version - elif channel == 'central': - return central_version - elif channel == 'esr': - return esr_version - - return (release_version, beta_version, central_version, esr_version) diff --git a/auto_nag/common.py b/auto_nag/common.py deleted file mode 100644 index d09b6cb12..000000000 --- a/auto_nag/common.py +++ /dev/null @@ -1,34 +0,0 @@ -import json -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - - -def loadJSON(url): - return json.load(urlopen(url)) - - -def getVersion(jsonVersion, key): - # In X.Y, we just need X - version = jsonVersion[key].split(".") - return version[0] - - -def get_current_versions(): - jsonContent = loadJSON("https://product-details.mozilla.org/1.0/firefox_versions.json") - esr_next_version = getVersion(jsonContent, "FIREFOX_ESR_NEXT") - if esr_next_version: - esr_version = esr_next_version - else: - # We are in a cycle where we don't have esr_next - # For example, with 52.6, esr_next doesn't exist - # But it will exist, once 60 is released - # esr_next will be 60 - esr_version = getVersion(jsonContent, "FIREFOX_ESR") - - return {"central": getVersion(jsonContent, "FIREFOX_NIGHTLY"), - "beta": getVersion(jsonContent, "LATEST_FIREFOX_DEVEL_VERSION"), - "esr": esr_version, - "release": getVersion(jsonContent, "LATEST_FIREFOX_VERSION"), - } diff --git a/auto_nag/escalation.py b/auto_nag/escalation.py index 19949a7de..fc26b571b 100644 --- a/auto_nag/escalation.py +++ b/auto_nag/escalation.py @@ -26,8 +26,6 @@ def is_in(self, x): @staticmethod def from_string(s): - # TODO: remove this line when switchin to python3 - s = s.encode('utf-8') mat = RANGE_PAT.match(s) if mat: m = int(mat.group(1)) diff --git a/auto_nag/next_release.py b/auto_nag/next_release.py index 31dc02085..e7c6a4795 100644 --- a/auto_nag/next_release.py +++ b/auto_nag/next_release.py @@ -29,7 +29,7 @@ def check_dates(dryrun=False): pat = re.compile(r'

(.*)

', re.DOTALL) url = 'https://wiki.mozilla.org/Template:NextReleaseDate' - template_page = str(requests.get(url).text.encode('utf-8')) + template_page = str(requests.get(url).text) m = pat.search(template_page) date = dateutil.parser.parse(m.group(1).strip()) date = pytz.utc.localize(date) diff --git a/auto_nag/scripts/attach.py b/auto_nag/scripts/attach.py deleted file mode 100644 index 0b52e11c2..000000000 --- a/auto_nag/scripts/attach.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python - -import base64 -import itertools -import os -import argparse - -from auto_nag.bugzilla.models import Attachment, Flag, Comment -from auto_nag.bugzilla.agents import BugzillaAgent -from auto_nag.bugzilla.utils import urljoin, get_credentials, FILE_TYPES - -REVIEW = 4 - - -class AttachmentAgent(BugzillaAgent): - """Stores credentials, navigates the site.""" - - def attach(self, bug_id, filename, description, patch=False, - reviewer=None, comment='', content_type='text/plain'): - """Create an attachment, add a comment, obsolete other attachments.""" - - print('Adding "%s" to %s' % (filename, bug_id)) - self._attach(bug_id, filename, description, patch, - reviewer, content_type) - - bug = self.get_bug(bug_id) - - if comment: - print('Adding the comment') - self._comment(bug_id, comment) - - print('Finding attachments to make obsolete...') - self.obsolete(bug) - - def _attach(self, bug_id, filename, description, is_patch=False, - reviewer=None, content_type='text/plain'): - """Create a new attachment.""" - fields = { - 'data': base64.b64encode(open(filename).read()), - 'encoding': 'base64', - 'file_name': filename, - 'content_type': content_type, - 'description': description, - 'is_patch': is_patch, - } - - if reviewer is not None: - fields['flags'] = [Flag(type_id=REVIEW, status='?', - requestee=reviewer)] - - url = urljoin(self.API_ROOT, 'bug/%s/attachment?%s' % (bug_id, self.qs())) - return Attachment(**fields).post_to(url) - - def _comment(self, bug_id, comment): - """Create a new comment.""" - url = urljoin(self.API_ROOT, 'bug/%s/comment?%s' % (bug_id, self.qs())) - return Comment(text=comment).post_to(url) - - def obsolete(self, bug): - """Ask what attachments should be obsoleted.""" - attachments = [a for a in bug.attachments - if not bool(int(a.is_obsolete))] - - if not attachments: - return - - print("What attachments do you want to obsolete?") - msg = '[{index}] {a.id}: "{a.description}" ({a.file_name})' - for index, a in enumerate(attachments): - print(msg.format(index=index, a=a)) - - numbers = raw_input('Enter the numbers (space-separated) of ' - 'attachments to make obsolete:\n').split() - - if not numbers: - return - - map_ = dict((str(index), a) for index, a in enumerate(attachments)) - for num, _ in itertools.groupby(sorted(numbers)): - try: - self._obsolete(map_[num]) - except KeyError: - pass - - def _obsolete(self, attachment): - """Mark an attachment obsolete.""" - print("Obsoleting", attachment) - attachment.is_obsolete = True - attachment._location += '?%s' % self.qs() - attachment.put() - - -def main(): - - # Script options - parser = argparse.ArgumentParser(description='Submit Bugzilla attachments') - - parser.add_argument('bug_id', - type=int, - metavar='BUG', - help='Bug number') - - parser.add_argument('filename', - metavar='FILE', - help='File to upload') - - parser.add_argument('--description', - help='Attachment description', - required=True) - - parser.add_argument('--patch', - action='store_true', - help='Is this a patch?') - - parser.add_argument('--reviewer', - help='Bugzilla name of someone to r?') - - parser.add_argument('--comment', - help='Comment for the attachment') - - parser.add_argument('--content_type', - choices=FILE_TYPES, - help="File's content_type") - - args = parser.parse_args() - - if args.content_type: - args.content_type = FILE_TYPES[args.content_type] - - # Get the API root, default to bugzilla.mozilla.org - API_ROOT = os.environ.get('BZ_API_ROOT', - 'https://api-dev.bugzilla.mozilla.org/latest/') - - # Authenticate - username, password = get_credentials() - - # Load the agent - bz = AttachmentAgent(API_ROOT, username, password) - - # Attach the file - bz.attach(**dict(args._get_kwargs())) - - -if __name__ == '__main__': - main() diff --git a/auto_nag/scripts/email_nag.py b/auto_nag/scripts/email_nag.py deleted file mode 100755 index 29c994547..000000000 --- a/auto_nag/scripts/email_nag.py +++ /dev/null @@ -1,489 +0,0 @@ -#!/usr/bin/env python -""" - -A script for automated nagging emails based on passed in queries -These can be collated into several 'queries' through the use of multiple query files with -a 'query_name' param set eg: 'Bugs tracked for Firefox Beta (13)' -Once the bugs have been collected from Bugzilla they are sorted into buckets cc: assignee manager -and to the assignee(s) or need-info? for each query - -""" -import sys -import os -import subprocess -import tempfile -import collections -from datetime import datetime -from argparse import ArgumentParser -from auto_nag.bugzilla.agents import BMOAgent -from auto_nag import mail, utils -import phonebook -from jinja2 import Environment, FileSystemLoader - -env = Environment(loader=FileSystemLoader('templates')) - -REPLY_TO_EMAIL = 'release-mgmt@mozilla.com' -DEFAULT_CC = ['release-mgmt@mozilla.com'] -EMAIL_SUBJECT = '' - -# TODO - Sort by who a bug is blocked on (thanks @dturner) -# TODO - write tests! -# TODO - look into knocking out duplicated bugs in queries -- perhaps print out if there are dupes in queries when queries > 1 -# TODO - should compare bugmail from API results to phonebook bugmail in to_lower() - - -def get_last_manager_comment(comments, manager, person): - # go through in reverse order to get most recent - for comment in comments[::-1]: - if person is not None: - if comment.creator == manager['mozillaMail'] or comment.creator == manager['bugzillaEmail']: - return comment.creation_time.replace(tzinfo=None) - return None - - -def get_last_assignee_comment(comments, person): - # go through in reverse order to get most recent - for comment in comments[::-1]: - if person is not None: - if comment.creator == person['mozillaMail'] or comment.creator == person['bugzillaEmail']: - return comment.creation_time.replace(tzinfo=None) - return None - - -def query_url_to_dict(url): - if (';')in url: - fields_and_values = url.split("?")[1].split(";") - else: - fields_and_values = url.split("?")[1].split("&") - d = collections.defaultdict(list) - - for pair in fields_and_values: - (key, val) = pair.split("=") - if key != "list_id": - d[key].append(val) - return d - - -def generateEmailOutput(subject, queries, template, people, show_comment=False, - manager_email=None, rollup=False, rollupEmail=None, cc_only=False): - cclist = [] - toaddrs = [] - template_params = {} - # stripping off the templates dir, just in case it gets passed in the args - template = env.get_template(template.replace('templates/', '', 1)) - - def addToAddrs(bug): - if bug.assigned_to.name in people.people_by_bzmail: - person = dict(people.people_by_bzmail[bug.assigned_to.name]) - if person['mozillaMail'] not in toaddrs: - toaddrs.append(person['mozillaMail']) - - for query in queries.keys(): - # Avoid dupes in the cclist from several queries - query_cc = queries[query].get('cclist', []) - for qcc in query_cc: - if qcc not in cclist: - cclist.append(qcc) - if query not in template_params: - template_params[query] = {'buglist': []} - if 'url' in queries[query]: - template_params[query]['query_url'] = queries[query]['url'] - if len(queries[query]['bugs']) != 0: - for bug in queries[query]['bugs']: - if bug.groups: - # probably a security bug, don't leak the summary - summary = '' - else: - summary = bug.summary - template_params[query]['buglist'].append( - { - 'id': bug.id, - 'summary': summary, - # 'comment': bug.comments[-1].creation_time.replace(tzinfo=None), - 'assignee': bug.assigned_to.real_name, - 'flags': bug.flags, - 'affected': bug.get_fx_affected_versions() - } - ) - # if needinfo? in flags, add the flag.requestee to the toaddrs instead of bug assignee - if bug.flags: - for flag in bug.flags: - if 'needinfo' in flag.name and flag.status == '?': - try: - person = dict(people.people_by_bzmail[str(flag.requestee)]) - if person['mozillaMail'] not in toaddrs: - toaddrs.append(person['mozillaMail']) - except KeyError: - if str(flag.requestee) not in toaddrs: - toaddrs.append(str(flag.requestee)) - else: - addToAddrs(bug) - else: - addToAddrs(bug) - - # Order by versions - if 'affected' in template_params[query]['buglist'][0]: - template_params[query]['buglist'] = sorted(template_params[query]['buglist'], key=lambda p: p['affected']) - - message_body = template.render(queries=template_params, show_comment=show_comment) - if manager_email is not None and manager_email not in cclist: - cclist.append(manager_email) - # no need to and cc the manager if more than one email - if len(toaddrs) > 1: - for email in toaddrs: - if email in cclist: - toaddrs.remove(email) - - if cclist == ['']: - cclist = None - if rollup: - joined_to = ",".join(rollupEmail) - else: - joined_to = ",".join(toaddrs) - if cc_only: - joined_to = ",".join(rollupEmail) - toaddrs = rollupEmail - - message = ( - "From: %s\r\n" % REPLY_TO_EMAIL + - "To: %s\r\n" % joined_to + - "CC: %s\r\n" % ",".join(cclist) + - "Subject: %s\r\n" % subject + - "\r\n" + - message_body) - - toaddrs = toaddrs + cclist - - return toaddrs, message - - -if __name__ == '__main__': - - parser = ArgumentParser(__doc__) - parser.set_defaults( - dryrun=False, - username=None, - password=None, - roll_up=False, - show_comment=False, - email_cc_list=None, - queries=[], - days_since_comment=-1, - verbose=False, - keywords=None, - email_subject=None, - no_verification=False, - cc_only=False - ) - parser.add_argument("-d", "--dryrun", dest="dryrun", action="store_true", - help="just do the query, and print emails to console without emailing anyone") - parser.add_argument("-m", "--mozilla-email", dest="mozilla_mail", - help="specify a specific address for sending email"), - parser.add_argument("-p", "--email-password", dest="email_password", - help="specify a specific password for sending email") - parser.add_argument("-b", "--bz-api-key", dest="bz_api_key", - help="Bugzilla API key") - parser.add_argument("-t", "--template", dest="template", - required=True, - help="template to use for the buglist output") - parser.add_argument("-e", "--email-cc-list", dest="email_cc_list", - action="append", - help="email addresses to include in cc when sending mail") - parser.add_argument("-q", "--query", dest="queries", - action="append", - required=True, - help="a file containing a dictionary of a bugzilla query") - parser.add_argument("-k", "--keyword", dest="keywords", - action="append", - help="keywords to collate buglists") - parser.add_argument("-s", "--subject", dest="email_subject", - required=True, - help="The subject of the email being sent") - parser.add_argument("-r", "--roll-up", dest="roll_up", action="store_true", - help="flag to get roll-up output in one email instead of creating multiple emails") - parser.add_argument("--show-comment", dest="show_comment", action="store_true", - help="flag to display last comment on a bug in the message output") - parser.add_argument("--days-since-comment", dest="days_since_comment", - help="threshold to check comments against to take action based on days since comment") - parser.add_argument("--verbose", dest="verbose", action="store_true", - help="turn on verbose output") - parser.add_argument("--no-verification", dest="no_verification", action="store_true", - help="don't wait for human verification of every email") - parser.add_argument("-c", "--cc-only", dest="cc_only", action="store_true", - help="Only email addresses in cc will receive the email") - - options, args = parser.parse_known_args() - - people = phonebook.PhonebookDirectory(dryrun=options.dryrun) - - try: - int(options.days_since_comment) - except ValueError: - if options.days_since_comment is not None: - parser.error("Need to provide a number for days \ - since last comment value") - if options.email_cc_list is None: - options.email_cc_list = DEFAULT_CC - - # Load our agent for BMO - bmo = BMOAgent(options.bz_api_key) - - # Get the buglist(s) - collected_queries = {} - for query in options.queries: - # import the query - if os.path.exists(query): - info = {} - execfile(query, info) - query_name = info['query_name'] - if query_name not in collected_queries: - collected_queries[query_name] = { - 'channel': info.get('query_channel', ''), - 'bugs': [], - 'cclist': options.email_cc_list, - } - if 'cc' in info: - for c in info.get('cc').split(','): - collected_queries[query_name]['cclist'].append(c) - if 'query_params' in info: - print("Gathering bugs from query_params in %s" % query) - collected_queries[query_name]['bugs'] = bmo.get_bug_list(info['query_params']) - elif 'query_url' in info: - print("Gathering bugs from query_url in %s" % query) - collected_queries[query_name]['bugs'] = bmo.get_bug_list(query_url_to_dict(info['query_url'])) - collected_queries[query_name]['url'] = info['query_url'] - # print "DEBUG: %d bug(s) found for query %s" % \ - # (len(collected_queries[query_name]['bugs']), info['query_url']) - else: - print("Error - no valid query params or url in the config file") - sys.exit(1) - else: - print("Not a valid path: %s" % query) - total_bugs = 0 - for channel in collected_queries.keys(): - total_bugs += len(collected_queries[channel]['bugs']) - - print("Found %s bugs total for %s queries" % (total_bugs, len(collected_queries.keys()))) - print("Queries to collect: %s" % collected_queries.keys()) - - managers = people.managers - manual_notify = {} - counter = 0 - - def add_to_managers(manager_email, query, info={}): - if manager_email not in managers: - managers[manager_email] = {} - managers[manager_email]['nagging'] = {query: {'bugs': [bug], - 'cclist': info.get('cclist', [])}, } - return - if 'nagging' in managers[manager_email]: - if query in managers[manager_email]['nagging']: - managers[manager_email]['nagging'][query]['bugs'].append(bug) - if options.verbose: - print("Adding %s to %s in nagging for %s" % - (bug.id, query, manager_email)) - else: - managers[manager_email]['nagging'][query] = { - 'bugs': [bug], - 'cclist': info.get('cclist', []) - } - if options.verbose: - print("Adding new query key %s for bug %s in nagging \ - and %s" % (query, bug.id, manager_email)) - else: - managers[manager_email]['nagging'] = {query: {'bugs': [bug], - 'cclist': info.get('cclist', [])}, } - if options.verbose: - print("Creating query key %s for bug %s in nagging and \ - %s" % (query, bug.id, manager_email)) - - for query, info in collected_queries.items(): - if len(collected_queries[query]['bugs']) != 0: - manual_notify[query] = {'bugs': []} - if 'url' in info: - manual_notify[query]['url'] = info['url'] - for b in collected_queries[query]['bugs']: - counter = counter + 1 - send_mail = True - bug = bmo.get_bug(b.id) - manual_notify[query]['bugs'].append(bug) - assignee = bug.assigned_to.name - if "@" not in assignee: - print("Error - expected email address, found %r instead in bug %s" % (assignee, b.id)) - print("Check that the authentication worked correctly") - sys.exit(1) - if assignee in people.people_by_bzmail: - person = dict(people.people_by_bzmail[assignee]) - else: - person = None - - # kick bug out if days since comment check is on - if options.days_since_comment != -1: - # try to get last_comment by assignee & manager - if person is not None: - last_comment = get_last_assignee_comment(bug.comments, person) - if 'manager' in person and person['manager'] is not None: - manager_email = person['manager']['dn'].split('mail=')[1].split(',')[0] - manager = people.people[manager_email] - last_manager_comment = get_last_manager_comment(bug.comments, - people.people_by_bzmail[manager['bugzillaEmail']], - person) - # set last_comment to the most recent of last_assignee and last_manager - if last_manager_comment is not None and last_comment is not None and last_manager_comment > last_comment: - last_comment = last_manager_comment - # otherwise just get the last comment - else: - last_comment = bug.comments[-1].creation_time.replace(tzinfo=None) - if last_comment is not None: - timedelta = datetime.now() - last_comment - if timedelta.days <= int(options.days_since_comment): - if options.verbose: - print("Skipping bug %s since it's had an assignee or manager comment within the past %s days" % (bug.id, options.days_since_comment)) - send_mail = False - counter = counter - 1 - manual_notify[query]['bugs'].remove(bug) - else: - if options.verbose: - print("This bug needs notification, it's been %s since last comment of note" % timedelta.days) - - if send_mail: - if 'nobody' in assignee: - if options.verbose: - print("No one assigned to: %s, will be in the manual notification list..." % bug.id) - else: - if bug.assigned_to.real_name is not None: - if person is not None: - # check if assignee is already a manager, add to their own list - if 'mozillaMail' in managers: - add_to_managers(person['mozillaMail'], query, info) - # otherwise we search for the assignee's manager - else: - # check for manager key first, a few people don't have them - if 'manager' in person and person['manager'] is not None: - manager_email = person['manager']['dn'].split('mail=')[1].split(',')[0] - if manager_email in managers: - add_to_managers(manager_email, query, info) - elif manager_email in people.vices: - # we're already at the highest level we'll go - if assignee in managers: - add_to_managers(assignee, query, info) - else: - if options.verbose: - print("%s has a V-level for a manager, and is not in the manager list" % assignee) - managers[person['mozillaMail']] = {} - add_to_managers(person['mozillaMail'], query, info) - else: - # try to go up one level and see if we find a manager - if manager_email in people.people: - person = dict(people.people[manager_email]) - manager_email = person['manager']['dn'].split('mail=')[1].split(',')[0] - if manager_email in managers: - add_to_managers(manager_email, query, info) - else: - print("Manager could not be found: %s" % manager_email) - # if you don't have a manager listed, but are an employee, we'll nag you anyway - else: - add_to_managers(person['mozillaMail'], query, info) - print("%s's entry doesn't list a manager! Let's ask them to update phonebook but in the meantime they get the email directly." % person['name']) - - if options.roll_up or options.cc_only: - # only send one email - toaddrs, msg = generateEmailOutput(subject=options.email_subject, - queries=manual_notify, - template=options.template, - people=people, - show_comment=options.show_comment, - rollup=options.roll_up, - rollupEmail=options.email_cc_list, - cc_only=options.cc_only) - if options.email_password is None or options.mozilla_mail is None: - print("Please supply a username/password (-m, -p) for sending email") - sys.exit(1) - if not options.dryrun: - print("SENDING EMAIL") - mail.sendMail(options.mozilla_mail, toaddrs, msg, - login={'ldap_username': options.mozilla_mail, - 'ldap_password': options.email_password}, - dryrun=options.dryrun) - else: - # Get yr nag on! - for email, info in managers.items(): - inp = '' - if 'nagging' in info: - if email in utils.get_config('auto_nag', 'no_mail_for_manager', default=[]): - email = None - toaddrs, msg = generateEmailOutput( - subject=options.email_subject, - manager_email=email, - queries=info['nagging'], - people=people, - template=options.template, - show_comment=options.show_comment, - cc_only=options.cc_only) - while True and not options.no_verification: - print("\nRelMan Nag is ready to send the following email:\n<------ MESSAGE BELOW -------->") - print(msg) - print("<------- END MESSAGE -------->\nWould you like to send now?") - inp = raw_input('\n Please select y/Y to send, v/V to edit, or n/N to skip and continue to next email: ') - - if inp != 'v' and inp != 'V': - break - - tempfilename = tempfile.mktemp() - temp_file = open(tempfilename, 'w') - temp_file.write(msg) - temp_file.close() - - subprocess.call(['vi', tempfilename]) - - temp_file = open(tempfilename, 'r') - msg = temp_file.read() - toaddrs = msg.split("To: ")[1].split("\r\n")[0].split(',') + msg.split("CC: ")[1].split("\r\n")[0].split(',') - os.remove(tempfilename) - - if inp == 'y' or inp == 'Y' or options.no_verification: - if options.email_password is None or options.mozilla_mail is None: - print("Please supply a username/password (-m, -p) for sending email") - sys.exit(1) - if not options.dryrun: - print("SENDING EMAIL") - mail.sendMail(options.mozilla_mail, toaddrs, msg, - login={'ldap_username': options.mozilla_mail, - 'ldap_password': options.email_password}, - dryrun=options.dryrun) - sent_bugs = 0 - for query, info in info['nagging'].items(): - sent_bugs += len(info['bugs']) - # take sent bugs out of manual notification list - for bug in info['bugs']: - manual_notify[query]['bugs'].remove(bug) - counter = counter - sent_bugs - - if not options.roll_up and not options.cc_only: - emailed_bugs = [] - # Send RelMan the manual notification list only when there are bugs that didn't go out - msg_body = """\n******************************************\nNo nag emails were generated for these bugs because - they are either assigned to no one or to non-employees (though ni? on non-employees will get nagged). - \nYou will need to look at the following bugs:\n******************************************\n\n""" - for k, v in manual_notify.items(): - if len(v['bugs']) != 0: - for bug in v['bugs']: - if bug.id not in emailed_bugs: - if k not in msg_body: - msg_body += "\n=== %s ===\n" % k - emailed_bugs.append(bug.id) - if bug.groups: - summary = '' - else: - summary = ' %s\n' % bug.summary - msg_body += "https://bugzilla.mozilla.org/%s -- assigned to: %s\n%s-- Last commented on: %s\n" % ( - bug.id, bug.assigned_to.real_name, summary, bug.comments[-1].creation_time.replace(tzinfo=None)) - msg = ("From: %s\r\n" % REPLY_TO_EMAIL + - "To: %s\r\n" % REPLY_TO_EMAIL + - "Subject: RelMan Attention Needed: %s\r\n" % options.email_subject + - "\r\n" + - msg_body) - mail.sendMail(options.mozilla_mail, ['release-mgmt@mozilla.com'], msg, - login={'ldap_username': options.mozilla_mail, - 'ldap_password': options.email_password}, - dryrun=options.dryrun) diff --git a/auto_nag/scripts/mismatch_priority_tracking_beta.py b/auto_nag/scripts/mismatch_priority_tracking_beta.py index 341c4a58e..fcce833a4 100644 --- a/auto_nag/scripts/mismatch_priority_tracking_beta.py +++ b/auto_nag/scripts/mismatch_priority_tracking_beta.py @@ -3,7 +3,7 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from auto_nag.bzcleaner import BzCleaner -from auto_nag.bugzilla.utils import getVersions +from auto_nag import utils class MismatchPrioTrackBeta(BzCleaner): @@ -20,7 +20,7 @@ def ignore_date(self): return True def get_bz_params(self, date): - _, beta_version, _, _ = getVersions() + beta_version = utils.get_versions(channel='beta') value = ','.join(['---', 'affected']) params = { 'resolution': [ diff --git a/auto_nag/scripts/mismatch_priority_tracking_esr.py b/auto_nag/scripts/mismatch_priority_tracking_esr.py index f54954262..9b27a2ab9 100644 --- a/auto_nag/scripts/mismatch_priority_tracking_esr.py +++ b/auto_nag/scripts/mismatch_priority_tracking_esr.py @@ -3,7 +3,6 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from auto_nag.bzcleaner import BzCleaner -from auto_nag.bugzilla.utils import getVersions from auto_nag import utils @@ -21,7 +20,7 @@ def ignore_date(self): return True def get_bz_params(self, date): - _, _, _, esr_version = getVersions() + esr_version = utils.get_versions(channel='esr') value = ','.join(['---', 'affected']) params = { 'resolution': [ diff --git a/auto_nag/scripts/mismatch_priority_tracking_nightly.py b/auto_nag/scripts/mismatch_priority_tracking_nightly.py index 876fb3d22..a3a5d5e34 100644 --- a/auto_nag/scripts/mismatch_priority_tracking_nightly.py +++ b/auto_nag/scripts/mismatch_priority_tracking_nightly.py @@ -3,7 +3,7 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from auto_nag.bzcleaner import BzCleaner -from auto_nag.bugzilla.utils import getVersions +from auto_nag import utils class MismatchPrioTrackNightly(BzCleaner): @@ -20,7 +20,7 @@ def ignore_date(self): return True def get_bz_params(self, date): - _, _, central_version, _ = getVersions() + central_version = utils.get_versions(channel='central') value = ','.join(['---', 'affected']) params = { 'resolution': [ diff --git a/auto_nag/scripts/mismatch_priority_tracking_release.py b/auto_nag/scripts/mismatch_priority_tracking_release.py index 20fe20f80..daed03287 100644 --- a/auto_nag/scripts/mismatch_priority_tracking_release.py +++ b/auto_nag/scripts/mismatch_priority_tracking_release.py @@ -3,7 +3,7 @@ # You can obtain one at http://mozilla.org/MPL/ from auto_nag.bzcleaner import BzCleaner -from auto_nag.bugzilla.utils import getVersions +from auto_nag import utils class MismatchPrioTrackRelease(BzCleaner): @@ -20,7 +20,7 @@ def ignore_date(self): return True def get_bz_params(self, date): - release_version, _, _, _ = getVersions() + release_version = utils.get_versions(channel='release') value = ','.join(['---', 'affected']) params = { 'resolution': [ diff --git a/auto_nag/scripts/missing_beta_status.py b/auto_nag/scripts/missing_beta_status.py index 6b2bb7ebe..203224714 100644 --- a/auto_nag/scripts/missing_beta_status.py +++ b/auto_nag/scripts/missing_beta_status.py @@ -64,13 +64,10 @@ def handle_bug(self, bug, data): return bug def get_bz_params(self, date): - self.status_central = utils.get_flag( - self.versions['central'], 'status', 'central' - ) - self.status_release = utils.get_flag( - self.versions['release'], 'status', 'release' - ) - self.status_beta = utils.get_flag(self.versions['beta'], 'status', 'beta') + release, beta, central, _ = utils.get_versions() + self.status_central = utils.get_flag(central, 'status', 'central') + self.status_release = utils.get_flag(release, 'status', 'release') + self.status_beta = utils.get_flag(beta, 'status', 'beta') fields = [self.status_central, self.status_release] params = { 'include_fields': fields, diff --git a/auto_nag/scripts/no_crashes.py b/auto_nag/scripts/no_crashes.py index ae44fbb68..d13b157fe 100644 --- a/auto_nag/scripts/no_crashes.py +++ b/auto_nag/scripts/no_crashes.py @@ -62,7 +62,7 @@ def chunkify(self, signatures): the total length of each chunk must be <= 1536""" total = sum(len(s) for s in signatures) M = 1536 - n = total / M + 1 + n = total // M + 1 res = [[M, []] for _ in range(n)] for s in signatures: L = len(s) diff --git a/auto_nag/scripts/phonebook.py b/auto_nag/scripts/phonebook.py deleted file mode 100644 index 9c607ee63..000000000 --- a/auto_nag/scripts/phonebook.py +++ /dev/null @@ -1,94 +0,0 @@ -import json - -from auto_nag.bugzilla.utils import get_project_root_path - -# NOTE: You must create a file for CONFIG_JSON with your LDAP auth in it like: -# { -# "username": "username", -# "password": "password" -# } -# -# In order to access the phonebook data - -BASE_URL = 'https://phonebook.mozilla.org' -PEOPLE_URL = '%s/search.php?query=*&format=fligtar' % BASE_URL - -''' -a single phonebook entry data looks like this when you pull it from JSON: -'email' = { - ims : [], - name : 'name', - title : 'title', - phones : 'string of numbers & assignments', - ext : XXX, - manager : {u'dn': u'mail=manager@mozilla.com,o=com,dc=mozilla', u'cn': u'Manager Name'}, - bugzillaEmail : 'email@example.com' - - ## this script adds in: - mozillaMail : 'email@mozilla.com' - } -''' - - -class PhonebookDirectory(): - def __init__(self, dryrun=False, isTest=False): - print("Fetching people from phonebook...") - if False and dryrun: - people_json = (get_project_root_path() + - '/auto_nag/tests/people.json') - with open(people_json, 'r') as pj: - self.people = json.load(pj) - else: - # when phonebook bug will be fixed: remove these lines and uncomment the following - people_json = (get_project_root_path() + - '/auto_nag/scripts/configs/people.json') - if isTest: - people_json = (get_project_root_path() + - '/auto_nag/tests/people.json') - with open(people_json, 'r') as pj: - self.people = {} - entries = json.load(pj) - for entry in entries: - self.people[entry['mail']] = entry - if 'title' not in entry: - entry['title'] = '' - # config = get_config_path() - # config = json.load(open(config, 'r')) - # self.people = json.loads(requests.get(PEOPLE_URL, - # auth=(config['ldap_username'], - # config['ldap_password'])).content) - self.people_by_bzmail = self.get_people_by_bzmail() - self.managers = self.get_managers() - self.vices = self.get_vices() - - def get_managers(self): - managers = {} - for email, info in self.people.items(): - if self.people[email]['title'] is not None: - if 'director' in self.people[email]['title'].lower() or 'manager' in self.people[email]['title'].lower(): - managers[email] = info - # HACK! don't have titles with manager/director or missing bugmail address - if email == 'dtownsend@mozilla.com' and email not in managers.keys(): - managers[email] = info - return managers - - def get_vices(self): - vices = {} - for email, info in self.people.items(): - if self.people[email]['title'] is not None: - if 'vice' in self.people[email]['title'].lower(): - vices[email] = info - return vices - - def get_people_by_bzmail(self): - temp = {} - for email, info in self.people.items(): - # if someone doesn't have a bugzillaEmail set, we'll try their mozilla mail instead - if info.get('bugzillaEmail'): - temp[info['bugzillaEmail']] = dict(info.items()) - temp[info['bugzillaEmail']].update({'mozillaMail': email}) - else: - temp[email] = dict(info.items()) - temp[email].update({'mozillaMail': email}) - - return temp diff --git a/auto_nag/scripts/tracked_needinfo.py b/auto_nag/scripts/tracked_needinfo.py index 5bdde33b2..7ce980a9f 100644 --- a/auto_nag/scripts/tracked_needinfo.py +++ b/auto_nag/scripts/tracked_needinfo.py @@ -32,7 +32,10 @@ def has_default_products(self): return False def get_extra_for_template(self): - return {'channel': self.channel, 'version': self.version} + return { + 'channel': 'nightly' if self.channel == 'central' else self.channel, + 'version': self.version, + } def get_extra_for_nag_template(self): return self.get_extra_for_template() diff --git a/auto_nag/scripts/tracking.py b/auto_nag/scripts/tracking.py index bfdaa48fb..cc73a811b 100644 --- a/auto_nag/scripts/tracking.py +++ b/auto_nag/scripts/tracking.py @@ -48,7 +48,7 @@ def has_assignee(self): def get_extra_for_template(self): return { - 'channel': self.channel, + 'channel': 'nightly' if self.channel == 'central' else self.channel, 'version': self.version, 'untouched': self.untouched, 'next_release': (utils.get_next_release_date() - self.nag_date).days, diff --git a/auto_nag/scripts/unlanded.py b/auto_nag/scripts/unlanded.py index 3eca57546..a91f80589 100644 --- a/auto_nag/scripts/unlanded.py +++ b/auto_nag/scripts/unlanded.py @@ -34,6 +34,15 @@ def get_config(self, entry, default=None): def nag_template(self): return self.template() + def get_extra_for_template(self): + return { + 'channel': 'nightly' if self.channel == 'central' else self.channel, + 'version': self.version, + } + + def get_extra_for_nag_template(self): + return self.get_extra_for_template() + def has_last_comment_time(self): return True diff --git a/auto_nag/tests/_get_credential.py b/auto_nag/tests/_get_credential.py deleted file mode 100644 index c4a3ec5f1..000000000 --- a/auto_nag/tests/_get_credential.py +++ /dev/null @@ -1,11 +0,0 @@ -# If relman-auto-nag not installed, add project root directory into -# PYTHONPATH -import os -import inspect -import sys -currentdir = os.path.dirname(os.path.abspath(__file__)) -project_root = os.path.dirname(os.path.dirname(currentdir)) -sys.path.insert(0, project_root) -from auto_nag.bugzilla import utils - -utils.get_credentials() diff --git a/auto_nag/tests/people.json b/auto_nag/tests/people.json deleted file mode 100644 index 7f7c6109c..000000000 --- a/auto_nag/tests/people.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "bugzillaEmail": "foo1@mozilla.com", - "mail": "foo1@mozilla.com", - "manager": { - "cn": "Sylvestre Ledru", - "dn": "mail=email@example.com,o=com" - }, - "name": "Calixte Denizet", - "title": "Release engineer" - }, - { - "bugzillaEmail": "email@example.com", - "mail": "email@example.com", - "name": "Sylvestre Ledru", - "title": "Release mgmt lead / manager" - } -] diff --git a/auto_nag/tests/templates/channel_meeting_wiki b/auto_nag/tests/templates/channel_meeting_wiki deleted file mode 100644 index 46e6a5d27..000000000 --- a/auto_nag/tests/templates/channel_meeting_wiki +++ /dev/null @@ -1,27 +0,0 @@ -= Triage = -{% for channel,info in channel_info.items() %} -{% if info.needs_attention %} -== {{ channel }} bugs needing attention == -# Keyworded with #relman/triage/defer-to-group using mediawiki bz extension - -{ - "id": {% for bug in info.needs_attention %}"{{ bug.id }},"{% endfor %} -} - -{% endif %} -{% if info.qawanted %} -== Tracking for {{ channel }} and QA Wanted for more than x days == - -{ - "id": {% for bug in info.qawanted %}"{{ bug.id }},"{% endfor %} -} - -{% endif %} -== Tracking {{ channel }} - {% if days_since_comment > -1 %}{{ days_since_comment }} days without assignee/manager comment{% endif %} == -{% for m in info.managers %} -{# note: escaping the wiki bug template with '{{' and '}}" #} -=== {{ m.name }} === - {% for bug in m.bugs %}* [https://bugzilla.mozilla.org/{{ bug.id }} {{ bug.id }}] - assignee : {{ bug.assigned_to.real_name }}{% if bug.keywords %} - keywords: {{ bug.keywords }}{% endif %}{% if bug.whiteboard %} - whiteboard: {{ bug.whiteboard }} {% endif %} - {% endfor %} -{% endfor %} -{% endfor %} diff --git a/auto_nag/tests/templates/daily_email b/auto_nag/tests/templates/daily_email deleted file mode 100644 index fcd1c3097..000000000 --- a/auto_nag/tests/templates/daily_email +++ /dev/null @@ -1,18 +0,0 @@ -NOTE: On Mondays, we send email to remind you of work tracked for upcoming releases. On Thursdays, we send email to remind you of tracked bugs that haven't been touched yet this week. Every day, we send email for unlanded approvals and bugs with need-info? unanswered. - -= Next Actions = -* If you don't believe a bug should be tracked for release or your investigation is blocked, please comment in the bug and optionally reply to this email -* If you're surprised that a bug is listed below, please make sure to check the version-specific status flags. -* Investigations for Beta should be prioritized over Aurora, etc. -* Stay abreast of where we are in the cycle by checking the calendar at https://bit.ly/1zUHw5G. Remember, beta 8 is the last opportunity for speculative fixes and we code freeze after beta 9. - -= Queries = -{% for name, results in queries.items() %}== {{ name }} == -{% for bug in results.buglist -%} -* https://bugzilla.mozilla.org/{{ bug.id }} - {{ bug.summary }}{% if bug.flags != None %} {% for flag in bug.flags %}{% if flag.name == 'needinfo' %}(needinfo? {{ flag.requestee }}){% endif %}{% endfor %}{% endif %} - (assigned to {{ bug.assignee }}) -{% endfor %} -{% endfor %} - - -Sincerely, -Release Management diff --git a/auto_nag/tests/test_agents.py b/auto_nag/tests/test_agents.py deleted file mode 100644 index f71b52d06..000000000 --- a/auto_nag/tests/test_agents.py +++ /dev/null @@ -1,78 +0,0 @@ - -try: - # If relman-auto-nag is installed - from bugzilla.agents import BMOAgent - from bugzilla.utils import os - from bugzilla.utils import get_config_path -except: - # If relman-auto-nag not installed, add project root directory into - # PYTHONPATH - import os - import sys - import inspect - currentdir = os.path.dirname(os.path.abspath( - inspect.getfile(inspect.currentframe()))) - parentdir = os.path.dirname(currentdir) - sys.path.insert(0, parentdir) - from bugzilla.agents import BMOAgent - from bugzilla.utils import os - from bugzilla.utils import get_config_path - -import json -from nose.tools import * - -class TestAgent: - # Test bugzilla agent methods - def test_get_bug_list(self): - # Set whatever REST API options we want - options = { - 'chfieldfrom': ['2012-12-24'], - 'chfieldto': ['2012-12-27'], - 'chfield': ['bug_status'], - 'chfieldvalue': ['RESOLVED'], - 'product': ['Firefox'], - 'resolution': ['FIXED'], - 'include_fields': ['attachments'], - } - # Load our agent for BMO - bmo = BMOAgent() - # Get the bugs from the api - buglist = bmo.get_bug_list(options) - assert buglist != [] - - - def test_get_bug(self): - # Load our agent for BMO - bmo = BMOAgent() - # Get the bugs from the api - bug = bmo.get_bug(656222) - assert bug != [] - - - @raises(Exception) - def test_get_bug_list_wrng_api_k(self): - """ Wrong API Key, it should raise an Error""" - # Set whatever REST API options we want - options = { - 'chfieldfrom': ['2012-12-24'], - 'chfieldto': ['2012-12-27'], - 'chfield': ['bug_status'], - 'chfieldvalue': ['RESOLVED'], - 'product': ['Firefox'], - 'resolution': ['FIXED'], - 'include_fields': ['attachments'], - } - # Load our agent for BMO - bmo = BMOAgent('wrong_api_key_test') - # Get the bugs from the api - bmo.get_bug_list(options) - - - @raises(Exception) - def test_get_bug_wrng_api_k(self): - """ Wrong API Key, it should raise an Error""" - # Load our agent for BMO - bmo = BMOAgent('wrong_api_key_test') - # Get the bugs from the api - bug = bmo.get_bug(656222) - print bug diff --git a/auto_nag/tests/test_bzcleaner.py b/auto_nag/tests/test_bzcleaner.py index 5e65b5bd9..b235179c0 100644 --- a/auto_nag/tests/test_bzcleaner.py +++ b/auto_nag/tests/test_bzcleaner.py @@ -48,7 +48,13 @@ def test_subject(self): assert 'Bug tracked' in TrackedBadSeverity().subject() def test_get_bz_params(self): - p = TrackedBadSeverity().get_bz_params(None) + tool = TrackedBadSeverity() + if not tool.has_enough_data(): + # we've non-following versions in product-details + # so cheat on versions. + tool.versions = {'central': 1, 'beta': 2, 'release': 3} + + p = tool.get_bz_params(None) assert p['f1'] == 'OP' assert 'cf_tracking_firefox' in p['f3'] assert 'enhancement' in p['bug_severity'] diff --git a/auto_nag/tests/test_email_nag.py b/auto_nag/tests/test_email_nag.py deleted file mode 100644 index 7bfaefeb5..000000000 --- a/auto_nag/tests/test_email_nag.py +++ /dev/null @@ -1,62 +0,0 @@ - -import datetime - -from auto_nag.bugzilla.utils import get_project_root_path, getVersions -from auto_nag.bugzilla.agents import BMOAgent -from auto_nag.bugzilla.models import Bug -from auto_nag.scripts.phonebook import PhonebookDirectory -from auto_nag.scripts.email_nag import (get_last_manager_comment, - get_last_assignee_comment, - generateEmailOutput) - - -class TestEmailNag: - def setUp(self): - bug_number = 489656 - bmo = BMOAgent() - self.bug = bmo.get_bug(bug_number) - self.people = PhonebookDirectory(dryrun=True, isTest=True) - hidden_dict = { - 'id': 123456, - 'summary': 'hide me', - 'assigned_to_detail': {'real_name': 'nobody'}, - 'groups': [{'name': 'security group'}]} - hidden_dict.update({'cf_status_firefox' + version: '---' - for version in getVersions()}) - self.hidden_bug = Bug.from_dict(hidden_dict) - assignee = 'email@example.com' - self.manager = {'mozillaMail': 'test@mozilla.com', - 'bugzillaEmail': 'demo@bugzilla.com'} - self.person = {'mozillaMail': 'test@mozilla.com', - 'bugzillaEmail': 'demo@bugzilla.com'} - - def test_get_last_manager_comment(self): - self.bug.comments[-1].creator = 'test@mozilla.com' - last_mgr_comnt = get_last_manager_comment(self.bug.comments, - self.manager, - self.person) - assert isinstance(last_mgr_comnt, (datetime.datetime)) - - def test_get_last_assignee_comment(self): - self.bug.comments[-1].creator = 'test@mozilla.com' - lac = get_last_assignee_comment(self.bug.comments, self.person) - assert isinstance(lac, (datetime.datetime)) - - def test_generateEmailOutput(self): - query = {'Test': {'bugs': [self.bug]}} - toaddrs, message = generateEmailOutput('Test', query, 'daily_email', - self.people, False, - 'Test@test.com') - contents = ('From', 'To', 'CC', 'Subject:', '= Next Actions =', - '= Queries =', 'Sincerely,', self.bug.summary) - assert '@' in toaddrs[0] - assert all(content in message for content in contents) - query = {'Test': {'bugs': [self.hidden_bug]}} - toaddrs, message = generateEmailOutput('Test', query, 'daily_email', - self.people, False, - 'Test@test.com') - contents = ('From', 'To', 'CC', 'Subject:', '= Next Actions =', - '= Queries =', 'Sincerely,') - assert '@' in toaddrs[0] - assert all(content in message for content in contents) - assert self.hidden_bug.summary not in message diff --git a/auto_nag/tests/test_mail.py b/auto_nag/tests/test_mail.py index 90c25563a..a1de6658e 100644 --- a/auto_nag/tests/test_mail.py +++ b/auto_nag/tests/test_mail.py @@ -11,15 +11,15 @@ class TestMail(unittest.TestCase): def test_replaceUnicode(self): - s = 'some letters and a é and a è, what else ?...'.decode('utf-8') + s = 'some letters and a é and a è, what else ?...' r = replaceUnicode(s) assert r == 'some letters and a é and a è, what else ?...' - s = 'some letters and a é and a è'.decode('utf-8') + s = 'some letters and a é and a è' r = replaceUnicode(s) assert r == 'some letters and a é and a è' - s = 'some letters with no accents, just pure ascii'.decode('utf-8') + s = 'some letters with no accents, just pure ascii' r = replaceUnicode(s) assert r == s diff --git a/auto_nag/tests/test_models.py b/auto_nag/tests/test_models.py deleted file mode 100644 index ab5f5430d..000000000 --- a/auto_nag/tests/test_models.py +++ /dev/null @@ -1,168 +0,0 @@ - -from auto_nag.bugzilla.agents import BMOAgent - -bug_number = 656222 -# Load our agent for BMO -bmo = BMOAgent() -bug = bmo.get_bug(bug_number) - - -class TestModels: - # Test bugzilla agent methods - def test_BugSearch(self): - # Set whatever REST API options we want - options = { - 'chfieldfrom': ['2012-12-24'], - 'chfieldto': ['2012-12-27'], - 'chfield': ['bug_status'], - 'chfieldvalue': ['RESOLVED'], - 'product': ['Firefox'], - 'resolution': ['FIXED'], - 'include_fields': ['attachments'], - } - # Load our agent for BMO - bmo = BMOAgent() - # Get the bugs from the api - buglist = bmo.get_bug_list(options) - assert buglist != [] - - def test_BugAttributes(self): - attrs = ['summary', 'id', 'assigned_to', 'creator', 'target_milestone', - 'attachments', 'comments', 'history', 'keywords', 'status', - 'resolution', 'cf_blocking_20', 'cf_blocking_fennec', - 'cf_crash_signature', 'creation_time', 'flags', 'blocks', - 'depends_on', 'url', 'cc', 'keywords', 'whiteboard', 'op_sys', - 'platform', 'priority', 'product', 'qa_contact', 'severity', - 'see_also', 'version', 'alias', 'classification', 'component', - 'is_cc_accessible', 'is_everconfirmed', - 'is_creator_accessible', 'last_change_time', 'ref', 'token', - 'actual_time', 'deadline', 'estimated_time', - 'percentage_complete', 'remaining_time', 'work_time'] - - for attr in attrs: - assert hasattr(bug, attr) - - def test_Bugrepr(self): - assert '=2017.4.17 -chardet<3.1.0,>=3.0.2 -httplib2>=0.9.1 humanize>=0.5.1 libmozdata>=0.1.54 -path.py==11.5.2 pep8==1.7.1 pyflakes==2.1.1 python-dateutil>=2.5.2 -remoteobjects==1.2.1 requests>=2.20.0 -pexpect==4.6.0 python-Levenshtein>=0.12.0 pytz>=2018.9 whatthepatch>=0.0.5 diff --git a/runauto_nag_daily.sh b/runauto_nag_daily.sh index 41c1828b9..2f0d23a89 100755 --- a/runauto_nag_daily.sh +++ b/runauto_nag_daily.sh @@ -4,5 +4,114 @@ set -e export PYTHONPATH=. ./runauto_nag_common.sh -./runauto_nag_daily_py2.sh -./runauto_nag_daily_py3.sh + +. venv/bin/activate + +# force the update of dependencies +pip install -r requirements.txt + +# Clean the log files +python -m auto_nag.log --clean + +# Not up-to-date release date +# Daily +python -m auto_nag.next_release + +# Code freeze week information for release managers +# Daily (but really runs during the soft freeze week) +python -m auto_nag.scripts.code_freeze_week -D yesterday + +# Send a todo list to set priority +# Daily +python -m auto_nag.scripts.to_triage + +# Nag triage fallback to update calendar +# Daily +python -m auto_nag.round_robin_fallback + +# What is fixed in nightly but affecting beta or release +# Daily +python -m auto_nag.scripts.missed_uplifts + +# Top crash with an incorrect severity +# Pretty rare +python -m auto_nag.scripts.topcrash_bad_severity + +# Bug with both the regression and feature keywords +# Pretty rare +python -m auto_nag.scripts.feature_regression + +# Detect one word summary +# a bit rare +python -m auto_nag.scripts.one_two_word_summary + +# Bugs where the reporter has a needinfo +# Pretty common +python -m auto_nag.scripts.reporter_with_ni + +# Unconfirmed bugs with an assignee (with autofix) +# Pretty common +python -m auto_nag.scripts.assignee_but_unconfirmed + +# Notify bugs in untriaged with an important severity +python -m auto_nag.scripts.untriage_important_sev + +# Needinfo the assignee or the triage owner when a bug has leave-open keyword an no activty +# Pretty common +python -m auto_nag.scripts.leave_open_no_activity + +# Needinfo the triage owner or the assignee when we find meta bugs not depending on bugs and no activity +# Pretty common +python -m auto_nag.scripts.meta_no_deps_no_activity + +# Several tools here +# 1) has an unlanded patch or some flags not up-to-date +# Pretty rare +# 2) Tracked bugs +# 3) Tracked bugs with needinfos +python -m auto_nag.scripts.multi_nag + +# has a r+ patch, is open, has no activity for few weeks +# Pretty common +python -m auto_nag.scripts.not_landed + +# New workflow +# https://docs.google.com/document/d/1EHuWa-uR-7Sq63X1ZiDN1mvJ9gQtWiqYrCifkySJyW0/edit# +# https://docs.google.com/drawings/d/1oZA-AUvkOxGMNhZNofL8Wlfk6ol3o5ATQCV5DJJKbwM/edit +python -m auto_nag.scripts.workflow.multi_nag + +# Defect or task with the "feature" keyword +python -m auto_nag.scripts.feature_but_type_defect_task + +# Defect with the "meta" keyword +python -m auto_nag.scripts.meta_defect + +# reporter has a needinfo and no activity for the last X weeks +# Pretty common +python -m auto_nag.scripts.newbie_with_ni -d + +# Bug caused several regressions recently reported +# Pretty rare +python -m auto_nag.scripts.warn_regressed_by + +# Defect starting with please or enable in the title +python -m auto_nag.scripts.defect_with_please_or_enable + +# Regressions without regressed_by and some dependencies (blocks, depends_on) +# Pretty rare +python -m auto_nag.scripts.regression_without_regressed_by + +# Try to detect potential regressions using bugbug +python -m auto_nag.scripts.regression + +# Suggest components for untriaged bugs (daily, full list without confidence threshold) +python -m auto_nag.scripts.component --frequency daily + +# Try to detect potential wrong bug types using bugbug +python -m auto_nag.scripts.defectenhancementtask + +# Send a mail if the logs are not empty +# MUST ALWAYS BE THE LAST COMMAND +python -m auto_nag.log --send + +deactivate diff --git a/runauto_nag_daily_py2.sh b/runauto_nag_daily_py2.sh deleted file mode 100755 index f29921ea4..000000000 --- a/runauto_nag_daily_py2.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -set -e - -. venv2/bin/activate - -# force the update of dependencies -pip install -r requirements.txt - -# Clean the log files -python -m auto_nag.log --clean - -# Not up-to-date release date -# Daily -python -m auto_nag.next_release - -# Code freeze week information for release managers -# Daily (but really runs during the soft freeze week) -python -m auto_nag.scripts.code_freeze_week -D yesterday - -# Send a todo list to set priority -# Daily -python -m auto_nag.scripts.to_triage - -# Nag triage fallback to update calendar -# Daily -python -m auto_nag.round_robin_fallback - -# What is fixed in nightly but affecting beta or release -# Daily -python -m auto_nag.scripts.missed_uplifts - -# Top crash with an incorrect severity -# Pretty rare -python -m auto_nag.scripts.topcrash_bad_severity - -# Bug with both the regression and feature keywords -# Pretty rare -python -m auto_nag.scripts.feature_regression - -# Detect one word summary -# a bit rare -python -m auto_nag.scripts.one_two_word_summary - -# Bugs where the reporter has a needinfo -# Pretty common -python -m auto_nag.scripts.reporter_with_ni - -# Unconfirmed bugs with an assignee (with autofix) -# Pretty common -python -m auto_nag.scripts.assignee_but_unconfirmed - -# Notify bugs in untriaged with an important severity -python -m auto_nag.scripts.untriage_important_sev - -# Needinfo the assignee or the triage owner when a bug has leave-open keyword an no activty -# Pretty common -python -m auto_nag.scripts.leave_open_no_activity - -# Needinfo the triage owner or the assignee when we find meta bugs not depending on bugs and no activity -# Pretty common -python -m auto_nag.scripts.meta_no_deps_no_activity - -# Several tools here -# 1) has an unlanded patch or some flags not up-to-date -# Pretty rare -# 2) Tracked bugs -# 3) Tracked bugs with needinfos -python -m auto_nag.scripts.multi_nag - -# has a r+ patch, is open, has no activity for few weeks -# Pretty common -python -m auto_nag.scripts.not_landed - -# New workflow -# https://docs.google.com/document/d/1EHuWa-uR-7Sq63X1ZiDN1mvJ9gQtWiqYrCifkySJyW0/edit# -# https://docs.google.com/drawings/d/1oZA-AUvkOxGMNhZNofL8Wlfk6ol3o5ATQCV5DJJKbwM/edit -python -m auto_nag.scripts.workflow.multi_nag - -# Defect or task with the "feature" keyword -python -m auto_nag.scripts.feature_but_type_defect_task - -# Defect with the "meta" keyword -python -m auto_nag.scripts.meta_defect - -# reporter has a needinfo and no activity for the last X weeks -# Pretty common -python -m auto_nag.scripts.newbie_with_ni -d - -# Bug caused several regressions recently reported -# Pretty rare -python -m auto_nag.scripts.warn_regressed_by - -# Defect starting with please or enable in the title -python -m auto_nag.scripts.defect_with_please_or_enable - -# Regressions without regressed_by and some dependencies (blocks, depends_on) -# Pretty rare -python -m auto_nag.scripts.regression_without_regressed_by - -# Send a mail if the logs are not empty -# MUST ALWAYS BE THE LAST COMMAND -python -m auto_nag.log --send - -deactivate diff --git a/runauto_nag_daily_py3.sh b/runauto_nag_daily_py3.sh deleted file mode 100755 index 39fcf509a..000000000 --- a/runauto_nag_daily_py3.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -e - -. venv3/bin/activate - -# force the update of dependencies -pip install -r requirements.txt - -# Clean the log files -python -m auto_nag.log --clean - -# Try to detect potential regressions using bugbug -python -m auto_nag.scripts.regression - -# Suggest components for untriaged bugs (daily, full list without confidence threshold) -python -m auto_nag.scripts.component --frequency daily - -# Try to detect potential wrong bug types using bugbug -python -m auto_nag.scripts.defectenhancementtask - -# Send a mail if the logs are not empty -# MUST ALWAYS BE THE LAST COMMAND -python -m auto_nag.log --send - -deactivate diff --git a/runauto_nag_hourly.sh b/runauto_nag_hourly.sh index 704d84da2..f71a07928 100755 --- a/runauto_nag_hourly.sh +++ b/runauto_nag_hourly.sh @@ -4,5 +4,92 @@ set -e export PYTHONPATH=. ./runauto_nag_common.sh -./runauto_nag_hourly_py2.sh -./runauto_nag_hourly_py3.sh + +. venv/bin/activate + +# Clean the log files +python -m auto_nag.log --clean + +# Bug fixed without assignee +# very common +python -m auto_nag.scripts.no_assignee + +# Bug closed with the leave open keyword +# very common +python -m auto_nag.scripts.leave_open + +# has a STR without flag has_str +# common +# python -m auto_nag.scripts.has_str_no_hasstr + +# Closes crash bug without any crashes for the last 12 weeks +# pretty common +python -m auto_nag.scripts.no_crashes + +# Unconfirmed bugs with an assignee (with autofix) +# Pretty common +python -m auto_nag.scripts.assignee_but_unconfirmed + +# List bug with the meta keyword but not [meta] in the title +# Pretty common +python -m auto_nag.scripts.meta_summary_missing + +# List bug without the meta keyword with [meta] in the title (with autofix) +# Pretty common +python -m auto_nag.scripts.summary_meta_missing + +# List reopened bugs with invalid nightly status flag +# Pretty common +python -m auto_nag.scripts.nightly_reopened + +# Bug closed with the stalled keyword +# Pretty rare +python -m auto_nag.scripts.stalled + +# Bugs with missing beta status +# Pretty rare +python -m auto_nag.scripts.missing_beta_status + +# Bugs with STR and no regression-range +# Pretty rare +python -m auto_nag.scripts.has_str_no_range + +# Notify bugs tracked (+ or blocking) +# with P3, P4 or P5 priorities for the ongoing releases +# Pretty common +python -m auto_nag.scripts.mismatch_priority_tracking_esr +python -m auto_nag.scripts.mismatch_priority_tracking_release +python -m auto_nag.scripts.mismatch_priority_tracking_beta +python -m auto_nag.scripts.mismatch_priority_tracking_nightly + +# Bug is tracked for a release but the bug severity is small +# pretty common +python -m auto_nag.scripts.tracked_bad_severity + +# Move info (signatures, product/component) from/to bugs & their dups +# Pretty common +python -m auto_nag.scripts.copy_duplicate_info + +# Enhancement or task with the "regression" keyword +python -m auto_nag.scripts.regression_but_type_enhancement_task + +# Move dupeme from whiteboard to keyword +# Pretty rare +python -m auto_nag.scripts.dupeme_whiteboard_keyword + +# Remove dupeme keyword when the bug is closed +# Pretty rare +python -m auto_nag.scripts.closed_dupeme + +# Suggest components for untriaged bugs (hourly, list only bugs on which we acted) +python -m auto_nag.scripts.component --frequency hourly + +# MUST ALWAYS BE AFTER COMPONENTS (to reset the priority if mandatory) +# Reset the priority if the product::component changed after the priority has been set +python -m auto_nag.scripts.prod_comp_changed_with_priority + +# Send a mail if the logs are not empty +# MUST ALWAYS BE THE LAST COMMAND +python -m auto_nag.log --send + +deactivate diff --git a/runauto_nag_hourly_py2.sh b/runauto_nag_hourly_py2.sh deleted file mode 100755 index 533cbe961..000000000 --- a/runauto_nag_hourly_py2.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -set -e - -. venv2/bin/activate - -# Clean the log files -python -m auto_nag.log --clean - -# Bug fixed without assignee -# very common -python -m auto_nag.scripts.no_assignee - -# Bug closed with the leave open keyword -# very common -python -m auto_nag.scripts.leave_open - -# has a STR without flag has_str -# common -# python -m auto_nag.scripts.has_str_no_hasstr - -# Closes crash bug without any crashes for the last 12 weeks -# pretty common -python -m auto_nag.scripts.no_crashes - -# Unconfirmed bugs with an assignee (with autofix) -# Pretty common -python -m auto_nag.scripts.assignee_but_unconfirmed - -# List bug with the meta keyword but not [meta] in the title -# Pretty common -python -m auto_nag.scripts.meta_summary_missing - -# List bug without the meta keyword with [meta] in the title (with autofix) -# Pretty common -python -m auto_nag.scripts.summary_meta_missing - -# List reopened bugs with invalid nightly status flag -# Pretty common -python -m auto_nag.scripts.nightly_reopened - -# Bug closed with the stalled keyword -# Pretty rare -python -m auto_nag.scripts.stalled - -# Bugs with missing beta status -# Pretty rare -python -m auto_nag.scripts.missing_beta_status - -# Bugs with STR and no regression-range -# Pretty rare -python -m auto_nag.scripts.has_str_no_range - -# Notify bugs tracked (+ or blocking) -# with P3, P4 or P5 priorities for the ongoing releases -# Pretty common -python -m auto_nag.scripts.mismatch_priority_tracking_esr -python -m auto_nag.scripts.mismatch_priority_tracking_release -python -m auto_nag.scripts.mismatch_priority_tracking_beta -python -m auto_nag.scripts.mismatch_priority_tracking_nightly - -# Bug is tracked for a release but the bug severity is small -# pretty common -python -m auto_nag.scripts.tracked_bad_severity - -# Move info (signatures, product/component) from/to bugs & their dups -# Pretty common -python -m auto_nag.scripts.copy_duplicate_info - -# Enhancement or task with the "regression" keyword -python -m auto_nag.scripts.regression_but_type_enhancement_task - -# Move dupeme from whiteboard to keyword -# Pretty rare -python -m auto_nag.scripts.dupeme_whiteboard_keyword - -# Remove dupeme keyword when the bug is closed -# Pretty rare -python -m auto_nag.scripts.closed_dupeme - -# Send a mail if the logs are not empty -# MUST ALWAYS BE THE LAST COMMAND -python -m auto_nag.log --send - -deactivate diff --git a/runauto_nag_hourly_py3.sh b/runauto_nag_hourly_py3.sh deleted file mode 100755 index a7dd8eb8e..000000000 --- a/runauto_nag_hourly_py3.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -e - -. venv3/bin/activate - -# Clean the log files -python -m auto_nag.log --clean - -# Suggest components for untriaged bugs (hourly, list only bugs on which we acted) -python -m auto_nag.scripts.component --frequency hourly - -# MUST ALWAYS BE AFTER COMPONENTS (to reset the priority if mandatory) -# Reset the priority if the product::component changed after the priority has been set -python -m auto_nag.scripts.prod_comp_changed_with_priority - -# Send a mail if the logs are not empty -# MUST ALWAYS BE THE LAST COMMAND -python -m auto_nag.log --send - -deactivate diff --git a/setup.py b/setup.py deleted file mode 100644 index f300ca8e8..000000000 --- a/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -import os - -from setuptools import setup, find_packages - - -root = os.path.abspath(os.path.dirname(__file__)) -path = lambda *p: os.path.join(root, *p) - - -setup( - name='bztools', - version=__import__('auto_nag.bugzilla').__version__, - description='Models and scripts to access the Bugzilla REST API.', - long_description=open(path('README.rst')).read(), - author='Jeff Balogh', - author_email='me@jeffbalogh.org', - url='https://github.com/mozilla/bztools', - license='BSD', - packages=find_packages(), - include_package_data=True, - zip_safe=False, - # install_requires=['remoteobjects>=1.1'], - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - entry_points={ - 'console_scripts': [ - 'bzattach = scripts.attach:main', - ], - }, -) diff --git a/templates/channel_meeting_wiki b/templates/channel_meeting_wiki deleted file mode 100644 index 1d9708c1a..000000000 --- a/templates/channel_meeting_wiki +++ /dev/null @@ -1,28 +0,0 @@ -= Triage = -{% for channel,info in channel_info.items() %} -{% if info.needs_attention %} -== {{ channel }} bugs needing attention == -# Keyworded with #relman/triage/defer-to-group using mediawiki bz extension - -{ - "id": {% for bug in info.needs_attention %}"{{ bug.id }},"{% endfor %} -} - -{% endif %} -{% if info.qawanted %} -== Tracking for {{ channel }} and QA Wanted for more than x days == - -{ - "id": {% for bug in info.qawanted %}"{{ bug.id }},"{% endfor %} -} - -{% endif %} -== Tracking {{ channel }} - {% if days_since_comment > -1 %}{{ days_since_comment }} days without assignee/manager comment{% endif %} == -{% for m in info.managers %} -{# note: escaping the wiki bug template with '{{' and '}}" #} -=== {{ m.name }} === - {% for bug in m.bugs %}* [https://bugzilla.mozilla.org/{{ bug.id }} {{ bug.id }}] - assignee : {{ bug.assigned_to.real_name }}{% if bug.keywords %} - keywords: {{ bug.keywords }}{% endif %}{% if bug.whiteboard %} - whiteboard: {{ bug.whiteboard }} {% endif %} - {% endfor %} -{% endfor %} -{% endfor %} -See the source / report issues on https://github.com/mozilla/relman-auto-nag/ diff --git a/templates/common.html b/templates/common.html index c0b9cfcf9..b517942e2 100644 --- a/templates/common.html +++ b/templates/common.html @@ -2,15 +2,6 @@ - {% if has_table -%} - - {% endif -%}

Hi there,

diff --git a/templates/daily_email b/templates/daily_email deleted file mode 100644 index d567699b9..000000000 --- a/templates/daily_email +++ /dev/null @@ -1,21 +0,0 @@ -NOTE: On Mondays, we send email to remind you of work tracked for upcoming releases. On Thursdays, we send email to remind you of tracked bugs that haven't been touched yet this week. Every day, we send email for unlanded approvals and bugs with need-info? unanswered. - -= Next Actions = -* If you don't believe a bug should be tracked for release or your investigation is blocked, please comment in the bug and optionally reply to this email -* If you're surprised that a bug is listed below, please make sure to check the version-specific status flags. -* Investigations for Beta should be prioritized over Aurora, etc. -* Stay abreast of where we are in the cycle by checking the calendar at https://bit.ly/1zUHw5G. Remember, beta 8 is the last opportunity for speculative fixes and we code freeze after beta 9. - -= Queries = -{% for name, results in queries.items() %}== {{ name }} == -{% for bug in results.buglist -%} -* https://bugzilla.mozilla.org/{{ bug.id }} - {{ bug.summary }}{% if bug.flags != None %} {% for flag in bug.flags %}{% if flag.name == 'needinfo' %}(needinfo? {{ flag.requestee }}){% endif %}{% endfor %}{% endif %} - (assigned to {{ bug.assignee }}) -{% endfor %} -{% endfor %} - - -Sincerely, -Release Management - -See the source / report issues on https://github.com/mozilla/relman-auto-nag/ - diff --git a/templates/tracked_affected_email b/templates/tracked_affected_email deleted file mode 100644 index d1c4e19dc..000000000 --- a/templates/tracked_affected_email +++ /dev/null @@ -1,16 +0,0 @@ -This email contains all the bugs which have been fixed in nightly but still affect other supported channels. - -= Queries = -{% for name, results in queries.items() %}== {{ name }} == -An up-to-date list can be found at {{ results.query_url }} - -{% for bug in results.buglist -%} -* https://bugzilla.mozilla.org/{{ bug.id }} - {{ bug.summary }} - - affected versions: {{ ', '.join(bug.affected) }} -{% endfor %} -{% endfor %} - -Sincerely, -Release Management - -See the source / report issues on https://github.com/mozilla/relman-auto-nag/ diff --git a/templates/unlanded.html b/templates/unlanded.html index d74932c68..eae6f3262 100644 --- a/templates/unlanded.html +++ b/templates/unlanded.html @@ -1,4 +1,4 @@ -

The following {{ plural('bug has', data, pword='bugs have') }} an unlanded patch or some flags not up-to-date{% if not nag %} (when the bug is red, then the assignee is a person with no manager){% endif %}. +

The following {{ plural('bug has', data, pword='bugs have') }} an unlanded patch in {{ extra['channel'] }} {{ extra['version'] }} or some flags not up-to-date{% if not nag %} (when the bug is red, then the assignee is a person with no manager){% endif %}.

If the patch has landed then the status flags are not up-to-date. diff --git a/tox.ini b/tox.ini index c68197611..8eaee9e8b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27 +envlist = py37 skipsdist = True [testenv]