From d086ee653e8bacbc52afb600f161b911c59592d8 Mon Sep 17 00:00:00 2001 From: Adrian Gaudebert Date: Fri, 21 Jun 2013 12:22:53 +0200 Subject: [PATCH] Fixes bug 867388 - Added basic middleware services for Bixie. --- socorro/external/postgresql/error.py | 39 ++-- socorro/external/postgresql/errors.py | 87 ++++++++ socorro/external/postgresql/products.py | 74 +++++-- .../external/postgresql/test_error.py | 24 +-- .../external/postgresql/test_errors.py | 196 ++++++++++++++++++ .../external/postgresql/test_products.py | 87 +++++++- 6 files changed, 451 insertions(+), 56 deletions(-) create mode 100644 socorro/external/postgresql/errors.py create mode 100644 socorro/unittest/external/postgresql/test_errors.py diff --git a/socorro/external/postgresql/error.py b/socorro/external/postgresql/error.py index 9b8b28ecfe..edbb77d2e5 100644 --- a/socorro/external/postgresql/error.py +++ b/socorro/external/postgresql/error.py @@ -2,20 +2,13 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import logging - from socorro.external import MissingOrBadArgumentError from socorro.external.postgresql.base import PostgreSQLBase from socorro.lib import datetimeutil, external_common -logger = logging.getLogger("webapi") - class Error(PostgreSQLBase): - - """ - Implement the /error service with PostgreSQL. - """ + """Implement the /error service with PostgreSQL. """ def get(self, **kwargs): """Return a single error report from its UUID. """ @@ -26,11 +19,10 @@ def get(self, **kwargs): if params.uuid is None: raise MissingOrBadArgumentError( - "Mandatory parameter 'uuid' is missing or empty") + "Mandatory parameter 'uuid' is missing or empty" + ) crash_date = datetimeutil.uuid_to_date(params.uuid) - logger.debug("Looking for error %s during day %s" % (params.uuid, - crash_date)) sql = """/* socorro.external.postgresql.error.Error.get */ SELECT @@ -40,26 +32,29 @@ def get(self, **kwargs): FROM bixie.crashes WHERE bixie.crashes.crash_id=%(uuid)s AND bixie.crashes.success IS NOT NULL - AND utc_day_is( bixie.crashes.processor_completed_datetime, %(crash_date)s) + AND utc_day_is( + bixie.crashes.processor_completed_datetime, + %(crash_date)s + ) """ sql_params = { - "uuid": params.uuid, "uuid": params.uuid, "crash_date": crash_date } - error_message = "Failed to retrieve crash data from PostgreSQL" + error_message = "Failed to retrieve error data from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) - crashes = [] + errors = [] for row in results: - crash = dict(zip(( - "product", - "error", - "signature"), row)) - crashes.append(crash) + error = dict(zip(( + "product", + "error", + "signature" + ), row)) + errors.append(error) return { - "hits": crashes, - "total": len(crashes) + "hits": errors, + "total": len(errors) } diff --git a/socorro/external/postgresql/errors.py b/socorro/external/postgresql/errors.py new file mode 100644 index 0000000000..6309c024d4 --- /dev/null +++ b/socorro/external/postgresql/errors.py @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import datetime + +from socorro.external import MissingOrBadArgumentError +from socorro.external.postgresql.base import PostgreSQLBase +from socorro.lib import datetimeutil, external_common, search_common + + +class Errors(PostgreSQLBase): + '''Implement the /errors services with PostgreSQL. ''' + + def get_signatures(self, **kwargs): + '''Return a list of errors aggregated by signatures. ''' + now = datetimeutil.utc_now() + lastweek = now - datetime.timedelta(7) + + filters = [ + ('signature', None, 'str'), + ('search_mode', 'is_exactly', 'str'), + ('product', None, 'str'), + ('start_date', lastweek, 'datetime'), + ('end_date', now, 'datetime'), + ] + params = external_common.parse_arguments(filters, kwargs) + + authorized_search_modes = ( + 'is_exactly', + 'contains', + 'starts_with', + 'ends_with' + ) + if params.search_mode not in authorized_search_modes: + search_mode = authorized_search_modes[0] + + fields = ('signature', 'count') + sql_fields = { + 'signature': 'signature', + 'count': 'count(crash_id) as total' + } + + sql = '''/* socorro.external.postgresql.error.Error.get */ + SELECT %s + FROM bixie.crashes + WHERE success IS NOT NULL + AND processor_completed_datetime BETWEEN + %%(start_date)s AND %%(end_date)s + ''' % ', '.join(sql_fields[x] for x in fields) + + sql_where = [sql] + if params.signature: + if params.search_mode == 'is_exactly': + sql_where.append('signature = %(signature)s') + else: + if params.search_mode == 'contains': + params.signature = '%%%s%%' % params.signature + elif params.search_mode == 'starts_with': + params.signature = '%%%s' % params.signature + elif params.search_mode == 'ends_with': + params.signature = '%s%%' % params.signature + + sql_where.append('signature LIKE %(signature)s') + + if params.product: + sql_where.append('product = %(product)s') + + sql = ' AND '.join(sql_where) + + sql_group = 'GROUP BY signature' + sql_order = 'ORDER BY total DESC, signature' + + sql = ' '.join((sql, sql_group, sql_order)) + + error_message = 'Failed to retrieve error data from PostgreSQL' + results = self.query(sql, params, error_message=error_message) + + errors = [] + for row in results: + error = dict(zip(fields, row)) + errors.append(error) + + return { + 'hits': errors, + 'total': len(errors) + } diff --git a/socorro/external/postgresql/products.py b/socorro/external/postgresql/products.py index 511972676c..5fa937cf3d 100644 --- a/socorro/external/postgresql/products.py +++ b/socorro/external/postgresql/products.py @@ -4,6 +4,7 @@ import logging +from socorro.external import MissingOrBadArgumentError from socorro.external.postgresql.base import add_param_to_dict, PostgreSQLBase from socorro.lib import datetimeutil, external_common @@ -16,27 +17,51 @@ def get(self, **kwargs): """ Return product information, or version information for one or more product:version combinations """ filters = [ - ("versions", None, ["list", "str"]) # for legacy, to be removed + ("versions", None, ["list", "str"]), # for legacy, to be removed + ("type", "desktop", "str"), ] params = external_common.parse_arguments(filters, kwargs) + accepted_types = ("desktop", "webapp") + if params.type not in accepted_types: + raise MissingOrBadArgumentError( + "Bad value for parameter 'type': got '%s', expected one of %s)" + % (params.type, accepted_types) + ) + if params.versions and params.versions[0]: return self._get_versions(params) - sql = """ - /* socorro.external.postgresql.products.Products.get */ - SELECT - product_name, - version_string, - start_date, - end_date, - throttle, - is_featured, - build_type, - has_builds - FROM product_info - ORDER BY product_sort, version_sort DESC, channel_sort - """ + if params.type == "desktop": + sql = """ + /* socorro.external.postgresql.products.Products.get */ + SELECT + product_name, + version_string, + start_date, + end_date, + throttle, + is_featured, + build_type, + has_builds + FROM product_info + ORDER BY product_sort, version_sort DESC, channel_sort + """ + elif params.type == "webapp": + sql = """ + /* socorro.external.postgresql.products.Products.get */ + SELECT + product_name, + version, + NULL as start_date, + NULL as end_date, + 1.0 as throttle, + FALSE as is_featured, + build_type, + FALSE as has_builds + FROM bixie.raw_product_releases + ORDER BY product_name, version DESC + """ error_message = "Failed to retrieve products/versions from PostgreSQL" results = self.query(sql, error_message=error_message) @@ -56,12 +81,19 @@ def get(self, **kwargs): 'has_builds', ), row)) - version['end_date'] = datetimeutil.date_to_string( - version['end_date'] - ) - version['start_date'] = datetimeutil.date_to_string( - version['start_date'] - ) + try: + version['end_date'] = datetimeutil.date_to_string( + version['end_date'] + ) + except TypeError: + pass + try: + version['start_date'] = datetimeutil.date_to_string( + version['start_date'] + ) + except TypeError: + pass + version['throttle'] = float(version['throttle']) product = version['product'] diff --git a/socorro/unittest/external/postgresql/test_error.py b/socorro/unittest/external/postgresql/test_error.py index d322668cf6..dcd477b8e2 100644 --- a/socorro/unittest/external/postgresql/test_error.py +++ b/socorro/unittest/external/postgresql/test_error.py @@ -82,6 +82,18 @@ def setUp(self): self.connection.commit() + #-------------------------------------------------------------------------- + def tearDown(self): + """Clean up the database, delete tables and functions. """ + cursor = self.connection.cursor() + cursor.execute(""" SET search_path TO bixie """) + cursor.execute(""" + TRUNCATE crashes + CASCADE + """) + self.connection.commit() + super(IntegrationTestError, self).tearDown() + #-------------------------------------------------------------------------- def test_get(self): """ Test GET for Bixie Errors """ @@ -115,15 +127,3 @@ def test_get(self): } self.assertEqual(res, res_expected) - - #-------------------------------------------------------------------------- - def tearDown(self): - """Clean up the database, delete tables and functions. """ - cursor = self.connection.cursor() - cursor.execute(""" SET search_path TO bixie """) - cursor.execute(""" - TRUNCATE crashes - CASCADE - """) - self.connection.commit() - super(IntegrationTestError, self).tearDown() diff --git a/socorro/unittest/external/postgresql/test_errors.py b/socorro/unittest/external/postgresql/test_errors.py new file mode 100644 index 0000000000..77037bc5c2 --- /dev/null +++ b/socorro/unittest/external/postgresql/test_errors.py @@ -0,0 +1,196 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import datetime + +from nose.plugins.attrib import attr + +from socorro.external.postgresql.errors import Errors +from socorro.lib import datetimeutil + +from unittestbase import PostgreSQLTestCase + + +@attr(integration='postgres') # for nosetests +class IntegrationTestErrors(PostgreSQLTestCase): + """Test socorro.external.postgresql.errors.Errors class. """ + + def setUp(self): + super(IntegrationTestErrors, self).setUp() + + cursor = self.connection.cursor() + + # Insert data + self.now = datetimeutil.utc_now() + last_month = self.now - datetime.timedelta(weeks=4) + uuid = "000aaf-98e0-4ece-a904-2573e2%s" % self.now.strftime("%y%m%d") + + # Insert valid data + cursor.execute(''' + INSERT INTO bixie.crashes + ( + crash_id, + signature, + error, + product, + processor_completed_datetime, + success + ) + VALUES + ( + '01%(uuid)s', + 'i_can_haz_crash()', + '{}', + 'ClockOClock', + '%(now)s', + TRUE + ), + ( + '02%(uuid)s', + 'i_can_haz_crash()', + '{}', + 'ClockOClock', + '%(now)s', + TRUE + ), + ( + '03%(uuid)s', + 'heyIJustMetYou', + '{}', + 'ClockOClock', + '%(now)s', + TRUE + ), + ( + '04%(uuid)s', + 'goGoGadgetCrash()', + '{}', + 'EmailApp', + '%(now)s', + TRUE + ) + ''' % {'now': self.now, 'uuid': uuid}) + + # Insert vinalid data + cursor.execute(''' + INSERT INTO bixie.crashes + ( + crash_id, + signature, + error, + product, + processor_completed_datetime, + success + ) + VALUES + ( + '11%(uuid)s', + 'i_can_haz_crash()', + '{}', + 'ClockOClock', + '%(last_month)s', + TRUE + ), + ( + '12%(uuid)s', + 'i_can_haz_crash()', + '{}', + 'ClockOClock', + '%(now)s', + NULL + ) + ''' % {'now': self.now, 'last_month': last_month, 'uuid': uuid}) + + self.connection.commit() + + def tearDown(self): + cursor = self.connection.cursor() + cursor.execute(""" + TRUNCATE bixie.crashes CASCADE + """) + self.connection.commit() + super(IntegrationTestErrors, self).tearDown() + + def test_get_signatures(self): + api = Errors(config=self.config) + + # Test no parameter + res = api.get_signatures() + res_expected = { + 'hits': [ + { + 'signature': 'i_can_haz_crash()', + 'count': 2, + }, + { + 'signature': 'goGoGadgetCrash()', + 'count': 1, + }, + { + 'signature': 'heyIJustMetYou', + 'count': 1, + }, + ], + 'total': 3 + } + self.assertEqual(res, res_expected) + + # Test signature parameter + res = api.get_signatures(signature='i_can_haz_crash()') + res_expected = { + 'hits': [ + { + 'signature': 'i_can_haz_crash()', + 'count': 2, + }, + ], + 'total': 1 + } + self.assertEqual(res, res_expected) + + # Test search_mode parameter + res = api.get_signatures(signature='et', search_mode='contains') + res_expected = { + 'hits': [ + { + 'signature': 'goGoGadgetCrash()', + 'count': 1, + }, + { + 'signature': 'heyIJustMetYou', + 'count': 1, + }, + ], + 'total': 2 + } + self.assertEqual(res, res_expected) + + # Test product parameter + res = api.get_signatures(product='EmailApp') + res_expected = { + 'hits': [ + { + 'signature': 'goGoGadgetCrash()', + 'count': 1, + }, + ], + 'total': 1 + } + self.assertEqual(res, res_expected) + + # Test date parameters + res = api.get_signatures( + start_date=self.now - datetime.timedelta(weeks=5), + end_date=self.now - datetime.timedelta(weeks=3) + ) + res_expected = { + 'hits': [ + { + 'signature': 'i_can_haz_crash()', + 'count': 1, + }, + ], + 'total': 1 + } + self.assertEqual(res, res_expected) diff --git a/socorro/unittest/external/postgresql/test_products.py b/socorro/unittest/external/postgresql/test_products.py index deb2bdbe85..e681fdcc4d 100644 --- a/socorro/unittest/external/postgresql/test_products.py +++ b/socorro/unittest/external/postgresql/test_products.py @@ -146,6 +146,44 @@ def setUp(self): ); """ % {'now': now, 'lastweek': lastweek}) + # insert bixie errors + cursor.execute(""" + INSERT INTO bixie.raw_product_releases + (id, product_name, version, build, build_type, platform, + repository, stability) + VALUES + ( + 1, + 'EmailApp', + '0.1', + 1234567890, + 'Release', + 'mobile', + 'repo', + 'stable' + ), + ( + 2, + 'EmailApp', + '0.2', + 1234567890, + 'Beta', + 'mobile', + 'repo', + 'stable' + ), + ( + 3, + 'ClockOClock', + '1.0.18', + 1234567890, + 'Release', + 'mobile', + 'repo', + 'stable' + ) + """) + self.connection.commit() #-------------------------------------------------------------------------- @@ -155,7 +193,8 @@ def tearDown(self): cursor.execute(""" TRUNCATE products, product_version_builds, product_versions, product_release_channels, release_channels, - product_versions + product_versions, + bixie.raw_product_releases CASCADE """) self.connection.commit() @@ -329,6 +368,52 @@ def test_get(self): res = products.get(**params) self.assertEqual(res['total'], 4) + def test_get_webapp_products(self): + api = Products(config=self.config) + + res = api.get(type='webapp') + res_expected = { + 'products': ['ClockOClock', 'EmailApp'], + 'hits': { + 'EmailApp': [ + { + "product": "EmailApp", + "version": "0.2", + "start_date": None, + "end_date": None, + "throttle": 1.0, + "featured": False, + "release": "Beta", + "has_builds": False + }, + { + "product": "EmailApp", + "version": "0.1", + "start_date": None, + "end_date": None, + "throttle": 1.0, + "featured": False, + "release": "Release", + "has_builds": False + } + ], + 'ClockOClock': [ + { + "product": "ClockOClock", + "version": "1.0.18", + "start_date": None, + "end_date": None, + "throttle": 1.0, + "featured": False, + "release": "Release", + "has_builds": False + } + ] + }, + 'total': 3 + } + self.assertEqual(res, res_expected) + def test_get_default_version(self): products = Products(config=self.config)