From a3004ee3744cb559e1a05ad4b607c258776d9221 Mon Sep 17 00:00:00 2001 From: K Lars Lohn Date: Fri, 2 Nov 2012 15:39:13 -0700 Subject: [PATCH 1/2] added missing __init__.py files, fixed unittests --- socorro/external/hbase/crashstorage.py | 11 +-- socorro/unittest/app/__init__.py | 0 .../test_generic_fetch_transform_save_app.py | 85 +++++++++++++------ .../unittest/external/filesystem/__init__.py | 0 .../external/filesystem/test_crashstorage.py | 14 --- socorro/unittest/external/hbase/__init__.py | 0 socorro/unittest/storage/__init__.py | 0 socorro/unittest/storage/testCrashstorage.py | 1 - 8 files changed, 65 insertions(+), 46 deletions(-) create mode 100644 socorro/unittest/app/__init__.py create mode 100644 socorro/unittest/external/filesystem/__init__.py create mode 100644 socorro/unittest/external/hbase/__init__.py create mode 100644 socorro/unittest/storage/__init__.py diff --git a/socorro/external/hbase/crashstorage.py b/socorro/external/hbase/crashstorage.py index 5a5bf3b4a4..f81d279ca4 100644 --- a/socorro/external/hbase/crashstorage.py +++ b/socorro/external/hbase/crashstorage.py @@ -8,6 +8,7 @@ CrashStorageBase, CrashIDNotFound) from socorro.external.hbase import hbase_client from socorro.database.transaction_executor import TransactionExecutor +from socorro.lib.util import DotDict from configman import Namespace, class_converter @@ -101,7 +102,7 @@ def save_raw_crash(self, raw_crash, dump, crash_id): def save_processed(self, processed_crash): sanitized_processed_crash = self.sanitize_processed_crash( processed_crash, - self.config.processed_crash_key_filter + self.config.forbidden_keys ) self._stringify_dates_in_dict(sanitized_processed_crash) self.transaction_executor( @@ -113,11 +114,11 @@ def save_processed(self, processed_crash): #-------------------------------------------------------------------------- def get_raw_crash(self, crash_id): - return self.transaction_executor( + return DotDict(self.transaction_executor( hbase_client.HBaseConnectionForCrashReports.get_json, crash_id, number_of_retries=self.config.number_of_retries - ) + )) #-------------------------------------------------------------------------- def get_raw_dump(self, crash_id): @@ -130,11 +131,11 @@ def get_raw_dump(self, crash_id): #-------------------------------------------------------------------------- def get_processed_crash(self, crash_id): try: - return self.transaction_executor( + return DotDict(self.transaction_executor( hbase_client.HBaseConnectionForCrashReports.get_processed_json, crash_id, number_of_retries=self.config.number_of_retries - ) + )) except hbase_client.OoidNotFoundException: # we want a consistent set of exceptions for the API raise CrashIDNotFound(crash_id) diff --git a/socorro/unittest/app/__init__.py b/socorro/unittest/app/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/socorro/unittest/app/test_generic_fetch_transform_save_app.py b/socorro/unittest/app/test_generic_fetch_transform_save_app.py index e1c25cd07a..7cae994274 100644 --- a/socorro/unittest/app/test_generic_fetch_transform_save_app.py +++ b/socorro/unittest/app/test_generic_fetch_transform_save_app.py @@ -5,7 +5,7 @@ import unittest from socorro.app.fetch_transform_save_app import FetchTransformSaveApp - +from socorro.lib.threaded_task_manager import ThreadedTaskManager from socorro.lib.util import DotDict, SilentFakeLogger @@ -38,8 +38,14 @@ def transform(self, anItem): 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, - 'source': DotDict({'crashstorage': None}), - 'destination': DotDict({'crashstorage': None}) + 'source': DotDict({'crashstorage_class': None}), + 'destination': DotDict({'crashstorage_class': None}), + 'producer_consumer': DotDict({'producer_consumer_class': + ThreadedTaskManager, + 'logger': logger, + 'number_of_threads': 1, + 'maximum_queue_size': 1} + ) }) fts_app = TestFTSAppClass(config) @@ -79,7 +85,7 @@ def get_raw_crash(self, ooid): def get_dump(self, ooid): return 'this is a fake dump' - def new_ooids(self): + def new_crashes(self): for k in self.store.keys(): yield k @@ -89,19 +95,25 @@ def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() - def save_raw_crash(self, raw_crash, dump): - self.store[raw_crash.ooid] = raw_crash - self.dumps[raw_crash.ooid] = dump + def save_raw_crash(self, raw_crash, dump, crash_id): + self.store[crash_id] = raw_crash + self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, - 'source': DotDict({'crashstorage': + 'source': DotDict({'crashstorage_class': FakeStorageSource}), - 'destination': DotDict({'crashstorage': - FakeStorageDestination}) + 'destination': DotDict({'crashstorage_class': + FakeStorageDestination}), + 'producer_consumer': DotDict({'producer_consumer_class': + ThreadedTaskManager, + 'logger': logger, + 'number_of_threads': 1, + 'maximum_queue_size': 1} + ) }) fts_app = NonInfiniteFTSAppClass(config) @@ -121,7 +133,7 @@ class FakeStorageSource(object): def __init__(self): self.first = True - def new_ooids(self): + def new_crashes(self): if self.first: self.first = False else: @@ -135,19 +147,25 @@ def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() - def save_raw_crash(self, raw_crash, dump): - self.store[raw_crash.ooid] = raw_crash - self.dumps[raw_crash.ooid] = dump + def save_raw_crash(self, raw_crash, dump, crash_id): + self.store[crash_id] = raw_crash + self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, - 'source': DotDict({'crashstorage': + 'source': DotDict({'crashstorage_class': FakeStorageSource}), - 'destination': DotDict({'crashstorage': - FakeStorageDestination}) + 'destination': DotDict({'crashstorage_class': + FakeStorageDestination}), + 'producer_consumer': DotDict({'producer_consumer_class': + ThreadedTaskManager, + 'logger': logger, + 'number_of_threads': 1, + 'maximum_queue_size': 1} + ) }) fts_app = FetchTransformSaveApp(config) @@ -177,19 +195,25 @@ def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() - def save_raw_crash(self, raw_crash, dump): - self.store[raw_crash.ooid] = raw_crash - self.dumps[raw_crash.ooid] = dump + def save_raw_crash(self, raw_crash, dump, crash_id): + self.store[crash_id] = raw_crash + self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, - 'source': DotDict({'crashstorage': + 'source': DotDict({'crashstorage_class': None}), - 'destination': DotDict({'crashstorage': - FakeStorageDestination}) + 'destination': DotDict({'crashstorage_class': + FakeStorageDestination}), + 'producer_consumer': DotDict({'producer_consumer_class': + ThreadedTaskManager, + 'logger': logger, + 'number_of_threads': 1, + 'maximum_queue_size': 1} + ) }) fts_app = FetchTransformSaveApp(config) @@ -223,15 +247,24 @@ def new_ooids(self): for k in self.store.keys(): yield k + + logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, - 'source': DotDict({'crashstorage': + 'source': DotDict({'crashstorage_class': FakeStorageSource}), - 'destination': DotDict({'crashstorage': - None}) + 'destination': DotDict({'crashstorage_class': + None}), + 'producer_consumer': DotDict({'producer_consumer_class': + ThreadedTaskManager, + 'logger': logger, + 'number_of_threads': 1, + 'maximum_queue_size': 1} + ) + }) fts_app = FetchTransformSaveApp(config) diff --git a/socorro/unittest/external/filesystem/__init__.py b/socorro/unittest/external/filesystem/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/socorro/unittest/external/filesystem/test_crashstorage.py b/socorro/unittest/external/filesystem/test_crashstorage.py index 138d5f9c50..dddc457e95 100644 --- a/socorro/unittest/external/filesystem/test_crashstorage.py +++ b/socorro/unittest/external/filesystem/test_crashstorage.py @@ -74,13 +74,6 @@ def _common_config_setup(self): def _common_basic_test(self, config, crashstorage): fake_dump = 'this is a fake dump' self.assertEqual(list(crashstorage.new_crashes()), []) - raw = {"name": "Peter", "legacy_processing": 0} - self.assertRaises( - CrashIDNotFound, - crashstorage.save_raw_crash, - raw, - fake_dump # as a stand in for the binary dump file - ) raw = {"name": "Peter", #"ooid":"114559a5-d8e6-428c-8b88-1c1f22120314", "legacy_processing": 0} @@ -130,13 +123,6 @@ def _common_throttle_test(self, config, crashstorage): fake_dump = 'this is a fake dump' crashstorage = FileSystemThrottledCrashStorage(config) self.assertEqual(list(crashstorage.new_crashes()), []) - raw = {"name": "Peter", "legacy_processing": 1} - self.assertRaises( - CrashIDNotFound, - crashstorage.save_raw_crash, - raw, - fake_dump # as a stand in for the binary dump file - ) raw = {"name": "Peter", #"ooid":"114559a5-d8e6-428c-8b88-1c1f22120314", "legacy_processing": 1} diff --git a/socorro/unittest/external/hbase/__init__.py b/socorro/unittest/external/hbase/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/socorro/unittest/storage/__init__.py b/socorro/unittest/storage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/socorro/unittest/storage/testCrashstorage.py b/socorro/unittest/storage/testCrashstorage.py index f1d8a16d4f..5afd225eee 100644 --- a/socorro/unittest/storage/testCrashstorage.py +++ b/socorro/unittest/storage/testCrashstorage.py @@ -159,7 +159,6 @@ def testCrashStorageSystem_makeJsonDictFromForm(): assert resultJson.a == '1' assert resultJson.b == 2 assert resultJson.c == '3' - assert 'd' not in resultJson assert resultJson.e == '5' def testCrashStorageSystem_save(): From f7e31124ad725456f4f44b73dd1a118d95ad12a2 Mon Sep 17 00:00:00 2001 From: Robert Helmer Date: Fri, 2 Nov 2012 15:29:35 -0700 Subject: [PATCH 2/2] bug 808208 - exit if not __init__.py in unittest dirs --- scripts/build.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/build.sh b/scripts/build.sh index b19ca270d3..f54fa89129 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -22,6 +22,21 @@ do done popd +errors=0 +while read d +do + if [ ! -f "$d/__init__.py" ] + then + echo "$d is missing an __init__.py file, tests will not run" + errors=$((errors+1)) + fi +done < <(find socorro/unittest/* -not -name logs -type d) + +if [ $errors != 0 ] +then + exit 1 +fi + # RHEL postgres 9 RPM installs pg_config here, psycopg2 needs it export PATH=$PATH:/usr/pgsql-9.0/bin/ # run unit tests