Skip to content

Commit

Permalink
Merge pull request #919 from twobraids/bug808208-unittest-hell
Browse files Browse the repository at this point in the history 
Bug808208 unittest hell
  • Loading branch information
@lonnen
lonnen committed Nov 5, 2012
2 parents f1c5c53 + f7e3112 commit 955b894
Show file tree
Hide file tree
Showing 9 changed files with 80 additions and 46 deletions.
15 changes: 15 additions & 0 deletions scripts/build.sh
View file
Expand Up @@ -22,6 +22,21 @@ do
done
popd

errors=0
while read d
do
if [ ! -f "$d/__init__.py" ]
then
echo "$d is missing an __init__.py file, tests will not run"
errors=$((errors+1))
fi
done < <(find socorro/unittest/* -not -name logs -type d)

if [ $errors != 0 ]
then
exit 1
fi

# RHEL postgres 9 RPM installs pg_config here, psycopg2 needs it
export PATH=$PATH:/usr/pgsql-9.0/bin/
# run unit tests
Expand Down
11 changes: 6 additions & 5 deletions socorro/external/hbase/crashstorage.py
View file
Expand Up @@ -8,6 +8,7 @@
CrashStorageBase, CrashIDNotFound)
from socorro.external.hbase import hbase_client
from socorro.database.transaction_executor import TransactionExecutor
from socorro.lib.util import DotDict
from configman import Namespace, class_converter


Expand Down Expand Up @@ -101,7 +102,7 @@ def save_raw_crash(self, raw_crash, dump, crash_id):
def save_processed(self, processed_crash):
sanitized_processed_crash = self.sanitize_processed_crash(
processed_crash,
self.config.processed_crash_key_filter
self.config.forbidden_keys
)
self._stringify_dates_in_dict(sanitized_processed_crash)
self.transaction_executor(
Expand All @@ -113,11 +114,11 @@ def save_processed(self, processed_crash):

#--------------------------------------------------------------------------
def get_raw_crash(self, crash_id):
return self.transaction_executor(
return DotDict(self.transaction_executor(
hbase_client.HBaseConnectionForCrashReports.get_json,
crash_id,
number_of_retries=self.config.number_of_retries
)
))

#--------------------------------------------------------------------------
def get_raw_dump(self, crash_id):
Expand All @@ -130,11 +131,11 @@ def get_raw_dump(self, crash_id):
#--------------------------------------------------------------------------
def get_processed_crash(self, crash_id):
try:
return self.transaction_executor(
return DotDict(self.transaction_executor(
hbase_client.HBaseConnectionForCrashReports.get_processed_json,
crash_id,
number_of_retries=self.config.number_of_retries
)
))
except hbase_client.OoidNotFoundException:
# we want a consistent set of exceptions for the API
raise CrashIDNotFound(crash_id)
Expand Down
Empty file added socorro/unittest/app/__init__.py
View file
Empty file.
85 changes: 59 additions & 26 deletions socorro/unittest/app/test_generic_fetch_transform_save_app.py
View file
Expand Up @@ -5,7 +5,7 @@
import unittest

from socorro.app.fetch_transform_save_app import FetchTransformSaveApp

from socorro.lib.threaded_task_manager import ThreadedTaskManager
from socorro.lib.util import DotDict, SilentFakeLogger


Expand Down Expand Up @@ -38,8 +38,14 @@ def transform(self, anItem):
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'source': DotDict({'crashstorage': None}),
'destination': DotDict({'crashstorage': None})
'source': DotDict({'crashstorage_class': None}),
'destination': DotDict({'crashstorage_class': None}),
'producer_consumer': DotDict({'producer_consumer_class':
ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1}
)
})

fts_app = TestFTSAppClass(config)
Expand Down Expand Up @@ -79,7 +85,7 @@ def get_raw_crash(self, ooid):
def get_dump(self, ooid):
return 'this is a fake dump'

def new_ooids(self):
def new_crashes(self):
for k in self.store.keys():
yield k

Expand All @@ -89,19 +95,25 @@ def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()

def save_raw_crash(self, raw_crash, dump):
self.store[raw_crash.ooid] = raw_crash
self.dumps[raw_crash.ooid] = dump
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump

logger = SilentFakeLogger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'source': DotDict({'crashstorage':
'source': DotDict({'crashstorage_class':
FakeStorageSource}),
'destination': DotDict({'crashstorage':
FakeStorageDestination})
'destination': DotDict({'crashstorage_class':
FakeStorageDestination}),
'producer_consumer': DotDict({'producer_consumer_class':
ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1}
)
})

fts_app = NonInfiniteFTSAppClass(config)
Expand All @@ -121,7 +133,7 @@ class FakeStorageSource(object):
def __init__(self):
self.first = True

def new_ooids(self):
def new_crashes(self):
if self.first:
self.first = False
else:
Expand All @@ -135,19 +147,25 @@ def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()

def save_raw_crash(self, raw_crash, dump):
self.store[raw_crash.ooid] = raw_crash
self.dumps[raw_crash.ooid] = dump
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump

logger = SilentFakeLogger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'source': DotDict({'crashstorage':
'source': DotDict({'crashstorage_class':
FakeStorageSource}),
'destination': DotDict({'crashstorage':
FakeStorageDestination})
'destination': DotDict({'crashstorage_class':
FakeStorageDestination}),
'producer_consumer': DotDict({'producer_consumer_class':
ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1}
)
})

fts_app = FetchTransformSaveApp(config)
Expand Down Expand Up @@ -177,19 +195,25 @@ def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()

def save_raw_crash(self, raw_crash, dump):
self.store[raw_crash.ooid] = raw_crash
self.dumps[raw_crash.ooid] = dump
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump

logger = SilentFakeLogger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'source': DotDict({'crashstorage':
'source': DotDict({'crashstorage_class':
None}),
'destination': DotDict({'crashstorage':
FakeStorageDestination})
'destination': DotDict({'crashstorage_class':
FakeStorageDestination}),
'producer_consumer': DotDict({'producer_consumer_class':
ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1}
)
})

fts_app = FetchTransformSaveApp(config)
Expand Down Expand Up @@ -223,15 +247,24 @@ def new_ooids(self):
for k in self.store.keys():
yield k



logger = SilentFakeLogger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'source': DotDict({'crashstorage':
'source': DotDict({'crashstorage_class':
FakeStorageSource}),
'destination': DotDict({'crashstorage':
None})
'destination': DotDict({'crashstorage_class':
None}),
'producer_consumer': DotDict({'producer_consumer_class':
ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1}
)

})

fts_app = FetchTransformSaveApp(config)
Expand Down
Empty file added socorro/unittest/external/filesystem/__init__.py
View file
Empty file.
14 changes: 0 additions & 14 deletions socorro/unittest/external/filesystem/test_crashstorage.py
View file
Expand Up @@ -74,13 +74,6 @@ def _common_config_setup(self):
def _common_basic_test(self, config, crashstorage):
fake_dump = 'this is a fake dump'
self.assertEqual(list(crashstorage.new_crashes()), [])
raw = {"name": "Peter", "legacy_processing": 0}
self.assertRaises(
CrashIDNotFound,
crashstorage.save_raw_crash,
raw,
fake_dump # as a stand in for the binary dump file
)
raw = {"name": "Peter",
#"ooid":"114559a5-d8e6-428c-8b88-1c1f22120314",
"legacy_processing": 0}
Expand Down Expand Up @@ -130,13 +123,6 @@ def _common_throttle_test(self, config, crashstorage):
fake_dump = 'this is a fake dump'
crashstorage = FileSystemThrottledCrashStorage(config)
self.assertEqual(list(crashstorage.new_crashes()), [])
raw = {"name": "Peter", "legacy_processing": 1}
self.assertRaises(
CrashIDNotFound,
crashstorage.save_raw_crash,
raw,
fake_dump # as a stand in for the binary dump file
)
raw = {"name": "Peter",
#"ooid":"114559a5-d8e6-428c-8b88-1c1f22120314",
"legacy_processing": 1}
Expand Down
Empty file added socorro/unittest/external/hbase/__init__.py
View file
Empty file.
Empty file added socorro/unittest/storage/__init__.py
View file
Empty file.
1 change: 0 additions & 1 deletion socorro/unittest/storage/testCrashstorage.py
View file
Expand Up @@ -159,7 +159,6 @@ def testCrashStorageSystem_makeJsonDictFromForm():
assert resultJson.a == '1'
assert resultJson.b == 2
assert resultJson.c == '3'
assert 'd' not in resultJson
assert resultJson.e == '5'

def testCrashStorageSystem_save():
Expand Down

0 comments on commit 955b894

Please sign in to comment.