bug 1507186: get all the webapp tests passing

docker/run_tests_python3.sh

@@ -8,55 +8,15 @@
 #
 # Usage: ./docker/run_tests_python3.sh
 
-# This is the list of known working tests by directory/filename. When you
-# have tests in a directory/file working, add it to this list as a new line.
-WORKING_TESTS=(
-    socorro/signature/tests/test_*.py
-    socorro/unittest/app/test_*.py
-    socorro/unittest/cron/test_*.py
-    socorro/unittest/cron/jobs/test_*.py
-    socorro/unittest/external/test_*.py
-    socorro/unittest/external/boto/test_*.py
-    socorro/unittest/external/es/test_*.py
-    socorro/unittest/external/fs/test_*.py
-    socorro/unittest/external/postgresql/test_*.py
-    socorro/unittest/external/rabbitmq/test_*.py
-    socorro/unittest/lib/test_*.py
-    socorro/unittest/processor/test_*.py
-    socorro/unittest/processor/rules/test_*.py
-    socorro/unittest/scripts/test_*.py
-)
-
-# This is the list of known working tests by directory/filename for the webapp.
-# The webapp is a separate test suite. When you have tests in a directory/file
-# working, add it to this list as a new line.
-WEBAPP_WORKING_TESTS=(
-    # crashstats/api/tests/test_*.py
-    # crashstats/authentication/tests/test_*.py
-    # crashstats/base/tests/test_*.py
-    # crashstats/crashstats/tests/test_*.py
-    # crashstats/documentation/tests/test_*.py
-    # crashstats/exploitability/tests/test_*.py
-    # crashstats/graphics/tests/test_*.py
-    # crashstats/manage/tests/test_*.py
-    # crashstats/monitoring/tests/test_*.py
-    # crashstats/profile/tests/test_*.py
-    # crashstats/signature/tests/test_*.py
-    # crashstats/sources/tests/test_*.py
-    # crashstats/status/tests/test_*.py
-    # crashstats/supersearch/tests/test_*.py
-    # crashstats/tokens/tests/test_*.py
-    # crashstats/topcrashers/tests/test_*.py
-)
+# Failures should cause this script to stop
+set -v -e -x
 
 # Run socorro tests
-pytest ${WORKING_TESTS[@]}
+pytest
 
-# If there are webapp tests to run, do this
-if [ ${#WEBAPP_WORKING_TESTS[@]} -gt 0 ]; then
-    # Collect static and run py.test in webapp
-    pushd webapp-django
-    "${WEBPACK_BINARY}" --mode=production --bail
-    python manage.py collectstatic --noinput
-    pytest ${WEBAPP_WORKING_TESTS[@]}
-fi
+# Collect static and run py.test in webapp
+pushd webapp-django
+"${WEBPACK_BINARY}" --mode=production --bail
+python manage.py collectstatic --noinput
+pytest
+popd

socorro/cron/jobs/archivescraper.py

@@ -66,13 +66,24 @@
 
 from configman import Namespace, class_converter
 import psycopg2
-from pyquery import PyQuery as pq
+import six
 from six.moves.urllib.parse import urljoin
 
 from socorro.cron.base import BaseCronApp
 from socorro.lib.requestslib import session_with_retries
 from socorro.lib.transaction import transaction_context
 
+# NOTE(willkg): We have to do this because lxml 4.2.5 has imports that kick up
+# warnings in Python 3. Once lxml puts out a new version with a fix, we can
+# stop doing this.
+if six.PY2:
+    from pyquery import PyQuery as pq
+else:
+    import warnings
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', category=ImportWarning)
+        from pyquery import PyQuery as pq
+
 
 # Substrings that indicate the thing is not a platform we want to traverse
 NON_PLATFORM_SUBSTRINGS = [

socorro/external/boto/connection_context.py

@@ -176,9 +176,9 @@ def _get_or_create_bucket(self, conn, bucket_name):
             return self._bucket_cache[bucket_name]
 
     def submit(self, id, name_of_thing, thing):
-        """Submit something to boto"""
-        # can only submit strings to boto
-        assert isinstance(thing, six.string_types), type(thing)
+        """submit something to boto"""
+        # can only submit binary to boto
+        assert isinstance(thing, six.binary_type), type(thing)
         try:
             start_time = time.time()
 

socorro/external/boto/crashstorage.py

@@ -61,10 +61,11 @@ def do_save_raw_crash(boto_connection, raw_crash, dumps, crash_id):
         if dumps is None:
             dumps = MemoryDumpsMapping()
 
-        raw_crash_as_string = boto_connection._convert_mapping_to_string(raw_crash)
-        boto_connection.submit(crash_id, 'raw_crash', raw_crash_as_string)
-        dump_names_as_string = boto_connection._convert_list_to_string(dumps.keys())
-        boto_connection.submit(crash_id, 'dump_names', dump_names_as_string)
+        raw_crash_data = boto_connection._convert_mapping_to_string(raw_crash).encode('utf-8')
+        boto_connection.submit(crash_id, 'raw_crash', raw_crash_data)
+
+        dump_names_data = boto_connection._convert_list_to_string(dumps.keys()).encode('utf-8')
+        boto_connection.submit(crash_id, 'dump_names', dump_names_data)
 
         # We don't know what type of dumps mapping we have. We do know,
         # however, that by calling the memory_dump_mapping method, we will get
@@ -88,8 +89,8 @@ def save_raw_crash(self, raw_crash, dumps, crash_id):
     @staticmethod
     def _do_save_processed(boto_connection, processed_crash):
         crash_id = processed_crash['uuid']
-        processed_crash_as_string = boto_connection._convert_mapping_to_string(processed_crash)
-        boto_connection.submit(crash_id, "processed_crash", processed_crash_as_string)
+        data = boto_connection._convert_mapping_to_string(processed_crash).encode('utf-8')
+        boto_connection.submit(crash_id, "processed_crash", data)
 
     def save_processed(self, processed_crash):
         retry(
@@ -264,8 +265,8 @@ def save_raw_and_processed(self, raw_crash, dumps, processed_crash, crash_id):
     def _do_save_processed(boto_connection, processed_crash):
         """Overriding this to change "name of thing" to crash_report"""
         crash_id = processed_crash['uuid']
-        processed_crash_as_string = boto_connection._convert_mapping_to_string(processed_crash)
-        boto_connection.submit(crash_id, "crash_report", processed_crash_as_string)
+        data = boto_connection._convert_mapping_to_string(processed_crash).encode('utf-8')
+        boto_connection.submit(crash_id, "crash_report", data)
 
     @staticmethod
     def _do_get_unredacted_processed(boto_connection, crash_id, json_object_hook):

socorro/processor/mozilla_transform_rules.py

@@ -469,9 +469,9 @@ class FlashVersionRule(Rule):
     # A regular expression to match Flash file names
     FLASH_RE = re.compile(
         r'NPSWF32_?(.*)\.dll|'
-        'FlashPlayerPlugin_?(.*)\.exe|'
-        'libflashplayer(.*)\.(.*)|'
-        'Flash ?Player-?(.*)'
+        r'FlashPlayerPlugin_?(.*)\.exe|'
+        r'libflashplayer(.*)\.(.*)|'
+        r'Flash ?Player-?(.*)'
     )
 
     def _get_flash_version(self, **kwargs):

socorro/signature/cmd_doc.py

@@ -27,7 +27,7 @@ def indent(text, prefix):
     return text.strip()
 
 
-LEAD_WHITESPACE = re.compile('^[ \t]*')
+LEAD_WHITESPACE = re.compile(r'^[ \t]*')
 
 
 def dedent_docstring(text):

socorro/signature/tests/test_rules.py

@@ -38,9 +38,9 @@ def setup_config_c_sig_tool(
 
     def test_c_config_tool_init(self):
         """test_C_config_tool_init: constructor test"""
-        exp_irrelevant_signature_re = re.compile('ignored1')
-        exp_prefix_signature_re = re.compile('pre1|pre2')
-        exp_signatures_with_line_numbers_re = re.compile('fnNeedNumber')
+        exp_irrelevant_signature_re = re.compile(r'ignored1')
+        exp_prefix_signature_re = re.compile(r'pre1|pre2')
+        exp_signatures_with_line_numbers_re = re.compile(r'fnNeedNumber')
         fixup_space = re.compile(r' (?=[\*&,])')
         fixup_comma = re.compile(r',(?! )')
 

socorro/unittest/external/boto/test_connection_context.py

@@ -43,7 +43,7 @@ class ConditionallyABadDeal(Exception):
         'db': 'same db',
     },
 }
-thing_as_str = json.dumps(a_thing)
+thing_as_binary = json.dumps(a_thing).encode('utf-8')
 
 
 def setup_mocked_s3_storage(cls=S3ConnectionContext, **extra):
@@ -103,7 +103,7 @@ def test_submit(self):
         conn.submit(
             'fff13cf0-5671-4496-ab89-47a922141114',
             'name_of_thing',
-            thing_as_str
+            thing_as_binary
         )
 
         # this should have happened
@@ -138,7 +138,7 @@ def test_submit_data_capture(self):
             conn.submit(
                 'fff13cf0-5671-4496-ab89-47a922141114',
                 'name_of_thing',
-                thing_as_str
+                thing_as_binary
             )
             # Do a failed submit
             conn._connect = mock.Mock()
@@ -147,7 +147,7 @@ def test_submit_data_capture(self):
                 conn.submit(
                     'fff13cf0-5671-4496-ab89-47a922141114',
                     'name_of_thing',
-                    thing_as_str
+                    thing_as_binary
                 )
 
             assert len(mm.filter_records(stat='processor.s3.submit',
@@ -163,7 +163,7 @@ def test_fetch(self):
             .get_bucket.return_value.get_key.return_value
             .get_contents_as_string
         )
-        mocked_get_contents_as_string.side_effect = [thing_as_str]
+        mocked_get_contents_as_string.side_effect = [thing_as_binary]
 
         # the tested call
         result = conn.fetch(
@@ -188,7 +188,7 @@ def test_fetch(self):
             ],
         )
 
-        assert result == thing_as_str
+        assert result == thing_as_binary
 
     def assert_regional_s3_connection_parameters(self, region, conn):
         kwargs = {

socorro/unittest/external/boto/test_crashstorage.py

@@ -124,10 +124,10 @@ def test_save_raw_crash_no_dumps(self, boto_helper):
 
         # Run save_raw_crash
         boto_s3_store.save_raw_crash(
-            {"submitted_timestamp": "2013-01-09T22:21:18.646733+00:00"},
+            {'submitted_timestamp': '2013-01-09T22:21:18.646733+00:00'},
             # This is an empty set of dumps--no dumps!
             MemoryDumpsMapping(),
-            "0bba929f-8721-460c-dead-a43c20071027"
+            '0bba929f-8721-460c-dead-a43c20071027'
         )
 
         # Verify the raw_crash made it to the right place and has the right

webapp-django/crashstats/api/cleaner.py

@@ -49,7 +49,7 @@ def _scrub(self, result, api_whitelist):
                 if result_key == self.ANY:
                     if isinstance(whitelist, (list, tuple)):
                         if isinstance(result, dict):
-                            for key, thing in result.iteritems():
+                            for key, thing in result.items():
                                 if isinstance(thing, dict):
                                     self._scrub_item(thing, whitelist)
                                 elif isinstance(thing, (list, tuple)):
@@ -67,7 +67,7 @@ def _scrub(self, result, api_whitelist):
 
     def _scrub_item(self, data, whitelist):
         matcher = SmartWhitelistMatcher(whitelist)
-        for key in data.keys():
+        for key in list(data.keys()):
             if key not in matcher:
                 # warnings.warn() never redirects the same message to
                 # the logger more than once in the same python
@@ -85,11 +85,9 @@ def _scrub_list(self, sequence, whitelist):
 
 
 class SmartWhitelistMatcher(object):
-
     def __init__(self, whitelist):
-
         def format(item):
-            return '^' + item.replace('*', '[\w-]*') + '$'
+            return '^' + item.replace('*', r'[\w-]*') + '$'
 
         items = [format(x) for x in whitelist]
         self.regex = re.compile('|'.join(items))

webapp-django/crashstats/api/templatetags/jinja_helpers.py

@@ -1,10 +1,10 @@
 import datetime
-import urllib
 import warnings
 
 from django_jinja import library
 import jinja2
 import six
+from six.moves.urllib.parse import quote
 
 from django.forms.widgets import RadioSelect
 
@@ -58,7 +58,7 @@ def make_test_input(parameter, defaults):
     if parameter['type'] is not six.text_type:
         classes.append('validate-%s' % parameter['type'].__name__)
     if defaults.get(parameter['name']):
-        data['value'] = urllib.quote(six.text_type(defaults.get(parameter['name'])))
+        data['value'] = quote(six.text_type(defaults.get(parameter['name'])))
     else:
         data['value'] = ''
 

webapp-django/crashstats/api/tests/test_views.py

@@ -4,6 +4,7 @@
 from django.contrib.auth.models import User, Permission
 from django.conf import settings
 from django.core.urlresolvers import reverse
+from django.utils.encoding import smart_text
 
 from markus.testing import MetricsMock
 import mock
@@ -113,11 +114,11 @@ def test_param_exceptions(self):
         url = reverse('api:model_wrapper', args=('NoOp',))
         response = self.client.get(url)
         assert response.status_code == 400
-        assert 'This field is required.' in response.content
+        assert 'This field is required.' in smart_text(response.content)
 
         response = self.client.get(url, {'product': 'bad'})
         assert response.status_code == 400
-        assert 'Bad value for parameter(s) \'Bad product\'' in response.content
+        assert 'Bad value for parameter(s) \'Bad product\'' in smart_text(response.content)
 
     def test_hit_or_not_hit_ratelimit(self):
         url = reverse('api:model_wrapper', args=('NoOp',))

webapp-django/crashstats/api/urls.py

@@ -5,11 +5,11 @@
 
 app_name = 'api'
 urlpatterns = [
-    url('^$', views.documentation, name='documentation'),
-    url('^CrashVerify/$',
+    url(r'^$', views.documentation, name='documentation'),
+    url(r'^CrashVerify/$',
         views.crash_verify,
         name='crash_verify'),
-    url('^(?P<model_name>\w+)/$',
+    url(r'^(?P<model_name>\w+)/$',
         views.model_wrapper,
         name='model_wrapper'),
 ]