Merge branch 'master' of git://github.com/mozilla/socorro

docs/installation.rst

@@ -116,7 +116,7 @@ As *root*:
   mkdir -p /data/socorro
   useradd socorro
   chown socorro:socorro /var/log/socorro
-  mkdir /home/socorro/primaryCrashStore /home/socorro/fallback
+  mkdir /home/socorro/primaryCrashStore /home/socorro/fallback /home/socorro/persistent
   chown apache /home/socorro/primaryCrashStore /home/socorro/fallback
   chmod 2775 /home/socorro/primaryCrashStore /home/socorro/fallback
 
@@ -213,7 +213,7 @@ Install Socorro cron jobs
 From inside the Socorro checkout, as the *root* user:
 ::
   ln -s /data/socorro/application/scripts/crons/socorrorc /etc/socorro/
-  crontab puppet/files/etc_crond/socorro
+  cp puppet/files/etc_crond/socorro /etc/cron.d/
 
 Socorro's cron jobs are moving to a new cronjob manager called :ref:`crontabber-chapter`.
 :ref:`crontabber-chapter` runs every 5 minutes from the system crontab, and looks inside
@@ -255,10 +255,21 @@ and versions into the system.
 
 Configure Apache
 ````````````
+Socorro uses three virtual hosts:
+
+* crash-stats   - the web UI for viewing crash reports
+* socorro-api   - the "middleware" used by the web UI 
+* crash-reports - receives reports from crashing clients (via HTTP POST)
+
 As *root*:
 ::
-  edit /etc/httpd/conf.d/socorro.conf
-  cp config/socorro.conf /etc/httpd/conf.d/socorro.conf
+  cp puppet/files/etc_apache2_sites-available/{crash-reports,crash-stats,socorro-api} /etc/httpd/conf.d/
+
+edit /etc/httpd/conf.d/{crash-reports,crash-stats,socorro-api} and customize
+as needed for your site
+
+As *root*
+::
   mkdir /var/log/httpd/{crash-stats,crash-reports,socorro-api}.example.com
   chown apache /data/socorro/htdocs/application/logs/
 

docs/populatepostgres.rst

@@ -21,7 +21,8 @@ Shut down all Socorro services, drop your database (if needed) and load
 the schema.
 From inside the Socorro checkout, as *postgres* user:
 ::
-  ./socorro/external/postgresql/setupdb_app.py --database_name=breakpad_rw
+  export PYTHONPATH=/data/socorro/application:/data/socorro/thirdparty
+  ./socorro/external/postgresql/setupdb_app.py --database_name=breakpad
 
 Customize CSVs, at minimum you probably need to bump the dates and build IDs in: 
   raw_adu.csv reports.csv releases_raw.csv
@@ -52,7 +53,7 @@ volume is low enough, you may want to modify this function
 (it is in breakpad_schema.sql referenced above).
 
 Normally this is run for the previous day by cron_daily_matviews.sh 
-but you can simply run the backfill function to bootstrap the system:
+but you can simply run the backfill_matviews() function to bootstrap the system.
 
 This is normally run by the import.sh, so take a look in there if
 you need to make adjustments.

puppet/manifests/classes/socorro-hbase.pp

@@ -14,7 +14,7 @@
     }
 
     exec { 'package-oracle-jdk':
-        command => '/usr/bin/wget https://raw.github.com/flexiondotorg/oab-java6/master/oab-java6.sh && bash oab-java6.sh',
+        command => '/usr/bin/wget https://raw.github.com/flexiondotorg/oab-java6/master/oab-java.sh && bash oab-java.sh',
         creates => '/etc/apt/sources.list.d/oab.list',
         cwd => '/home/socorro',
         timeout => 0

scripts/config/processorconfig.py.dist

@@ -119,31 +119,48 @@ irrelevantSignatureRegEx.doc = 'a regular expression matching frame signatures t
 irrelevantSignatureRegEx.default = '|'.join([
   '@0x[0-9a-fA-F]{2,}',
   '@0x[1-9a-fA-F]',
-  'RaiseException',
+  'ashmem',
+  'app_process@0x.*',
+  'core\.odex@0x.*',
   '_CxxThrowException',
-  'mozilla::ipc::RPCChannel::Call\(IPC::Message\*, IPC::Message\*\)',
+  'dalvik-heap',
+  'dalvik-jit-code-cache',
+  'dalvik-LinearAlloc',
+  'dalvik-mark-stack',
+  'data@app@org\.mozilla\.fennec-\d\.apk@classes\.dex@0x.*',
+  'framework\.odex@0x.*',
+  'google_breakpad::ExceptionHandler::HandleInvalidParameter.*',
   'KiFastSystemCallRet',
+  'libandroid_runtime\.so@0x.*',
+  'libbinder\.so@0x.*',
+  'libc\.so@.*',
+  'libc-2\.5\.so@.*',
+  'libEGL\.so@.*',
+  'libdvm\.so\s*@\s*0x.*',
+  'libgui\.so@0x.*',
+  'libicudata.so@.*',
+  'libMali\.so@0x.*',
+  'libutils\.so@0x.*',
+  'libz\.so@0x.*',
+  'linux-gate\.so@0x.*',
+  'mnt@asec@org\.mozilla\.fennec-\d@pkg\.apk@classes\.dex@0x.*',
+  'MOZ_Assert',
+  'MOZ_Crash',
+  'mozcrt19.dll@0x.*',
+  'mozilla::ipc::RPCChannel::Call\(IPC::Message\*, IPC::Message\*\)',
+  '_NSRaiseError',
   '(Nt|Zw)WaitForSingleObject(Ex)?',
   '(Nt|Zw)WaitForMultipleObjects(Ex)?',
+  'nvmap@0x.*',
+  'org\.mozilla\.fennec-\d\.apk@0x.*',
+  'RaiseException',
+  'RtlpAdjustHeapLookasideDepth',
+  'system@framework@*\.jar@classes\.dex@0x.*',
+  '___TERMINATING_DUE_TO_UNCAUGHT_EXCEPTION___',
   'WaitForSingleObjectExImplementation',
   'WaitForMultipleObjectsExImplementation',
-  '___TERMINATING_DUE_TO_UNCAUGHT_EXCEPTION___',
-  '_NSRaiseError',
-  'mozcrt19.dll@0x.*',
-  'linux-gate\.so@0x.*',
-  'libdvm\.so\s*@\s*0x.*',
-  'dalvik-LinearAlloc\s*@0x.*',
-  'dalvik-heap',
-  'data@app@org\.mozilla\.fennec-1\.apk@classes\.dex@0x.*',
-  'libc\.so@.*',
-  'libEGL\.so@.*',
-  'libc-2\.5\.so@.*',
-  'RtlpAdjustHeapLookasideDepth',
-  'google_breakpad::ExceptionHandler::HandleInvalidParameter.*',
-  'libicudata.so@.*',
   '_ZdlPv',
-  'MOZ_Crash',
-  'MOZ_Assert',
+  'zero',
   ])
 
 prefixSignatureRegEx = cm.Option()
@@ -152,6 +169,7 @@ prefixSignatureRegEx = '|'.join([
   '@0x0',
   '.*abort',
   '_alloca_probe.*',
+  '__android_log_assert',
   'arena_alloc',
   'arena_dalloc',
   'arena_dalloc_small',
@@ -169,19 +187,27 @@ prefixSignatureRegEx = '|'.join([
   '_chkstk',
   'CrashInJS',
   '__delayLoadHelper2',
-  'dvmAbort',
-  'dvmStringLen',
+  'dlmalloc',
+  'dlmalloc_trim',
+  'dvm.*',
   'EtwEventEnabled',
   'fastcopy_I',
   'fastzero_I',
+  '_files_getaddrinfo',
   '.*free',
   'GCGraphBuilder::NoteXPCOMChild',
+  'getanswer',
+  'huge_dalloc',
   'ialloc',
+  'init_library',
   'isalloc',
   'je_malloc',
   'je_realloc',
   'JNI_CreateJavaVM',
-  '_JNIEnv::CallObjectMethod',
+  '_JNIEnv.*',
+  'JNI_GetCreatedJavaVM.*',
+  'JS_DHashTableEnumerate',
+  'JS_DHashTableOperate',
   'kill',
   '__libc_android_abort',
   'libobjc.A.dylib@0x1568.',
@@ -190,11 +216,13 @@ prefixSignatureRegEx = '|'.join([
   'malloc',
   '_MD_.*',
   'memcmp',
+  '__memcmp16',
   'memcpy',
   'memmove',
   'memset',
   'mozalloc_abort.*',
   'mozalloc_handle_oom',
+  'moz_free',
   'mozilla::AndroidBridge::AutoLocalJNIFrame::~AutoLocalJNIFrame',
   'mozilla::ipc::RPCChannel::Call',
   'mozilla::ipc::RPCChannel::CxxStackFrame::CxxStackFrame',
@@ -255,13 +283,15 @@ prefixSignatureRegEx = '|'.join([
   'SocketWritev',
   'ssl_.*',
   'SSL_.*',
+  'strcat',
   'ssl3_.*',
   'strchr',
   'strcmp',
   'strcpy',
   '.*strdup',
   'strlen',
   'strncpy',
+  'strzcmp16',
   'strstr',
   '__swrite',
   'TouchBadMemory',

scripts/crons/cron_libraries.sh

@@ -21,7 +21,7 @@ do
   done
 done
 
-MANUAL_VERSION_OVERRIDE="13.0 14.0a2 15.0a1"
+MANUAL_VERSION_OVERRIDE="14.0 15.0a2 16.0a1"
 for I in Firefox
 do
   for J in $MANUAL_VERSION_OVERRIDE

socorro/unittest/cron/jobs/test_bugzilla.py

@@ -1,21 +1,17 @@
-import re
-import sys
 import datetime
 import shutil
 import os
 import json
 import unittest
 import tempfile
-from cStringIO import StringIO
 import mock
 import psycopg2
 from psycopg2.extensions import TRANSACTION_STATUS_IDLE
 from nose.plugins.attrib import attr
 from socorro.cron import crontabber
 from socorro.unittest.config.commonconfig import (
   databaseHost, databaseName, databaseUserName, databasePassword)
-from socorro.lib.datetimeutil import utc_now
-from configman import ConfigurationManager, Namespace
+from configman import ConfigurationManager
 
 DSN = {
   "database_host": databaseHost.default,
@@ -38,8 +34,6 @@
 ]
 
 
-
-
 class _TestCaseBase(unittest.TestCase):
 
     def setUp(self):
@@ -74,7 +68,6 @@ def _setup_config_manager(self, jobs_string, extra_value_source=None):
         return config_manager, json_file
 
 
-
 #==============================================================================
 class TestBugzilla(_TestCaseBase):
 
@@ -101,33 +94,30 @@ def setUp(self):
                'user=%(database_user)s password=%(database_password)s' % DSN)
         self.conn = psycopg2.connect(dsn)
         cursor = self.conn.cursor()
-#        cursor.execute("""
-#        DROP TABLE IF EXISTS test_cron_victim;
-#        CREATE TABLE test_cron_victim (
-#          id serial primary key,
-#          time timestamp DEFAULT current_timestamp
-#        );
-#        """)
-#        self.conn.commit()
+        cursor.execute("""
+        UPDATE crontabber_state SET state='{}';
+        """)
+        self.conn.commit()
         assert self.conn.get_transaction_status() == TRANSACTION_STATUS_IDLE
 
     def tearDown(self):
         super(TestFunctionalBugzilla, self).tearDown()
         self.conn.cursor().execute("""
+        UPDATE crontabber_state SET state='{}';
         TRUNCATE TABLE reports CASCADE;
         TRUNCATE TABLE bugs CASCADE;
         TRUNCATE TABLE bug_associations CASCADE;
         """)
         self.conn.commit()
 
     def _setup_config_manager(self, days_into_past):
-        datestring = ((datetime.datetime.utcnow() - datetime.timedelta(days=days_into_past))
+        datestring = ((datetime.datetime.utcnow() -
+                       datetime.timedelta(days=days_into_past))
                        .strftime('%Y-%m-%d'))
         filename = os.path.join(self.tempdir, 'sample-%s.csv' % datestring)
         with open(filename, 'w') as f:
             f.write('\n'.join(SAMPLE_CSV))
 
-#        print filename
         query = 'file://' + filename.replace(datestring, '%s')
 
         _super = super(TestFunctionalBugzilla, self)._setup_config_manager
@@ -140,7 +130,6 @@ def _setup_config_manager(self, days_into_past):
         )
         return config_manager, json_file
 
-
     def test_basic_run_job_without_reports(self):
         config_manager, json_file = self._setup_config_manager(3)
 
@@ -198,7 +187,6 @@ def test_basic_run_job_with_some_reports(self):
             assert information['bugzilla-associations']
             assert not information['bugzilla-associations']['last_error']
             assert information['bugzilla-associations']['last_success']
-#            print config.logger.info.mock_calls
 
         cursor.execute('select id from bugs order by id')
         bugs = cursor.fetchall()
@@ -217,6 +205,13 @@ def test_basic_run_job_with_reports_with_existing_bugs_different(self):
         config_manager, json_file = self._setup_config_manager(3)
 
         cursor = self.conn.cursor()
+        cursor.execute('select count(*) from bugs')
+        count, = cursor.fetchone()
+        assert not count
+        cursor.execute('select count(*) from bug_associations')
+        count, = cursor.fetchone()
+        assert not count
+
         # these are matching the SAMPLE_CSV above
         cursor.execute("""insert into reports
         (uuid,signature)
@@ -248,13 +243,13 @@ def test_basic_run_job_with_reports_with_existing_bugs_different(self):
             assert information['bugzilla-associations']
             assert not information['bugzilla-associations']['last_error']
             assert information['bugzilla-associations']['last_success']
-#            print config.logger.info.mock_calls
 
         cursor.execute('select id, short_desc from bugs where id = 8')
         bug = cursor.fetchone()
         self.assertEqual(bug[1], 'newlines in sigs')
 
-        cursor.execute('select signature from bug_associations where bug_id = 8')
+        cursor.execute(
+          'select signature from bug_associations where bug_id = 8')
         association = cursor.fetchone()
         self.assertEqual(association[0], 'legitimate(sig)')
 
@@ -294,13 +289,13 @@ def test_basic_run_job_with_reports_with_existing_bugs_same(self):
             assert information['bugzilla-associations']
             assert not information['bugzilla-associations']['last_error']
             assert information['bugzilla-associations']['last_success']
-#            print config.logger.info.mock_calls
 
         cursor.execute('select id, short_desc from bugs where id = 8')
         bug = cursor.fetchone()
         self.assertEqual(bug[1], 'newlines in sigs')
 
-        cursor.execute('select signature from bug_associations where bug_id = 8')
+        cursor.execute(
+          'select signature from bug_associations where bug_id = 8')
         association = cursor.fetchone()
         self.assertEqual(association[0], 'legitimate(sig)')
         cursor.execute('select * from bug_associations')

socorro/unittest/cron/jobs/test_ftpscraper.py

@@ -7,6 +7,7 @@
 from cStringIO import StringIO
 import mock
 import psycopg2
+from nose.plugins.attrib import attr
 from socorro.cron import crontabber
 from socorro.unittest.config.commonconfig import (
   databaseHost, databaseName, databaseUserName, databasePassword)
@@ -234,6 +235,7 @@ def mocked_urlopener(url):
 
 
 #==============================================================================
+@attr(integration='postgres')  # for nosetests
 class TestFunctionalFTPScraper(_TestCaseBase):
 
     def setUp(self):
@@ -243,12 +245,18 @@ def setUp(self):
         dsn = ('host=%(database_host)s dbname=%(database_name)s '
                'user=%(database_user)s password=%(database_password)s' % DSN)
         self.conn = psycopg2.connect(dsn)
+        cursor = self.conn.cursor()
+        cursor.execute("""
+        UPDATE crontabber_state SET state='{}';
+        """)
+        self.conn.commit()
         self.urllib2_patcher = mock.patch('urllib2.urlopen')
         self.urllib2 = self.urllib2_patcher.start()
 
     def tearDown(self):
         super(TestFunctionalFTPScraper, self).tearDown()
         self.conn.cursor().execute("""
+        UPDATE crontabber_state SET state='{}';
         TRUNCATE TABLE releases_raw CASCADE;
         """)
         self.conn.commit()