Permalink
Browse files

Merge branch 'stage' of github.com:mozilla/socorro into stage

  • Loading branch information...
2 parents 80d32b5 + 4c016f7 commit 55453353efe266123532fc2672d442948265bf0f @selenamarie selenamarie committed Dec 6, 2012
Showing with 5,584 additions and 2,120 deletions.
  1. +2 −1 .gitignore
  2. +4 −5 Makefile
  3. +0 −54 config/collector.ini
  4. +173 −0 config/collector.ini-dist
  5. +49 −0 config/common_database.ini-dist
  6. +42 −0 config/common_logging.ini-dist
  7. +0 −56 config/crontabber.ini
  8. +147 −0 config/crontabber.ini-dist
  9. +0 −37 config/middleware.ini
  10. +0 −96 config/monitor.ini
  11. +0 −269 config/processor.ini
  12. +850 −0 config/processor.ini-dist
  13. +49 −0 config/processor_database.ini-dist
  14. +41 −13 docs/generic_app.rst
  15. +1 −1 requirements/prod.txt
  16. +20 −2 scripts/build.sh
  17. +4 −0 scripts/config/commonconfig.py.dist
  18. +1 −0 scripts/config/monitorconfig.py.dist
  19. +8 −0 scripts/config/processorconfig.py.dist
  20. +1 −1 scripts/crons/cron_libraries.sh
  21. +2 −0 scripts/staging/dumpschema.sh
  22. +108 −0 scripts/update-socorro.sh
  23. +13 −1 socorro/app/generic_app.py
  24. +3 −0 socorro/collector/throttler.py
  25. +163 −0 socorro/cron/base.py
  26. +24 −151 socorro/cron/crontabber.py
  27. +5 −0 socorro/cron/dailyMatviews.py
  28. +5 −0 socorro/cron/dailyUrl.py
  29. +5 −0 socorro/cron/duplicates.py
  30. +6 −0 socorro/cron/ftpscraper.py
  31. +1 −1 socorro/cron/jobs/bugzilla.py
  32. +1 −1 socorro/cron/jobs/daily_url.py
  33. +6 −1 socorro/cron/jobs/ftpscraper.py
  34. +1 −1 socorro/cron/jobs/matviews.py
  35. +1 −1 socorro/cron/jobs/weekly_reports_partitions.py
  36. +5 −1 socorro/cron/reportsClean.py
  37. +5 −0 socorro/cron/updateADUs.py
  38. +6 −5 socorro/external/hbase/crashstorage.py
  39. +10 −5 socorro/external/postgresql/crash.py
  40. +27 −18 socorro/external/postgresql/setupdb_app.py
  41. +3 −0 socorro/lib/util.py
  42. +125 −17 socorro/processor/externalProcessor.py
  43. +6 −5 socorro/processor/processor.py
  44. 0 socorro/unittest/app/__init__.py
  45. +69 −0 socorro/unittest/app/test_generic_app.py
  46. +59 −26 socorro/unittest/app/test_generic_fetch_transform_save_app.py
  47. +2 −2 socorro/unittest/config/commonconfig.py.dist
  48. +2 −1 socorro/unittest/cron/jobs/test_matviews.py
  49. +21 −9 socorro/unittest/cron/test_crontabber.py
  50. 0 socorro/unittest/external/filesystem/__init__.py
  51. +0 −14 socorro/unittest/external/filesystem/test_crashstorage.py
  52. 0 socorro/unittest/external/hbase/__init__.py
  53. +12 −0 socorro/unittest/external/postgresql/test_base.py
  54. +15 −7 socorro/unittest/external/postgresql/test_bugs.py
  55. +10 −4 socorro/unittest/external/postgresql/test_crash.py
  56. +92 −2 socorro/unittest/external/postgresql/test_setupdb_app.py
  57. +6 −0 socorro/unittest/external/postgresql/test_signature_urls.py
  58. +291 −12 socorro/unittest/processor/testProcessor.py
  59. 0 socorro/unittest/storage/__init__.py
  60. +0 −1 socorro/unittest/storage/testCrashstorage.py
  61. +1 −1 socorro/webapi/webapiService.py
  62. +144 −1,034 sql/schema.sql
  63. +11 −4 sql/upgrade/26.0/README.rst
  64. +31 −0 sql/upgrade/26.0/add_flash_process_dump.sql
  65. +0 −26 sql/upgrade/26.0/add_webrtmobile.sql
  66. 0 sql/upgrade/{25.0 → 26.0}/backfill_crashes_by_user.sql
  67. +20 −0 sql/upgrade/26.0/exploitability_column.sql
  68. 0 sql/upgrade/{25.0 → 26.0}/update_adu.sql
  69. 0 sql/upgrade/{25.0 → 26.0}/update_crashes_by_user.sql
  70. +0 −223 sql/upgrade/26.0/update_product_versions.sql
  71. +16 −9 sql/upgrade/26.0/upgrade.sh
  72. +22 −0 sql/upgrade/29.0/README.rst
  73. +23 −0 sql/upgrade/29.0/backfill_daily_crashes.sql
  74. +50 −0 sql/upgrade/29.0/backfill_one_day_319793.sql
  75. +46 −0 sql/upgrade/29.0/backfill_one_day_319794.sql
  76. +90 −0 sql/upgrade/29.0/backfill_reports_duplicates.sql
  77. +102 −0 sql/upgrade/29.0/citext-updates.sql
  78. +20 −0 sql/upgrade/29.0/functions.txt
  79. +10 −0 sql/upgrade/29.0/generate_function_defs.sh
  80. +88 −0 sql/upgrade/29.0/param_cleanup.txt
  81. +88 −0 sql/upgrade/29.0/recreate-views.sql
  82. +140 −0 sql/upgrade/29.0/update_adu.sql
  83. +118 −0 sql/upgrade/29.0/update_build_adu.sql
  84. +97 −0 sql/upgrade/29.0/update_correlations.sql
  85. +103 −0 sql/upgrade/29.0/update_crashes_by_user.sql
  86. +142 −0 sql/upgrade/29.0/update_crashes_by_user_build.sql
  87. +102 −0 sql/upgrade/29.0/update_daily_crashes.sql
  88. +258 −0 sql/upgrade/29.0/update_explosiveness.sql
  89. +101 −0 sql/upgrade/29.0/update_home_page_graph.sql
  90. +135 −0 sql/upgrade/29.0/update_home_page_graph_build.sql
  91. +65 −0 sql/upgrade/29.0/update_nightly_builds.sql
  92. +75 −0 sql/upgrade/29.0/update_os_versions.sql
  93. +69 −0 sql/upgrade/29.0/update_os_versions_new_reports.sql
  94. +75 −0 sql/upgrade/29.0/update_rank_compare.sql
  95. +373 −0 sql/upgrade/29.0/update_reports_clean.sql
  96. +97 −0 sql/upgrade/29.0/update_reports_duplicates.sql
  97. +107 −0 sql/upgrade/29.0/update_signatures.sql
  98. +88 −0 sql/upgrade/29.0/update_tcbs.sql
  99. +108 −0 sql/upgrade/29.0/update_tcbs_build.sql
  100. +54 −0 sql/upgrade/29.0/upgrade.sh
  101. +6 −1 webapp-php/application/config/codebases.php-dist
  102. +7 −1 webapp-php/application/views/report/index.php
View
@@ -1,6 +1,6 @@
# exclude config overrides to prevent accidental check-ins
webapp-php/application/config/*.php
-socorro/unittest/config/*.py
+socorro/unittest/config/*.py
*.pyc
*.swp
distribute*.tar.gz
@@ -20,3 +20,4 @@ cover/
.vagrant
socorro/cron/crontabber.json
vagrantconfig_local.yaml
+/config/*.ini
View
@@ -11,23 +11,22 @@ SETUPDB = $(VIRTUALENV)/bin/python ./socorro/external/postgresql/setupdb_app.py
COVEROPTS = --with-coverage --cover-package=socorro
COVERAGE = $(VIRTUALENV)/bin/coverage
PYLINT = $(VIRTUALENV)/bin/pylint
-CITEXT="/usr/share/postgresql/9.0/contrib/citext.sql"
.PHONY: all test install reinstall install-socorro install-web virtualenv coverage lint clean minidump_stackwalk analysis thirdparty
all: test
setup-test: virtualenv
- PYTHONPATH=$(PYTHONPATH) $(SETUPDB) --database_name=socorro_integration_test --database_username=$(DB_USER) --database_hostname=$(DB_HOST) --database_password=$(DB_PASSWORD) --database_port=$(DB_PORT) --citext=$(CITEXT) --dropdb
- PYTHONPATH=$(PYTHONPATH) $(SETUPDB) --database_name=socorro_test --database_username=$(DB_USER) --database_hostname=$(DB_HOST) --database_password=$(DB_PASSWORD) --database_port=$(DB_PORT) --citext=$(CITEXT) --dropdb --no_schema
+ PYTHONPATH=$(PYTHONPATH) $(SETUPDB) --database_name=socorro_integration_test --database_username=$(DB_USER) --database_hostname=$(DB_HOST) --database_password=$(DB_PASSWORD) --database_port=$(DB_PORT) --dropdb
+ PYTHONPATH=$(PYTHONPATH) $(SETUPDB) --database_name=socorro_test --database_username=$(DB_USER) --database_hostname=$(DB_HOST) --database_password=$(DB_PASSWORD) --database_port=$(DB_PORT) --dropdb --no_schema
cd socorro/unittest/config; for file in *.py.dist; do if [ ! -f `basename $$file .dist` ]; then cp $$file `basename $$file .dist`; fi; done
test: setup-test
PYTHONPATH=$(PYTHONPATH) $(NOSE)
thirdparty:
- [ -d $(VIRTUALENV) ] || virtualenv $(VIRTUALENV)
+ [ -d $(VIRTUALENV) ] || virtualenv -p python2.6 $(VIRTUALENV)
# install production dependencies
$(VIRTUALENV)/bin/pip install --use-mirrors --download-cache=pip-cache/ --ignore-installed --install-option="--prefix=`pwd`/thirdparty" --install-option="--install-lib=`pwd`/thirdparty" -r requirements/prod.txt
@@ -66,7 +65,7 @@ install-web:
cd $(PREFIX)/htdocs; cp htaccess-dist .htaccess
virtualenv:
- [ -e $(VIRTUALENV) ] || virtualenv $(VIRTUALENV)
+ [ -e $(VIRTUALENV) ] || virtualenv -p python2.6 $(VIRTUALENV)
$(VIRTUALENV)/bin/pip install --use-mirrors --download-cache=./pip-cache -r requirements/dev.txt
coverage: setup-test
View
@@ -1,54 +0,0 @@
-# name: application
-# doc: the fully qualified module or class of the application
-# converter: configman.converters.class_converter
-#application='CollectorApp'
-
-[collector]
-
- # name: dump_field
- # doc: the name of the form field containing the raw dump
- # converter: str
- dump_field='upload_file_minidump'
-
- # name: dump_id_prefix
- # doc: the prefix to return to the client in front of the OOID
- # converter: str
- dump_id_prefix='bp-'
-
-[logging]
- +include /etc/socorro/logging.ini
-
- # name: stderr_error_logging_level
- # doc: logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)
- # converter: int
- stderr_error_logging_level='10'
-
- # name: syslog_error_logging_level
- # doc: logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)
- # converter: int
- syslog_error_logging_level='40'
-
-[storage]
- # name: crashstorage_class
- # doc: the source storage class
- # converter: configman.converters.class_converter
- crashstorage_class='socorro.external.filesystem.crashstorage.FileSystemCrashStorage'
-
- +include /etc/socorro/filesystem_storage.ini
-
-[throttler]
-
- +include /etc/socorro/throttle_rules.ini
-
- # name: throttler_class
- # doc: the class that implements the throttling action
- # converter: configman.converters.class_converter
- throttler_class='socorro.collector.throttler.LegacyThrottler'
-
-[web_server]
-
- # name: wsgi_server_class
- # doc: a class implementing a wsgi web server
- # converter: configman.converters.class_converter
- wsgi_server_class='socorro.webapi.servers.ApacheModWSGI'
-
View
@@ -0,0 +1,173 @@
+# name: application
+# doc: the fully qualified module or class of the application
+# converter: configman.converters.class_converter
+application='CollectorApp'
+
+[collector]
+
+ # name: dump_field
+ # doc: the name of the form field containing the raw dump
+ # converter: str
+ # OPS - set once
+ #dump_field='upload_file_minidump'
+
+ # name: dump_id_prefix
+ # doc: the prefix to return to the client in front of the OOID
+ # converter: str
+ # OPS - set once
+ #dump_id_prefix='bp-'
+
+[logging]
+
+ # if you'd like to have one common logging configuration for the
+ # all or a subset of the apps, uncomment the +include line below
+ # and edit the 'common_logging.ini' file instead of using the individual
+ # values below.
+ #+include common_logging.ini
+
+ # name: stderr_error_logging_level
+ # doc: logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)
+ # converter: int
+ #stderr_error_logging_level='10'
+
+ # name: syslog_error_logging_level
+ # doc: logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)
+ # converter: int
+ #syslog_error_logging_level='40'
+
+ # name: stderr_line_format_string
+ # doc: python logging system format for logging to stderr
+ # converter: str
+ #stderr_line_format_string='{asctime} {levelname} - {threadName} - {message}'
+
+ # name: syslog_facility_string
+ # doc: syslog facility string ("user", "local0", etc)
+ # converter: str
+ # OPS - set once
+ #syslog_facility_string='user'
+
+ # name: syslog_host
+ # doc: syslog hostname
+ # converter: str
+ # OPS - set once
+ #syslog_host='localhost'
+
+ # name: syslog_line_format_string
+ # doc: python logging system format for syslog entries
+ # converter: str
+ #syslog_line_format_string='crontabber (pid {process}): {asctime} {levelname} - {threadName} - {message}'
+
+ # name: syslog_port
+ # doc: syslog port
+ # converter: int
+ # OPS - set once
+ #syslog_port='514'
+
+[storage]
+
+ # name: crashstorage_class
+ # doc: the source storage class
+ # converter: configman.converters.class_converter
+ #crashstorage_class='socorro.external.filesystem.crashstorage.FileSystemRawCrashStorage'
+
+ # name: dir_permissions
+ # doc: a number used for permissions for directories in the local file system
+ # converter: int
+ # OPS - set once
+ #dir_permissions='504'
+
+ # name: dump_dir_count
+ # doc: the number of dumps to be stored in a single directory in the local file system
+ # converter: int
+ # OPS - set once
+ #dump_dir_count='1024'
+
+ # name: dump_file_suffix
+ # doc: the suffix used to identify a dump file
+ # converter: str
+ #dump_file_suffix='.dump'
+
+ # name: dump_gid
+ # doc: the group ID for saved crashes in local file system (optional)
+ # converter: str
+ # OPS - set once
+ #dump_gid=''
+
+ # name: dump_permissions
+ # doc: a number used for permissions crash dump files in the local file system
+ # converter: int
+ # OPS - set once
+ #dump_permissions='432'
+
+ # name: json_file_suffix
+ # doc: the suffix used to identify a json file
+ # converter: str
+ #json_file_suffix='.json'
+
+ # name: std_fs_root
+ # doc: a path to a local file system
+ # converter: str
+ # OPS - set once
+ #std_fs_root='/home/socorro/primaryCrashStore'
+
+[throttler]
+
+ # name: minimal_version_for_understanding_refusal
+ # doc: ignore the Thottleable protocol
+ # converter: eval
+ # OPS - volatile
+ #minimal_version_for_understanding_refusal='{"Firefox": "3.5.4"}'
+
+ # name: never_discard
+ # doc: ignore the Thottleable protocol
+ # converter: configman.converters.boolean_converter
+ #never_discard='True'
+
+ # name: throttle_conditions
+ # doc: the throttling rules
+ # converter: eval
+ # OPS - volatile
+ #throttle_conditions='''
+#[
+ # drop browser hangs
+ #("*", lambda d: "HangID" in d and
+ #d.get("ProcessType", "browser") == "browser", None),
+ #("Comments", lambda x: x, 100), # 100% of crashes with comments
+ #("ReleaseChannel", lambda x: x in ("aurora", "beta", "esr"), 100),
+ #("ReleaseChannel", lambda x: x.startswith('nightly'), 100),
+ #("ProductName", 'Firefox', 10), # 10% of Firefox
+ #("ProductName", 'Fennec', 100), # 100% of Fennec
+ ## 100% of all alpha, beta or special
+ #("Version", re.compile(r'\..*?[a-zA-Z]+'), 100),
+ ## 100% of Thunderbird, SeaMonkey & Camino
+ #("ProductName", lambda x: x[0] in 'TSC', 100),
+ #(None, True, 0) # reject everything else
+#]
+#'''
+
+ # name: throttler_class
+ # doc: the class that implements the throttling action
+ # converter: configman.converters.class_converter
+ #throttler_class='socorro.collector.throttler.LegacyThrottler'
+
+[web_server]
+
+ # name: wsgi_server_class
+ # doc: a class implementing a wsgi web server
+ # converter: configman.converters.class_converter
+ # CHOOSE - you can use either a local standalone server OR
+ # Apache. If you choose Apache, leave the 'ip_address' & 'port'
+ # values commented out.
+ #wsgi_server_class='socorro.webapi.servers.CherryPy'
+ #wsgi_server_class='socorro.webapi.servers.ApacheModWSGI'
+
+ # name: ip_address
+ # doc: the IP address from which to accept submissions
+ # converter: str
+ #ip_address='127.0.0.1'
+
+ # name: port
+ # doc: the port to listen to for submissions
+ # converter: int
+ #port='8882'
+
@@ -0,0 +1,49 @@
+# name: database_host
+# doc: the hostname of the database
+# converter: str
+# OPS important line
+#database_host='localhost'
+
+# name: database_name
+# doc: the name of the database
+# converter: str
+# OPS set once
+#database_name='breakpad'
+
+# name: database_password
+# doc: the user's database password
+# converter: str
+# OPS set once
+#database_password='aPassword'
+
+# name: database_port
+# doc: the port for the database
+# converter: int
+# OPS set once
+#database_port='5432'
+
+# name: database_user
+# doc: the name of the user within the database
+# converter: str
+# OPS set once
+#database_user='breakpad_rw'
+
+# name: database_class
+# doc: None
+# converter: configman.converters.class_converter
+#database_class='socorro.external.postgresql.connection_context.ConnectionContext'
+
+# name: backoff_delays
+# doc: delays in seconds between retries
+# converter: eval
+#backoff_delays="10, 30, 60, 120, 300"
+
+# name: wait_log_interval
+# doc: seconds between log during retries
+# converter: int
+#wait_log_interval='1'
+
+# name: transaction_executor_class
+# doc: a class that will execute transactions
+# converter: configman.converters.class_converter
+#transaction_executor_class='socorro.database.transaction_executor.TransactionExecutorWithLimitedBackoff'
@@ -0,0 +1,42 @@
+# name: stderr_error_logging_level
+# doc: logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)
+# converter: int
+# OPS - may change
+#stderr_error_logging_level='10'
+
+# name: syslog_error_logging_level
+# doc: logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)
+# converter: int
+# OPS - may change
+#syslog_error_logging_level='40'
+
+# name: stderr_line_format_string
+# doc: python logging system format for logging to stderr
+# converter: str
+# OPS set once
+#stderr_line_format_string='{asctime} {levelname} - {threadName} - {message}'
+
+# name: syslog_facility_string
+# doc: syslog facility string ("user", "local0", etc)
+# converter: str
+# OPS set once
+#syslog_facility_string='user'
+
+# name: syslog_host
+# doc: syslog hostname
+# converter: str
+# OPS set once
+#syslog_host='localhost'
+
+# name: syslog_line_format_string
+# doc: python logging system format for syslog entries
+# converter: str
+# OPS set once
+#syslog_line_format_string='crontabber (pid {process}): {asctime} {levelname} - {threadName} - {message}'
+
+# name: syslog_port
+# doc: syslog port
+# converter: int
+# OPS set once
+#syslog_port='514'
+
Oops, something went wrong.

0 comments on commit 5545335

Please sign in to comment.