Skip to content
Browse files

Merge branch 'master' of github.com:mozilla/socorro

  • Loading branch information...
2 parents 90ec720 + 837848d commit 01f80019b987b73a26dacc2fe9ed42c90f548e92 @jberkus committed Oct 12, 2012
Showing with 3,930 additions and 52,330 deletions.
  1. +6 −0 .gitmodules
  2. +14 −10 Makefile
  3. +1 −0 akela
  4. +0 −37 analysis/.classpath
  5. +0 −17 analysis/.project
  6. +0 −33 analysis/bin/correlation_reports.sh
  7. +0 −57 analysis/bin/modulelist.sh
  8. +0 −26 analysis/bin/run-jetty.sh
  9. +0 −37 analysis/build.properties
  10. +0 −83 analysis/build.xml
  11. BIN analysis/lib/akela-0.2-SNAPSHOT.jar
  12. BIN analysis/lib/aopalliance.jar
  13. BIN analysis/lib/asm-3.1.jar
  14. BIN analysis/lib/commons-lang-2.5.jar
  15. BIN analysis/lib/commons-logging-1.1.1.jar
  16. BIN analysis/lib/commons-math-2.1.jar
  17. BIN analysis/lib/guice-2.0.jar
  18. BIN analysis/lib/guice-servlet-2.0.jar
  19. BIN analysis/lib/guiceyfruit-2.0.jar
  20. BIN analysis/lib/hadoop-core-0.20.2-cdh3u0.jar
  21. BIN analysis/lib/hazelcast-1.9.jar
  22. BIN analysis/lib/hbase-0.90.1-cdh3u0.jar
  23. BIN analysis/lib/jackson-core-asl-1.5.5.jar
  24. BIN analysis/lib/jackson-mapper-asl-1.5.5.jar
  25. BIN analysis/lib/jersey-bundle-1.3.jar
  26. BIN analysis/lib/jersey-guice-1.1.5.jar
  27. BIN analysis/lib/jersey-multipart-1.4.jar
  28. BIN analysis/lib/jetty-continuation-7.1.6.v20100715.jar
  29. BIN analysis/lib/jetty-http-7.1.6.v20100715.jar
  30. BIN analysis/lib/jetty-io-7.1.6.v20100715.jar
  31. BIN analysis/lib/jetty-security-7.1.6.v20100715.jar
  32. BIN analysis/lib/jetty-server-7.1.6.v20100715.jar
  33. BIN analysis/lib/jetty-servlet-7.1.6.v20100715.jar
  34. BIN analysis/lib/jetty-util-7.1.6.v20100715.jar
  35. BIN analysis/lib/jsr311-api-1.1.jar
  36. BIN analysis/lib/log4j-1.2.16.jar
  37. BIN analysis/lib/mimepull-1.3.jar
  38. BIN analysis/lib/pig-0.8.0-cdh3u0.jar
  39. BIN analysis/lib/servlet-api-2.5.jar
  40. BIN analysis/lib/slf4j-api-1.6.1.jar
  41. BIN analysis/lib/trove-2.1.0.jar
  42. +0 −36 analysis/pig/crashstats.pig
  43. +0 −14 analysis/pig/dumpsizetrends.pig
  44. +0 −17 analysis/pig/modulelist.pig
  45. +0 −20 analysis/pig/stack_frames.pig
  46. +0 −90 analysis/src/java/com/mozilla/socorro/CorrelationReport.java
  47. +0 −89 analysis/src/java/com/mozilla/socorro/Module.java
  48. +0 −126 analysis/src/java/com/mozilla/socorro/OperatingSystem.java
  49. +0 −143 analysis/src/java/com/mozilla/socorro/RawDumpSizeScan.java
  50. +0 −143 analysis/src/java/com/mozilla/socorro/Signature.java
  51. +0 −20 analysis/src/java/com/mozilla/socorro/dao/CrashCountDao.java
  52. +0 −492 analysis/src/java/com/mozilla/socorro/dao/hbase/HbaseCrashCountDao.java
  53. +0 −72 analysis/src/java/com/mozilla/socorro/dao/hbase/HbaseCrashReportDao.java
  54. +0 −259 analysis/src/java/com/mozilla/socorro/hadoop/CrashCount.java
  55. +0 −408 analysis/src/java/com/mozilla/socorro/hadoop/CrashCount2.java
  56. +0 −275 analysis/src/java/com/mozilla/socorro/hadoop/CrashCountToHbase.java
  57. +0 −345 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportDataMatrix.java
  58. +0 −299 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportFeatureIndex.java
  59. +0 −119 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportJob.java
  60. +0 −223 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportModuleList.java
  61. +0 −251 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportStats.java
  62. +0 −307 analysis/src/java/com/mozilla/socorro/hadoop/DumpSizeTrends.java
  63. +0 −243 analysis/src/java/com/mozilla/socorro/hadoop/HardwareAccel.java
  64. +0 −94 analysis/src/java/com/mozilla/socorro/hadoop/KeysForDateRange.java
  65. +0 −279 analysis/src/java/com/mozilla/socorro/hadoop/PerCrashCoreCount.java
  66. +0 −362 analysis/src/java/com/mozilla/socorro/hadoop/PerCrashInterestingModules.java
  67. +0 −207 analysis/src/java/com/mozilla/socorro/hadoop/RawDumpSize.java
  68. +0 −51 analysis/src/java/com/mozilla/socorro/pig/eval/FrameBag.java
  69. +0 −49 analysis/src/java/com/mozilla/socorro/pig/eval/ModuleBag.java
  70. +0 −44 analysis/src/java/com/mozilla/socorro/web/CorrelationReportServer.java
  71. +0 −331 analysis/src/java/com/mozilla/socorro/web/CorrelationReportService.java
  72. +0 −33 analysis/src/java/com/mozilla/socorro/web/GuiceConfig.java
  73. +0 −17 analysis/src/java/com/mozilla/util/MapValueComparator.java
  74. +0 −139 analysis/src/java/org/apache/hadoop/hbase/rest/CollectorResource.java
  75. +0 −98 analysis/src/java/org/apache/hadoop/hbase/rest/HazelcastQueueResource.java
  76. +0 −42 analysis/src/java/org/apache/hadoop/hbase/rest/model/QueueStringModel.java
  77. +0 −41,910 analysis/src/python/addonids-amo.json
  78. +0 −122 analysis/src/python/addonids-local.json
  79. +0 −71 analysis/src/python/addonids.py
  80. +0 −98 analysis/src/python/crash_stats.py
  81. +0 −137 analysis/src/python/dump_size_trends.py
  82. +0 −32 analysis/src/python/jsondb.py
  83. +0 −55 analysis/src/python/macdebugids.json
  84. +0 −45 analysis/src/python/macdebugids.py
  85. +0 −117 analysis/src/python/per-crash-core-count-hadoop.py
  86. +0 −180 analysis/src/python/per-crash-interesting-modules-hadoop.py
  87. +0 −47 analysis/src/python/raw_dump_size.py
  88. +0 −47 analysis/src/python/threads_vs_dumpsize.py
  89. +2 −4 config/crontabber.ini
  90. +37 −0 config/middleware.ini
  91. +2 −2 docs/installation.rst
  92. +5 −6 docs/middleware.rst
  93. +1 −1 puppet/manifests/classes/socorro-base.pp
  94. +1 −1 puppet/manifests/classes/socorro-hbase.pp
  95. +1 −0 requirements/dev.txt
  96. +38 −0 scripts/crons/cron_correlations.sh
  97. +1 −1 scripts/crons/cron_libraries.sh
  98. +16 −5 scripts/crons/cron_modulelist.sh
  99. +11 −2 scripts/crons/socorrorc
  100. +4 −4 scripts/staging/dumpschema.sh
  101. +1 −0 socorro-toolbox
  102. +10 −3 socorro/collector/wsgi_collector.py
  103. +7 −0 socorro/collector/wsgicollector.py
  104. +19 −2 socorro/cron/crontabber.py
  105. +3 −1 socorro/cron/dailyMatviews.py
  106. +0 −25 socorro/cron/jobs/duplicates.py
  107. +40 −6 socorro/cron/jobs/matviews.py
  108. +0 −19 socorro/cron/jobs/nightly_builds.py
  109. +0 −20 socorro/cron/jobs/reports_clean.py
  110. +16 −5 socorro/external/elasticsearch/base.py
  111. +13 −3 socorro/external/elasticsearch/search.py
  112. +19 −7 socorro/external/postgresql/base.py
  113. +10 −8 socorro/external/postgresql/bugs.py
  114. +2 −5 socorro/external/postgresql/crash.py
  115. +93 −44 socorro/external/postgresql/crashes.py
  116. +1 −3 socorro/external/postgresql/job.py
  117. +5 −4 socorro/external/postgresql/priorityjobs.py
  118. +16 −5 socorro/external/postgresql/products.py
  119. +1 −10 socorro/external/postgresql/products_builds.py
  120. +6 −3 socorro/external/postgresql/releases.py
  121. +20 −4 socorro/external/postgresql/report.py
  122. +37 −7 socorro/external/postgresql/search.py
  123. +8 −2 socorro/external/postgresql/server_status.py
  124. +15 −8 socorro/external/postgresql/setupdb_app.py
  125. +0 −6 socorro/external/postgresql/signature_summary.py
  126. +2 −9 socorro/external/postgresql/signature_urls.py
  127. +5 −3 socorro/external/postgresql/tcbs.py
  128. +0 −3 socorro/external/postgresql/util.py
  129. +0 −7 socorro/lib/ConfigurationManager.py
  130. +3 −3 socorro/lib/datetimeutil.py
  131. +0 −6 socorro/lib/search_common.py
  132. +2 −0 socorro/middleware/crashes_comments_service.py
  133. +2 −0 socorro/middleware/crashes_frequency_service.py
  134. +2 −0 socorro/middleware/crashes_paireduuid_service.py
  135. +415 −0 socorro/middleware/middleware_app.py
  136. +2 −1 socorro/middleware/report_list_service.py
  137. +2 −2 socorro/middleware/search_service.py
  138. +22 −0 socorro/middleware/service.py
  139. +8 −0 socorro/middleware/signature_summary_service.py
  140. +2 −0 socorro/middleware/signature_urls_service.py
  141. +6 −4 socorro/processor/externalProcessor.py
  142. +8 −4 socorro/processor/legacy_processor.py
  143. +1 −2 socorro/storage/crashstorage.py
  144. +29 −24 socorro/unittest/collector/test_wsgi_collector.py
  145. +0 −56 socorro/unittest/cron/jobs/test_duplicates.py
  146. +62 −0 socorro/unittest/cron/jobs/test_matviews.py
  147. +0 −60 socorro/unittest/cron/jobs/test_reports_clean.py
  148. +12 −0 socorro/unittest/cron/testDailyMatviews.py
  149. +132 −8 socorro/unittest/cron/test_crontabber.py
  150. +7 −0 socorro/unittest/external/postgresql/test_base.py
  151. +58 −54 socorro/unittest/external/postgresql/test_crashes.py
  152. +11 −12 socorro/unittest/external/postgresql/test_job.py
  153. +53 −9 socorro/unittest/external/postgresql/test_products.py
  154. +5 −3 socorro/unittest/external/postgresql/test_server_status.py
  155. +2 −2 socorro/unittest/external/postgresql/test_signature_urls.py
  156. +1 −4 socorro/unittest/external/postgresql/unittestbase.py
  157. +2 −2 socorro/unittest/lib/test_datetimeutil.py
  158. +2 −7 socorro/unittest/lib/test_search_common.py
  159. +12 −0 socorro/unittest/middleware/crash.py
  160. +999 −0 socorro/unittest/middleware/test_middleware_app.py
  161. +11 −7 socorro/unittest/middleware/test_service.py
  162. +13 −4 socorro/webapi/servers.py
  163. +925 −2,054 sql/schema.sql
  164. +19 −0 sql/upgrade/20.0/README.rst
  165. +7 −0 sql/upgrade/20.0/insert_os_name_matches.sql
  166. +22 −0 sql/upgrade/20.0/upgrade.sh
  167. +1 −1 webapp-php/application/config/daily.php-dist
  168. +3 −0 webapp-php/application/config/routes.php-dist
  169. +5 −27 webapp-php/application/controllers/crash_trends.php
  170. +44 −75 webapp-php/application/controllers/daily.php
  171. +19 −11 webapp-php/application/controllers/products.php
  172. +20 −5 webapp-php/application/controllers/status.php
  173. +11 −0 webapp-php/application/libraries/MY_Model.php
  174. +0 −1 webapp-php/application/libraries/MY_SearchReportHelper.php
  175. +246 −215 webapp-php/application/models/daily.php
  176. +2 −2 webapp-php/application/models/report.php
  177. +1 −2 webapp-php/application/models/search.php
  178. +1 −1 webapp-php/application/models/signature_summary.php
  179. +1 −1 webapp-php/application/views/admin/branch_data_sources/non_current_entries.php
  180. +1 −1 webapp-php/application/views/admin/branch_data_sources/products.php
  181. +2 −1 webapp-php/application/views/crash_trends/index.php
  182. +2 −3 webapp-php/application/views/daily/daily_crash_data_by_os.php
  183. +0 −264 webapp-php/application/views/daily/daily_crash_data_by_report_type.php
  184. +10 −18 webapp-php/application/views/daily/daily_crash_data_by_version.php
  185. +5 −7 webapp-php/application/views/daily/daily_csv_by_os.php
  186. +11 −16 webapp-php/application/views/daily/daily_csv_by_version.php
  187. +61 −87 webapp-php/application/views/daily/daily_search.php
  188. +3 −39 webapp-php/application/views/daily/index.php
  189. +1 −0 webapp-php/application/views/layout.php
  190. +2 −2 webapp-php/application/views/status/index.php
  191. +6 −2 webapp-php/css/crash_trends.css
  192. +1 −0 webapp-php/css/screen.css
  193. +109 −46 webapp-php/js/socorro/crash_trends.js
  194. +10 −25 webapp-php/js/socorro/daily.js
  195. +1 −1 webapp-php/js/socorro/socorro.dashboard.js
  196. +3 −3 webapp-php/js/socorro/utils.js
View
6 .gitmodules
@@ -0,0 +1,6 @@
+[submodule "socorro-toolbox"]
+ path = socorro-toolbox
+ url = git://github.com/mozilla-metrics/socorro-toolbox.git
+[submodule "akela"]
+ path = akela
+ url = git://github.com/mozilla-metrics/akela.git
View
24 Makefile
@@ -13,7 +13,7 @@ COVERAGE = $(VIRTUALENV)/bin/coverage
PYLINT = $(VIRTUALENV)/bin/pylint
CITEXT="/usr/share/postgresql/9.0/contrib/citext.sql"
-.PHONY: all test install reinstall install-socorro install-web virtualenv coverage lint clean minidump_stackwalk java_analysis thirdparty
+.PHONY: all test install reinstall install-socorro install-web virtualenv coverage lint clean minidump_stackwalk analysis thirdparty
all: test
@@ -27,11 +27,11 @@ test: setup-test
PYTHONPATH=$(PYTHONPATH) $(NOSE)
thirdparty:
- virtualenv $(VIRTUALENV)
+ [ -d $(VIRTUALENV) ] || virtualenv $(VIRTUALENV)
# install production dependencies
$(VIRTUALENV)/bin/pip install --use-mirrors --download-cache=pip-cache/ --ignore-installed --install-option="--prefix=`pwd`/thirdparty" --install-option="--install-lib=`pwd`/thirdparty" -r requirements/prod.txt
-install: java_analysis thirdparty reinstall
+install: analysis thirdparty reinstall
# this a dev-only option, `make install` needs to be run at least once in the checkout (or after `make clean`)
reinstall: install-socorro install-web
@@ -54,8 +54,7 @@ install-socorro:
rsync -a wsgi $(PREFIX)/application
rsync -a stackwalk $(PREFIX)/
rsync -a scripts/stackwalk.sh $(PREFIX)/stackwalk/bin/
- rsync -a analysis/build/lib/socorro-analysis-job.jar $(PREFIX)/analysis/
- rsync -a analysis/bin/modulelist.sh $(PREFIX)/analysis/
+ rsync -a analysis $(PREFIX)/
# copy default config files
cd $(PREFIX)/application/scripts/config; for file in *.py.dist; do cp $$file `basename $$file .dist`; done
@@ -67,7 +66,7 @@ install-web:
cd $(PREFIX)/htdocs; cp htaccess-dist .htaccess
virtualenv:
- virtualenv $(VIRTUALENV)
+ [ -e $(VIRTUALENV) ] || virtualenv $(VIRTUALENV)
$(VIRTUALENV)/bin/pip install --use-mirrors --download-cache=./pip-cache -r requirements/dev.txt
coverage: setup-test
@@ -83,14 +82,19 @@ clean:
rm -rf ./thirdparty/*
rm -rf ./google-breakpad/ ./builds/ ./breakpad/ ./stackwalk ./pip-cache
rm -rf ./breakpad.tar.gz
- cd analysis && ant clean
minidump_stackwalk:
svn co http://google-breakpad.googlecode.com/svn/trunk google-breakpad
cd google-breakpad && ./configure --prefix=`pwd`/../stackwalk/
cd google-breakpad && make install
cd google-breakpad && svn info | grep Revision | cut -d' ' -f 2 > ../stackwalk/revision.txt
-java_analysis:
- cd analysis && ant hadoop-jar
-
+analysis:
+ git submodule update --init socorro-toolbox akela
+ cd akela && mvn package
+ cd akela && mvn package
+ cd socorro-toolbox && mvn package
+ mkdir -p analysis
+ rsync socorro-toolbox/target/*.jar analysis/
+ rsync akela/target/*.jar analysis/
+ rsync -a socorro-toolbox/src/main/pig/ analysis/
1 akela
@@ -0,0 +1 @@
+Subproject commit a82d41a3e64df59ae12fef88c4fa76bbcda30fd8
View
37 analysis/.classpath
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" path="src/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="lib" path="lib/commons-lang-2.5.jar"/>
- <classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
- <classpathentry kind="lib" path="lib/trove-2.1.0.jar"/>
- <classpathentry kind="lib" path="lib/asm-3.1.jar"/>
- <classpathentry kind="lib" path="lib/jersey-bundle-1.3.jar"/>
- <classpathentry kind="lib" path="lib/jsr311-api-1.1.jar"/>
- <classpathentry kind="lib" path="lib/guice-2.0.jar"/>
- <classpathentry kind="lib" path="lib/guice-servlet-2.0.jar"/>
- <classpathentry kind="lib" path="lib/jetty-server-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/jetty-util-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/servlet-api-2.5.jar"/>
- <classpathentry kind="lib" path="lib/guiceyfruit-2.0.jar"/>
- <classpathentry kind="lib" path="lib/jersey-guice-1.1.5.jar"/>
- <classpathentry kind="lib" path="lib/slf4j-api-1.6.1.jar"/>
- <classpathentry kind="lib" path="lib/jackson-core-asl-1.5.5.jar"/>
- <classpathentry kind="lib" path="lib/jackson-mapper-asl-1.5.5.jar"/>
- <classpathentry kind="lib" path="lib/aopalliance.jar"/>
- <classpathentry kind="lib" path="lib/jetty-servlet-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/log4j-1.2.16.jar"/>
- <classpathentry kind="lib" path="lib/commons-math-2.1.jar"/>
- <classpathentry kind="lib" path="lib/hazelcast-1.9.jar"/>
- <classpathentry kind="lib" path="lib/jersey-multipart-1.4.jar"/>
- <classpathentry kind="lib" path="lib/jetty-continuation-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/jetty-http-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/jetty-io-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/jetty-security-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/mimepull-1.3.jar"/>
- <classpathentry kind="lib" path="lib/hadoop-core-0.20.2-cdh3u0.jar"/>
- <classpathentry kind="lib" path="lib/hbase-0.90.1-cdh3u0.jar"/>
- <classpathentry kind="lib" path="lib/pig-0.8.0-cdh3u0.jar"/>
- <classpathentry kind="lib" path="lib/akela-0.2-SNAPSHOT.jar"/>
- <classpathentry kind="output" path="build/classes"/>
-</classpath>
View
17 analysis/.project
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>socorro-analysis</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
View
33 analysis/bin/correlation_reports.sh
@@ -1,33 +0,0 @@
-#! /bin/sh
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-if [ "$#" -ne 3 ]
-then
- echo "Usage: $0 <product> <release> <yyyyMMdd>"
- exit 1
-fi
-
-PRODUCT=$1
-RELEASE=$2
-DATE=$3
-PYTHON="/usr/bin/python26"
-HADOOP="/usr/lib/hadoop/bin/hadoop"
-REPORTS_HOME="/usr/local/socorro/correlation-reports"
-
-${HADOOP} jar ${REPORTS_HOME}/crash-reports-job.jar com.mozilla.socorro.hadoop.PerCrashCoreCount -Dproduct.filter=${PRODUCT} -Drelease.filter=${RELEASE} -Dstart.date=${DATE} -Dend.date=${DATE} ${DATE}-${PRODUCT}-${RELEASE}-core-counts
-${HADOOP} fs -getmerge ${DATE}-${PRODUCT}-${RELEASE}-core-counts /tmp/${DATE}-${PRODUCT}-${RELEASE}-core-counts.data
-${PYTHON} ${REPORTS_HOME}/per-crash-core-count-hadoop.py /tmp/${DATE}-${PRODUCT}-${RELEASE}-core-counts.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-core-counts.txt
-${HADOOP} jar ${REPORTS_HOME}/crash-reports-job.jar com.mozilla.socorro.hadoop.PerCrashInterestingModules -Dproduct.filter=${PRODUCT} -Drelease.filter=${RELEASE} -Dstart.date=${DATE} -Dend.date=${DATE} ${DATE}-${PRODUCT}-${RELEASE}-interesting-modules
-${HADOOP} fs -getmerge ${DATE}-${PRODUCT}-${RELEASE}-interesting-modules /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-modules.data
-${PYTHON} ${REPORTS_HOME}/per-crash-interesting-modules-hadoop.py /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-modules.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-interesting-modules.txt
-${PYTHON} ${REPORTS_HOME}/per-crash-interesting-modules-hadoop.py -v /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-modules.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-interesting-modules-with-versions.txt
-${HADOOP} jar ${REPORTS_HOME}/crash-reports-job.jar com.mozilla.socorro.hadoop.PerCrashInterestingModules -Dproduct.filter=${PRODUCT} -Drelease.filter=${RELEASE} -Dstart.date=${DATE} -Dend.date=${DATE} -Daddons=true ${DATE}-${PRODUCT}-${RELEASE}-interesting-addons
-${HADOOP} fs -getmerge ${DATE}-${PRODUCT}-${RELEASE}-interesting-addons /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-addons.data
-${PYTHON} ${REPORTS_HOME}/per-crash-interesting-modules-hadoop.py -a /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-addons.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-interesting-addons.txt
-${PYTHON} ${REPORTS_HOME}/per-crash-interesting-modules-hadoop.py -a -v /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-addons.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-interesting-addons-with-versions.txt
-
-rm /tmp/*.data
-find /tmp -name ${DATE}\* -type f -size +500k | xargs gzip -9
View
57 analysis/bin/modulelist.sh
@@ -1,57 +0,0 @@
-#! /bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-if [ "$#" -ne 3 ]
-then
- echo "Usage: $0 <product> <os> <yyyyMMdd>"
- exit 1
-fi
-
-PRODUCT=$1
-OS=$2
-DATE=$3
-HOME_DIRECTORY=$( cd "$( dirname "$0" )" && pwd )
-HADOOP="/usr/lib/hadoop/bin/hadoop"
-
-function fatal {
- if [ "$#" != "2" ]
- then
- echo "syntax: fatal <exit_code> <message>"
- return 1
- fi
-
- exit_code=$1
- message=$2
-
- if [ "$exit_code" != 0 ]
- then
- echo "Fatal exit code: $exit_code"
- echo $message
- exit $exit_code
- fi
-}
-
-
-# build the list, sort it, and then upload the result
-${HADOOP} jar $HOME_DIRECTORY/socorro-analysis-job.jar com.mozilla.socorro.hadoop.CrashReportModuleList -Dproduct.filter="${PRODUCT}" -Dos.filter="${OS}" -Dstart.date=${DATE} -Dend.date=${DATE} ${DATE}-modulelist-out
-fatal $? "Hadoop run failed"
-${HADOOP} fs -getmerge ${DATE}-modulelist-out /tmp/${DATE}-modulelist.txt
-fatal $? "fs getmerge failed"
-/bin/sort /tmp/${DATE}-modulelist.txt -o /tmp/${DATE}-modulelist.sorted.txt
-fatal $? "sort failed"
-
-mkdir -p /mnt/crashanalysis/crash_analysis/modulelist
-fatal $? "could not create output dir"
-cp /tmp/${DATE}-modulelist.sorted.txt /mnt/crashanalysis/crash_analysis/modulelist/${DATE}-modulelist.txt
-fatal $? "could not copy output file to output dir"
-
-# cleanup
-${HADOOP} fs -rmr ${DATE}-modulelist-out
-fatal $? "could not remove modulelist from hdfs"
-rm /tmp/${DATE}-modulelist.txt
-fatal $? "could not remove unsorted modulelist"
-rm /tmp/${DATE}-modulelist.sorted.txt
-fatal $? "could not remove sorted modulelist"
View
26 analysis/bin/run-jetty.sh
@@ -1,26 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-function usage() {
- echo "Usage: $0 <jar> <server-port>"
-}
-
-# Print usage if incorrect number of args
-[[ $# -ne 2 ]] && usage
-
-MAIN_JAR=$1
-SERVER_PORT=$2
-SERVER_CLASS_NAME="com.mozilla.socorro.web.CorrelationReportServer"
-CLASSPATH=$MAIN_JAR
-
-for lib in `ls lib/*.jar`;
-do
- CLASSPATH=$CLASSPATH:$lib
-done
-
-echo $CLASSPATH
-
-java -Dserver.port=$SERVER_PORT -cp $CLASSPATH $SERVER_CLASS_NAME
View
37 analysis/build.properties
@@ -1,37 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-project.name = socorro-analysis
-
-basedir = .
-src.dir = ${basedir}/src
-lib.dir = ${basedir}/lib
-conf.dir = ${basedir}/conf
-bin.dir = ${basedir}/bin
-build.dir = ${basedir}/build
-dist.dir = ${basedir}/dist
-
-build.classes = ${build.dir}/classes
-build.javadoc = ${build.dir}/javadoc
-build.encoding = ISO-8859-1
-build.lib = ${build.dir}/lib
-
-webapp.dir = ${basedir}/webapp
-webapp.webinf.dir = ${basedir}/webapp/WEB-INF
-webapp.metainf.dir = ${basedir}/webapp/META-INF
-
-# Java
-java.src.dir = ${src.dir}/java
-javadoc.packages=*
-
-debug=on
-optimize=on
-deprecation=on
-build.compiler=modern
-
-java.src.version=1.6
-java.target.version=1.6
-
-#Tomcat
-tomcat.dir=/usr/local/tomcat
View
83 analysis/build.xml
@@ -1,83 +0,0 @@
-<?xml version="1.0"?>
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
- - License, v. 2.0. If a copy of the MPL was not distributed with this
- - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-
-<project name="Mozilla Crash-Reports" default="compile" basedir=".">
-
- <!-- Load all properties -->
- <property file="${basedir}/build.properties"/>
-
- <path id="classpath">
- <pathelement location="${build.classes}"/>
- <pathelement location="."/>
- <fileset dir="${lib.dir}">
- <include name="*.jar"/>
- </fileset>
- </path>
-
- <target name="clean">
- <delete includeEmptyDirs="true" failonerror="false">
- <fileset dir="build"/>
- </delete>
- </target>
-
- <target name="init" depends="clean">
- <mkdir dir="${build.dir}"/>
- <mkdir dir="${build.classes}"/>
- <mkdir dir="${build.javadoc}"/>
- <mkdir dir="${build.lib}"/>
- </target>
-
- <target name="compile-java" depends="init">
- <javac encoding="${build.encoding}" srcdir="${java.src.dir}" includes="**/*.java" destdir="${build.classes}" debug="${debug}" source="${java.src.version}" optimize="${optimize}" deprecation="${deprecation}" includeAntRuntime="false">
- <classpath refid="classpath"/>
- <compilerarg line="-Xlint:unchecked"/>
- </javac>
- </target>
- <target name="compile" depends="compile-java"/>
- <target name="ensure-class-name" unless="class">
- <fail message="You must run this target with -Dclass=ClassName"/>
- </target>
-
- <target name="hadoop-jar" depends="compile">
- <delete file="${build.lib}/${project.name}-job.jar"/>
- <jar destfile="${build.lib}/${project.name}-job.jar">
- <fileset dir="${build.classes}"/>
- <fileset dir=".">
- <include name="lib/*.jar"/>
- <exclude name="lib/hadoop*.jar"/>
- <exclude name="lib/hbase*.jar"/>
- <exclude name="lib/zookeeper*.jar"/>
- <exclude name="lib/pig*.jar"/>
- </fileset>
- </jar>
- </target>
-
- <target name="jar" depends="compile">
- <jar destfile="${build.lib}/${project.name}.jar">
- <fileset dir="${build.classes}"/>
- </jar>
- </target>
-
- <target name="war" depends="compile" description="--> Create the WAR file">
- <mkdir dir="${dist.dir}"/>
- <!-- Copy Log4j config to build so it gets included in classes dir -->
- <copy todir="${build.dir}/classes">
- <fileset dir="${webapp.webinf.dir}/config" includes="log4j.*"/>
- </copy>
- <war destfile="${dist.dir}/${project.name}.war" webxml="${webapp.webinf.dir}/web.xml">
- <webinf dir="${webapp.webinf.dir}">
- <exclude name="web.xml"/>
- </webinf>
- <metainf dir="${webapp.metainf.dir}"/>
- <lib dir="${lib.dir}"/>
- <classes dir="${build.dir}/classes"/>
- </war>
- </target>
-
- <target name="deploy" depends="war" description="--> Deploy the WAR file">
- <copy file="${dist.dir}/${project.name}.war" todir="${tomcat.dir}/webapps" overwrite="true"/>
- </target>
-
-</project>
View
BIN analysis/lib/akela-0.2-SNAPSHOT.jar
Binary file not shown.
View
BIN analysis/lib/aopalliance.jar
Binary file not shown.
View
BIN analysis/lib/asm-3.1.jar
Binary file not shown.
View
BIN analysis/lib/commons-lang-2.5.jar
Binary file not shown.
View
BIN analysis/lib/commons-logging-1.1.1.jar
Binary file not shown.
View
BIN analysis/lib/commons-math-2.1.jar
Binary file not shown.
View
BIN analysis/lib/guice-2.0.jar
Binary file not shown.
View
BIN analysis/lib/guice-servlet-2.0.jar
Binary file not shown.
View
BIN analysis/lib/guiceyfruit-2.0.jar
Binary file not shown.
View
BIN analysis/lib/hadoop-core-0.20.2-cdh3u0.jar
Binary file not shown.
View
BIN analysis/lib/hazelcast-1.9.jar
Binary file not shown.
View
BIN analysis/lib/hbase-0.90.1-cdh3u0.jar
Binary file not shown.
View
BIN analysis/lib/jackson-core-asl-1.5.5.jar
Binary file not shown.
View
BIN analysis/lib/jackson-mapper-asl-1.5.5.jar
Binary file not shown.
View
BIN analysis/lib/jersey-bundle-1.3.jar
Binary file not shown.
View
BIN analysis/lib/jersey-guice-1.1.5.jar
Binary file not shown.
View
BIN analysis/lib/jersey-multipart-1.4.jar
Binary file not shown.
View
BIN analysis/lib/jetty-continuation-7.1.6.v20100715.jar
Binary file not shown.
View
BIN analysis/lib/jetty-http-7.1.6.v20100715.jar
Binary file not shown.
View
BIN analysis/lib/jetty-io-7.1.6.v20100715.jar
Binary file not shown.
View
BIN analysis/lib/jetty-security-7.1.6.v20100715.jar
Binary file not shown.
View
BIN analysis/lib/jetty-server-7.1.6.v20100715.jar
Binary file not shown.
View
BIN analysis/lib/jetty-servlet-7.1.6.v20100715.jar
Binary file not shown.
View
BIN analysis/lib/jetty-util-7.1.6.v20100715.jar
Binary file not shown.
View
BIN analysis/lib/jsr311-api-1.1.jar
Binary file not shown.
View
BIN analysis/lib/log4j-1.2.16.jar
Binary file not shown.
View
BIN analysis/lib/mimepull-1.3.jar
Binary file not shown.
View
BIN analysis/lib/pig-0.8.0-cdh3u0.jar
Binary file not shown.
View
BIN analysis/lib/servlet-api-2.5.jar
Binary file not shown.
View
BIN analysis/lib/slf4j-api-1.6.1.jar
Binary file not shown.
View
BIN analysis/lib/trove-2.1.0.jar
Binary file not shown.
View
36 analysis/pig/crashstats.pig
@@ -1,36 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-register './akela-0.1.jar'
-register './socorro-analysis.jar'
-register '/usr/lib/hbase/hbase-0.90.1-cdh3u0.jar'
-
-raw = LOAD 'hbase://crash_reports' USING com.mozilla.pig.load.HBaseMultiScanLoader('$start_date', '$end_date', 'meta_data:json,processed_data:json,raw_data:dump') AS (meta_json:chararray,processed_json:chararray,raw_dump:bytearray);
-gen_meta_map = FOREACH raw GENERATE com.mozilla.pig.eval.json.JsonMap(meta_json) AS meta_json_map:map[];
-product_filtered = FILTER gen_meta_map BY meta_json_map#'ProductName' == 'Firefox';
-
-/* in seconds need to compare to start_date and stop_date */
-start_date_millis = com.mozilla.pig.eval.date.ParseDate('yyyyMMdd', $start_date);
-end_date_millis = com.mozilla.pig.eval.date.ParseDate('yyyyMMdd', $end_date);
-time_filtered = FILTER product_filtered BY ((meta_json_map#'CrashTime' * 1000) >= start_date_millis AND (meta_json_map#'CrashTime' * 1000) <= end_date_millis);
-
-/* count and output submission */
-submissions = FOREACH time_filtered GENERATE (com.mozilla.pig.eval.date.FormatDate('yyyyMMdd', (meta_json_map#'CrashTime'*1000)), meta_json_map#'ProductName', meta_json_map#'Version', 'submissions');
-STORE submissions INTO '$start_date-$end_date-submissions' USING PigStorage();
-
-/* count and output hang */
-hang_filtered = FILTER time_filtered BY meta_json_map#'HangId' is not null;
-gen_hangs = FOREACH hang_filtered GENERATE (com.mozilla.pig.eval.date.FormatDate('yyyyMMdd', (meta_json_map#'CrashTime'*1000)), meta_json_map#'ProductName', meta_json_map#'Version', 'hangs');
-STORE gen_hangs INTO '$start_date-$end_date-hangs' USING PigStorage();
-
-/* count and output oopp */
-oopp_filtered = FILTER time_filtered BY meta_json_map#'ProcessType' is not null AND meta_json_map#'ProcessType' == 'plugin';
-gen_oopp = FOREACH oopp_filtered GENERATE (com.mozilla.pig.eval.date.FormatDate('yyyyMMdd', (meta_json_map#'CrashTime'*1000)), meta_json_map#'ProductName', meta_json_map#'Version', 'oopp');
-STORE gen_oopp INTO '$start_date-$end_date-oopps' USING PigStorage();
-
-/* count and output processed */
-gen_processed_map = FOREACH oopp_filtered GENERATE com.mozilla.hadoop.pig.eval.json.JsonMap(processed_json) AS processed_json_map:map[];
-processed_filtered = FILTER gen_processed_map BY processed_json_map is not null;
-
-STORE processed_filtered INTO '$start_date-$end_date-processed' USING PigStorage();
View
14 analysis/pig/dumpsizetrends.pig
@@ -1,14 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-register './akela-0.1.jar'
-register './socorro-analysis.jar'
-register '/usr/lib/hbase/hbase-0.90.1-cdh3u0.jar'
-
-raw = LOAD 'hbase://crash_reports' USING com.mozilla.pig.load.HBaseMultiScanLoader('$start_date', '$end_date', 'meta_data:json,processed_data:json,raw_data:dump') AS (meta_json:chararray,processed_json:chararray,raw_dump:bytearray);
-gen_meta_map = FOREACH raw GENERATE com.mozilla.pig.eval.json.JsonMap(meta_json) AS meta_json_map:map[];
-gen_processed_map = FOREACH raw GENERATE com.mozilla.pig.eval.json.JsonMap(processed_json) AS processed_json_map:map[];
-
-sizes = FOREACH raw GENERATE com.mozilla.pig.eval.BytesSize(raw_dump) AS raw_size:long, com.mozilla.pig.eval.BytesSize(meta_json) AS meta_size:long, com.mozilla.pig.eval.BytesSize(processed_json) AS processed_size:long;
-STORE sizes INTO '$start_date-$end_date-dumpsizes' USING PigStorage();
View
17 analysis/pig/modulelist.pig
@@ -1,17 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-register './akela-0.1.jar'
-register './socorro-analysis.jar'
-/* Not sure why we have to register this JAR when it's already in Pig's classpath but we do */
-register '/usr/lib/hbase/hbase-0.90.1-cdh3u0.jar'
-
-raw = LOAD 'hbase://crash_reports' USING com.mozilla.pig.load.HBaseMultiScanLoader('$start_date', '$end_date', 'processed_data:json') AS (k:chararray, processed_json:chararray);
-genmap = FOREACH raw GENERATE com.mozilla.pig.eval.json.JsonMap(processed_json) AS processed_json_map:map[];
-product_filtered = FILTER genmap BY processed_json_map#'product' == 'Firefox' AND processed_json_map#'os_name' == 'Windows NT';
-module_bag = FOREACH product_filtered GENERATE com.mozilla.socorro.pig.eval.ModuleBag(processed_json_map#'dump') AS modules:bag{module_tuple:tuple(f1:chararray, f2:chararray, f3:chararray, f4:chararray, f5:chararray, f6:chararray, f7:chararray, f8:chararray)};
-filtered_modules = FILTER module_bag BY modules is not null AND modules.f1 is not null AND modules.f3 is not null AND modules.f4 is not null;
-flat_modules = FOREACH filtered_modules GENERATE FLATTEN(modules);
-modules_list = FOREACH flat_modules GENERATE f1, f3, f4;
-STORE modules_list INTO '$start_date-$end_date-module-list' USING PigStorage(',');
View
20 analysis/pig/stack_frames.pig
@@ -1,20 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-register './akela-0.1.jar'
-register './socorro-analysis.jar'
-/* Not sure why we have to register this JAR when it's already in Pig's classpath but we do */
-register '/usr/lib/hbase/hbase-0.90.1-cdh3u0.jar'
-
-raw = LOAD 'hbase://crash_reports' USING com.mozilla.pig.load.HBaseMultiScanLoader('$start_date', '$end_date', 'processed_data:json') AS (k:chararray, processed_json:chararray);
-genmap = FOREACH raw GENERATE k,com.mozilla.pig.eval.json.JsonMap(processed_json) AS processed_json_map:map[];
-product_filtered = FILTER genmap BY processed_json_map#'product' == 'Firefox' AND processed_json_map#'os_name' == 'Windows NT';
-stack_bag = FOREACH product_filtered GENERATE k,com.mozilla.socorro.pig.eval.FrameBag(processed_json_map#'dump') AS frames:bag{frame:tuple(f1:chararray, f2:chararray, f3:chararray, f4:chararray, f5:chararray, f6:chararray, f7:chararray)};
-flat_stack = FOREACH stack_bag GENERATE k,FLATTEN(frames);
-STORE flat_stack INTO '$start_date-$end_date-stackframes' USING PigStorage();
-
-method_sigs = FOREACH flat_stack GENERATE $5 AS (method_sig:chararray);
-grouped_sigs = GROUP method_sigs BY method_sig;
-distinct_sigs = FOREACH grouped_sigs GENERATE group, COUNT(method_sigs.method_sig);
-STORE distinct_sigs INTO '$start_date-$end_date-method-signatures' USING PigStorage();
View
90 analysis/src/java/com/mozilla/socorro/CorrelationReport.java
@@ -1,90 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro;
-
-import java.util.Map;
-
-public class CorrelationReport {
-
- private String product = null;
- private String productVersion = null;
- private OperatingSystem os = null;
-
- public CorrelationReport(String product, String productVersion, String os) {
- this.product = product;
- this.productVersion = productVersion;
- this.os = new OperatingSystem(os);
- }
-
- public CorrelationReport(String product, String productVersion, String os, String signature) {
- this(product, productVersion, os);
- this.os.addSignature(signature, new Signature(signature));
- }
-
- public String getProduct() {
- return product;
- }
-
- public void setProduct(String product) {
- this.product = product;
- }
-
- public String getProductVersion() {
- return productVersion;
- }
-
- public void setProductVersion(String productVersion) {
- this.productVersion = productVersion;
- }
-
- public OperatingSystem getOs() {
- return os;
- }
-
- public void setOs(OperatingSystem os) {
- this.os = os;
- }
-
- public void calculateModuleRatios() {
- Map<String, Signature> signatures = os.getSignatures();
- Map<String, Module> osModuleMap = os.getModuleCounts();
- Map<String, Module> osAddonMap = os.getAddonCounts();
- for (Map.Entry<String, Signature> sigEntry : signatures.entrySet()) {
- Signature sig = sigEntry.getValue();
-
- Map<String, Module> modules = sig.getModuleCounts();
- for (Map.Entry<String, Module> moduleEntry : modules.entrySet()) {
- Module m = moduleEntry.getValue();
- float sigRatio = sig.getCount() > 0 ? (float)m.getCount() / (float)sig.getCount() : 0.0f;
- int osCount = osModuleMap.get(moduleEntry.getKey()).getCount();
- float osRatio = os.getCount() > 0 ? (float)osCount / (float)os.getCount() : 0.0f;
-
- m.setSigRatio(sigRatio);
- m.setOsRatio(osRatio);
-
- modules.put(moduleEntry.getKey(), m);
- }
- sig.setModuleCounts(modules);
-
- Map<String, Module> addons = sig.getAddonCounts();
- for (Map.Entry<String, Module> addonEntry : addons.entrySet()) {
- Module m = addonEntry.getValue();
- float sigRatio = sig.getCount() > 0 ? (float)m.getCount() / (float)sig.getCount() : 0.0f;
- int osCount = osAddonMap.get(addonEntry.getKey()).getCount();
- float osRatio = os.getCount() > 0 ? (float)osCount / (float)os.getCount() : 0.0f;
-
- m.setSigRatio(sigRatio);
- m.setOsRatio(osRatio);
-
- addons.put(addonEntry.getKey(), m);
- }
- sig.setAddonCounts(addons);
-
- signatures.put(sigEntry.getKey(), sig);
- }
-
- os.setSignatures(signatures);
- }
-}
View
89 analysis/src/java/com/mozilla/socorro/Module.java
@@ -1,89 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.mozilla.util.MapValueComparator;
-
-public class Module {
-
- private String name;
- private int count;
- private float sigRatio = 0.0f;
- private float osRatio = 0.0f;
- private Map<String, Integer> versionCounts = new HashMap<String, Integer>();
-
- public Module(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
-
- public int getCount() {
- return count;
- }
-
- public void setCount(int count) {
- this.count = count;
- }
-
- public Map<String, Integer> getVersionCounts() {
- return versionCounts;
- }
-
- public List<Map.Entry<String, Integer>> getSortedVersionCounts() {
- List<Map.Entry<String, Integer>> versionPairs = new ArrayList<Map.Entry<String, Integer>>(versionCounts.entrySet());
- Collections.sort(versionPairs, Collections.reverseOrder(new MapValueComparator()));
- return versionPairs;
- }
-
- public void incrementVersionCount(String moduleVersion, int count) {
- int existingCount = 0;
- if (versionCounts.containsKey(moduleVersion)) {
- existingCount = versionCounts.get(moduleVersion);
- }
- versionCounts.put(moduleVersion, existingCount + count);
- }
-
- public void setVersionCounts(Map<String, Integer> versionCounts) {
- this.versionCounts = versionCounts;
- }
-
- public float getSigRatio() {
- return sigRatio;
- }
-
- public void setSigRatio(float sigRatio) {
- this.sigRatio = sigRatio;
- }
-
- public float getOsRatio() {
- return osRatio;
- }
-
- public void setOsRatio(float osRatio) {
- this.osRatio = osRatio;
- }
-
- public static class ModuleComparator implements Comparator<Module> {
-
- public int compare(Module o1, Module o2) {
- float diffRatio1 = o1.getSigRatio() - o1.getOsRatio();
- float diffRatio2 = o2.getSigRatio() - o2.getOsRatio();
-
- return diffRatio1 < diffRatio2 ? -1 : diffRatio1 > diffRatio2 ? 1 : 0;
- }
-
- }
-
-}
View
126 analysis/src/java/com/mozilla/socorro/OperatingSystem.java
@@ -1,126 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.mozilla.util.MapValueComparator;
-
-public class OperatingSystem {
-
- private String name = null;
- private int count = 0;
- private Map<String, Signature> signatures = new HashMap<String, Signature>();
- private Map<String, Integer> coreCounts = new HashMap<String, Integer>();
- private Map<String, Module> moduleCounts = new HashMap<String, Module>();
- private Map<String, Module> addonCounts = new HashMap<String, Module>();
-
- public OperatingSystem(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
-
- public int getCount() {
- return count;
- }
-
- public void setCount(int count) {
- this.count = count;
- }
-
- public Map<String, Signature> getSignatures() {
- return signatures;
- }
-
- public void addSignature(String name, Signature signature) {
- this.signatures.put(name, signature);
- }
-
- public void setSignatures(Map<String, Signature> signatures) {
- this.signatures = signatures;
- }
-
- public Map<String, Integer> getCoreCounts() {
- return coreCounts;
- }
-
- public List<Map.Entry<String, Integer>> getSortedCoreCounts() {
- List<Map.Entry<String, Integer>> coreCountPairs = new ArrayList<Map.Entry<String,Integer>>(coreCounts.entrySet());
- Collections.sort(coreCountPairs, Collections.reverseOrder(new MapValueComparator()));
- return coreCountPairs;
- }
-
- public void incrementCoreCount(String arch, int count) {
- int existingCount = 0;
- if (coreCounts.containsKey(arch)) {
- existingCount = coreCounts.get(arch);
- }
- coreCounts.put(arch, existingCount + count);
- }
-
- public void setCoreCounts(Map<String, Integer> coreCounts) {
- this.coreCounts = coreCounts;
- }
-
- public Map<String, Module> getModuleCounts() {
- return moduleCounts;
- }
-
- public List<Module> getSortedModuleCounts() {
- List<Module> modules = new ArrayList<Module>(moduleCounts.values());
- Collections.sort(modules, Collections.reverseOrder(new Module.ModuleComparator()));
- return modules;
- }
-
- public void incrementModuleCount(String moduleName, String moduleVersion, int count) {
- Module module = null;
- if (moduleCounts.containsKey(moduleName)) {
- module = moduleCounts.get(moduleName);
- } else {
- module = new Module(moduleName);
- }
- module.setCount(module.getCount() + count);
- module.incrementVersionCount(moduleVersion, count);
- moduleCounts.put(moduleName, module);
- }
-
- public void setModuleCounts(Map<String, Module> moduleCounts) {
- this.moduleCounts = moduleCounts;
- }
-
- public Map<String, Module> getAddonCounts() {
- return addonCounts;
- }
-
- public List<Module> getSortedAddonCounts() {
- List<Module> addons = new ArrayList<Module>(addonCounts.values());
- Collections.sort(addons, Collections.reverseOrder(new Module.ModuleComparator()));
- return addons;
- }
-
- public void incrementAddonCount(String addonName, String addonVersion, int count) {
- Module module = null;
- if (addonCounts.containsKey(addonName)) {
- module = addonCounts.get(addonName);
- } else {
- module = new Module(addonName);
- }
- module.setCount(module.getCount() + count);
- module.incrementVersionCount(addonVersion, count);
- addonCounts.put(addonName, module);
- }
-
- public void setAddonCounts(Map<String, Module> addonCounts) {
- this.addonCounts = addonCounts;
- }
-
-}
View
143 analysis/src/java/com/mozilla/socorro/RawDumpSizeScan.java
@@ -1,143 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import com.mozilla.util.DateUtil;
-
-public class RawDumpSizeScan {
-
- private static final String TABLE_NAME_CRASH_REPORTS = "crash_reports";
- private static final byte[] RAW_DATA_BYTES = "raw_data".getBytes();
- private static final byte[] DUMP_BYTES = "dump".getBytes();
-
- /**
- * Generates an array of scans for different salted ranges for the given dates
- * @param startDate
- * @param endDate
- * @return
- */
- public static Scan[] generateScans(Calendar startCal, Calendar endCal) {
- SimpleDateFormat rowsdf = new SimpleDateFormat("yyMMdd");
-
- ArrayList<Scan> scans = new ArrayList<Scan>();
- String[] salts = new String[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f" };
-
- long endTime = DateUtil.getEndTimeAtResolution(endCal.getTimeInMillis(), Calendar.DATE);
-
- while (startCal.getTimeInMillis() < endTime) {
- int d = Integer.parseInt(rowsdf.format(startCal.getTime()));
-
- for (int i=0; i < salts.length; i++) {
- Scan s = new Scan();
- s.setCaching(4);
- // disable block caching
- s.setCacheBlocks(false);
- s.addColumn(RAW_DATA_BYTES, DUMP_BYTES);
-
- s.setStartRow(Bytes.toBytes(salts[i] + String.format("%06d", d)));
- s.setStopRow(Bytes.toBytes(salts[i] + String.format("%06d", d + 1)));
- System.out.println("Adding start-stop range: " + salts[i] + String.format("%06d", d) + " - " + salts[i] + String.format("%06d", d + 1));
-
- scans.add(s);
- }
-
- startCal.add(Calendar.DATE, 1);
- }
-
- return scans.toArray(new Scan[scans.size()]);
- }
-
- public static void main(String[] args) throws ParseException {
- String startDateStr = args[0];
- String endDateStr = args[1];
-
- // Set both start/end time and start/stop row
- Calendar startCal = Calendar.getInstance();
- Calendar endCal = Calendar.getInstance();
-
- SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
-
- if (!StringUtils.isBlank(startDateStr)) {
- startCal.setTime(sdf.parse(startDateStr));
- }
- if (!StringUtils.isBlank(endDateStr)) {
- endCal.setTime(sdf.parse(endDateStr));
- }
-
- DescriptiveStatistics stats = new DescriptiveStatistics();
- long numNullRawBytes = 0L;
- HTable table = null;
- Map<String,Integer> rowValueSizeMap = new HashMap<String, Integer>();
- try {
- table = new HTable(TABLE_NAME_CRASH_REPORTS);
- Scan[] scans = generateScans(startCal, endCal);
- for (Scan s : scans) {
- ResultScanner rs = table.getScanner(s);
- Iterator<Result> iter = rs.iterator();
- while(iter.hasNext()) {
- Result r = iter.next();
- ImmutableBytesWritable rawBytes = r.getBytes();
- //length = r.getValue(RAW_DATA_BYTES, DUMP_BYTES);
- if (rawBytes != null) {
- int length = rawBytes.getLength();
- if (length > 20971520) {
- rowValueSizeMap.put(new String(r.getRow()), length);
- }
- stats.addValue(length);
- } else {
- numNullRawBytes++;
- }
-
- if (stats.getN() % 10000 == 0) {
- System.out.println("Processed " + stats.getN());
- System.out.println(String.format("Min: %.02f Max: %.02f Mean: %.02f", stats.getMin(), stats.getMax(), stats.getMean()));
- System.out.println(String.format("1st Quartile: %.02f 2nd Quartile: %.02f 3rd Quartile: %.02f", stats.getPercentile(25.0d), stats.getPercentile(50.0d), stats.getPercentile(75.0d)));
- System.out.println("Number of large entries: " + rowValueSizeMap.size());
- }
- }
- rs.close();
- }
-
- System.out.println("Finished Processing!");
- System.out.println(String.format("Min: %.02f Max: %.02f Mean: %.02f", stats.getMin(), stats.getMax(), stats.getMean()));
- System.out.println(String.format("1st Quartile: %.02f 2nd Quartile: %.02f 3rd Quartile: %.02f", stats.getPercentile(25.0d), stats.getPercentile(50.0d), stats.getPercentile(75.0d)));
-
- for (Map.Entry<String, Integer> entry : rowValueSizeMap.entrySet()) {
- System.out.println(String.format("RowId: %s => Length: %d", entry.getKey(), entry.getValue()));
- }
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } finally {
- if (table != null) {
- try {
- table.close();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- }
- }
-
-}
View
143 analysis/src/java/com/mozilla/socorro/Signature.java
@@ -1,143 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.mozilla.util.MapValueComparator;
-
-public class Signature {
-
- private String raw = null;
- private String name = null;
- private String reason = null;
-
- private int count = 0;
- private Map<String, Integer> coreCounts = new HashMap<String, Integer>();
- private Map<String, Module> moduleCounts = new HashMap<String, Module>();
- private Map<String, Module> addonCounts = new HashMap<String, Module>();
-
- public Signature(String raw) {
- this.raw = raw;
- int lastIdx = raw.lastIndexOf("|");
- if (lastIdx != -1) {
- name = raw.substring(0, lastIdx);
- if ((lastIdx+1) <= (raw.length()-1)) {
- reason = raw.substring(lastIdx+1);
- } else {
- reason = "";
- }
- } else {
- name = raw;
- reason = "";
- }
- }
-
- public String getRaw() {
- return raw;
- }
-
- public String getName() {
- return name;
- }
-
- public String getReason() {
- return reason;
- }
-
- public int getCount() {
- return count;
- }
-
- public void setCount(int count) {
- this.count = count;
- }
-
- public Map<String, Integer> getCoreCounts() {
- return coreCounts;
- }
-
- public List<Map.Entry<String, Integer>> getSortedCoreCounts() {
- List<Map.Entry<String, Integer>> coreCountPairs = new ArrayList<Map.Entry<String,Integer>>(coreCounts.entrySet());
- Collections.sort(coreCountPairs, Collections.reverseOrder(new MapValueComparator()));
- return coreCountPairs;
- }
-
- public void incrementCoreCount(String arch, int count) {
- int existingCount = 0;
- if (coreCounts.containsKey(arch)) {
- existingCount = coreCounts.get(arch);
- }
- coreCounts.put(arch, existingCount + count);
- }
-
- public void setCoreCounts(Map<String, Integer> coreCounts) {
- this.coreCounts = coreCounts;
- }
-
- public Map<String, Module> getModuleCounts() {
- return moduleCounts;
- }
-
- public List<Module> getSortedModuleCounts() {
- List<Module> modules = new ArrayList<Module>(moduleCounts.values());
- Collections.sort(modules, Collections.reverseOrder(new Module.ModuleComparator()));
- return modules;
- }
-
- public void incrementModuleCount(String moduleName, String moduleVersion, int count) {
- Module module = null;
- if (moduleCounts.containsKey(moduleName)) {
- module = moduleCounts.get(moduleName);
- } else {
- module = new Module(moduleName);
- }
- module.setCount(module.getCount() + count);
- module.incrementVersionCount(moduleVersion, count);
- moduleCounts.put(moduleName, module);
- }
-
- public void setModuleCounts(Map<String, Module> moduleCounts) {
- this.moduleCounts = moduleCounts;
- }
-
- public Map<String, Module> getAddonCounts() {
- return addonCounts;
- }
-
- public List<Module> getSortedAddonCounts() {
- List<Module> addons = new ArrayList<Module>(addonCounts.values());
- Collections.sort(addons, Collections.reverseOrder(new Module.ModuleComparator()));
- return addons;
- }
-
- public void incrementAddonCount(String addonName, String addonVersion, int count) {
- Module module = null;
- if (addonCounts.containsKey(addonName)) {
- module = addonCounts.get(addonName);
- } else {
- module = new Module(addonName);
- }
- module.setCount(module.getCount() + count);
- module.incrementVersionCount(addonVersion, count);
- addonCounts.put(addonName, module);
- }
-
- public void setAddonCounts(Map<String, Module> addonCounts) {
- this.addonCounts = addonCounts;
- }
-
- public static class SignatureCountComparator implements Comparator<Signature> {
- public int compare(Signature o1, Signature o2) {
- return o1.getCount() < o2.getCount() ? -1 : o1.getCount() > o2.getCount() ? 1 : 0;
- }
- }
-
-}
View
20 analysis/src/java/com/mozilla/socorro/dao/CrashCountDao.java
@@ -1,20 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro.dao;
-
-import java.io.IOException;
-import java.util.Map;
-
-import com.mozilla.socorro.CorrelationReport;
-
-public interface CrashCountDao {
-
- public void incrementCounts(String date, String product, String version, String os, String signature, String arch, Map<String,String> moduleVersions, Map<String,String> addonVersions) throws IOException;
-
- public CorrelationReport getReport(String date, String product, String version, String os, String signature) throws IOException;
-
- public CorrelationReport getTopCrashers(String date, String product, String version, String os) throws IOException;
-
-}
View
492 analysis/src/java/com/mozilla/socorro/dao/hbase/HbaseCrashCountDao.java
@@ -1,492 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro.dao.hbase;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.net.URLDecoder;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.HTablePool;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.RegexStringComparator;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.eclipse.jetty.util.log.Log;
-
-import com.google.inject.Singleton;
-import com.mozilla.socorro.CorrelationReport;
-import com.mozilla.socorro.OperatingSystem;
-import com.mozilla.socorro.Signature;
-import com.mozilla.socorro.dao.CrashCountDao;
-
-@Singleton
-public class HbaseCrashCountDao implements CrashCountDao {
-
- private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(HbaseCrashCountDao.class);
-
- public static final String TABLE_NAME = "crash_counts";
-
- // Table Column Families
- private static final byte[] DATE = Bytes.toBytes("date");
- private static final byte[] PRODUCT = Bytes.toBytes("product");
- private static final byte[] PRODUCT_VERSION = Bytes.toBytes("product_version");
- private static final byte[] OS = Bytes.toBytes("os");
- private static final byte[] SIGNATURE = Bytes.toBytes("signature");
- private static final byte[] ARCH = Bytes.toBytes("arch");
- private static final byte[] MODULE_WITH_VERSION = Bytes.toBytes("module_with_version");
- private static final byte[] ADDON_WITH_VERSION = Bytes.toBytes("addon_with_version");
-
- // Table Column Qualifiers
- private static final byte[] NAME = Bytes.toBytes("name");
- private static final byte[] COUNT = Bytes.toBytes("count");
-
- // Safe delimiter for appending/splitting module names with versions
- private static final String MODULE_INFO_DELIMITER = "\u0002";
-
- private final HTablePool pool;
- private final Pattern shortKeyPattern;
- private final Map<String,String> productShortNameMap;
-
- public HbaseCrashCountDao() throws IOException {
- pool = new HTablePool();
-
- shortKeyPattern = Pattern.compile("\\p{Punct}|\\s");
-
- productShortNameMap = new HashMap<String,String>();
- productShortNameMap.put("Firefox", "FF");
- productShortNameMap.put("Thunderbird", "TB");
- productShortNameMap.put("SeaMonkey", "SM");
- productShortNameMap.put("Camino", "CM");
- }
-
- public byte[] makeRowKey(String date, String product, String version, String os, String signature, boolean hash) throws IOException {
- // calculate total length to use as a consistent hash char
- int totalLength = date.length() + product.length() + version.length() + os.length();
- if (signature != null) {
- totalLength += signature.length();
- }
- int hexMod = totalLength % 16;
-
- if (productShortNameMap.containsKey(product)) {
- product = productShortNameMap.get(product);
- }
-
- version = shortKeyPattern.matcher(version).replaceAll("");
- os = shortKeyPattern.matcher(os).replaceAll("");
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- if (hash) {
- baos.write(Bytes.toBytes(Integer.toHexString(hexMod)));
- }
- baos.write(Bytes.toBytes(date));
- baos.write(Bytes.toBytes(product));
- baos.write(Bytes.toBytes(version));
- baos.write(Bytes.toBytes(os));
- if (!StringUtils.isBlank(signature)) {
- signature = shortKeyPattern.matcher(signature).replaceAll("");
- baos.write(Bytes.toBytes(signature));
- }
-
- return baos.toByteArray();
- }
-
- public void checkSignatureExists(byte[] sigRowKey, String product, String version, String os, String signature) throws IOException {
- HTableInterface table = null;
- try {
- table = pool.getTable(TABLE_NAME);
- if (!table.exists(new Get(sigRowKey))) {
- Put put = new Put(sigRowKey);
-// put.add(DATE, System.currentTimeMillis(), Bytes.toBytes(date));
- put.add(PRODUCT, Bytes.toBytes(product), Bytes.toBytes(true));
- put.add(PRODUCT_VERSION, Bytes.toBytes(version), Bytes.toBytes(true));
- put.add(OS, Bytes.toBytes(os), Bytes.toBytes(true));
- put.add(SIGNATURE, NAME, Bytes.toBytes(signature));
-
- table.put(put);
- }
- } finally {
- if (table != null) {
- pool.putTable(table);
- }
- }
- }
-
- public void checkOsExists(byte[] osRowKey, String product, String version, String os) throws IOException {
- HTableInterface table = null;
- try {
- table = pool.getTable(TABLE_NAME);
- if (!table.exists(new Get(osRowKey))) {
- Put put = new Put(osRowKey);
-// put.add(DATE, System.currentTimeMillis(), Bytes.toBytes(date));
- put.add(PRODUCT, Bytes.toBytes(product), Bytes.toBytes(true));
- put.add(PRODUCT_VERSION, Bytes.toBytes(version), Bytes.toBytes(true));
- put.add(OS, Bytes.toBytes(os), Bytes.toBytes(true));
-
- table.put(put);
- }
- } finally {
- if (table != null) {
- pool.putTable(table);
- }
- }
- }
-
- public void incrementCounts(String date, String product, String version, String os, String signature, String arch, Map<String,String> moduleVersions, Map<String,String> addonVersions) throws IOException {
- HTableInterface table = null;
- try {
- table = pool.getTable(TABLE_NAME);
-
- byte[] osRowKey = makeRowKey(date, product, version, os, null, true);
- byte[] sigRowKey = makeRowKey(date, product, version, os, signature, true);
-
- // Fill in main info for OS/signature key to allow better scans by different dimensions
- // only need to do this once for efficiency
- checkOsExists(osRowKey, product, version, os);
- checkSignatureExists(sigRowKey, product, version, os, signature);
-
- // increment os count
- table.incrementColumnValue(osRowKey, OS, COUNT, 1L);
- // increment os -> cpu info
- table.incrementColumnValue(osRowKey, ARCH, Bytes.toBytes(arch), 1L);
- // increment os -> signature count
- table.incrementColumnValue(sigRowKey, SIGNATURE, COUNT, 1L);
- // increment os -> sig -> cpu info
- table.incrementColumnValue(sigRowKey, ARCH, Bytes.toBytes(arch), 1L);
-
- for (Map.Entry<String, String> entry : moduleVersions.entrySet()) {
- String module = entry.getKey();
- String moduleVersion = entry.getValue();
- byte[] moduleQualifier = null;
- if (StringUtils.isBlank(moduleVersion)) {
- moduleQualifier = Bytes.toBytes(module);
- } else {
- moduleQualifier = Bytes.toBytes(module + MODULE_INFO_DELIMITER + moduleVersion);
- }
-
- // increment os -> module -> version
- table.incrementColumnValue(osRowKey, MODULE_WITH_VERSION, moduleQualifier, 1L);
-
- // increment os -> sig -> module -> version
- table.incrementColumnValue(sigRowKey, MODULE_WITH_VERSION, moduleQualifier, 1L);
- }
-
- for (Map.Entry<String, String> entry : addonVersions.entrySet()) {
- String addon = entry.getKey();
- String addonVersion = entry.getValue();
- byte[] addonQualifier = null;
- if (StringUtils.isBlank(addonVersion)) {
- addonQualifier = Bytes.toBytes(addon);
- } else {
- addonQualifier = Bytes.toBytes(addon + MODULE_INFO_DELIMITER + addonVersion);
- }
-
- // increment os -> addon -> version
- table.incrementColumnValue(osRowKey, ADDON_WITH_VERSION, addonQualifier, 1L);
-
- // increment os -> sig -> addon -> version
- table.incrementColumnValue(sigRowKey, ADDON_WITH_VERSION, addonQualifier, 1L);
- }
- } finally {
- if (table != null) {
- pool.putTable(table);
- }
- }
- }
-
- public CorrelationReport getTopCrashers(String date, String product, String version, String os) throws IOException {
- Scan scan = new Scan();
- FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
-
- String rowKeyExpr = "^[a-zA-Z0-9]{1}" + new String(makeRowKey(date, product, version, os, null, false)) + ".+";
- RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(rowKeyExpr));
- filterList.addFilter(rowFilter);
-
- SingleColumnValueFilter productFilter = new SingleColumnValueFilter(PRODUCT, Bytes.toBytes(product), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(true));
- SingleColumnValueFilter productVersionFilter = new SingleColumnValueFilter(PRODUCT_VERSION, Bytes.toBytes(version), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(true));
- SingleColumnValueFilter osFilter = new SingleColumnValueFilter(OS, Bytes.toBytes(os), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(true));
- filterList.addFilter(productFilter);
- filterList.addFilter(productVersionFilter);
- filterList.addFilter(osFilter);
-
- scan.setFilter(filterList);
-
- // Want the values of these columns for filtered rows
- scan.addFamily(SIGNATURE);
- scan.addFamily(ARCH);
- scan.addFamily(MODULE_WITH_VERSION);
- scan.addFamily(ADDON_WITH_VERSION);
-
- CorrelationReport report = new CorrelationReport(date, product, version, os);
-
- HTableInterface table = null;
- ResultScanner scanner = null;
- try {
- table = pool.getTable(TABLE_NAME);
- scanner = table.getScanner(scan);
- long resultCount = 0;
- for (Result r : scanner) {
- OperatingSystem osys = report.getOs();
-
- resultCount++;
- if (Log.isDebugEnabled()) {
- LOG.debug("Returned row: " + new String(r.getRow()));
- }
-
- byte[] sigBytes = r.getValue(SIGNATURE, NAME);
- byte[] sigCountBytes = r.getValue(SIGNATURE, COUNT);
- if (sigBytes != null && sigCountBytes != null) {
- String rawSignature = new String(sigBytes);
- Signature sig = null;
- if (osys.getSignatures().containsKey(rawSignature)) {
- sig = osys.getSignatures().get(rawSignature);
- } else {
- sig = new Signature(rawSignature);
- }
-
- sig.setCount(sig.getCount() + (int)Bytes.toLong(sigCountBytes));
-
- NavigableMap<byte[],byte[]> nm = r.getFamilyMap(ARCH);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String archCores = new String(entry.getKey());
- long count = Bytes.toLong(entry.getValue());
-
- sig.incrementCoreCount(archCores, (int)count);
- }
-
- nm = r.getFamilyMap(MODULE_WITH_VERSION);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String moduleVersion = new String(entry.getKey());
- String[] splits = moduleVersion.split(MODULE_INFO_DELIMITER);
- long count = Bytes.toLong(entry.getValue());
-
- if (splits.length == 2) {
- sig.incrementModuleCount(splits[0], splits[1], (int)count);
- } else {
- sig.incrementModuleCount(moduleVersion, "", (int)count);
- }
- }
-
- nm = r.getFamilyMap(ADDON_WITH_VERSION);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String addonVersion = new String(entry.getKey());
- String[] splits = addonVersion.split(MODULE_INFO_DELIMITER);
- long count = Bytes.toLong(entry.getValue());
-
- if (splits.length == 2) {
- sig.incrementAddonCount(splits[0], splits[1], (int)count);
- } else {
- sig.incrementAddonCount(addonVersion, "", (int)count);
- }
- }
-
- osys.addSignature(sig.getName(), sig);
- report.setOs(osys);
- }
- }
-
- if (Log.isDebugEnabled()) {
- LOG.debug("Result Count: " + resultCount);
- }
-
- byte[] osRowKey = makeRowKey(date, product, version, os, null, true);
- Get osGet = new Get(osRowKey);
- Result osResult = table.get(osGet);
- if (osResult != null && !osResult.isEmpty()) {
- OperatingSystem osys = report.getOs();
-
- int osCount = (int)Bytes.toLong(osResult.getValue(OS, COUNT));
- osys.setCount(osCount);
-
- NavigableMap<byte[],byte[]> nm = osResult.getFamilyMap(ARCH);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String archCores = new String(entry.getKey());
- long count = Bytes.toLong(entry.getValue());
-
- osys.incrementCoreCount(archCores, (int)count);
- }
-
- nm = osResult.getFamilyMap(MODULE_WITH_VERSION);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String moduleVersion = new String(entry.getKey());
- String[] splits = moduleVersion.split(MODULE_INFO_DELIMITER);
- long count = Bytes.toLong(entry.getValue());
-
- if (splits.length == 2) {
- osys.incrementModuleCount(splits[0], splits[1], (int)count);
- } else {
- osys.incrementModuleCount(moduleVersion, "", (int)count);
- }
- }
-
- nm = osResult.getFamilyMap(ADDON_WITH_VERSION);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String addonVersion = new String(entry.getKey());
- String[] splits = addonVersion.split(MODULE_INFO_DELIMITER);
- long count = Bytes.toLong(entry.getValue());
-
- if (splits.length == 2) {
- osys.incrementAddonCount(splits[0], splits[1], (int)count);
- } else {
- osys.incrementAddonCount(addonVersion, "", (int)count);
- }
- }
-
- report.setOs(osys);
- }
- } finally {
- if (scanner != null) {
- scanner.close();
- }
-
- if (table != null) {
- pool.putTable(table);
- }
- }
-
- report.calculateModuleRatios();
-
- return report;
- }
-
- public CorrelationReport getReport(String date, String product, String version, String os, String signature) throws IOException {
- os = URLDecoder.decode(os, "UTF-8");
- signature = URLDecoder.decode(signature, "UTF-8");
-
- CorrelationReport report = new CorrelationReport(product, version, os);
- OperatingSystem osys = report.getOs();
-
- HTableInterface table = null;
- try {
- byte[] osRowKey = makeRowKey(date, product, version, os, null, true);
- byte[] sigRowKey = makeRowKey(date, product, version, os, signature, true);
-
- if (LOG.isInfoEnabled()) {
- LOG.debug("Date: " + date);
- LOG.debug("Product: " + product);
- LOG.debug("Version: " + version);
- LOG.debug("OS: " + os);
- LOG.debug("Signature: " + signature);
- LOG.info("Sig Row Key: " + new String(sigRowKey));
- }
-
- table = pool.getTable(TABLE_NAME);
- Get sigGet = new Get(sigRowKey);
- Result sigResult = table.get(sigGet);
- if (sigResult != null && !sigResult.isEmpty()) {
- Signature sig = new Signature(signature);
-
- int sigCount = (int)Bytes.toLong(sigResult.getValue(SIGNATURE, COUNT));
- sig.setCount(sigCount);
-
- NavigableMap<byte[],byte[]> nm = sigResult.getFamilyMap(ARCH);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String archCores = new String(entry.getKey());
- long count = Bytes.toLong(entry.getValue());
-
- sig.incrementCoreCount(archCores, (int)count);
- }
-
- nm = sigResult.getFamilyMap(MODULE_WITH_VERSION);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String moduleVersion = new String(entry.getKey());
- String[] splits = moduleVersion.split(MODULE_INFO_DELIMITER);
- long count = Bytes.toLong(entry.getValue());
-
- if (splits.length == 2) {
- sig.incrementModuleCount(splits[0], splits[1], (int)count);
- } else {
- sig.incrementModuleCount(moduleVersion, "", (int)count);
- }
- }
-
- nm = sigResult.getFamilyMap(ADDON_WITH_VERSION);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String addonVersion = new String(entry.getKey());
- String[] splits = addonVersion.split(MODULE_INFO_DELIMITER);
- long count = Bytes.toLong(entry.getValue());
-
- if (splits.length == 2) {
- sig.incrementAddonCount(splits[0], splits[1], (int)count);
- } else {
- sig.incrementAddonCount(addonVersion, "", (int)count);
- }
- }
-
- osys.addSignature(signature, sig);
- } else {
- LOG.warn("Signature result was empty for params: " + String.format("%s, %s, %s, %s, %s", date, product, version, os, signature));
- }
-
- Get osGet = new Get(osRowKey);
- Result osResult = table.get(osGet);
- if (osResult != null && !osResult.isEmpty()) {
-
- int osCount = (int)Bytes.toLong(osResult.getValue(OS, COUNT));
- osys.setCount(osCount);
-
- NavigableMap<byte[],byte[]> nm = osResult.getFamilyMap(ARCH);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String archCores = new String(entry.getKey());
- long count = Bytes.toLong(entry.getValue());
-
- osys.incrementCoreCount(archCores, (int)count);
- }
-
- nm = osResult.getFamilyMap(MODULE_WITH_VERSION);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String moduleVersion = new String(entry.getKey());
- String[] splits = moduleVersion.split(MODULE_INFO_DELIMITER);
- long count = Bytes.toLong(entry.getValue());
-
- if (splits.length == 2) {
- osys.incrementModuleCount(splits[0], splits[1], (int)count);
- } else {
- osys.incrementModuleCount(moduleVersion, "", (int)count);
- }
- }
-
- nm = osResult.getFamilyMap(ADDON_WITH_VERSION);
- for (Map.Entry<byte[], byte[]> entry : nm.entrySet()) {
- String addonVersion = new String(entry.getKey());
- String[] splits = addonVersion.split(MODULE_INFO_DELIMITER);
- long count = Bytes.toLong(entry.getValue());
-
- if (splits.length == 2) {
- osys.incrementAddonCount(splits[0], splits[1], (int)count);
- } else {
- osys.incrementAddonCount(addonVersion, "", (int)count);
- }
- }
-
- report.setOs(osys);
- } else {
- LOG.warn("OS result was empty for params: " + String.format("%s, %s, %s, %s", date, product, version, os));
- }
- } finally {
- if (table != null) {
- pool.putTable(table);
- }
- }
-
- // calculate module ratios for proper sorting
- report.calculateModuleRatios();
-
- return report;
- }
-
-}
View
72 analysis/src/java/com/mozilla/socorro/dao/hbase/HbaseCrashReportDao.java
@@ -1,72 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro.dao.hbase;
-
-import java.io.IOException;
-import java.util.Calendar;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.HTablePool;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-
-public class HbaseCrashReportDao {
-
- private static final int DEFAULT_DEPTH = 2;
-
- private HTablePool pool;
-
- public HbaseCrashReportDao(HTablePool pool) {
- this.pool = pool;
- }
-
- public String generateOOID(long millis) {
- return generateOOID(millis, DEFAULT_DEPTH);
- }
-
- public String generateOOID(long millis, int depth) {
- if (depth < 1 || depth > 4) {
- depth = DEFAULT_DEPTH;
- }
-
- Calendar cal = Calendar.getInstance();
- cal.setTimeInMillis(millis);
-
- String uuid = UUID.randomUUID().toString();
- String dateStr = String.format("%d%d%d", new Object[] { cal.get(Calendar.YEAR) % 100, cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DATE) });
-
- StringBuilder sb = new StringBuilder();
- sb.append(uuid.substring(0, 1));
- sb.append(dateStr);
- sb.append(uuid.substring(0, uuid.length() - 7));
- sb.append(depth);
- sb.append(dateStr);
-
- return sb.toString();
- }
-
- public String insert(Map<String, String> fields, byte[] dump) throws IOException {
- String ooid = null;
- HTableInterface table = null;
- try {
- table = pool.getTable("crash_reports_test");
- ooid = generateOOID(System.currentTimeMillis());
- Put p = new Put(Bytes.toBytes(ooid));
- for (Map.Entry<String, String> field : fields.entrySet()) {
- p.add(Bytes.toBytes("meta_data"), Bytes.toBytes(field.getKey()), Bytes.toBytes(field.getValue()));
- }
- p.add(Bytes.toBytes("raw_data"), Bytes.toBytes("dump"), dump);
-
- table.put(p);
- } finally {
- pool.putTable(table);
- }
-
- return ooid;
- }
-
-}
View
259 analysis/src/java/com/mozilla/socorro/hadoop/CrashCount.java
@@ -1,259 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package com.mozilla.socorro.hadoop;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;