Permalink
Browse files

bug 765001 - switch to new way of doing mapreduce jobs, starting with…

… modulelist

This reverts commit 438fa4e.
  • Loading branch information...
1 parent e7bb40a commit 57a5b2de77cca0ff8b03084a09d8014ab02d803c @rhelmer rhelmer committed Oct 3, 2012
Showing with 38 additions and 48,905 deletions.
  1. +6 −0 .gitmodules
  2. +12 −8 Makefile
  3. +1 −0 akela
  4. +0 −37 analysis/.classpath
  5. +0 −17 analysis/.project
  6. +0 −33 analysis/bin/correlation_reports.sh
  7. +0 −57 analysis/bin/modulelist.sh
  8. +0 −26 analysis/bin/run-jetty.sh
  9. +0 −37 analysis/build.properties
  10. +0 −83 analysis/build.xml
  11. BIN analysis/lib/akela-0.2-SNAPSHOT.jar
  12. BIN analysis/lib/aopalliance.jar
  13. BIN analysis/lib/asm-3.1.jar
  14. BIN analysis/lib/commons-lang-2.5.jar
  15. BIN analysis/lib/commons-logging-1.1.1.jar
  16. BIN analysis/lib/commons-math-2.1.jar
  17. BIN analysis/lib/guice-2.0.jar
  18. BIN analysis/lib/guice-servlet-2.0.jar
  19. BIN analysis/lib/guiceyfruit-2.0.jar
  20. BIN analysis/lib/hadoop-core-0.20.2-cdh3u0.jar
  21. BIN analysis/lib/hazelcast-1.9.jar
  22. BIN analysis/lib/hbase-0.90.1-cdh3u0.jar
  23. BIN analysis/lib/jackson-core-asl-1.5.5.jar
  24. BIN analysis/lib/jackson-mapper-asl-1.5.5.jar
  25. BIN analysis/lib/jersey-bundle-1.3.jar
  26. BIN analysis/lib/jersey-guice-1.1.5.jar
  27. BIN analysis/lib/jersey-multipart-1.4.jar
  28. BIN analysis/lib/jetty-continuation-7.1.6.v20100715.jar
  29. BIN analysis/lib/jetty-http-7.1.6.v20100715.jar
  30. BIN analysis/lib/jetty-io-7.1.6.v20100715.jar
  31. BIN analysis/lib/jetty-security-7.1.6.v20100715.jar
  32. BIN analysis/lib/jetty-server-7.1.6.v20100715.jar
  33. BIN analysis/lib/jetty-servlet-7.1.6.v20100715.jar
  34. BIN analysis/lib/jetty-util-7.1.6.v20100715.jar
  35. BIN analysis/lib/jsr311-api-1.1.jar
  36. BIN analysis/lib/log4j-1.2.16.jar
  37. BIN analysis/lib/mimepull-1.3.jar
  38. BIN analysis/lib/pig-0.8.0-cdh3u0.jar
  39. BIN analysis/lib/servlet-api-2.5.jar
  40. BIN analysis/lib/slf4j-api-1.6.1.jar
  41. BIN analysis/lib/trove-2.1.0.jar
  42. +0 −36 analysis/pig/crashstats.pig
  43. +0 −14 analysis/pig/dumpsizetrends.pig
  44. +0 −17 analysis/pig/modulelist.pig
  45. +0 −20 analysis/pig/stack_frames.pig
  46. +0 −90 analysis/src/java/com/mozilla/socorro/CorrelationReport.java
  47. +0 −89 analysis/src/java/com/mozilla/socorro/Module.java
  48. +0 −126 analysis/src/java/com/mozilla/socorro/OperatingSystem.java
  49. +0 −143 analysis/src/java/com/mozilla/socorro/RawDumpSizeScan.java
  50. +0 −143 analysis/src/java/com/mozilla/socorro/Signature.java
  51. +0 −20 analysis/src/java/com/mozilla/socorro/dao/CrashCountDao.java
  52. +0 −492 analysis/src/java/com/mozilla/socorro/dao/hbase/HbaseCrashCountDao.java
  53. +0 −72 analysis/src/java/com/mozilla/socorro/dao/hbase/HbaseCrashReportDao.java
  54. +0 −259 analysis/src/java/com/mozilla/socorro/hadoop/CrashCount.java
  55. +0 −408 analysis/src/java/com/mozilla/socorro/hadoop/CrashCount2.java
  56. +0 −275 analysis/src/java/com/mozilla/socorro/hadoop/CrashCountToHbase.java
  57. +0 −345 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportDataMatrix.java
  58. +0 −299 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportFeatureIndex.java
  59. +0 −119 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportJob.java
  60. +0 −223 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportModuleList.java
  61. +0 −251 analysis/src/java/com/mozilla/socorro/hadoop/CrashReportStats.java
  62. +0 −307 analysis/src/java/com/mozilla/socorro/hadoop/DumpSizeTrends.java
  63. +0 −243 analysis/src/java/com/mozilla/socorro/hadoop/HardwareAccel.java
  64. +0 −94 analysis/src/java/com/mozilla/socorro/hadoop/KeysForDateRange.java
  65. +0 −279 analysis/src/java/com/mozilla/socorro/hadoop/PerCrashCoreCount.java
  66. +0 −362 analysis/src/java/com/mozilla/socorro/hadoop/PerCrashInterestingModules.java
  67. +0 −207 analysis/src/java/com/mozilla/socorro/hadoop/RawDumpSize.java
  68. +0 −51 analysis/src/java/com/mozilla/socorro/pig/eval/FrameBag.java
  69. +0 −49 analysis/src/java/com/mozilla/socorro/pig/eval/ModuleBag.java
  70. +0 −44 analysis/src/java/com/mozilla/socorro/web/CorrelationReportServer.java
  71. +0 −331 analysis/src/java/com/mozilla/socorro/web/CorrelationReportService.java
  72. +0 −33 analysis/src/java/com/mozilla/socorro/web/GuiceConfig.java
  73. +0 −17 analysis/src/java/com/mozilla/util/MapValueComparator.java
  74. +0 −139 analysis/src/java/org/apache/hadoop/hbase/rest/CollectorResource.java
  75. +0 −98 analysis/src/java/org/apache/hadoop/hbase/rest/HazelcastQueueResource.java
  76. +0 −42 analysis/src/java/org/apache/hadoop/hbase/rest/model/QueueStringModel.java
  77. +0 −41,910 analysis/src/python/addonids-amo.json
  78. +0 −122 analysis/src/python/addonids-local.json
  79. +0 −71 analysis/src/python/addonids.py
  80. +0 −98 analysis/src/python/crash_stats.py
  81. +0 −137 analysis/src/python/dump_size_trends.py
  82. +0 −32 analysis/src/python/jsondb.py
  83. +0 −55 analysis/src/python/macdebugids.json
  84. +0 −45 analysis/src/python/macdebugids.py
  85. +0 −117 analysis/src/python/per-crash-core-count-hadoop.py
  86. +0 −180 analysis/src/python/per-crash-interesting-modules-hadoop.py
  87. +0 −47 analysis/src/python/raw_dump_size.py
  88. +0 −47 analysis/src/python/threads_vs_dumpsize.py
  89. +2 −2 docs/installation.rst
  90. +1 −1 puppet/manifests/classes/socorro-base.pp
  91. +1 −1 puppet/manifests/classes/socorro-hbase.pp
  92. +14 −5 scripts/crons/cron_modulelist.sh
  93. +1 −0 socorro-toolbox
View
@@ -0,0 +1,6 @@
+[submodule "socorro-toolbox"]
+ path = socorro-toolbox
+ url = git://github.com/mozilla-metrics/socorro-toolbox.git
+[submodule "akela"]
+ path = akela
+ url = git://github.com/mozilla-metrics/akela.git
View
@@ -13,7 +13,7 @@ COVERAGE = $(VIRTUALENV)/bin/coverage
PYLINT = $(VIRTUALENV)/bin/pylint
CITEXT="/usr/share/postgresql/9.0/contrib/citext.sql"
-.PHONY: all test install reinstall install-socorro install-web virtualenv coverage lint clean minidump_stackwalk java_analysis thirdparty
+.PHONY: all test install reinstall install-socorro install-web virtualenv coverage lint clean minidump_stackwalk analysis thirdparty
all: test
@@ -31,7 +31,7 @@ thirdparty:
# install production dependencies
$(VIRTUALENV)/bin/pip install --use-mirrors --download-cache=pip-cache/ --ignore-installed --install-option="--prefix=`pwd`/thirdparty" --install-option="--install-lib=`pwd`/thirdparty" -r requirements/prod.txt
-install: java_analysis thirdparty reinstall
+install: analysis thirdparty reinstall
# this a dev-only option, `make install` needs to be run at least once in the checkout (or after `make clean`)
reinstall: install-socorro install-web
@@ -54,8 +54,7 @@ install-socorro:
rsync -a wsgi $(PREFIX)/application
rsync -a stackwalk $(PREFIX)/
rsync -a scripts/stackwalk.sh $(PREFIX)/stackwalk/bin/
- rsync -a analysis/build/lib/socorro-analysis-job.jar $(PREFIX)/analysis/
- rsync -a analysis/bin/modulelist.sh $(PREFIX)/analysis/
+ rsync -a analysis $(PREFIX)/
# copy default config files
cd $(PREFIX)/application/scripts/config; for file in *.py.dist; do cp $$file `basename $$file .dist`; done
@@ -83,14 +82,19 @@ clean:
rm -rf ./thirdparty/*
rm -rf ./google-breakpad/ ./builds/ ./breakpad/ ./stackwalk ./pip-cache
rm -rf ./breakpad.tar.gz
- cd analysis && ant clean
minidump_stackwalk:
svn co http://google-breakpad.googlecode.com/svn/trunk google-breakpad
cd google-breakpad && ./configure --prefix=`pwd`/../stackwalk/
cd google-breakpad && make install
cd google-breakpad && svn info | grep Revision | cut -d' ' -f 2 > ../stackwalk/revision.txt
-java_analysis:
- cd analysis && ant hadoop-jar
-
+analysis:
+ git submodule update --init socorro-toolbox akela
+ cd akela && mvn package
+ cd akela && mvn package
+ cd socorro-toolbox && mvn package
+ mkdir -p analysis
+ rsync socorro-toolbox/target/*.jar analysis/
+ rsync akela/target/*.jar analysis/
+ rsync -a socorro-toolbox/src/main/pig/ analysis/
1 akela
Submodule akela added at a82d41
View
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" path="src/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="lib" path="lib/commons-lang-2.5.jar"/>
- <classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
- <classpathentry kind="lib" path="lib/trove-2.1.0.jar"/>
- <classpathentry kind="lib" path="lib/asm-3.1.jar"/>
- <classpathentry kind="lib" path="lib/jersey-bundle-1.3.jar"/>
- <classpathentry kind="lib" path="lib/jsr311-api-1.1.jar"/>
- <classpathentry kind="lib" path="lib/guice-2.0.jar"/>
- <classpathentry kind="lib" path="lib/guice-servlet-2.0.jar"/>
- <classpathentry kind="lib" path="lib/jetty-server-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/jetty-util-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/servlet-api-2.5.jar"/>
- <classpathentry kind="lib" path="lib/guiceyfruit-2.0.jar"/>
- <classpathentry kind="lib" path="lib/jersey-guice-1.1.5.jar"/>
- <classpathentry kind="lib" path="lib/slf4j-api-1.6.1.jar"/>
- <classpathentry kind="lib" path="lib/jackson-core-asl-1.5.5.jar"/>
- <classpathentry kind="lib" path="lib/jackson-mapper-asl-1.5.5.jar"/>
- <classpathentry kind="lib" path="lib/aopalliance.jar"/>
- <classpathentry kind="lib" path="lib/jetty-servlet-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/log4j-1.2.16.jar"/>
- <classpathentry kind="lib" path="lib/commons-math-2.1.jar"/>
- <classpathentry kind="lib" path="lib/hazelcast-1.9.jar"/>
- <classpathentry kind="lib" path="lib/jersey-multipart-1.4.jar"/>
- <classpathentry kind="lib" path="lib/jetty-continuation-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/jetty-http-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/jetty-io-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/jetty-security-7.1.6.v20100715.jar"/>
- <classpathentry kind="lib" path="lib/mimepull-1.3.jar"/>
- <classpathentry kind="lib" path="lib/hadoop-core-0.20.2-cdh3u0.jar"/>
- <classpathentry kind="lib" path="lib/hbase-0.90.1-cdh3u0.jar"/>
- <classpathentry kind="lib" path="lib/pig-0.8.0-cdh3u0.jar"/>
- <classpathentry kind="lib" path="lib/akela-0.2-SNAPSHOT.jar"/>
- <classpathentry kind="output" path="build/classes"/>
-</classpath>
View
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>socorro-analysis</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
@@ -1,33 +0,0 @@
-#! /bin/sh
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-if [ "$#" -ne 3 ]
-then
- echo "Usage: $0 <product> <release> <yyyyMMdd>"
- exit 1
-fi
-
-PRODUCT=$1
-RELEASE=$2
-DATE=$3
-PYTHON="/usr/bin/python26"
-HADOOP="/usr/lib/hadoop/bin/hadoop"
-REPORTS_HOME="/usr/local/socorro/correlation-reports"
-
-${HADOOP} jar ${REPORTS_HOME}/crash-reports-job.jar com.mozilla.socorro.hadoop.PerCrashCoreCount -Dproduct.filter=${PRODUCT} -Drelease.filter=${RELEASE} -Dstart.date=${DATE} -Dend.date=${DATE} ${DATE}-${PRODUCT}-${RELEASE}-core-counts
-${HADOOP} fs -getmerge ${DATE}-${PRODUCT}-${RELEASE}-core-counts /tmp/${DATE}-${PRODUCT}-${RELEASE}-core-counts.data
-${PYTHON} ${REPORTS_HOME}/per-crash-core-count-hadoop.py /tmp/${DATE}-${PRODUCT}-${RELEASE}-core-counts.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-core-counts.txt
-${HADOOP} jar ${REPORTS_HOME}/crash-reports-job.jar com.mozilla.socorro.hadoop.PerCrashInterestingModules -Dproduct.filter=${PRODUCT} -Drelease.filter=${RELEASE} -Dstart.date=${DATE} -Dend.date=${DATE} ${DATE}-${PRODUCT}-${RELEASE}-interesting-modules
-${HADOOP} fs -getmerge ${DATE}-${PRODUCT}-${RELEASE}-interesting-modules /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-modules.data
-${PYTHON} ${REPORTS_HOME}/per-crash-interesting-modules-hadoop.py /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-modules.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-interesting-modules.txt
-${PYTHON} ${REPORTS_HOME}/per-crash-interesting-modules-hadoop.py -v /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-modules.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-interesting-modules-with-versions.txt
-${HADOOP} jar ${REPORTS_HOME}/crash-reports-job.jar com.mozilla.socorro.hadoop.PerCrashInterestingModules -Dproduct.filter=${PRODUCT} -Drelease.filter=${RELEASE} -Dstart.date=${DATE} -Dend.date=${DATE} -Daddons=true ${DATE}-${PRODUCT}-${RELEASE}-interesting-addons
-${HADOOP} fs -getmerge ${DATE}-${PRODUCT}-${RELEASE}-interesting-addons /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-addons.data
-${PYTHON} ${REPORTS_HOME}/per-crash-interesting-modules-hadoop.py -a /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-addons.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-interesting-addons.txt
-${PYTHON} ${REPORTS_HOME}/per-crash-interesting-modules-hadoop.py -a -v /tmp/${DATE}-${PRODUCT}-${RELEASE}-interesting-addons.data > /tmp/${DATE}_${PRODUCT}_${RELEASE}-interesting-addons-with-versions.txt
-
-rm /tmp/*.data
-find /tmp -name ${DATE}\* -type f -size +500k | xargs gzip -9
View
@@ -1,57 +0,0 @@
-#! /bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-if [ "$#" -ne 3 ]
-then
- echo "Usage: $0 <product> <os> <yyyyMMdd>"
- exit 1
-fi
-
-PRODUCT=$1
-OS=$2
-DATE=$3
-HOME_DIRECTORY=$( cd "$( dirname "$0" )" && pwd )
-HADOOP="/usr/lib/hadoop/bin/hadoop"
-
-function fatal {
- if [ "$#" != "2" ]
- then
- echo "syntax: fatal <exit_code> <message>"
- return 1
- fi
-
- exit_code=$1
- message=$2
-
- if [ "$exit_code" != 0 ]
- then
- echo "Fatal exit code: $exit_code"
- echo $message
- exit $exit_code
- fi
-}
-
-
-# build the list, sort it, and then upload the result
-${HADOOP} jar $HOME_DIRECTORY/socorro-analysis-job.jar com.mozilla.socorro.hadoop.CrashReportModuleList -Dproduct.filter="${PRODUCT}" -Dos.filter="${OS}" -Dstart.date=${DATE} -Dend.date=${DATE} ${DATE}-modulelist-out
-fatal $? "Hadoop run failed"
-${HADOOP} fs -getmerge ${DATE}-modulelist-out /tmp/${DATE}-modulelist.txt
-fatal $? "fs getmerge failed"
-/bin/sort /tmp/${DATE}-modulelist.txt -o /tmp/${DATE}-modulelist.sorted.txt
-fatal $? "sort failed"
-
-mkdir -p /mnt/crashanalysis/crash_analysis/modulelist
-fatal $? "could not create output dir"
-cp /tmp/${DATE}-modulelist.sorted.txt /mnt/crashanalysis/crash_analysis/modulelist/${DATE}-modulelist.txt
-fatal $? "could not copy output file to output dir"
-
-# cleanup
-${HADOOP} fs -rmr ${DATE}-modulelist-out
-fatal $? "could not remove modulelist from hdfs"
-rm /tmp/${DATE}-modulelist.txt
-fatal $? "could not remove unsorted modulelist"
-rm /tmp/${DATE}-modulelist.sorted.txt
-fatal $? "could not remove sorted modulelist"
View
@@ -1,26 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-function usage() {
- echo "Usage: $0 <jar> <server-port>"
-}
-
-# Print usage if incorrect number of args
-[[ $# -ne 2 ]] && usage
-
-MAIN_JAR=$1
-SERVER_PORT=$2
-SERVER_CLASS_NAME="com.mozilla.socorro.web.CorrelationReportServer"
-CLASSPATH=$MAIN_JAR
-
-for lib in `ls lib/*.jar`;
-do
- CLASSPATH=$CLASSPATH:$lib
-done
-
-echo $CLASSPATH
-
-java -Dserver.port=$SERVER_PORT -cp $CLASSPATH $SERVER_CLASS_NAME
View
@@ -1,37 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-project.name = socorro-analysis
-
-basedir = .
-src.dir = ${basedir}/src
-lib.dir = ${basedir}/lib
-conf.dir = ${basedir}/conf
-bin.dir = ${basedir}/bin
-build.dir = ${basedir}/build
-dist.dir = ${basedir}/dist
-
-build.classes = ${build.dir}/classes
-build.javadoc = ${build.dir}/javadoc
-build.encoding = ISO-8859-1
-build.lib = ${build.dir}/lib
-
-webapp.dir = ${basedir}/webapp
-webapp.webinf.dir = ${basedir}/webapp/WEB-INF
-webapp.metainf.dir = ${basedir}/webapp/META-INF
-
-# Java
-java.src.dir = ${src.dir}/java
-javadoc.packages=*
-
-debug=on
-optimize=on
-deprecation=on
-build.compiler=modern
-
-java.src.version=1.6
-java.target.version=1.6
-
-#Tomcat
-tomcat.dir=/usr/local/tomcat
View
@@ -1,83 +0,0 @@
-<?xml version="1.0"?>
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
- - License, v. 2.0. If a copy of the MPL was not distributed with this
- - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-
-<project name="Mozilla Crash-Reports" default="compile" basedir=".">
-
- <!-- Load all properties -->
- <property file="${basedir}/build.properties"/>
-
- <path id="classpath">
- <pathelement location="${build.classes}"/>
- <pathelement location="."/>
- <fileset dir="${lib.dir}">
- <include name="*.jar"/>
- </fileset>
- </path>
-
- <target name="clean">
- <delete includeEmptyDirs="true" failonerror="false">
- <fileset dir="build"/>
- </delete>
- </target>
-
- <target name="init" depends="clean">
- <mkdir dir="${build.dir}"/>
- <mkdir dir="${build.classes}"/>
- <mkdir dir="${build.javadoc}"/>
- <mkdir dir="${build.lib}"/>
- </target>
-
- <target name="compile-java" depends="init">
- <javac encoding="${build.encoding}" srcdir="${java.src.dir}" includes="**/*.java" destdir="${build.classes}" debug="${debug}" source="${java.src.version}" optimize="${optimize}" deprecation="${deprecation}" includeAntRuntime="false">
- <classpath refid="classpath"/>
- <compilerarg line="-Xlint:unchecked"/>
- </javac>
- </target>
- <target name="compile" depends="compile-java"/>
- <target name="ensure-class-name" unless="class">
- <fail message="You must run this target with -Dclass=ClassName"/>
- </target>
-
- <target name="hadoop-jar" depends="compile">
- <delete file="${build.lib}/${project.name}-job.jar"/>
- <jar destfile="${build.lib}/${project.name}-job.jar">
- <fileset dir="${build.classes}"/>
- <fileset dir=".">
- <include name="lib/*.jar"/>
- <exclude name="lib/hadoop*.jar"/>
- <exclude name="lib/hbase*.jar"/>
- <exclude name="lib/zookeeper*.jar"/>
- <exclude name="lib/pig*.jar"/>
- </fileset>
- </jar>
- </target>
-
- <target name="jar" depends="compile">
- <jar destfile="${build.lib}/${project.name}.jar">
- <fileset dir="${build.classes}"/>
- </jar>
- </target>
-
- <target name="war" depends="compile" description="--> Create the WAR file">
- <mkdir dir="${dist.dir}"/>
- <!-- Copy Log4j config to build so it gets included in classes dir -->
- <copy todir="${build.dir}/classes">
- <fileset dir="${webapp.webinf.dir}/config" includes="log4j.*"/>
- </copy>
- <war destfile="${dist.dir}/${project.name}.war" webxml="${webapp.webinf.dir}/web.xml">
- <webinf dir="${webapp.webinf.dir}">
- <exclude name="web.xml"/>
- </webinf>
- <metainf dir="${webapp.metainf.dir}"/>
- <lib dir="${lib.dir}"/>
- <classes dir="${build.dir}/classes"/>
- </war>
- </target>
-
- <target name="deploy" depends="war" description="--> Deploy the WAR file">
- <copy file="${dist.dir}/${project.name}.war" todir="${tomcat.dir}/webapps" overwrite="true"/>
- </target>
-
-</project>
Binary file not shown.
Binary file not shown.
View
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
View
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,36 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-register './akela-0.1.jar'
-register './socorro-analysis.jar'
-register '/usr/lib/hbase/hbase-0.90.1-cdh3u0.jar'
-
-raw = LOAD 'hbase://crash_reports' USING com.mozilla.pig.load.HBaseMultiScanLoader('$start_date', '$end_date', 'meta_data:json,processed_data:json,raw_data:dump') AS (meta_json:chararray,processed_json:chararray,raw_dump:bytearray);
-gen_meta_map = FOREACH raw GENERATE com.mozilla.pig.eval.json.JsonMap(meta_json) AS meta_json_map:map[];
-product_filtered = FILTER gen_meta_map BY meta_json_map#'ProductName' == 'Firefox';
-
-/* in seconds need to compare to start_date and stop_date */
-start_date_millis = com.mozilla.pig.eval.date.ParseDate('yyyyMMdd', $start_date);
-end_date_millis = com.mozilla.pig.eval.date.ParseDate('yyyyMMdd', $end_date);
-time_filtered = FILTER product_filtered BY ((meta_json_map#'CrashTime' * 1000) >= start_date_millis AND (meta_json_map#'CrashTime' * 1000) <= end_date_millis);
-
-/* count and output submission */
-submissions = FOREACH time_filtered GENERATE (com.mozilla.pig.eval.date.FormatDate('yyyyMMdd', (meta_json_map#'CrashTime'*1000)), meta_json_map#'ProductName', meta_json_map#'Version', 'submissions');
-STORE submissions INTO '$start_date-$end_date-submissions' USING PigStorage();
-
-/* count and output hang */
-hang_filtered = FILTER time_filtered BY meta_json_map#'HangId' is not null;
-gen_hangs = FOREACH hang_filtered GENERATE (com.mozilla.pig.eval.date.FormatDate('yyyyMMdd', (meta_json_map#'CrashTime'*1000)), meta_json_map#'ProductName', meta_json_map#'Version', 'hangs');
-STORE gen_hangs INTO '$start_date-$end_date-hangs' USING PigStorage();
-
-/* count and output oopp */
-oopp_filtered = FILTER time_filtered BY meta_json_map#'ProcessType' is not null AND meta_json_map#'ProcessType' == 'plugin';
-gen_oopp = FOREACH oopp_filtered GENERATE (com.mozilla.pig.eval.date.FormatDate('yyyyMMdd', (meta_json_map#'CrashTime'*1000)), meta_json_map#'ProductName', meta_json_map#'Version', 'oopp');
-STORE gen_oopp INTO '$start_date-$end_date-oopps' USING PigStorage();
-
-/* count and output processed */
-gen_processed_map = FOREACH oopp_filtered GENERATE com.mozilla.hadoop.pig.eval.json.JsonMap(processed_json) AS processed_json_map:map[];
-processed_filtered = FILTER gen_processed_map BY processed_json_map is not null;
-
-STORE processed_filtered INTO '$start_date-$end_date-processed' USING PigStorage();
Oops, something went wrong.

0 comments on commit 57a5b2d

Please sign in to comment.