bug 765001 - switch to new way of doing mapreduce jobs, starting with…

… modulelist This reverts commit 438fa4e.
mozilla-services · Oct 3, 2012 · 57a5b2d · 57a5b2d
1 parent e7bb40a
commit 57a5b2d
Show file tree

Hide file tree

Showing 93 changed files with 38 additions and 48,905 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "socorro-toolbox"]
+	path = socorro-toolbox
+	url = git://github.com/mozilla-metrics/socorro-toolbox.git
+[submodule "akela"]
+	path = akela
+	url = git://github.com/mozilla-metrics/akela.git
diff --git a/Makefile b/Makefile
@@ -13,7 +13,7 @@ COVERAGE = $(VIRTUALENV)/bin/coverage
 PYLINT = $(VIRTUALENV)/bin/pylint
 CITEXT="/usr/share/postgresql/9.0/contrib/citext.sql"
 
-.PHONY: all test install reinstall install-socorro install-web virtualenv coverage lint clean minidump_stackwalk java_analysis thirdparty
+.PHONY: all test install reinstall install-socorro install-web virtualenv coverage lint clean minidump_stackwalk analysis thirdparty
 
 
 all:	test
@@ -31,7 +31,7 @@ thirdparty:
 	# install production dependencies
 	$(VIRTUALENV)/bin/pip install --use-mirrors --download-cache=pip-cache/ --ignore-installed --install-option="--prefix=`pwd`/thirdparty" --install-option="--install-lib=`pwd`/thirdparty" -r requirements/prod.txt
 
-install: java_analysis thirdparty reinstall
+install: analysis thirdparty reinstall
 
 # this a dev-only option, `make install` needs to be run at least once in the checkout (or after `make clean`)
 reinstall: install-socorro install-web
@@ -54,8 +54,7 @@ install-socorro:
 	rsync -a wsgi $(PREFIX)/application
 	rsync -a stackwalk $(PREFIX)/
 	rsync -a scripts/stackwalk.sh $(PREFIX)/stackwalk/bin/
-	rsync -a analysis/build/lib/socorro-analysis-job.jar $(PREFIX)/analysis/
+	rsync -a analysis $(PREFIX)/
-	rsync -a analysis/bin/modulelist.sh $(PREFIX)/analysis/
 	# copy default config files
 	cd $(PREFIX)/application/scripts/config; for file in *.py.dist; do cp $$file `basename $$file .dist`; done
 
@@ -83,14 +82,19 @@ clean:
 	rm -rf ./thirdparty/*
 	rm -rf ./google-breakpad/ ./builds/ ./breakpad/ ./stackwalk ./pip-cache
 	rm -rf ./breakpad.tar.gz
-	cd analysis && ant clean
 
 minidump_stackwalk:
 	svn co http://google-breakpad.googlecode.com/svn/trunk google-breakpad
 	cd google-breakpad && ./configure --prefix=`pwd`/../stackwalk/
 	cd google-breakpad && make install
 	cd google-breakpad && svn info | grep Revision | cut -d' ' -f 2 > ../stackwalk/revision.txt
 
-java_analysis:
+analysis:
-	cd analysis && ant hadoop-jar
+	git submodule update --init socorro-toolbox akela
-
+	cd akela && mvn package
+	cd akela && mvn package
+	cd socorro-toolbox && mvn package
+	mkdir -p analysis
+	rsync socorro-toolbox/target/*.jar analysis/
+	rsync akela/target/*.jar analysis/
+	rsync -a socorro-toolbox/src/main/pig/ analysis/
diff --git a/akela b/akela
diff --git a/analysis/.classpath b/analysis/.classpath
diff --git a/analysis/.project b/analysis/.project
diff --git a/analysis/bin/correlation_reports.sh b/analysis/bin/correlation_reports.sh
diff --git a/analysis/bin/modulelist.sh b/analysis/bin/modulelist.sh
diff --git a/analysis/bin/run-jetty.sh b/analysis/bin/run-jetty.sh
diff --git a/analysis/build.properties b/analysis/build.properties
diff --git a/analysis/build.xml b/analysis/build.xml
diff --git a/analysis/lib/akela-0.2-SNAPSHOT.jar b/analysis/lib/akela-0.2-SNAPSHOT.jar
diff --git a/analysis/lib/aopalliance.jar b/analysis/lib/aopalliance.jar
diff --git a/analysis/lib/asm-3.1.jar b/analysis/lib/asm-3.1.jar
diff --git a/analysis/lib/commons-lang-2.5.jar b/analysis/lib/commons-lang-2.5.jar
diff --git a/analysis/lib/commons-logging-1.1.1.jar b/analysis/lib/commons-logging-1.1.1.jar
diff --git a/analysis/lib/commons-math-2.1.jar b/analysis/lib/commons-math-2.1.jar
diff --git a/analysis/lib/guice-2.0.jar b/analysis/lib/guice-2.0.jar
diff --git a/analysis/lib/guice-servlet-2.0.jar b/analysis/lib/guice-servlet-2.0.jar
diff --git a/analysis/lib/guiceyfruit-2.0.jar b/analysis/lib/guiceyfruit-2.0.jar
diff --git a/analysis/lib/hadoop-core-0.20.2-cdh3u0.jar b/analysis/lib/hadoop-core-0.20.2-cdh3u0.jar
diff --git a/analysis/lib/hazelcast-1.9.jar b/analysis/lib/hazelcast-1.9.jar
diff --git a/analysis/lib/hbase-0.90.1-cdh3u0.jar b/analysis/lib/hbase-0.90.1-cdh3u0.jar
diff --git a/analysis/lib/jackson-core-asl-1.5.5.jar b/analysis/lib/jackson-core-asl-1.5.5.jar
diff --git a/analysis/lib/jackson-mapper-asl-1.5.5.jar b/analysis/lib/jackson-mapper-asl-1.5.5.jar
diff --git a/analysis/lib/jersey-bundle-1.3.jar b/analysis/lib/jersey-bundle-1.3.jar
diff --git a/analysis/lib/jersey-guice-1.1.5.jar b/analysis/lib/jersey-guice-1.1.5.jar
diff --git a/analysis/lib/jersey-multipart-1.4.jar b/analysis/lib/jersey-multipart-1.4.jar
diff --git a/analysis/lib/jetty-continuation-7.1.6.v20100715.jar b/analysis/lib/jetty-continuation-7.1.6.v20100715.jar
diff --git a/analysis/lib/jetty-http-7.1.6.v20100715.jar b/analysis/lib/jetty-http-7.1.6.v20100715.jar
diff --git a/analysis/lib/jetty-io-7.1.6.v20100715.jar b/analysis/lib/jetty-io-7.1.6.v20100715.jar
diff --git a/analysis/lib/jetty-security-7.1.6.v20100715.jar b/analysis/lib/jetty-security-7.1.6.v20100715.jar
diff --git a/analysis/lib/jetty-server-7.1.6.v20100715.jar b/analysis/lib/jetty-server-7.1.6.v20100715.jar
diff --git a/analysis/lib/jetty-servlet-7.1.6.v20100715.jar b/analysis/lib/jetty-servlet-7.1.6.v20100715.jar
diff --git a/analysis/lib/jetty-util-7.1.6.v20100715.jar b/analysis/lib/jetty-util-7.1.6.v20100715.jar
diff --git a/analysis/lib/jsr311-api-1.1.jar b/analysis/lib/jsr311-api-1.1.jar
diff --git a/analysis/lib/log4j-1.2.16.jar b/analysis/lib/log4j-1.2.16.jar
diff --git a/analysis/lib/mimepull-1.3.jar b/analysis/lib/mimepull-1.3.jar
diff --git a/analysis/lib/pig-0.8.0-cdh3u0.jar b/analysis/lib/pig-0.8.0-cdh3u0.jar
diff --git a/analysis/lib/servlet-api-2.5.jar b/analysis/lib/servlet-api-2.5.jar
diff --git a/analysis/lib/slf4j-api-1.6.1.jar b/analysis/lib/slf4j-api-1.6.1.jar
diff --git a/analysis/lib/trove-2.1.0.jar b/analysis/lib/trove-2.1.0.jar
diff --git a/analysis/pig/crashstats.pig b/analysis/pig/crashstats.pig