Merge branch 'master' into mos-shuffle-tracked

Conflicts: .gitignore core/src/main/scala/spark/LocalFileShuffle.scala src/scala/spark/BasicLocalFileShuffle.scala src/scala/spark/Broadcast.scala src/scala/spark/LocalFileShuffle.scala
mesos · Apr 27, 2011 · ac7e066 · ac7e066
2 parents dfbc5af + 9d2d533
commit ac7e066
Show file tree

Hide file tree

Showing 304 changed files with 103,106 additions and 4,645 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,26 @@
 *~
 *.swp
-build
+*.iml
-work
+.idea/
+/build/
+work/
+out/
 .DS_Store
 third_party/libmesos.so
 third_party/libmesos.dylib
+conf/java-opts
+conf/spark-env.sh
+conf/log4j.properties
+target/
+reports/
+.project
+.classpath
+.scala_dependencies
+lib_managed/
+src_managed/
+project/boot/
+project/plugins/project/build.properties
+project/build/target/
+project/plugins/target/
+project/plugins/lib_managed/
+project/plugins/src_managed/
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2010, Regents of the University of California.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the University of California, Berkeley nor the
+      names of its contributors may be used to endorse or promote
+      products derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/Makefile b/Makefile
diff --git a/README b/README
@@ -1,24 +1,32 @@
+ONLINE DOCUMENTATION
+
+You can find the latest Spark documentation, including a programming guide,
+on the project wiki at http://github.com/mesos/spark/wiki. This file only
+contains basic setup instructions.
+
+
+
 BUILDING
 
-Spark requires Scala 2.8. This version has been tested with 2.8.0.final.
+Spark requires Scala 2.8. This version has been tested with 2.8.1.final.
 
-To build and run Spark, you will need to have Scala's bin in your $PATH,
+The project is built using Simple Build Tool (SBT), which is packaged with it.
-or you will need to set the SCALA_HOME environment variable to point
+To build Spark and its example programs, run sbt/sbt compile.
-to where you've installed Scala. Scala must be accessible through one
-of these methods on Mesos slave nodes as well as on the master.
 
-To build Spark and the example programs, run make.
+To run Spark, you will need to have Scala's bin in your $PATH, or you
+will need to set the SCALA_HOME environment variable to point to where
+you've installed Scala. Scala must be accessible through one of these
+methods on Mesos slave nodes as well as on the master.
 
 To run one of the examples, use ./run <class> <params>. For example,
-./run SparkLR will run the Logistic Regression example. Each of the
+./run spark.examples.SparkLR will run the Logistic Regression example.
-example programs prints usage help if no params are given.
+Each of the example programs prints usage help if no params are given.
 
 All of the Spark samples take a <host> parameter that is the Mesos master
 to connect to. This can be a Mesos URL, or "local" to run locally with one
 thread, or "local[N]" to run locally with N threads.
 
-Tip: If you are building Spark and examples repeatedly, export USE_FSC=1
+
-to have the Makefile use the fsc compiler daemon instead of scalac.
 
 CONFIGURATION
 

diff --git a/alltests b/alltests
diff --git a/conf/java-opts b/conf/java-opts
@@ -1,4 +1,4 @@
--Dspark.shuffle.class=spark.TrackedCustomBlockedInMemoryShuffle
+-Dspark.shuffle.class=spark.CustomBlockedInMemoryShuffle
 -Dspark.shuffle.masterHostAddress=127.0.0.1
 -Dspark.shuffle.masterTrackerPort=22222 
 -Dspark.shuffle.trackerStrategy=spark.BalanceRemainingShuffleTrackerStrategy 

diff --git a/conf/spark-env.sh b/conf/spark-env.sh
@@ -10,4 +10,4 @@
 #   be in the same format as the JVM's -Xmx option, e.g. 300m or 1g).
 # - SPARK_LIBRARY_PATH, to add extra search paths for native libraries.
 
-MESOS_HOME=/home/mosharaf/Work/mesos
+MESOS_HOME=/Users/mosharaf/Work/mesos
diff --git a/...arty/apache-log4j-1.2.16/log4j-1.2.16.jar → .../lib/apache-log4j-1.2.16/log4j-1.2.16.jar b/...arty/apache-log4j-1.2.16/log4j-1.2.16.jar → .../lib/apache-log4j-1.2.16/log4j-1.2.16.jar
diff --git a/third_party/asm-3.2/.DS_Store → core/lib/asm-3.2/.DS_Store b/third_party/asm-3.2/.DS_Store → core/lib/asm-3.2/.DS_Store
diff --git a/third_party/asm-3.2/lib/all/README.txt → core/lib/asm-3.2/lib/all/README.txt b/third_party/asm-3.2/lib/all/README.txt → core/lib/asm-3.2/lib/all/README.txt
diff --git a/third_party/asm-3.2/lib/all/asm-all-3.2.jar → core/lib/asm-3.2/lib/all/asm-all-3.2.jar b/third_party/asm-3.2/lib/all/asm-all-3.2.jar → core/lib/asm-3.2/lib/all/asm-all-3.2.jar
diff --git a/third_party/asm-3.2/lib/all/asm-all-3.2.pom → core/lib/asm-3.2/lib/all/asm-all-3.2.pom b/third_party/asm-3.2/lib/all/asm-all-3.2.pom → core/lib/asm-3.2/lib/all/asm-all-3.2.pom
diff --git a/...rty/asm-3.2/lib/all/asm-debug-all-3.2.jar → ...lib/asm-3.2/lib/all/asm-debug-all-3.2.jar b/...rty/asm-3.2/lib/all/asm-debug-all-3.2.jar → ...lib/asm-3.2/lib/all/asm-debug-all-3.2.jar
diff --git a/...rty/asm-3.2/lib/all/asm-debug-all-3.2.pom → ...lib/asm-3.2/lib/all/asm-debug-all-3.2.pom b/...rty/asm-3.2/lib/all/asm-debug-all-3.2.pom → ...lib/asm-3.2/lib/all/asm-debug-all-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-3.2.jar → core/lib/asm-3.2/lib/asm-3.2.jar b/third_party/asm-3.2/lib/asm-3.2.jar → core/lib/asm-3.2/lib/asm-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-3.2.pom → core/lib/asm-3.2/lib/asm-3.2.pom b/third_party/asm-3.2/lib/asm-3.2.pom → core/lib/asm-3.2/lib/asm-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-analysis-3.2.jar → core/lib/asm-3.2/lib/asm-analysis-3.2.jar b/third_party/asm-3.2/lib/asm-analysis-3.2.jar → core/lib/asm-3.2/lib/asm-analysis-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-analysis-3.2.pom → core/lib/asm-3.2/lib/asm-analysis-3.2.pom b/third_party/asm-3.2/lib/asm-analysis-3.2.pom → core/lib/asm-3.2/lib/asm-analysis-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-commons-3.2.jar → core/lib/asm-3.2/lib/asm-commons-3.2.jar b/third_party/asm-3.2/lib/asm-commons-3.2.jar → core/lib/asm-3.2/lib/asm-commons-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-commons-3.2.pom → core/lib/asm-3.2/lib/asm-commons-3.2.pom b/third_party/asm-3.2/lib/asm-commons-3.2.pom → core/lib/asm-3.2/lib/asm-commons-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-parent-3.2.pom → core/lib/asm-3.2/lib/asm-parent-3.2.pom b/third_party/asm-3.2/lib/asm-parent-3.2.pom → core/lib/asm-3.2/lib/asm-parent-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-tree-3.2.jar → core/lib/asm-3.2/lib/asm-tree-3.2.jar b/third_party/asm-3.2/lib/asm-tree-3.2.jar → core/lib/asm-3.2/lib/asm-tree-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-tree-3.2.pom → core/lib/asm-3.2/lib/asm-tree-3.2.pom b/third_party/asm-3.2/lib/asm-tree-3.2.pom → core/lib/asm-3.2/lib/asm-tree-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-util-3.2.jar → core/lib/asm-3.2/lib/asm-util-3.2.jar b/third_party/asm-3.2/lib/asm-util-3.2.jar → core/lib/asm-3.2/lib/asm-util-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-util-3.2.pom → core/lib/asm-3.2/lib/asm-util-3.2.pom b/third_party/asm-3.2/lib/asm-util-3.2.pom → core/lib/asm-3.2/lib/asm-util-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-xml-3.2.jar → core/lib/asm-3.2/lib/asm-xml-3.2.jar b/third_party/asm-3.2/lib/asm-xml-3.2.jar → core/lib/asm-3.2/lib/asm-xml-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-xml-3.2.pom → core/lib/asm-3.2/lib/asm-xml-3.2.pom b/third_party/asm-3.2/lib/asm-xml-3.2.pom → core/lib/asm-3.2/lib/asm-xml-3.2.pom
diff --git a/third_party/colt.jar → core/lib/colt.jar b/third_party/colt.jar → core/lib/colt.jar
diff --git a/core/lib/compress-lzf-0.6.0/LICENSE b/core/lib/compress-lzf-0.6.0/LICENSE
@@ -0,0 +1,11 @@
+Copyright 2009-2010 Ning, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.
diff --git a/core/lib/compress-lzf-0.6.0/compress-lzf-0.6.0.jar b/core/lib/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
diff --git a/third_party/guava-r07/COPYING → core/lib/guava-r07/COPYING b/third_party/guava-r07/COPYING → core/lib/guava-r07/COPYING
diff --git a/third_party/guava-r07/README → core/lib/guava-r07/README b/third_party/guava-r07/README → core/lib/guava-r07/README
diff --git a/third_party/guava-r07/guava-r07.jar → core/lib/guava-r07/guava-r07.jar b/third_party/guava-r07/guava-r07.jar → core/lib/guava-r07/guava-r07.jar