Permalink
Browse files

HADDOP-18: Introducing SBT for build management

  • Loading branch information...
1 parent 8dac711 commit 0b4ea84894851060c30c26a6b788aa3744f886b2 @bwmcadams bwmcadams committed Feb 6, 2012
Showing with 590 additions and 65 deletions.
  1. +8 −0 build.sbt
  2. +37 −8 core/pom.xml
  3. +0 −14 core/src/mongo-hadoop.iml
  4. +23 −10 pig/pom.xml
  5. +6 −33 pom.xml
  6. +97 −0 project/MongoHadoopBuild.scala
  7. +419 −0 sbt
View
@@ -0,0 +1,8 @@
+
+name := "mongo-hadoop"
+
+version := "1.0.0"
+
+scalaVersion := "2.9.1"
+
+hadoopRelease := "0.21"
View
@@ -7,14 +7,44 @@
<version>1.0-SNAPSHOT</version>
<name>MongoDB Hadoop Core Support</name>
<url>http://github.com/mongodb/mongo-hadoop</url>
+ <properties>
+ <scala.version>2.9.1</scala.version>
+ </properties>
<parent>
<groupId>org.mongodb</groupId>
<artifactId>mongo-hadoop</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
+ <repositories>
+ <repository>
+ <id>scala-tools.org</id>
+ <name>Scala-tools Maven2 Repository</name>
+ <url>http://scala-tools.org/repo-releases</url>
+ </repository>
+ </repositories>
+ <pluginRepositories>
+ <pluginRepository>
+ <id>scala-tools.org</id>
+ <name>Scala-tools Maven2 Repository</name>
+ <url>http://scala-tools.org/repo-releases</url>
+ </pluginRepository>
+ </pluginRepositories>
<dependencies>
<dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongo-java-driver</artifactId>
+ <version>2.7.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.specs2</groupId>
+ <artifactId>specs2_${scala.version}</artifactId>
+ <version>1.7.1</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>${hadoop.baseVersion}</version>
@@ -26,17 +56,13 @@
<version>4.8.1</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>2.9.1</version>
+ <version>${scala.version}</version>
+ <scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.specs2</groupId>
- <artifactId>specs2_2.9.1</artifactId>
- <version>1.7.1</version>
-<!-- <scope>test</scope>-->
- </dependency>
</dependencies>
<build>
<resources>
@@ -50,7 +76,7 @@
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
- <version>2.9.1</version>
+ <version>${scala.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -80,6 +106,9 @@
</goals>
</execution>
</executions>
+ <configuration>
+ <scalaVersion>${scala.version}</scalaVersion>
+ </configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
View
@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="JAVA_MODULE" version="4">
- <component name="NewModuleRootManager" inherit-compiler-output="true">
- <exclude-output />
- <content url="file://$MODULE_DIR$">
- <sourceFolder url="file://$MODULE_DIR$/main" isTestSource="false" />
- </content>
- <orderEntry type="inheritedJdk" />
- <orderEntry type="sourceFolder" forTests="false" />
- <orderEntry type="library" name="mongo-java-driver" level="project" />
- <orderEntry type="library" name="hadoop-core" level="project" />
- </component>
-</module>
-
View
@@ -4,25 +4,38 @@
<groupId>org.mongodb</groupId>
<artifactId>mongo-pig</artifactId>
<packaging>jar</packaging>
- <version>1.0-SNAPSHOT</version>
+ <version>1.0.0-RC1</version>
<name>MongoDB Pig Support</name>
<url>http://github.com/mongodb/mongo-hadoop</url>
- <parent>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-hadoop</artifactId>
- <version>1.0-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
+ <!--
+ -<parent>
+ - <groupId>org.mongodb</groupId>
+ - <artifactId>mongo-hadoop</artifactId>
+ - <version>1.0.0-RC1</version>
+ - <relativePath>../</relativePath>
+ -</parent>
+ -->
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongo-hadoop</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
</dependency>
<dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-hadoop-core</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongo-hadoop-core</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
<build>
View
39 pom.xml
@@ -20,42 +20,15 @@
<!-- <module>streaming</module> -->
<module>examples</module>
</modules>
- <repositories>
- <repository>
- <id>scala-tools.org</id>
- <name>Scala-tools Maven2 Repository</name>
- <url>http://scala-tools.org/repo-releases</url>
- </repository>
- </repositories>
- <pluginRepositories>
- <pluginRepository>
- <id>scala-tools.org</id>
- <name>Scala-tools Maven2 Repository</name>
- <url>http://scala-tools.org/repo-releases</url>
- </pluginRepository>
- </pluginRepositories>
- <dependencies>
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-java-driver</artifactId>
- <version>2.7.3</version>
- </dependency>
- <dependency>
- <groupId>org.specs2</groupId>
- <artifactId>specs2_2.9.1</artifactId>
- <version>1.7.1</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
<profiles>
<profile>
<id>apache-hadoop-1.0</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>hadoop.release</name>
- <value>1.0</value>
+ <value>1.0.0</value>
</property>
</activation>
<modules>
@@ -69,7 +42,7 @@
</modules>
<properties>
<hadoop.baseVersion>1.0.0</hadoop.baseVersion>
- <mongo-hadoop.version>1.0_${hadoop.baseVersion}-SNAPSHOT</mongo-hadoop.version>
+ <mongo-hadoop.version>${project.version}_${hadoop.baseVersion}</mongo-hadoop.version>
<pig.version>0.9.1</pig.version>
</properties>
<repositories>
@@ -104,7 +77,7 @@
</modules>
<properties>
<hadoop.baseVersion>0.20.205.0</hadoop.baseVersion>
- <mongo-hadoop.version>1.0_${hadoop.baseVersion}-SNAPSHOT</mongo-hadoop.version>
+ <mongo-hadoop.version>${project.version}_${hadoop.baseVersion}</mongo-hadoop.version>
<pig.version>0.9.1</pig.version>
</properties>
</profile>
@@ -127,7 +100,7 @@
</modules>
<properties>
<hadoop.baseVersion>0.21.0</hadoop.baseVersion>
- <mongo-hadoop.version>1.0_${hadoop.baseVersion}-SNAPSHOT</mongo-hadoop.version>
+ <mongo-hadoop.version>${project.version}_${hadoop.baseVersion}</mongo-hadoop.version>
<pig.version>0.9.1</pig.version>
</properties>
</profile>
@@ -150,7 +123,7 @@
</modules>
<properties>
<hadoop.baseVersion>0.20.2-cdh3u3</hadoop.baseVersion>
- <mongo-hadoop.version>1.0_${hadoop.baseVersion}-SNAPSHOT</mongo-hadoop.version>
+ <mongo-hadoop.version>${project.version}_${hadoop.baseVersion}</mongo-hadoop.version>
<pig.version>0.8.1-cdh3u3</pig.version>
</properties>
<repositories>
@@ -184,7 +157,7 @@
</modules>
<properties>
<hadoop.baseVersion>0.20.2-cdh3u3</hadoop.baseVersion>
- <mongo-hadoop.version>1.0_${hadoop.baseVersion}-SNAPSHOT</mongo-hadoop.version>
+ <mongo-hadoop.version>${project.version}_${hadoop.baseVersion}</mongo-hadoop.version>
<pig.version>0.8.1-cdh3u3</pig.version>
</properties>
<repositories>
@@ -0,0 +1,97 @@
+import sbt._
+import Keys._
+import Reference._
+
+object MongoHadoopBuild extends Build {
+
+ /** The version of Hadoop to build against. */
+ lazy val hadoopRelease = SettingKey[String]("hadoop-release", "Hadoop Target Release Distro/Version")
+
+ private val stockPig = "0.9.1"
+ private val cdhRel = "cdh3u3"
+ private val cdhHadoop = "0.20.2-%s".format(cdhRel) // current "base" version they patch against
+ private val cdhPig = "0.8.1-%s".format(cdhRel)
+
+ private val coreHadoopMap = Map("0.20" -> hadoopDependency("0.20.205.0", false),
+ "0.20.x" -> hadoopDependency("0.20.205.0", false),
+ "0.21" -> hadoopDependency("0.21.0", true),
+ "0.21.x" -> hadoopDependency("0.21.0", true),
+ "1.0" -> hadoopDependency("1.0.0", false),
+ "1.0.x" -> hadoopDependency("1.0.0", false),
+ "cdh" -> hadoopDependency(cdhHadoop, true),
+ "cdh3" -> hadoopDependency(cdhHadoop, true),
+ "cloudera" -> hadoopDependency(cdhHadoop, true)
+ )
+
+ lazy val root = Project( id = "mongo-hadoop",
+ base = file("."),
+ aggregate = subProjects,
+ settings = baseSettings )
+
+ lazy val core = Project( id = "mongo-hadoop-core",
+ base = file("core"),
+ settings = coreSettings )
+
+ lazy val streaming = Project( id = "mongo-hadoop-streaming",
+ base = file("streaming"),
+ settings = streamingSettings ) dependsOn( core )
+
+
+
+ private val subProjects: Seq[ProjectReference] = scala.collection.mutable.Seq(projectToRef(core))
+
+
+ lazy val baseSettings = Defaults.defaultSettings ++ Seq(
+ hadoopRelease := "1.0"
+ )
+
+ lazy val parentSettings = baseSettings ++ Seq(
+ publishArtifact := false
+ )
+
+ lazy val streamingSettings = baseSettings ++ Seq(
+ libraryDependencies <++= (scalaVersion, libraryDependencies, hadoopRelease) { (sv, deps, hr: String) =>
+
+ val streamingDeps = coreHadoopMap.getOrElse(hr, sys.error("Hadoop Release '%s' is an invalid/unsupported release. Valid entries are in %s".format(hr, coreHadoopMap.keySet)))
+ streamingDeps._1.getOrElse(() => Seq.empty[ModuleID])()
+ })
+
+ lazy val coreSettings: Seq[sbt.Project.Setting[_]] = baseSettings ++ Seq(
+ libraryDependencies <++= (scalaVersion, libraryDependencies, hadoopRelease) { (sv, deps, hr: String) =>
+
+ val hadoopDeps = coreHadoopMap.getOrElse(hr, sys.error("Hadoop Release '%s' is an invalid/unsupported release. Valid entries are in %s".format(hr, coreHadoopMap.keySet)))
+ if (hadoopDeps._1.isDefined) { subProjects :+ projectToRef(streaming) }
+ hadoopDeps._2()
+ })
+
+
+
+def hadoopDependency(hadoopVersion: String, useStreaming: Boolean): (Option[() => Seq[ModuleID]], () => Seq[ModuleID]) = {
+ (if (useStreaming) Some(streamingDependency(hadoopVersion)) else None, () => {
+ println("*** Adding Hadoop Dependencies for Hadoop '%s'".format(hadoopVersion))
+
+ Seq("org.apache.hadoop" % "hadoop-core" % hadoopVersion)
+ })
+ }
+
+ def pigDependency(pigVersion: String): () => Seq[ModuleID] = {
+ () => {
+ println("*** Adding Pig Dependency for Version '%s'".format(pigVersion))
+
+ Seq(
+ "org.apache.pig" % "pig" % pigVersion
+ )
+ }
+ }
+
+ def streamingDependency(hadoopVersion: String): () => Seq[ModuleID] = {
+ () => {
+ println("Enabling build of Hadoop Streaming.")
+ Seq("org.apache.hadoop" % "hadoop-streaming" % hadoopVersion)
+ }
+ }
+
+
+}
+
+// vim: set ts=2 sw=2 sts=2 et:
Oops, something went wrong.

0 comments on commit 0b4ea84

Please sign in to comment.