build.gradle

import org.apache.tools.ant.filters.ReplaceTokens

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'download-task'
apply plugin: 'versions'
apply from: 'gradle/functions.gradle'

ext.configDir = new File(rootDir, 'config')
ext.hadoopBinaries = "${System.getProperty('user.home')}/hadoop-binaries"
ext.branchVersion = project.getProperties().get('clusterVersion', '2.4').toLowerCase()
ext.hiveVersionMap = [
        '1.0' : '0.10.0',
        '1.1' : '0.10.0',
        'cdh4': '0.10.0-cdh4.6.0',
        'cdh5': '0.12.0-cdh5.0.0'
].withDefault {
    '0.12.0'
}
ext.hadoopClusters = [
        '0.23': '0.23.10',
        '1.0' : '1.0.4',
        '1.1' : '1.1.2',
        '2.2' : '2.2.0',
        '2.3' : '2.3.0',
        '2.4' : '2.4.0',
        'cdh4': '2.0.0-cdh4.6.0',
        'cdh5': '2.3.0-cdh5.0.0',
].withDefault {
    throw new GradleException("Unknown hadoop version: ${it}")
}

ext.clusterVersion = hadoopClusters[branchVersion]
ext.hiveVersion = hiveVersionMap[branchVersion]
ext.javaDriverVersion = '2.12.2'
ext.hadoopVersion = '2.4.0'

ext.hadoopHome = "${hadoopBinaries}/hadoop-${clusterVersion}"
ext.hiveHome = "${hadoopBinaries}/hive-${hiveVersion}"
ext.dataHome = "${hadoopBinaries}/examples/data"

buildscript {
    repositories {
        mavenLocal()
        mavenCentral()
        maven { url 'https://oss.sonatype.org/content/repositories/snapshots' }
        jcenter()
        maven { url "https://github.com/ben-manes/gradle-versions-plugin/raw/mvnrepo" }
    }
    dependencies {
        classpath 'me.trnl:clirr-gradle-plugin:0.4'
        classpath 'com.antwerkz.github:github-release-gradle-plugin:1.1.0'
        classpath 'de.undercouch:gradle-download-task:1.0'
        classpath 'com.github.ben-manes:gradle-versions-plugin:0.5-beta-2'
    }
}

task wrapper(type: Wrapper) {
    gradleVersion = '1.11'
}

configure(subprojects) {
    apply plugin: 'java'
    apply plugin: 'idea'
    apply plugin: 'eclipse'
    apply plugin: 'checkstyle'
    apply plugin: 'findbugs'

    repositories {
        mavenCentral()
        mavenLocal()
        maven { url "https://repository.cloudera.com/artifactory/cloudera-repos/" }
    }

    group = 'org.mongodb'
    version = '1.3.0'

    sourceCompatibility = JavaVersion.VERSION_1_6
    targetCompatibility = JavaVersion.VERSION_1_6

    dependencies {
        compile "org.mongodb:mongo-java-driver:${javaDriverVersion}"

        testCompile 'junit:junit:4.11'
        testCompile 'org.hamcrest:hamcrest-all:1.3'
        testCompile 'org.zeroturnaround:zt-exec:1.6'
        testCompile 'com.jayway.awaitility:awaitility:1.6.0'
        testCompile 'commons-daemon:commons-daemon:1.0.15'

        testCompile "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}"
        testCompile "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}:tests"
        testCompile "org.apache.hadoop:hadoop-common:${hadoopVersion}:tests"
        testCompile "org.apache.hadoop:hadoop-yarn-server-tests:${hadoopVersion}:tests"
        testCompile "org.apache.hadoop:hadoop-mapreduce-client-jobclient:${hadoopVersion}:tests"
    }

    /* Compiling */
    tasks.withType(AbstractCompile) {
        options.encoding = 'ISO-8859-1'
        options.fork = true
        options.debug = true
        options.compilerArgs = [/*'-Xlint:deprecation',*/ '-Xlint:-options']
    }

    project.ext.buildingWith = { n ->
        project.hasProperty(n) && project.property(n).toBoolean()
    }

    /* Testing */
    tasks.withType(Test) {
        maxParallelForks = 1
        systemProperty 'project.version', project.version
        systemProperty 'cluster.version', hadoopClusters[branchVersion]
        systemProperty 'hadoop.version', branchVersion
        systemProperties << System.getProperties()

        beforeTest { descr ->
            logger.info("[Test ${descr.className} > ${descr.name}]")
        }
    }

    test.dependsOn ':configureCluster'
    compileJava.dependsOn ':generateScripts'
    clean.dependsOn ':cleanHadoop'

    /* Code quality */
    checkstyle {
        configFile = new File("$configDir/checkstyle.xml")
    }

    checkstyleTest {
        classpath += configurations.compile
        classpath += configurations.testCompile
    }

    checkstyleMain {
        classpath += configurations.compile
    }

    findbugs {
        excludeFilter = new File("$configDir/findbugs-exclude.xml")
        sourceSets = [sourceSets.main]
    }

    tasks.withType(FindBugs) {
        reports {
            xml.enabled = project.buildingWith('xmlReports.enabled')
            html.enabled = !project.buildingWith('xmlReports.enabled')
        }
    }

    javadoc {
        options.version = true
        options.links 'http://docs.oracle.com/javase/6/docs/api/'
        options.links 'http://hadoop.apache.org/docs/r2.4.0/api'
        options.links 'http://api.mongodb.org/java/2.12.0/'
    }

    task testJar(type: Jar, dependsOn: testClasses) {
        from sourceSets.test.output
    }

    gradle.taskGraph.whenReady { taskGraph ->
        testJar {
            baseName = "${archivesBaseName}-test"
            mustRunAfter jar
        }
    }

    configurations {
        tests
    }

    artifacts {
        tests testJar
    }

    test.dependsOn ':configureCluster', ':downloadEnronEmails'
}

configure(subprojects.findAll { it.name.contains('examples/') }) {
    def exampleName = project.name.split('/')[1]
    jar {
        baseName = exampleName
    }
    group += ".mongo-hadoop-examples"

}

project(":core") {
    archivesBaseName = "mongo-hadoop-core"

    dependencies {
        compile "org.apache.hadoop:hadoop-common:${hadoopVersion}"
        compile("org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}")
        compile("org.apache.hadoop:hadoop-mapreduce-client-common:${hadoopVersion}")
        compile("org.apache.hadoop:hadoop-mapreduce-client-shuffle:${hadoopVersion}")
        compile("org.apache.hadoop:hadoop-mapreduce-client-app:${hadoopVersion}")
        compile("org.apache.hadoop:hadoop-mapreduce-client-jobclient:${hadoopVersion}")
    }
}

project(":hive") {
    archivesBaseName = "mongo-hadoop-hive"
    dependencies {
        compile "${group}:mongo-hadoop-core:${version}"

        compile "org.apache.hive:hive-exec:${hiveVersion}"
        compile "org.apache.hive:hive-serde:${hiveVersion}"

        testCompile "org.apache.hive:hive-cli:${hiveVersion}"
        testCompile "org.apache.hive:hive-service:${hiveVersion}"

        testCompile project(path: ':core', configuration: 'tests')
        testCompile("com.nitayjoffe.thirdparty.com.jointhegrid:hive_test:4.0.0") {
            exclude group: 'org.apache.derby', module: 'derby'
            exclude group: 'org.apache.hadoop', module: 'hadoop-core'
        }
    }
}

project(":pig") {
    archivesBaseName = "mongo-hadoop-pig"
    dependencies {
        compile "${group}:mongo-hadoop-core:${version}"
        compile "org.apache.pig:pig:0.12.1"

        testCompile "org.antlr:antlr:3.5.2"
    }

    jar {
        from project(':core').sourceSets.main.output
        from sourceSets.main.output

        configurations.compile.filter {
            it.name.startsWith('mongo-java-driver')
        }.each {
            from zipTree(it)
        }
    }

}

project(":streaming") {
    archivesBaseName = "mongo-hadoop-streaming"

    dependencies {
        compile "${group}:mongo-hadoop-core:${version}"
        compile "org.apache.hadoop:hadoop-streaming:${hadoopVersion}"
    }

    jar {
        from project(':core').sourceSets.main.output
        from sourceSets.main.output

        configurations.compile.filter {
            it.name.startsWith('mongo-java-driver')
        }.each {
            from zipTree(it)
        }
    }
}

project(":flume") {
    dependencies {
        compile "${group}:mongo-hadoop-core:${version}"
        compile("com.cloudera:flume-core:0.9.4-cdh3u3") {
            exclude group: 'org.apache.hadoop', module: 'hadoop-core'
            exclude group: 'com.cloudera.cdh', module: 'hadoop-ant'
        }
    }
}

project(":examples/treasury_yield") {
    uploadArchives.onlyIf { false }
    dependencies {
        compile "org.mongodb:mongo-hadoop-core:${version}"
        testCompile project(path: ':core', configuration: 'tests')
        testCompile project(':streaming')
        testCompile 'org.slf4j:slf4j-jdk14:1.7.7'
    }
}

project(":examples/enron") {
    uploadArchives.onlyIf { false }
    dependencies {
        compile "org.mongodb:mongo-hadoop-core:${version}"
    }
}

project(":examples/sensors") {
    uploadArchives.onlyIf { false }
    dependencies {
        compile "org.mongodb:mongo-hadoop-core:${version}"
    }
}

project(":integration-tests") {
    uploadArchives.onlyIf { false }
    dependencies {
        testCompile "${group}:mongo-hadoop-core:${version}"
        testCompile "${group}:mongo-hadoop-streamin:${version}"
        testCompile 'org.slf4j:slf4j-jdk14:1.7.7'
    }
}

task cleanHadoop(type: Delete) {
    delete hadoopHome, hiveHome
}

task installHadoop() << {
    new File(hadoopHome).getParentFile().mkdirs()

    def url
    switch (branchVersion) {
        case ("cdh4"):
            url = "http://archive.cloudera.com/cdh4/cdh/4/hadoop-${clusterVersion}.tar.gz"
            break
        case ("cdh5"):
            url = "http://archive.cloudera.com/cdh5/cdh/5/hadoop-${clusterVersion}.tar.gz"
            break
        default:
            url = "http://archive.apache.org/dist/hadoop/common/hadoop-${clusterVersion}/hadoop-${clusterVersion}.tar.gz"
            break
    }
    
    download(url, hadoopBinaries, hadoopHome)
}

def download(url, destination, target) {
    def file = new URL(url).getPath()
    file = "${destination}/${file.substring(file.lastIndexOf('/') + 1)}"
    def count = 0;
    while (!(new File(target + "/bin").exists())) {
        try {
            if(!new File(file).exists()) {
                println "${target} not found.  Downloading from ${url} to ${file}"
            }
            download {
                src url
                dest destination
                onlyIfNewer true
            }

            println "Extracting ${file}"
            copy {
                from(tarTree(resources.gzip(file)))
                into destination
            }
        } catch (Exception e) {
            println "Extraction failed: " + e.getMessage()
            println "Trying again"
            new File(target).deleteDir()
            new File(new URL(url).getFile()).delete()
        }
        if(count++ > 3) {
            throw new GradleException();
        }
    }
}

task installHive() << {
    def url
    switch (branchVersion) {
        case ("cdh4"):
            url = "http://archive.cloudera.com/cdh4/cdh/4/hive-${hiveVersion}.tar.gz"
            break
        case ("cdh5"):
            url = "http://archive.cloudera.com/cdh5/cdh/5/hive-${hiveVersion}.tar.gz"
            break
        default:
            url = "https://archive.apache.org/dist/hive/hive-${hiveVersion}/hive-${hiveVersion}.tar.gz"
            break
    }

    download(url, hadoopBinaries, hiveHome)

}

task copyFiles(dependsOn: [installHadoop, installHive]) << {
    def hadoopEtc
    def hadoopLib
    if (clusterVersion.startsWith("1")) {
        hadoopLib = "${hadoopHome}/lib"
        hadoopEtc = "${hadoopHome}/conf"
    } else {
        hadoopLib = "${hadoopHome}/share/hadoop/common"
        hadoopEtc = "${hadoopHome}/etc/hadoop"
    }

    println "Updating mongo jars"


    safeCopy("core/build/libs/mongo-hadoop-core-${project(':core').version}.jar", hadoopLib, "mongo-hadoop-core.jar")
    safeCopy("streaming/build/libs/mongo-hadoop-streaming-${project(':core').version}.jar", hadoopLib, "mongo-hadoop-streaming.jar")
    safeCopy("hive/build/libs/mongo-hadoop-hive-${project(':core').version}.jar", hiveHome + '/lib', "mongo-hadoop-hive.jar")

    download {
        src "http://central.maven.org/maven2/org/mongodb/mongo-java-driver/${javaDriverVersion}/mongo-java-driver-${javaDriverVersion}.jar"
        dest "${hadoopLib}/mongo-java-driver.jar"
        onlyIfNewer true
    }
    println "Updating cluster configuration"
    copy {
        from 'clusterConfigs'
        into hadoopEtc
    }
}

task configureCluster(dependsOn: copyFiles) << {
    println "Configuring ${branchVersion} cluster"
    exec {
        commandLine "build/hadoop-${branchVersion}.sh"
        args "start", "-format"
    }
}

task shutdownCluster << {
    println "Shutting down ${branchVersion} cluster"
    exec {
        commandLine "build/hadoop-${branchVersion}.sh"
        args "shutdown"
    }
}

task historicalYield(dependsOn: configureCluster) << {
    exec() {
        commandLine "mongoimport", "-d", "mongo_hadoop", "-c", "yield_historical.in", "--drop",
                    "examples/treasury_yield/src/main/resources/yield_historical_in.json"
    }

    hadoop("examples/treasury_yield/build/libs/treasury_yield-${project(':core').version}.jar",
           "com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig", [
            "mongo.input.uri=mongodb://localhost:27017/mongo_hadoop.yield_historical.in",
            "mongo.output.uri=mongodb://localhost:27017/mongo_hadoop.yield_historical.out"
    ])
}

task sensorData(dependsOn: 'configureCluster') << {
    hadoop("examples/sensors/build/libs/sensors-${project(':core').version}.jar",
           "com.mongodb.hadoop.examples.sensors.Devices", [])

    hadoop("examples/sensors/build/libs/sensors-${project(':core').version}.jar",
           "com.mongodb.hadoop.examples.sensors.Logs", ["io.sort.mb=100"])
}

task downloadEnronEmails() {
    if (!new File("${dataHome}/dump").exists()) {
        println "Downloading Enron email test data to ${dataHome}"
        new File(dataHome).mkdirs()
        def attempts = 0
        while (!new File(dataHome, "enron_mongo.tar.bz2").exists() && attempts < 10) {
            try {
                download {
                    src 'https://s3.amazonaws.com/mongodb-enron-email/enron_mongo.tar.bz2'
                    dest dataHome
                    onlyIfNewer true
                }
            } catch (Exception e) {
                attempts++
                if (attempts == 10) {
                    throw e;
                }
            }
        }
        println "extracting email data"
        copy {
            from(tarTree("${dataHome}/enron_mongo.tar.bz2"))
            into dataHome
        }
    }
}

task enronEmails(dependsOn: [downloadEnronEmails, configureCluster]) << {
    exec() {
        commandLine "mongorestore", "-v", "-d", "mongo_hadoop", "--drop", "${dataHome}/dump/enron_mail"
    }

    hadoop("examples/enron/build/libs/enron-${project(':core').version}.jar",
           "com.mongodb.hadoop.examples.enron.EnronMail", ["mongo.input.split_size=64"])
}

def hadoop(jar, className, args) {
    def line = ["${hadoopHome}/bin/hadoop",
                "jar", jar, className,
                //Split settings
                "-Dmongo.input.split_size=8",
                "-Dmongo.job.verbose=true",
    ]
    args.each {
        line << "-D${it}"
    }
    println "Executing hadoop job:\n ${line.join(' \\\n\t')}"
    def hadoopEnv = [:]
    if (clusterVersion.startsWith("cdh")) {
        hadoopEnv.MAPRED_DIR = 'share/hadoop/mapreduce2'
    }
    exec() {
        environment << hadoopEnv
        commandLine line
    }
}

def safeCopy(fromPath, toPath, newName) {
    def copied = copy {
        from fromPath
        into toPath
        rename { newName }
    }.didWork

    if (!copied) {
        throw new GradleScriptException("Failed to copy a file", new FileNotFoundException(fromPath))
    }
}

task generateScripts() << {
    hadoopClusters.keySet().each { version ->
        copy {
            from 'bin/templates/hadoop-template.sh'
            into "build/"
            filter ReplaceTokens, tokens: [
                    HADOOP_HOME    : "${hadoopBinaries}/hadoop-${hadoopClusters[version]}".toString(),
                    HADOOP_BINARIES: hadoopBinaries.toString(),
                    HADOOP_VERSION : hadoopClusters[version].toString(),
                    HIVE_HOME      : "${hadoopBinaries}/hive-${hiveVersionMap[version]}".toString(),
                    PROJECT_HOME   : project.rootDir.getAbsolutePath(),
                    BIN            : version.startsWith("1.") ? 'hadoop' : 'hdfs',
                    BIN_PATH       : project.rootDir.getAbsolutePath() + '/bin'
            ]
            rename { "hadoop-${version}.sh" }
        }
    }
}

apply from: 'gradle/maven-deployment.gradle'