build.gradle

// Copyright (C) 2014-2016 LinkedIn Corp. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at  http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied.
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.locks.ReentrantLock

buildscript {
  repositories {
    maven {
      url "https://plugins.gradle.org/m2/"
    }
  }
  dependencies {
    classpath 'gradle.plugin.org.inferred:gradle-processors:1.1.2'
    classpath 'org.kt3k.gradle.plugin:coveralls-gradle-plugin:1.0.2'
    classpath 'gradle.plugin.com.palantir:jacoco-coverage:0.3.0'
  }
}

apply plugin: 'org.inferred.processors'
apply plugin: 'idea'

apply from: rootProject.projectDir.path + '/gradle/scripts/jacoco-coveralls-support.gradle'

idea.project {
  ext.languageLevel = JavaVersion.VERSION_1_7
  languageLevel = JavaVersion.VERSION_1_7
}

ext.build_script_dir = "${projectDir.path}/build_script"
ext.isDefaultEnvironment = !project.hasProperty('overrideBuildEnvironment')

File getEnvironmentScript()
{
  final File env = file(isDefaultEnvironment ? 'defaultEnvironment.gradle' : project.overrideBuildEnvironment)
  assert env.isFile() : "The environment script [$env] does not exists or is not a file."
  return env
}

apply from: environmentScript

// Maven POM generation is not thread safe, so serialize all the Upload tasks we can use `--parallel`.
// https://issues.gradle.org/browse/GRADLE-2492
// When we start building with 2.3 and later we should remove this and just add a common output dir for all tasks and let Gradle serialize them
def lock = new ReentrantLock()
def available = lock.newCondition()
def busy = new AtomicBoolean()
def serializedTasks = []
allprojects {
  apply from: rootProject.projectDir.path + '/gradle/scripts/hadoop-version.gradle'
  tasks.matching { it.name == "generatePom" || it instanceof Upload }.all {
    serializedTasks << it
    doFirst {
      lock.lock()
      while (busy.get()) {
        available.await()
      }
      busy.set(true)
    }
  }
}

gradle.taskGraph.afterTask {
  if (it in serializedTasks && lock.heldByCurrentThread) {
    busy.set(false)
    available.signal()
    lock.unlock()
  }
}

ext.publishToMaven = project.hasProperty('publishToMaven')
if (ext.publishToMaven) {
    plugins.apply('maven')
    // Workaround for a bug in gradle's "maven" plugin. See https://discuss.gradle.org/t/error-in-parallel-build/7215/3
    project.setProperty("org.gradle.parallel", "false")
}

ext.signArtifacts = !project.hasProperty('doNotSignArtifacts')

if (!project.hasProperty('group') || project.group.length() == 0) {
    project.group = 'com.linkedin.gobblin'
}

if (!project.hasProperty('artifactRepository') || project.artifactRepository.length() == 0) {
    ext.artifactRepository = "https://oss.sonatype.org/service/local/staging/deploy/maven2/"
}

if (!project.hasProperty('artifactSnapshotRepository') || project.artifactSnapshotRepository.length() == 0) {
    ext.artifactSnapshotRepository = "https://oss.sonatype.org/content/repositories/snapshots/"
}

if (!project.hasProperty('version') || project.version == 'unspecified') {
    exec {
        commandLine 'git', 'fetch', '-t', 'https://github.com/linkedin/gobblin.git', 'master'
    }
    def versionOut = new ByteArrayOutputStream()
    exec {
        commandLine 'git', 'describe', '--tags', '--always'
        standardOutput versionOut
    }
    def tagStr = versionOut.toString().trim()
    println 'Using latest tag for version: ' + tagStr
    if (tagStr.startsWith("gobblin_")) {
        project.version = tagStr.substring(8)
    }
    else {
        project.version = tagStr
    }
    if (!useHadoop2) {
      project.version = project.version + "-hadoop1"
    }
}

println "name=" + project.name + " group=" + project.group
println "project.version=" + project.version

if (!project.hasProperty('hiveVersion')) {
  ext.hiveVersion = '1.0.1'
}

if (!project.hasProperty('pegasusVersion')) {
  ext.pegasusVersion = '1.15.9'
}

if (!project.hasProperty('bytemanVersion')) {
  ext.bytemanVersion = '2.2.1'
}

ext.avroVersion = '1.7.7'
ext.dropwizardMetricsVersion = '3.1.0'
ext.findBugsVersion = '3.0.0'

if (useHadoop2) {
  println "Using Hadoop 2 adaptor."
  ext.gobblinHadoop = ":gobblin-hadoop2"
} else {
  println "Using Hadoop 1 adaptor."
  ext.gobblinHadoop = ":gobblin-hadoop1"
}

ext.internalDependency = [
  "gobblin-hadoop": project(ext.gobblinHadoop)
]

ext.externalDependency = [
  "antlrRuntime": "org.antlr:antlr-runtime:3.5.2",
  "avro": "org.apache.avro:avro:" + avroVersion,
  "avroMapredH1": "org.apache.avro:avro-mapred:" + avroVersion + ":hadoop1",
  "avroMapredH2": "org.apache.avro:avro-mapred:" + avroVersion + ":hadoop2",
  "commonsCli": "commons-cli:commons-cli:1.3.1",
  "commonsCodec": "commons-codec:commons-codec:1.10",
  "commonsDbcp": "commons-dbcp:commons-dbcp:1.4",
  "commonsEmail": "org.apache.commons:commons-email:1.4",
  "commonsLang": "commons-lang:commons-lang:2.6",
  "commonsLang3": "org.apache.commons:commons-lang3:3.4",
  "commonsConfiguration": "commons-configuration:commons-configuration:1.10",
  "commonsIo": "commons-io:commons-io:2.4",
  "commonsMath": "org.apache.commons:commons-math3:3.5",
  "commonsHttpClient": "commons-httpclient:commons-httpclient:3.1",
  "commonsCompress":"org.apache.commons:commons-compress:1.10",
  "commonsPool": "org.apache.commons:commons-pool2:2.4.2",
  "datanucleusCore": "org.datanucleus:datanucleus-core:3.2.10",
  "datanucleusRdbms": "org.datanucleus:datanucleus-rdbms:3.2.9",
  "guava": "com.google.guava:guava:15.0",
  "gson": "com.google.code.gson:gson:2.6.2",
  "findBugsAnnotations": "com.google.code.findbugs:jsr305:" + findBugsVersion,
  "hadoop": "org.apache.hadoop:hadoop-core:" + hadoopVersion,
  "hadoopCommon": "org.apache.hadoop:hadoop-common:" + hadoopVersion,
  "hadoopClientCore": "org.apache.hadoop:hadoop-mapreduce-client-core:" + hadoopVersion,
  "hadoopClientCommon": "org.apache.hadoop:hadoop-mapreduce-client-common:" + hadoopVersion,
  "hadoopHdfs": "org.apache.hadoop:hadoop-hdfs:" + hadoopVersion,
  "hadoopAuth": "org.apache.hadoop:hadoop-auth:" + hadoopVersion,
  "hadoopYarnApi": "org.apache.hadoop:hadoop-yarn-api:" + hadoopVersion,
  "hadoopYarnCommon": "org.apache.hadoop:hadoop-yarn-common:" + hadoopVersion,
  "hadoopYarnClient": "org.apache.hadoop:hadoop-yarn-client:" + hadoopVersion,
  "hadoopYarnMiniCluster": "org.apache.hadoop:hadoop-minicluster:" + hadoopVersion,
  "hadoopAnnotations": "org.apache.hadoop:hadoop-annotations:" + hadoopVersion,
  "hadoopAws": "org.apache.hadoop:hadoop-aws:2.6.0",
  "hiveCommon": "org.apache.hive:hive-common:" + hiveVersion,
  "hiveService": "org.apache.hive:hive-service:" + hiveVersion,
  "hiveJdbc": "org.apache.hive:hive-jdbc:" + hiveVersion,
  "hiveMetastore": "org.apache.hive:hive-metastore:" + hiveVersion,
  "hiveExec": "org.apache.hive:hive-exec:" + hiveVersion + ":core",
  "hiveSerDe": "org.apache.hive:hive-serde:" + hiveVersion,
  "httpclient": "org.apache.httpcomponents:httpclient:4.5.2",
  "httpcore": "org.apache.httpcomponents:httpcore:4.4.4",
  "kafka": "org.apache.kafka:kafka_2.11:1.1.0",
  "kafkaTest": "org.apache.kafka:kafka_2.11:1.1.0:test",
  "kafkaClient": "org.apache.kafka:kafka-clients:1.1.0",
  "quartz": "org.quartz-scheduler:quartz:2.2.3",
  "testng": "org.testng:testng:6.9.10",
  "mockserver":"org.mock-server:mockserver-netty:3.10.4",
  "jacksonCore": "org.codehaus.jackson:jackson-core-asl:1.9.13",
  "jacksonMapper": "org.codehaus.jackson:jackson-mapper-asl:1.9.13",
  "jasypt": "org.jasypt:jasypt:1.9.2",
  "slf4j": "org.slf4j:slf4j-api:1.7.25",
  "log4j": "log4j:log4j:1.2.17",
  "log4jextras": "log4j:apache-log4j-extras:1.2.17",
  "slf4jLog4j": "org.slf4j:slf4j-log4j12:1.7.25",
  "jodaTime": "joda-time:joda-time:2.9.3",
  "metricsCore": "io.dropwizard.metrics:metrics-core:" + dropwizardMetricsVersion,
  "metricsJvm": "io.dropwizard.metrics:metrics-jvm:" + dropwizardMetricsVersion,
  "metricsGraphite": "io.dropwizard.metrics:metrics-graphite:" + dropwizardMetricsVersion,
  "jsch": "com.jcraft:jsch:0.1.53",
  "jdo2": "javax.jdo:jdo2-api:2.1",
  "azkaban": "com.linkedin.azkaban:azkaban:2.5.0",
  "commonsVfs": "org.apache.commons:commons-vfs2:2.0",
  "mysqlConnector": "mysql:mysql-connector-java:6.0.2",
  "javaxInject": "javax.inject:javax.inject:1",
  "guice": "com.google.inject:guice:4.0",
  "derby": "org.apache.derby:derby:10.12.1.1",
  "mockito": "org.mockito:mockito-core:1.10.19",
  "salesforceWsc": "com.force.api:force-wsc:37.0",
  "salesforcePartner": "com.force.api:force-partner-api:36.0.0",
  "scala": "org.scala-lang:scala-library:2.11.12",
  "influxdbJava": "org.influxdb:influxdb-java:2.2",
  "libthrift":"org.apache.thrift:libthrift:0.9.3",
  "lombok":"org.projectlombok:lombok:1.16.8",
  "mockRunnerJdbc":"com.mockrunner:mockrunner-jdbc:1.0.8",
  "xerces":"xerces:xercesImpl:2.11.0",
  "typesafeConfig": "com.typesafe:config:1.3.3",
  "byteman": "org.jboss.byteman:byteman:" + bytemanVersion,
  "bytemanBmunit": "org.jboss.byteman:byteman-bmunit:" + bytemanVersion,
  "bcpgJdk15on": "org.bouncycastle:bcpg-jdk15on:1.52",
  "bcprovJdk15on": "org.bouncycastle:bcprov-jdk15on:1.52",
  "calciteCore": "org.apache.calcite:calcite-core:1.2.0-incubating",
  "calciteAvatica": "org.apache.calcite:calcite-avatica:1.2.0-incubating",
  "jhyde": "org.pentaho:pentaho-aggdesigner-algorithm:5.1.5-jhyde",
  "curatorFramework": "org.apache.curator:curator-framework:2.10.0",
  "curatorRecipes": "org.apache.curator:curator-recipes:2.10.0",
  "curatorClient": "org.apache.curator:curator-client:2.10.0",
  "curatorTest": "org.apache.curator:curator-test:2.10.0",
  "hamcrest": "org.hamcrest:hamcrest-all:1.3",
  "joptSimple": "net.sf.jopt-simple:jopt-simple:4.9",
  "protobuf": "com.google.protobuf:protobuf-java:2.5.0",
  "pegasus" : [
    "data" : "com.linkedin.pegasus:data:" + pegasusVersion,
    "generator" : "com.linkedin.pegasus:generator:" + pegasusVersion,
    "restliClient" : "com.linkedin.pegasus:restli-client:" + pegasusVersion,
    "restliServer" : "com.linkedin.pegasus:restli-server:" + pegasusVersion,
    "restliTools" : "com.linkedin.pegasus:restli-tools:" + pegasusVersion,
    "pegasusCommon" : "com.linkedin.pegasus:pegasus-common:" + pegasusVersion,
    "restliCommon" : "com.linkedin.pegasus:restli-common:" + pegasusVersion,
    "r2" : "com.linkedin.pegasus:r2:" + pegasusVersion,
    "d2" : "com.linkedin.pegasus:d2:" + pegasusVersion,
    "restliNettyStandalone" : "com.linkedin.pegasus:restli-netty-standalone:" + pegasusVersion
  ],
  "jetty": [
          "org.eclipse.jetty:jetty-server:9.2.14.v20151106",
          "org.eclipse.jetty:jetty-servlet:9.2.14.v20151106"
  ],
  "servlet-api": "javax.servlet:servlet-api:3.1.0",
  "reflections" : "org.reflections:reflections:0.9.10"
];

if (!isDefaultEnvironment)
{
  ext.externalDependency.each { overrideDepKey, overrideDepValue ->
    if (externalDependency[overrideDepKey] != null)
    {
      externalDependency[overrideDepKey] = overrideDepValue
    }
  }
}

task wrapper(type: Wrapper) { gradleVersion = '1.12' }

import javax.tools.ToolProvider

task javadocTarball(type: Tar) {
  baseName = "gobblin-javadoc-all"
  destinationDir = new File(project.buildDir, baseName)
  compression = Compression.GZIP
  extension = 'tgz'
  description = "Generates a tar-ball with all javadocs to ${destinationDir}/${archiveName}"
}

javadocTarball << {
  def indexFile = new File(destinationDir, "index.md")
  def version = rootProject.ext.javadocVersion
  indexFile << """----
layout: page
title: Gobblin Javadoc packages ${version}
permalink: /javadoc/${version}/
----

"""
  rootProject.ext.javadocPackages.each {
    indexFile << "* [${it}](${it})\n"
  }
}

// Javadoc initialization for subprojects
ext.javadocVersion = null != project.version ? project.version.toString() : "latest"
if (ext.javadocVersion.indexOf('-') > 0) {
  // Remove any "-" addons from the version
  ext.javadocVersion = javadocVersion.substring(0, javadocVersion.indexOf('-'))
}

ext.javadocPackages = new HashSet<String>()
subprojects.each{Project pr ->
  if (file(pr.projectDir.absolutePath + "/src/main/java").exists()) {
    rootProject.ext.javadocPackages += pr.name
  }
}

subprojects {
  plugins.withType(JavaPlugin) {

    // Sometimes generating javadocs can lead to OOM. This may needs to be increased.
    // Also force javadocs to pick up system proxy settings if available
    javadoc {
      options.jFlags('-Xmx256m', '-Djava.net.useSystemProxies=true');
    }

    rootProject.tasks.javadocTarball.dependsOn project.tasks.javadoc
    if ( rootProject.ext.javadocPackages.contains(project.name)) {
      rootProject.tasks.javadocTarball.into(project.name){from(fileTree(dir: "${project.buildDir}/docs/javadoc/"))}
    }
  }
}

ext.pomAttributes = {
  name "${project.name}"
  packaging 'jar'
  // optionally artifactId can be defined here
  description 'Gobblin Ingestion Framework'
  url 'https://github.com/linkedin/gobblin/'

  scm {
    connection 'scm:git:git@github.com:linkedin/gobblin.git'
    developerConnection 'scm:git:git@github.com:linkedin/gobblin.git'
    url 'git@github.com:linkedin/gobblin.git'
  }

  licenses {
    license {
      name 'The Apache License, Version 2.0'
      url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
    }
  }

  developers {
    developer {
      name 'Abhishek Tiwari'
      organization 'LinkedIn'
    }
    developer {
      name 'Chavdar Botev'
      organization 'LinkedIn'
    }
    developer {
      name 'Issac Buenrostro'
      organization 'LinkedIn'
    }
    developer {
      name 'Min Tu'
      organization 'LinkedIn'
    }
    developer {
      name 'Narasimha Veeramreddy'
      organization 'LinkedIn'
    }
    developer {
      name 'Pradhan Cadabam'
      organization 'LinkedIn'
    }
    developer {
      name 'Sahil Takiar'
      organization 'LinkedIn'
    }
    developer {
      name 'Shirshanka Das'
      organization 'LinkedIn'
    }
    developer {
      name 'Yinan Li'
      organization 'LinkedIn'
    }
    developer {
      name 'Ying Dai'
      organization 'LinkedIn'
    }
    developer {
      name 'Ziyang Liu'
      organization 'LinkedIn'
    }
  }
}

def getAllDependentProjects(project) {
  def projectDependencies = project.configurations.runtime.getAllDependencies().withType(ProjectDependency)
  def dependentProjects = projectDependencies*.dependencyProject
  if (dependentProjects.size() > 0) {
      dependentProjects.each { dependentProjects += getAllDependentProjects(it) }
  }
  return dependentProjects.unique()
}

subprojects {
  plugins.withType(JavaPlugin) {
    plugins.apply('idea')
    plugins.apply('eclipse')
    plugins.apply('maven')

    sourceCompatibility = JavaVersion.VERSION_1_7

    plugins.apply('findbugs')
    findbugs {
      toolVersion = findBugsVersion
      ignoreFailures = true
      effort = "max"
      // The exclude filter file must be under "ligradle/findbugs/"
      excludeFilter = file(rootProject.projectDir.path + "/ligradle/findbugs/findbugsExclude.xml")
    }

    test {
      if (project.hasProperty("printTestOutput")) {
        testLogging.showStandardStreams = true
      }
      useTestNG () {
        excludeGroups 'ignore', 'performance'
        if (project.hasProperty('skipTestGroup')) {
          excludeGroups skipTestGroup
        }
        if (useHadoop2) {
          excludeGroups 'Hadoop1Only'
        }
      }
    }

    configurations {
      compile
      dependencies {
        if (useHadoop2) {
          compile(externalDependency.hadoopCommon) {
            exclude module: 'servlet-api'
          }
          compile externalDependency.hadoopClientCore
          compile externalDependency.hadoopAnnotations
          if (project.name.equals('gobblin-runtime') || project.name.equals('gobblin-test')) {
            compile externalDependency.hadoopClientCommon
          }
        } else {
          compile externalDependency.hadoop
        }
        compile(externalDependency.guava) {
          force = true
        }

        // Required to add JDK's tool jar, which is required to run byteman tests.
        testCompile (files(((URLClassLoader) ToolProvider.getSystemToolClassLoader()).getURLs()))
      }
    }

    if (isDefaultEnvironment) {
      task sourcesJar(type: Jar, dependsOn: classes) {
        from sourceSets.main.allSource
        classifier = 'sources'
      }
      task javadocJar(type: Jar) {
        from javadoc
        classifier = 'javadoc'
      }
      artifacts { archives sourcesJar, javadocJar }
    }

    plugins.apply('maven')

    project.version = rootProject.version
    project.group = rootProject.group

    install {
      repositories {
        mavenInstaller {
          mavenLocal()
          pom.project {
            name "${project.name}"
            packaging 'jar'
            description 'Gobblin Ingestion Framework'
            url 'https://github.com/linkedin/gobblin/'
          }
        }
      }
    }

    // Publishing of maven artifacts for subprojects
    if (rootProject.ext.publishToMaven) {
      if (rootProject.ext.signArtifacts) {
        plugins.apply('signing')
      }

      uploadArchives {
        repositories {
          mavenDeployer {
            beforeDeployment { MavenDeployment deployment ->
              if (rootProject.ext.signArtifacts) {
                signing.signPom(deployment)
              }
            }

            repository(url: rootProject.artifactRepository) {
              authentication(userName: ossrhUsername, password: ossrhPassword)
            }

            snapshotRepository(url: rootProject.artifactSnapshotRepository) {
              authentication(userName: ossrhUsername, password: ossrhPassword)
            }

            pom.project pomAttributes
          }
        }
      }

      if (rootProject.ext.signArtifacts) {
        signing {
          sign configurations.archives
        }
      }
    }

    // Configure the IDEA plugin to (1) add the codegen as source dirs and (2) work around
    // an apparent bug in the plugin which doesn't set the outputDir/testOutputDir as documented
    idea.project {
      ext.languageLevel = JavaVersion.VERSION_1_7
      languageLevel = JavaVersion.VERSION_1_7
    }
    idea.module {
      // Gradle docs claim the two settings below are the default, but
      // the actual defaults appear to be "out/production/$MODULE_NAME"
      // and "out/test/$MODULE_NAME". Changing it so IDEA and gradle share
      // the class output directory.

      outputDir = sourceSets.main.output.classesDir
      testOutputDir = sourceSets.test.output.classesDir
    }

    // Add standard javadoc repositories so we can reference classes in them using @link
    tasks.javadoc.options.links "http://typesafehub.github.io/config/latest/api/",
                                "https://docs.oracle.com/javase/7/docs/api/",
                                "http://docs.guava-libraries.googlecode.com/git-history/v15.0/javadoc/",
                                "http://hadoop.apache.org/docs/r${rootProject.ext.hadoopVersion}/api/",
                                "https://hive.apache.org/javadocs/r${rootProject.ext.hiveVersion}/api/",
                                "http://avro.apache.org/docs/${avroVersion}/api/java/",
                                "https://dropwizard.github.io/metrics/${dropwizardMetricsVersion}/apidocs/"

    afterEvaluate {
      // add the standard pegasus dependencies wherever the plugin is used
      if (project.plugins.hasPlugin('pegasus')) {
        dependencies {
          dataTemplateCompile externalDependency.pegasus.data
          restClientCompile externalDependency.pegasus.restliClient,externalDependency.pegasus.restliCommon,externalDependency.pegasus.restliTools
        }
      }
    }
  }
}

gradle.projectsEvaluated {
  subprojects {
    plugins.withType(JavaPlugin) {
      getAllDependentProjects(project).each {
        tasks.javadoc.options.linksOffline "http://linkedin.github.io/gobblin/javadoc/${javadocVersion}/${it.name}/",
                                         "${rootProject.buildDir}/${it.name}/docs/javadoc/"
      }
    }
  }
}

//Turn off javadoc lint for Java 8+
if (JavaVersion.current().isJava8Compatible()) {
  allprojects {
    tasks.withType(Javadoc) {
      options.addStringOption('Xdoclint:none', '-quiet')
    }
  }
}

task dotProjectDependencies(description: 'List of gobblin project dependencies in dot format') << {
  println "// ========= Start of project dependency graph ======= "
  println "digraph project_dependencies {"
  subprojects.each { Project project ->
    def project_node_name = project.name.replaceAll("-","_")
    if (project.configurations.findByName("compile") != null) {
      project.configurations.compile.dependencies.each { Dependency dep ->
        if (dep instanceof ProjectDependency) {
          def dep_node_name = dep.dependencyProject.name.replaceAll("-","_")
          println "\t${project_node_name} -> ${dep_node_name};"
        }
      }
    }
  }
  println "}"
  println "// ========= End of project dependency graph ======= "
}