build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

buildscript {
  repositories {
      jcenter()
      maven { url  "http://palantir.bintray.com/releases" }
  }

  dependencies {
    classpath 'com.github.jengelman.gradle.plugins:shadow:5.0.0'
    classpath 'com.netflix.nebula:gradle-aggregate-javadocs-plugin:2.2.+'
    classpath 'com.netflix.nebula:nebula-publishing-plugin:9.5.0'
    classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4'
  }
}

plugins {
  id 'com.palantir.git-version' version '0.9.1'
  id 'com.palantir.consistent-versions' version '1.9.2'
}

if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
  throw new GradleException("This build must be run with Java 8")
}

allprojects {
  group = "com.netflix.iceberg"
  apply plugin: 'idea'
  version = gitVersion()
  repositories {
    maven { url  "http://palantir.bintray.com/releases" }
    mavenCentral()
  }
}

apply plugin: 'nebula-aggregate-javadocs'

subprojects {
  apply plugin: 'nebula.javadoc-jar'
  apply plugin: 'nebula.source-jar'
  apply plugin: 'java'
  apply plugin: 'maven' // make pom files for deployment
  apply plugin: 'nebula.maven-base-publish'

  configurations {
    testCompile.extendsFrom compileOnly
    all {
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }

  sourceCompatibility = '1.8'
  targetCompatibility = '1.8'

  dependencies {
    compileOnly 'org.slf4j:slf4j-api'
    compileOnly 'com.google.guava:guava'

    testCompile 'junit:junit'
    testCompile 'org.slf4j:slf4j-simple'
    testCompile 'org.mockito:mockito-all'
  }

  apply from: "${rootProject.projectDir}/gradle/publish.gradle"
}

project(':iceberg-api') {
  dependencies {
    testCompile "org.apache.avro:avro"
    testCompile 'joda-time:joda-time'
  }
}

project(':iceberg-common') {
}

project(':iceberg-core') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')

    compile "org.apache.avro:avro"

    compile "com.fasterxml.jackson.core:jackson-databind"
    compile "com.fasterxml.jackson.core:jackson-core"
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }
}

project(':iceberg-data') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')
    compileOnly project(':iceberg-parquet')

    testCompile("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }
}

project(':iceberg-hive') {
  dependencies {
    compile project(':iceberg-core')

    compileOnly "org.apache.avro:avro"

    compileOnly("org.apache.hive:hive-standalone-metastore") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testCompile("org.apache.hive:hive-exec::core") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testCompile("org.apache.hive:hive-metastore") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }
}

project(':iceberg-orc') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')

    compile("org.apache.orc:orc-core::nohive") {
      exclude group: 'org.apache.hadoop', module: 'hadoop-common'
    }


    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

  }
}

project(':iceberg-parquet') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')

    compile "org.apache.parquet:parquet-avro"

    compileOnly "org.apache.avro:avro"
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

project(':iceberg-spark') {
  apply plugin: 'scala'

  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')
    compile project(':iceberg-core')
    compile project(':iceberg-parquet')

    compileOnly "org.apache.avro:avro"
    compileOnly("org.apache.spark:spark-hive_2.11") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testCompile "org.apache.hadoop:hadoop-hdfs::tests"
    testCompile "org.apache.hadoop:hadoop-common::tests"
    testCompile("org.apache.hadoop:hadoop-minicluster") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

project(':iceberg-pig') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')
    compile project(':iceberg-core')
    compile project(':iceberg-parquet')

    compileOnly("org.apache.pig:pig")
    compileOnly("org.apache.hadoop:hadoop-mapreduce-client-core")
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testCompile "org.apache.hadoop:hadoop-hdfs::tests"
    testCompile "org.apache.hadoop:hadoop-common::tests"
    testCompile("org.apache.hadoop:hadoop-minicluster") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

// the runtime jar is a self-contained artifact for testing in a notebook
project(':iceberg-runtime') {
  apply plugin: 'com.github.johnrengelman.shadow'

  configurations {
    shadow
    compileOnly.extendsFrom shadow
  }

  shadowJar {
    configurations = [project.configurations.shadow]
    from(project.sourceSets.main.output)
  }

  tasks.build.dependsOn tasks.shadowJar
  tasks.install.dependsOn tasks.shadowJar
  tasks.javadocJar.dependsOn tasks.shadowJar

  dependencies {
    shadow project(':iceberg-api')
    shadow project(':iceberg-common')
    shadow project(':iceberg-core')
    shadow project(':iceberg-parquet')
    shadow project(':iceberg-spark')
    shadow project(':iceberg-pig')
    shadow project(':iceberg-hive')

    shadow "org.apache.avro:avro"
    shadow "org.apache.parquet:parquet-avro"
  }

  publishing {
    publications {
      withType(IvyPublication) {
        configurations {
          'shadow' { extend 'runtime' }
        }
      }
    }
  }

  shadowJar {
    zip64 true

    // Relocate dependencies to avoid conflicts
    // relocate 'com.google.common', 'com.netflix.bdp.shaded.com.google.common'
    relocate 'com.fasterxml', 'com.netflix.iceberg.shaded.com.fasterxml'
    relocate 'org.apache.avro', 'com.netflix.iceberg.shaded.org.apache.avro'
    relocate 'org.apache.parquet.avro', 'com.netflix.iceberg.shaded.org.apache.parquet.avro'

    archiveName = "iceberg-runtime-${version}.${extension}"
  }
}

// Runtime jar for presto. Presto depends on hive fat jar that does not shade avro and bunch
// of other dependencies. This artifact is used by presto-iceberg plugin.
project(':iceberg-presto-runtime') {
    apply plugin: 'com.github.johnrengelman.shadow'

    configurations {
        shadow
        compileOnly.extendsFrom shadow
    }

    dependencies {
        shadow project(':iceberg-api')
        shadow project(':iceberg-core')
        shadow project(':iceberg-parquet')
        shadow project(':iceberg-hive')

        shadow "org.apache.parquet:parquet-avro"
        shadow "org.apache.avro:avro"
        shadow ("org.apache.hive:hive-standalone-metastore") {
            exclude group: 'org.apache.hadoop', module: 'hadoop-common'
//            exclude group: 'org.apache.orc', module: 'orc-core'
        }
    }

    tasks.build.dependsOn tasks.shadowJar
    tasks.install.dependsOn tasks.shadowJar
    tasks.javadocJar.dependsOn tasks.shadowJar

    shadowJar {
        zip64 true
        exclude 'org/htrace/**'
        exclude 'org/objectweb/**'
        exclude 'org/apache/thrift/**'
        exclude 'META-INF/services/org.apache.hadoop**'
        exclude 'com/thoughtworks/**'
        exclude 'org/slf4j/**'
        exclude 'org/znerd/**'
        exclude 'org/xerial/**'
        exclude 'org/tukaani/**'
        exclude 'org/objectweb/**'
        exclude 'org/apache/commons/**'
        exclude 'it/unimi/**'
        exclude 'io/airlift/**'
        exclude 'com/google/**'
        exclude 'com/fasterxml/**'
        exclude 'org/apache/log4j/**'
        exclude 'org/apache/http/**'
        exclude 'org/apache/http/version.properties'
        exclude 'mozilla/public-suffix-list.txt'
        exclude 'org/apache/http/client/version.properties'
        exclude 'common-version-info.properties'
        exclude 'core-default.xml'
        exclude 'org/apache/hadoop/application-classloader.properties'
        exclude '**application-classloader.properties'

        configurations = [project.configurations.shadow]
        from(project.sourceSets.main.output)

        relocate 'org.apache.avro', 'com.netflix.iceberg.shaded.org.apache.avro'
        relocate 'org.apache.parquet.avro', 'com.netflix.iceberg.shaded.org.apache.parquet.avro'
        relocate 'org.apache.hadoop.hive', 'com.netflix.iceberg.shaded.org.apache.hadoop.hive'
        relocate 'org.apache.hive', 'com.netflix.iceberg.shaded.org.apache.hive'
        archiveName = "iceberg-presto-runtime-${version}.${extension}"
    }
}