Skip to content
This repository has been archived by the owner on Apr 5, 2022. It is now read-only.

Commit

Permalink
Add build support for HDP 2.6 [SHDP-583]
Browse files Browse the repository at this point in the history
- fix whitespace issues
  • Loading branch information
trisberg committed Jun 16, 2017
1 parent 7f493e2 commit 251618e
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 34 deletions.
55 changes: 27 additions & 28 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,10 @@ switch (hadoopDistro) {
sparkVersion = phd30SparkVersion
break;

// Hortonworks Data Platform 2.4
case "hdp24":
hadoopVersion = hdp24Version
println "Using Hortonworks Data Platform 2.4 [$hadoopVersion]"
// Hortonworks Data Platform 2.6
case "hdp26":
hadoopVersion = hdp26Version
println "Using Hortonworks Data Platform 2.6 [$hadoopVersion]"
hadoopArtifacts = ["org.apache.hadoop:hadoop-common:$hadoopVersion",
"org.apache.hadoop:hadoop-hdfs:$hadoopVersion",
"org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion",
Expand All @@ -164,14 +164,14 @@ switch (hadoopDistro) {
"org.apache.hadoop:hadoop-mapreduce-client-hs:$hadoopVersion",
"org.apache.hadoop:hadoop-yarn-server-tests:$hadoopVersion",
"org.apache.hadoop:hadoop-yarn-server-tests:$hadoopVersion:tests"]
hbaseVersion = hdp24HbaseVersion
hbaseVersion = hdp26HbaseVersion
hbaseArtifacts = ["org.apache.hbase:hbase:$hbaseVersion",
"org.apache.hbase:hbase-common:$hbaseVersion",
"org.apache.hbase:hbase-client:$hbaseVersion"]
hiveVersion = hdp24HiveVersion
pigVersion = hdp24PigVersion
hiveVersion = hdp26HiveVersion
pigVersion = hdp26PigVersion
pigQualifier = ':h2'
sparkVersion = hdp24SparkVersion
sparkVersion = hdp26SparkVersion
break;

// Hortonworks Data Platform 2.5
Expand Down Expand Up @@ -239,7 +239,7 @@ switch (hadoopDistro) {
println "Using Apache Hadoop 2.7.x - [$hadoopVersion]"
} else {
println "Failing build: $hadoopDistro is not a supported distro"
println "Supported distros: hadoop26, hadoop27[*], hdp24, hdp25, cdh5 and phd30"
println "Supported distros: hadoop26, hadoop27[*], hdp25, hdp26, cdh5 and phd30"
println "* default"
throw new InvalidUserDataException("$hadoopDistro is not a supported distro")
}
Expand Down Expand Up @@ -370,7 +370,7 @@ configure(javaProjects()) {

javadoc {
ext.srcDir = file("${projectDir}/docs/src/api")

configure(options) {
stylesheetFile = file("${rootProject.projectDir}/docs/src/api/stylesheet.css")
overview = "${rootProject.projectDir}/docs/src/api/overview.html"
Expand All @@ -383,7 +383,7 @@ configure(javaProjects()) {
// groups = [
// 'Spring Data Hadoop' : ['org.springframework.data.hadoop*'],
// ]

links = [
"http://docs.spring.io/spring/docs/4.0.x/javadoc-api/",
"http://docs.oracle.com/javase/6/docs/api/",
Expand Down Expand Up @@ -453,7 +453,7 @@ configure(hadoopProjects()) {
testRuntime "org.apache.hadoop:hadoop-mapreduce-client-jobclient:$hadoopVersion"
testRuntime "dk.brics.automaton:automaton:1.11-8"
}
break;
break;

// Pivotal HD 3.0
case "phd30":
Expand All @@ -464,8 +464,8 @@ configure(hadoopProjects()) {
}
break;

// Hortonworks Data Platform 2.4
case "hdp24":
// Hortonworks Data Platform 2.6
case "hdp26":
dependencies {
testCompile "org.apache.hadoop:hadoop-mapreduce-examples:$hadoopVersion"
testRuntime "org.apache.hadoop:hadoop-mapreduce-client-jobclient:$hadoopVersion"
Expand Down Expand Up @@ -503,19 +503,19 @@ configure(hadoopProjects()) {
dependencies {
hadoopArtifacts.each {
compile(it) { dep ->
if (it.contains("hadoop-common") ||
if (it.contains("hadoop-common") ||
it.contains("hadoop-yarn-common") ||
it.contains("hadoop-mapreduce-client-core") ||
it.contains("hadoop-mapreduce-client-app") ||
it.contains("hadoop-mapreduce-client-hs") ||
it.contains("hadoop-mapreduce-client-jobclient") ||
it.contains("hive-service")) {
it.contains("hive-service")) {
exclude group: "org.slf4j", module: "slf4j-log4j12"
}
if (it.contains("hadoop-common") ||
it.contains("hadoop-hdfs") ||
it.contains("hadoop-mapreduce-client-core") ||
it.contains("hadoop-mapreduce-client-jobclient") ||
it.contains("hadoop-mapreduce-client-jobclient") ||
it.contains("hadoop-yarn-common") ||
it.contains("hadoop-yarn-client") ||
it.contains("hadoop-yarn-server-tests") ||
Expand All @@ -536,23 +536,23 @@ configure(hadoopProjects()) {
optional("org.springframework:spring-tx:$springVersion")
// used by Hive package
optional("org.springframework:spring-jdbc:$springVersion")

// Missing dependency in Hadoop 1.0.3
testRuntime "commons-io:commons-io:$commonsioVersion"
testRuntime "org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion"
testRuntime "cglib:cglib-nodep:$cglibVersion"

// Hive
optional("$hiveGroup:hive-service:$hiveVersion")

// needed by JDBC test
testRuntime "$hiveGroup:hive-jdbc:$hiveVersion"

// Pig
optional("org.apache.pig:pig:$pigVersion$pigQualifier") { dep ->
exclude group: "junit", module: "junit"
}

// HBase
hbaseArtifacts.each {
optional(it)
Expand Down Expand Up @@ -943,7 +943,7 @@ project('spring-data-hadoop-build-tests') {
testRuntime "org.codehaus.groovy:groovy:$groovyVersion"
testRuntime "org.jruby:jruby:$jrubyVersion"
testRuntime "org.python:jython-standalone:$jythonVersion"

// specify a version of antlr that works with both hive and pig
testRuntime "org.antlr:antlr-runtime:$antlrVersion"

Expand Down Expand Up @@ -1048,7 +1048,7 @@ project('spring-data-hadoop-build-tests') {
ext.msg += "HBase "
excludes.add("**/hbase/**")
}

if (project.ext.skipHive) {
ext.msg += "Hive "
excludes.add("**/hive/**")
Expand All @@ -1071,18 +1071,18 @@ project('spring-data-hadoop-build-tests') {

if (!msg.trim().isEmpty())
println "Skipping [$msg] Tests";

// check prefix for hd.fs
// first copy the properties since we can't change them
ext.projProps = project.properties

if (projProps.containsKey("hd.fs")) {
String hdfs = projProps["hd.fs"].toString()
if (!hdfs.contains("://")) {
projProps.put("hd.fs", "hdfs://" + hdfs)
}
}

// due to GRADLE-2475, set the system properties manually
projProps.each { k,v ->
if (k.toString().startsWith("hd.")) {
Expand Down Expand Up @@ -1182,7 +1182,7 @@ project('spring-yarn:spring-yarn-core') {
projProps.put("hd.fs", "hdfs://" + hdfs)
}
}

// due to GRADLE-2475, set the system properties manually
projProps.each { k,v ->
if (k.toString().startsWith("hd.")) {
Expand Down Expand Up @@ -1373,4 +1373,3 @@ task wrapper(type: Wrapper) {
description = "Generates gradlew[.bat] scripts"
gradleVersion = "2.4"
}

12 changes: 6 additions & 6 deletions gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,12 @@ hdp25HiveVersion = 1.2.1000.2.5.0.0-1245
hdp25PigVersion = 0.16.0.2.5.0.0-1245
hdp25SparkVersion = 1.6.2.2.5.0.0-1245

## Hortonworks Data Platform 2.4
hdp24Version = 2.7.1.2.4.4.1-5
hdp24HbaseVersion = 1.1.2.2.4.4.1-5
hdp24HiveVersion = 1.2.1000.2.4.4.1-5
hdp24PigVersion = 0.15.0.2.4.4.1-5
hdp24SparkVersion = 1.6.2.2.4.4.1-5
## Hortonworks Data Platform 2.6
hdp26Version = 2.7.3.2.6.0.3-8
hdp26HbaseVersion = 1.1.2.2.6.0.3-8
hdp26HiveVersion = 1.2.1000.2.6.0.3-8
hdp26PigVersion = 0.16.0.2.6.0.3-8
hdp26SparkVersion = 1.6.3.2.6.0.3-8

## Defaults for common hadoop libraries - each distro should override as necessary
defaultHiveVersion = 0.10.0
Expand Down

0 comments on commit 251618e

Please sign in to comment.