build.gradle

//
// Sparkling Water build file.
// This is a multi-module build file.
//

// The build script settings to fetch plugins and put them on
// classpath
buildscript {
  repositories {
    mavenCentral()
    gradlePluginPortal()
  }

  dependencies {
    classpath 'com.bmuschko:gradle-docker-plugin:6.7.0'
    classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
    classpath 'net.researchgate:gradle-release:3.0.0'
    classpath 'kr.motd.gradle:sphinx-gradle-plugin:2.10.1'
    classpath 'ru.vyarus:gradle-use-python-plugin:2.3.0'
    classpath 'com.github.jk1:gradle-license-report:2.0'
    classpath "com.diffplug.spotless:spotless-plugin-gradle:4.4.0"
  }
}

apply plugin: 'base'

// For multiproject setup we have to apply release plugin here (we share same release number cross all modules)
if (project.hasProperty("doRelease")) {
  apply from: 'gradle/release.gradle'
}

//
// Common configuration
//
ext {
  // Published projects
  publishedProjects = [
    project(':sparkling-water-api-generation'),
    project(':sparkling-water-repl'),
    project(':sparkling-water-core'),
    project(':sparkling-water-examples'),
    project(':sparkling-water-ml'),
    project(':sparkling-water-package'),
    project(':sparkling-water-scoring-package'),
    project(":sparkling-water-doc"),
    project(':sparkling-water-utils'),
    project(':sparkling-water-scoring'),
    project(':sparkling-water-extensions')
  ]
  // Projects with integration tests
  integTestProjects = [
    project(':sparkling-water-core'),
    project(':sparkling-water-examples'),
    project(':sparkling-water-ml')
  ]

  // Projects with benchmarks
  benchProjects = [project(':sparkling-water-core'), project(':sparkling-water-benchmarks')]

  // Scala projects
  scalaProjects = [
    project(':sparkling-water-api-generation'),
    project(':sparkling-water-repl'),
    project(':sparkling-water-core'),
    project(':sparkling-water-examples'),
    project(':sparkling-water-ml'),
    project(':sparkling-water-package'),
    project(':sparkling-water-scoring-package'),
    project(':sparkling-water-benchmarks'),
    project(':sparkling-water-macros'),
    project(':sparkling-water-utils'),
    project(':sparkling-water-scoring'),
    project(':sparkling-water-extensions'),
    project(':sparkling-water-doc'),
    project(':sparkling-water-booklet')
  ]
  pythonProjects = [project(':sparkling-water-py'), project(':sparkling-water-py-scoring')]
  rProjects = [project(':sparkling-water-r'), project(':sparkling-water-r-cran')]
  docProjects = [project(':sparkling-water-doc')]

  scalaTestVersions = ['2.11': '2.2.1', '2.12': '3.0.8']
  numpyVersions = ['3.6': '1.19.5', '3.7': '1.19.5', 'default': '1.23.5']
}

def versionSpecificProps = new Properties()
file("$rootDir/gradle-spark${spark}.properties").withInputStream { versionSpecificProps.load(it) }

def loadVersionSpecificProp(propName, versionSpecificProps) {
  // first check if the property was specified/override via -P
  if (project.hasProperty(propName)) {
    return project.properties.get(propName)
  } else {
    return versionSpecificProps.getProperty(propName)
  }
}

def loadVersion(versionSpecificProps) {
  def version = project.properties.get("version")
  def sparkMajorVersion = getSparkMajorVersion(versionSpecificProps)
  if (version.endsWith("-${sparkMajorVersion}")) {
    return version
  } else {
    supportedSparkVersions.split(" ").each { majorVersion ->
      if (version.endsWith("-${majorVersion}")) {
        throw new RuntimeException("You are specifying version suffix for Spark ${majorVersion}, but build is for Spark ${sparkMajorVersion} ")
      }
    }
    return "${version}-${sparkMajorVersion}"
  }
}


def getSparkMajorVersion(versionSpecificProps) {
  def sparkVersion = loadVersionSpecificProp("sparkVersion", versionSpecificProps)
  return sparkVersion.count(".") == 1 ? sparkVersion : sparkVersion.substring(0, sparkVersion.lastIndexOf('.'))
}
//
// For all projects (this and all subprojects) specify common properties and tasks
//
def loadedVersion = loadVersion(versionSpecificProps).replace("-SNAPSHOT", "")

configure(allprojects) { project ->
  apply plugin: 'idea'
  apply plugin: 'eclipse'
  apply from: "$rootDir/gradle/artifacts.gradle"
  apply plugin: "com.diffplug.gradle.spotless"
  // Version of main components
  ext {
    scalaVersion = loadVersionSpecificProp("scalaVersion", versionSpecificProps)
    scalaBaseVersion = scalaVersion.substring(0, 4)
    scalaTestVersion = scalaTestVersions[scalaBaseVersion]
    // h2oBuild property is defined in gradle.properties
    h2oVersion = "$h2oMajorVersion.$h2oBuild"
    sparkVersion = loadVersionSpecificProp("sparkVersion", versionSpecificProps)
    if (!sparkVersion.startsWith(spark)) {
      throw new IllegalArgumentException("Inconsistent Spark configuration. Both params spark and sparkVersion specified, but" +
      " with different major Spark versions.")
    }
    sparkMajorVersion = sparkVersion.count(".") == 1 ? sparkVersion : sparkVersion.substring(0, sparkVersion.lastIndexOf('.'))
    junitVersion = '4.11'
    minSupportedJavaVersion = loadVersionSpecificProp("minSupportedJavaVersion", versionSpecificProps)
    supportedEmrVersion = loadVersionSpecificProp("supportedEmrVersion", versionSpecificProps)
    unsupportedMinorSparkVersions = loadVersionSpecificProp("unsupportedMinorSparkVersions", versionSpecificProps)
    executorOverheadMemoryOption = loadVersionSpecificProp("executorOverheadMemoryOption", versionSpecificProps)
    driverOverheadMemoryOption = loadVersionSpecificProp("driverOverheadMemoryOption", versionSpecificProps)
    fabricK8sClientVersion = loadVersionSpecificProp("fabricK8sClientVersion", versionSpecificProps)
    supportedPythonVersions = loadVersionSpecificProp("supportedPythonVersions", versionSpecificProps)
  }
  // All project inherits the same versioning number
  version = loadedVersion
  spotless {
    groovyGradle {
      greclipse().configFile("$rootDir/greclipse.properties")
    }
  }
}

//
// Common configuration for all subprojects
//
configure(subprojects) { project ->
  repositories {
    // Should be enabled only in development mode
    if (project.hasProperty('useMavenLocal') || h2oBuild.endsWith("-SNAPSHOT")) {
      mavenLocal()
    }

    mavenCentral()

    // Public Sonatype repository
    maven {
      url "https://oss.sonatype.org/content/repositories/releases/"
    }

    // Snapshot repository of H2O builds
    maven {
      url "https://h2o-release.s3.amazonaws.com/h2o/master/$h2oBuild/maven/repo/"
    }

    if (sparkVersion.endsWith("-SNAPSHOT")) {
      maven {
        url "https://repository.apache.org/content/repositories/snapshots/"
      }
    }
  }

  if (project in scalaProjects) {
    apply from: "$rootDir/gradle/scala.gradle"
  }

  if (project in pythonProjects) {
    apply from: "$rootDir/gradle/python.gradle"
  }

  if (project in rProjects) {
    apply from: "$rootDir/gradle/r.gradle"
  }

  // All subprojects needs Spark support
  apply from: "$rootDir/gradle/spark.gradle"

  if (project in docProjects) {
    apply plugin: 'java'
    apply plugin: 'kr.motd.sphinx'

    configurations {
      publishArchives
    }
  }

  // Publish artifacts
  if (project in publishedProjects) {
    apply from: "$rootDir/gradle/publish.gradle"
  }

  if (project in integTestProjects) {
    apply from: "$rootDir/gradle/itest.gradle"
  }

  if (project in benchProjects) {
    apply from: "$rootDir/gradle/bench.gradle"
  }

  spotless {
    format 'misc', {
      target '**/*.sh', '**/*.gitignore', '**/*.properties', '**/*.R', '**/*.tf', '**/*.xml', "**/*.groovy"
      targetExclude 'src-gen/**/*.R', 'build/src/R/**/*.R'
      trimTrailingWhitespace()
      endWithNewline()
    }
  }
}

spotless {
  format 'misc', {
    target 'bin/*.sh', '.gitignore', '*.properties', 'gradle/publish/*.*'
    trimTrailingWhitespace()
    endWithNewline()
  }
}

clean.doFirst {
  delete file("build/")
  delete file("h2ologs/")
  delete file("null/")
  delete file("metastore_db/")
}

def createStagingId(def username, def password) {
  def proc = ['./createStagingId.sh', username, password].execute([], file("${rootDir.toString()}/gradle/publish"))
  return proc.text
}

def uploadFiles(def username, def password, def repoDir, def stagingId) {
  def proc = ['./uploadToNexus.sh'
    , username, password, repoDir, stagingId].execute([], file("${rootDir.toString()}/gradle/publish"))
  proc.waitFor()
}

def closeBucket(def username, def password, def stagingId) {
  def proc = ['./closeBucket.sh'
    , username, password, stagingId].execute([], file("${rootDir.toString()}/gradle/publish"))
  proc.waitFor()
}

task publishToNexus {
  doLast {
    def stagingId = createStagingId(project.findProperty('nexusUsername'), project.findProperty('nexusPassword')).replace('\n', '')
    uploadFiles(project.findProperty('nexusUsername'), project.findProperty('nexusPassword'), "${rootDir}/dist/build/dist/maven/repo", stagingId)
    closeBucket(project.findProperty('nexusUsername'), project.findProperty('nexusPassword'), stagingId)
  }
}