build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

buildscript {
  repositories {
    mavenCentral()
  }
}

plugins {
  id "de.undercouch.download" version "3.2.0"
  id "org.nosphere.apache.rat" version "0.2.0"
}

//        Generated by  http://patorjk.com/software/taag/#p=display&f=Bloody&t=Apache%20Bigtop
def final ASCII_bigtop = '''
                                                     :aass..                                    
                                                       =XS22nai,>__. .                            
                                                       =n-- +!!!""^--                             
                                                      .vX>       .                                
                                                     .)e<o;.                                      
                                                   ._v2`-{S> .                                    
                                                 ..<de~..;)Sa,                                    
                                              .._aoX}:===>=-?Xo>, .  .                            
                                        . .__aaoZe!`=><i=s+s;~*XXos,,_ .   .  .                   
                  ...........______=iisaaoXXZY!"~._v(=d=:nc-1s,-~?SX#Xouass,,_____.:..........    
                 =XXoXXXSXXXXXXXXZUZXX21?!"^-.._au*`=u2` ]X>.+*a>,.-"!Y1XSSX##ZZXXXXXXXXXXXoXXc   
                 .{XXXXXX2*?!!"!"^~--- ...__aa2!^- =dX( .+XXc. ~!1nas,,.---~~^""!"!!!?YSXXXXX2+   
                  -"YSXXXo=.   ._=sssaaav1!!~-  ._aXXe`   )SXo>.  -~"?Yoouass_s,,    _vXXXX2}~    
                      -{XXZoai%%*XXSSSX>..    .<uXXX2~  .  {XXXXs,. .  .=dXXXZX2lii%uXXXXe-       
                      .<XXXXX%-  -<XXXXX1|==%vdXXXXXo;:. ._vXXXXXXos_=i|*XXXXX>   -<XXXXX`        
                       =SXXXZc   ..nXXX2> ---=2XXX2^-"|||}"--~{ZXXX1-- .:XXXXo;  . )XXXX2`        
                     . =XXXXZc     nXXXX>    =XXXXe..__s=>_...)XXXX1 . .:SXXXo;   .)XXXX2..       
                       <SXXXXc .  .nXXXS>    =XXXXosummmmBmma,)ZXXX1    :XXXX2;    )XXXXX.        
                      .<XXXXX(    :nXXXS;    <XXXXXm#mmmWmmmmmoZXXX1   ..3XXXo;. . )XXXXX;        
                     . nXXXXX;.   :XXXXX;   .=XXXXXmmmBmmmWmB#XXXXX1  . .nXXXX>    :XXXXXc .      
                    . =oZXXXe;   .<XXXX2` .  )XXXXZmBmBmWmmmW#2XXXX1 .  .vXXXXc     vXXXXo;       
                      +Y3S2Xz__...vXXXXe .  .)ZXXXZmmWmBmmBBm#XXXXXo..   {XXXXz:.___vSS2Y1=       
                    .     ---+"""*!!*Y1s|=_==uXSSSXZUXUXUXUXUXS2XX2n|_=||%Y*??!"""^~---           
                                         .--- - ---------------- - - -.       
      .o.                                  oooo                  oooooooooo.  o8o               .                     
     .888.                                 `888                  `888'   `Y8b `"'             .o8                     
    .8"888.    oo.ooooo.  .oooo.   .ooooo.  888 .oo.   .ooooo.    888     888oooo  .oooooooo.o888oo .ooooo. oo.ooooo. 
   .8' `888.    888' `88b`P  )88b d88' `"Y8 888P"Y88b d88' `88b   888oooo888'`888 888' `88b   888  d88' `88b 888' `88b
  .88ooo8888.   888   888 .oP"888 888       888   888 888ooo888   888    `88b 888 888   888   888  888   888 888   888
 .8'     `888.  888   888d8(  888 888   .o8 888   888 888    .o   888    .88P 888 `88bod8P'   888 .888   888 888   888
o88o     o8888o 888bod8P'`Y888""8o`Y8bod8P'o888o o888o`Y8bod8P'  o888bood8P' o888o`8oooooo.   "888"`Y8bod8P' 888bod8P'
                888                                                               d"     YD                  888      
               o888o                                                              "Y88888P'                 o888o     
'''

def final langLevel = "1.8"

rat {
  excludes = [
      ".git/**",
      ".idea/**",
       "**/.gitignore",
       /* Test data with rigid structure and/or binary */
       "bigtop-tests/test-artifacts/**/resources/**",
       "bigtop-tests/smoke-tests/alluxio/datafile",
       "bigtop-tests/smoke-tests/flink/test.data",
       "bigtop-tests/smoke-tests/hive/passwd.ql",
       "bigtop-tests/smoke-tests/phoenix/*.csv",
       "bigtop-tests/smoke-tests/phoenix/*.sql",
       "bigtop-tests/smoke-tests/ignite-hadoop/*.data",
       "bigtop-tests/smoke-tests/tajo/table1/*.csv",
       "bigtop-tests/smoke-tests/odpi-runtime/src/test/resources/*.json",
       "bigtop-tests/smoke-tests/odpi-runtime/src/test/resources/*.list",
       "**/target/**",
       "**/build/**",
       "**/.gradle/**",
       "**/*.iml",
       ".gradle",
       "output/**",
       "MAINTAINERS.txt",
       "buildSrc/build/**",
       "bigtop-data-generators/**/input_data/**",
       "docker/**/*.yaml",
       "bigtop-deploy/puppet/**/*.yaml",
       "bigtop-deploy/puppet/modules/hadoop/files/*/id_hdfsuser*",
       "bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock",
       "bigtop-deploy/puppet/modules/zeppelin/templates/interpreter.json",
       /* Debian package files that have a rigid structure
          (* matches package name) */
       "bigtop-packages/src/deb/*/*.dirs",
       "bigtop-packages/src/deb/*/dirs",
       "bigtop-packages/src/deb/*/*.install",
       "bigtop-packages/src/deb/*/*.include",
       "bigtop-packages/src/deb/**/include-binaries",
       "bigtop-packages/src/deb/**/format",
       "bigtop-packages/src/deb/*/*.manpages",
       "bigtop-packages/src/deb/*/*.docs",
       "bigtop-packages/src/deb/*/docs",
       "bigtop-packages/src/deb/*/compat",
       "bigtop-packages/src/deb/*/source/format",
       "bigtop-packages/src/deb/*/*.lintian-overrides",
       "bigtop-packages/src/deb/*/shlibs.local",
       "bigtop-packages/src/**/*.diff",
       "bigtop-packages/src/common/*/*.json",
       "bigtop-packages/src/common/**/*.default",
       "bigtop-repos/apt/distributions",
       /* Juju charm files with rigid structure */
       "bigtop-packages/src/charm/**/wheelhouse.txt",
       "bigtop-packages/src/charm/**/*.json",
       "bigtop-packages/src/charm/**/*.svg",
       "bigtop-packages/src/charm/**/*.yaml",
       "bigtop-packages/src/charm/**/resources/**/*.txt",
       "bigtop-deploy/juju/**/*.yaml",
       "bigtop-tests/cloud-weather-report/**/*.yaml",
       /* Misc individual files */
       "src/site/resources/bigtop.rdf",
       "src/site/resources/images/bigtop-logo.ai",
       /* File with FreeBSD documentation license */
       "src/site/resources/css/freebsd_docbook.css",
       "test/MANIFEST.txt",
       "bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg",
       "bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg",
       "bigtop-deploy/puppet/README.md",
       "bigtop-deploy/puppet/modules/qfs/README.md",
       "bigtop-deploy/puppet/config/site.csv.example",
       "dl/**",
       "bigtop-deploy/puppet/modules/gpdb/templates/**",
       "provisioner/docker/config/hiera.yaml",
       "provisioner/docker/config/hieradata/PLACEHOLDER",
       "provisioner/docker/config/hosts",
       "bigtop_toolchain/files/*.patch",
  ]
}


allprojects {
  apply plugin: 'java'
  apply plugin: 'maven'

  group = 'org.apache.bigtop'
  version = '1.5.0-SNAPSHOT'

  description = """Bigtop"""

  sourceCompatibility = langLevel
  targetCompatibility = langLevel

  repositories {
    mavenCentral()
  }
}

subprojects {
  apply plugin: 'groovy'
}

project(':itest-common') {
  description = """iTest: system and integration testing in the cloud"""

  dependencies {
    compile group: 'org.codehaus.groovy', name: 'groovy-all', version:'2.4.10'
    compile group: 'junit', name: 'junit', version:'4.11'
    compile group: 'commons-logging', name: 'commons-logging', version:'1.1'
    compile group: 'org.apache.ant', name: 'ant', version:'1.8.2'
    compile group: 'org.apache.ant', name: 'ant-junit', version:'1.8.2'
  }

  task packageITest(type: Jar) {
    from sourceSets.main.output
    //classifier = 'tests'
  }

  artifacts.archives packageITest

  test.doFirst {
    if (System.getenv('JAVA_HOME') == null)
      throw new GradleException("JAVA_HOME is required to test ${project.name}")
  }
  test {
    exclude '**/Dummy*'
  }
}

/**
 * checkSmokeTestProjects() configures smoke-tests:<component>:test task only
 * in if the project has smoke.tests properties set.
 * This is done to avoid running cluster smoke tests during the normal life-cycle
 * of the project development
 */
def checkClusterTestProjects = { property ->
  def suiteName = property.replaceAll('.', '-')
  FileTree fTree = fileTree(dir: "bigtop-tests/$suiteName", include: '*/build.gradle')
  fTree.each() { smokeProject ->
    def parent = smokeProject.getParentFile().name
    project (":bigtop-tests:$suiteName:$parent") {
      test {
        onlyIf {project.hasProperty(suiteName)}
      }
    }
  }
}

def TESTARTIFACTS_GROUP = 'test artifacts'
def DEVENV_GROUP = 'development tools'
def DEPLOY_GROUP = 'deployment'
def DOCKERBUILD_GROUP = 'docker build'

// All packaging logic is separated into its own build module
apply from: 'packages.gradle'

// Wrapping all release logic to simplify the release process
apply from: 'release.gradle'

task toolchain(type:Exec,
    description: 'Setup dev. env via toolchain; Requires: Puppet, sudo',
    group: DEVENV_GROUP) {
  def command = [
      'sudo', 'puppet', 'apply', '-d', '--parser', 'future',
      "--modulepath=${projectDir.absolutePath}:/etc/puppet/modules", '-e',
      'include bigtop_toolchain::installer'
  ]
  workingDir '.'
  commandLine command
}
task "toolchain-puppetmodules"(type:Exec,
    description: 'Setup puppet modules via toolchain; Requires: Puppet, sudo',
    group: DEVENV_GROUP) {
  def command = [
      'sudo', 'puppet', 'apply', '-d',
      "--modulepath=${projectDir.absolutePath}", '-e',
      'include bigtop_toolchain::puppet-modules'
  ]
  workingDir '.'
  commandLine command
}
task "toolchain-devtools"(type:Exec,
    description: 'Setup additional dev. tools like Groovy SDK via toolchain; Requires: Puppet, sudo',
    group: DEVENV_GROUP) {
  def command = [
      'sudo', 'puppet', 'apply', '-d',
      "--modulepath=${projectDir.absolutePath}", '-e',
      'include bigtop_toolchain::development-tools'
  ]
  workingDir '.'
  commandLine command
}

task "docker-provisioner"(type:Exec,
    description: '''
Provision a Bigtop stack cluster on Docker container(s). Default to CentOS and 1 node.
Properties:
  -Pconfig=[CONFIG_FILE] (located under provisioner/docker/)
  -Penable_local_repo
  -Pimage=[DOCKER_IMAGE] (overwrites -POS and -Pprefix)
  -Pmemory=[4g|8g|...]
  -Pnum_instances=[NUM_INSTANCES]
  -Pnexus=[NEXUS_URL] (NEXUS_URL is optional)
  -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3]
  -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...]
  -Prepository=[REPO_URL]
  -Prun_smoke_tests (run test components defined in config file)
  -Psmoke_tests=[COMPONENTS]
  -Pstack=[COMPONENTS]

For detailed explaination, please refer to the helper message:
  $ provisioner/docker/docker-hadoop.sh --help

Examples:
Create a 3 node cluster:
  $ ./gradlew -Pnum_instances=3 docker-provisioner
Run smoke tests on 3 node cluster:
  $ ./gradlew -Pnum_instances=3 -Prun_smoke_tests docker-provisioner
Specify custom configuration file:
  $ ./gradlew -Pconfig=config_debian-9.yaml docker-provisioner
''',
    group: DEPLOY_GROUP) {
  def _config = project.hasProperty("config") ? config : "config.yaml"
  def _num_instances = project.hasProperty("num_instances") ? num_instances : 1
  def _OS = project.hasProperty("OS") ? OS : 'centos-7'
  def _prefix = project.hasProperty("prefix") ? prefix : 'trunk'
  def command = [
      './docker-hadoop.sh',
      '-C', _config,
      '--create', _num_instances,
  ]
  if ( project.hasProperty("stack") ) {
    command.add('--stack')
    command.add("${stack}")
  }
  if ( project.hasProperty("run_smoke_tests") ) {
    command.add("--smoke-tests")
  }
  if ( project.hasProperty("smoke_tests") ) {
    command.add('--smoke-tests')
    command.add("${smoke_tests}")
  }
  if ( project.hasProperty("nexus") ) {
    command.add('--nexus')
    command.add("${nexus}")
  }
  if ( project.hasProperty("enable_local_repo") ) {
    command.add('--enable-local-repo')
  }
  if (project.hasProperty("OS") || project.hasProperty("prefix")) {
    command.add('--image')
    command.add("bigtop/puppet:${_prefix}-${_OS}")
  }
  if (project.hasProperty("image")) {
    command.add('--image')
    command.add("${image}")
  }
  if (project.hasProperty("repository")) {
    command.add('--repo')
    command.add("${repository}")
  }
  if (project.hasProperty("memory")) {
    command.add('--memory')
    command.add("${memory}")
  }
  workingDir 'provisioner/docker'
  commandLine command
}
task "docker-provisioner-ssh"(type:Exec,
    description: 'Show command to get in to the provisioned container',
    group: DEPLOY_GROUP) {
  workingDir 'provisioner/docker'
  def message = "To get in to the container, do: \n\ndocker exec -ti <CONTAIER_NAME> bash \n\nYou can get container name by ./gradlew docker-provisioner-status"
  def command = [
      '/bin/echo',
      message
  ]
  commandLine command
}
task "docker-provisioner-status"(type:Exec,
    description: 'Show status of Bigtop Docker cluster',
    group: DEPLOY_GROUP) {
  def command = [
      './docker-hadoop.sh',
      '--list',
  ]
  workingDir 'provisioner/docker'
  commandLine command
}
task "docker-provisioner-destroy"(type:Exec,
    description: 'Destroy provisioned Bigtop Docker cluster',
    group: DEPLOY_GROUP) {
  def command = [
      './docker-hadoop.sh',
      '--destroy'
  ]
  workingDir 'provisioner/docker'
  commandLine command
}

task installTopLevel(type:Exec) {
  workingDir "."
  commandLine 'mvn clean install -f pom.xml'.split(" ")
}

task installiTest(dependsOn: installTopLevel, type:Exec) {
  workingDir "."
  commandLine 'mvn clean install -f bigtop-test-framework/pom.xml -DskipTests'.split(" ")
}
task installTestArtifacts(dependsOn: installiTest, type:Exec) {
  workingDir "."
  commandLine 'mvn clean install -f bigtop-tests/test-artifacts/pom.xml'.split(" ")
}
task installConf(type:Exec) {
  workingDir "."
  commandLine 'mvn clean install -f bigtop-tests/test-execution/conf/pom.xml'.split(" ")
}
task installCommon(type:Exec) {
  workingDir "."
  commandLine 'mvn clean install -f bigtop-tests/test-execution/common/pom.xml'.split(" ")
}
task installAllLocalArtifacts (
    description: "Prepare and locally install all test artifacts",
    group: TESTARTIFACTS_GROUP) {
}

/**
 * Allows user to specify which artifacts to install by dynamically generating tasks.
 */
def artifactToInstall = {
  def final BASE_DIR = projectDir.absolutePath
  def final TEST_DIR = "$BASE_DIR/bigtop-tests/test-artifacts"

  def project = new XmlSlurper().parse("$TEST_DIR/pom.xml")
  project.modules.module.each { artifact ->
    task "install-${artifact}" (description: "Installs ${artifact} artifact with Maven",
        group: TESTARTIFACTS_GROUP,
        dependsOn: installiTest
    ) doLast {
      def final PATH = "${TEST_DIR}/$artifact/pom.xml"
      def final WRAPPER = "mvn clean install -f " + PATH
      exec {
        workingDir '.'
        commandLine WRAPPER.split(" ")
      }
    }
  }
}


def _NEXUS_URL = project.hasProperty("NEXUS_URL") ? NEXUS_URL : "http://localhost:8081/nexus"

def generate_nexus_tasks = { name, url, snapshot ->
  task "configure-nexus-${name}"(type: Exec) {
    def req = [ data : [ id: name, name: name, repoType: "proxy", repoPolicy : snapshot, browseable: true, "indexable": true, "notFoundCacheTTL": 1440,
			  "artifactMaxAge": 1440, "metadataMaxAge": 1440, "itemMaxAge": 1440, "provider": "maven2",
			  "providerRole": "org.sonatype.nexus.proxy.repository.Repository", "downloadRemoteIndexes": true,
			  "autoBlockActive": true, "fileTypeValidation": true, "exposed": true, "checksumPolicy": "WARN",
			  "remoteStorage": ["remoteStorageUrl": url, "authentication": null,
					    "connectionSettings": null]]]
    def root = new groovy.json.JsonBuilder(req)
    def p = root.toString()
    def _NEXUS_USERPASS = project.hasProperty("NEXUS_USERPASS") ? NEXUS_USERPASS : "admin:admin123"

    File.createTempFile("temp",".tmp").with {
      deleteOnExit()

      write root.toString()

      workingDir '.'
      commandLine  "curl", "-o", "/dev/null", "-s", "-w", "%{http_code}", "-X", "POST", "-d", "@$absolutePath",
        "--header", "Content-Type: application/json", "-u", "$_NEXUS_USERPASS", "$_NEXUS_URL/service/local/repositories"
      standardOutput = new ByteArrayOutputStream()

      doLast {
        def httpCode = standardOutput.toString()
        switch (httpCode) {
          case "200":
            println "Nexus proxy ${name} successfully configured"
            break
          case "400":
            println "Skipped because Nexus proxy ${name} already configured"
            break
          default:
            throw new GradleException("Failed to configure Nexus proxy ${name} with http code ${httpCode} returned. " +
             "Run with --info option to see the executed command")
            break
        }
      }
    }
  }
}

def repos = [ [ "name": "conjars", "url" : "http://conjars.org", 'snapshot':'RELEASE'],
          [ "name": "repository.jboss.org", "url": "http://repository.jboss.org/nexus/content/groups/public/", 'snapshot': 'RELEASE'],
          [ "name": "apache.snapshots.https", "url": "https://repository.apache.org/content/repositories/snapshots", "snapshot": 'SNAPSHOT'],
          [ "name": "apache.snapshots", "url": "https://repository.apache.org/content/repositories/snapshots", "snapshot": 'SNAPSHOT'],
          [ "name": "maven2-repository.atlassian", "url": "https://maven.atlassian.com/repository/public", "snapshot": 'RELEASE']]

repos.each { r->
  generate_nexus_tasks( r.name, r.url, r.snapshot)
}

task "configure-nexus"(dependsOn: tasks.findAll { alltask -> alltask.name.startsWith("configure-nexus-")}*.name,
      description: 'configure build to use a sonatype nexus server.\n' +
        '  Use -PNEXUS_URL=http://server:8081 to overwrite default server http://localhost:8081/nexus\n' +
        '  Use -PNEXUS_USERPASS=user:passwd to overwrite default username passwd admin:admin123') doLast {
   def m2Dir = System.getProperty("user.home") + "/.m2"
   mkdir(m2Dir)
   def m2Writer = new File(m2Dir + "/settings.xml")
   def writeMirrorLine = { name ->
     m2Writer.append("<mirror><name>$name</name><url>$_NEXUS_URL/content/repositories/$name/</url><mirrorOf>$name</mirrorOf></mirror>")
   }
   m2Writer.text = "<settings><mirrors>"
   writeMirrorLine( "central")
   repos.each{ r->
     writeMirrorLine( r.name)
   }
   m2Writer.append("</mirrors></settings>")

    def gradleDir = System.getProperty("user.home") + "/.gradle/init.d"
    mkdir(gradleDir)
    def gradleWriter = new File(gradleDir + "/bigtop_nexus.gradle")
    gradleWriter.text = """
allprojects {
    repositories {
        maven {
            url "$_NEXUS_URL/content/repositories/central"
        }
    }
}
    """
}

task "bigtop-puppet"(type:Exec,
    description: '''
Build bigtop/puppet images
Usage:
  $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] bigtop-puppet
Example:
  $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-puppet
  The built image name: bigtop/puppet:1.0.0-debian-9
      ''',
    group: DOCKERBUILD_GROUP) {
  def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
  def _OS = project.hasProperty("OS") ? OS : "centos-7"
  def command = [
      './build.sh', _prefix + '-' + _OS,
  ]
  workingDir './docker/bigtop-puppet'
  commandLine command
}

task "bigtop-slaves"(type:Exec,
    description: '''
Build bigtop/slaves images
Usage:
  $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] bigtop-slaves
Example:
  $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-slaves
  The built image name: bigtop/slaves:1.0.0-debian-9
      ''',
    group: DOCKERBUILD_GROUP) {
  def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
  def _OS = project.hasProperty("OS") ? OS : "centos-7"
  def command = [
      './build.sh', _prefix + '-' + _OS,
  ]
  workingDir './docker/bigtop-slaves'
  commandLine command
}

project.afterEvaluate {
  checkClusterTestProjects("smoke.tests")
  artifactToInstall(dependsOn: [installTopLevel, installCommon, installConf, installiTest])
}

installAllLocalArtifacts.dependsOn installTopLevel, installCommon, installConf, installiTest, installTestArtifacts

help.doFirst {
  println ASCII_bigtop
}