Skip to content

Commit

Permalink
Build: Consolidate archives and packages configuration (elastic#28760)
Browse files Browse the repository at this point in the history
This commit moves the distribution specific tasks into the respective
archives and packages builds. The collocation of common and distribution
specific tasks make it much easier to reason about what is expected in a
particular distribution.
  • Loading branch information
rjernst committed Feb 22, 2018
1 parent 0a7022e commit 299f2b2
Show file tree
Hide file tree
Showing 37 changed files with 467 additions and 612 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ class VagrantTestPlugin implements Plugin<Project> {
} else {
it = "packages:${it}"
}
project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'archives'))
project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
}

UPGRADE_FROM_ARCHIVES.each {
Expand Down
187 changes: 156 additions & 31 deletions distribution/archives/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -17,70 +17,195 @@
* under the License.
*/

import org.apache.tools.ant.filters.FixCrLfFilter
import org.apache.tools.ant.taskdefs.condition.Os
import org.apache.tools.ant.filters.FixCrLfFilter
import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.EmptyDirTask
import org.elasticsearch.gradle.ConcatFilesTask
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.NoticeTask
import org.elasticsearch.gradle.precommit.DependencyLicensesTask
import org.elasticsearch.gradle.precommit.UpdateShasTask
import org.elasticsearch.gradle.test.RunTask
import org.elasticsearch.gradle.plugin.PluginBuildPlugin

subprojects {
// CopySpec does not make it easy to create an empty director so we create the directory that we want, and then point CopySpec to its
// parent to copy to the root of the distribution
File logs = new File(buildDir, 'logs-hack/logs')
task createLogDir(type: EmptyDirTask) {
dir "${logs}"
dirMode 0755
}
File plugins = new File(buildDir, 'plugins-hack/plugins')
task createPluginsDir(type: EmptyDirTask) {
dir "${plugins}"
dirMode 0755
}
project.ext.archivesFiles = copySpec {
// need this so Zip/Tar tasks get basic defaults...
apply plugin: 'base'

// CopySpec does not make it easy to create an empty directory so we
// create the directory that we want, and then point CopySpec to its
// parent to copy to the root of the distribution
ext.logsDir = new File(buildDir, 'logs-hack/logs')
task createLogsDir(type: EmptyDirTask) {
dir "${logsDir}"
dirMode 0755
}
ext.pluginsDir= new File(buildDir, 'plugins-hack/plugins')
task createPluginsDir(type: EmptyDirTask) {
dir "${pluginsDir}"
dirMode 0755
}

CopySpec archiveFiles(CopySpec... innerFiles) {
return copySpec {
into("elasticsearch-${version}") {
with libFiles
into('config') {
dirMode 0750
fileMode 0660
with configFiles
with configFiles('def')
}
into('bin') {
with copySpec {
with binFiles
from('../../src/main/resources/bin') {
with binFiles('def')
from('../src/bin') {
include '*.bat'
filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
}
MavenFilteringHack.filter(it, expansions)
MavenFilteringHack.filter(it, expansionsForDistribution('def'))
}
}
into('') {
from {
dirMode 0755
logs.getParent()
logsDir.getParent()
}
}
into('') {
from {
dirMode 0755
plugins.getParent()
pluginsDir.getParent()
}
}
with commonFiles
with noticeFile
from('../../src/main/resources') {
from('../src') {
include 'bin/*.exe'
}
if (project.name != 'integ-test-zip') {
with modulesFiles
} else {
with transportModulesFiles
for (CopySpec files : innerFiles) {
with files
}
}
}
}

task buildIntegTestZip(type: Zip) {
dependsOn createLogsDir, createPluginsDir
destinationDir = file('integ-test-zip/build/distributions')
baseName = 'elasticsearch'
with archiveFiles(transportModulesFiles)
}

task buildZip(type: Zip) {
dependsOn createLogsDir, createPluginsDir
destinationDir = file('zip/build/distributions')
baseName = 'elasticsearch'
with archiveFiles(modulesFiles)
}

task buildTar(type: Tar) {
dependsOn createLogsDir, createPluginsDir
destinationDir = file('tar/build/distributions')
baseName = 'elasticsearch'
extension = 'tar.gz'
compression = Compression.GZIP
dirMode 0755
fileMode 0644
with archiveFiles(modulesFiles)
}

// This configures the default artifact for the distribution specific
// subprojects. We have subprojects for two reasons:
// 1. Gradle project substitutions can only bind to the default
// configuration of a project
// 2. The integ-test-zip and zip distributions have the exact same
// filename, so they must be placed in different directories.
subprojects {
apply plugin: 'distribution'

archivesBaseName = 'elasticsearch'

String buildTask = "build${it.name.replaceAll(/-[a-z]/) { it.substring(1).toUpperCase() }.capitalize()}"
ext.buildDist = parent.tasks.getByName(buildTask)
artifacts {
'default' buildDist
}
}

/*****************************************************************************
* Rest test config *
*****************************************************************************/
subprojects {
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'

if (project.name == 'integ-test-zip') {
integTest {
includePackaged true
}
}

integTestCluster {
dependsOn assemble
distribution = project.name
}
integTestRunner {
if (Os.isFamily(Os.FAMILY_WINDOWS) && System.getProperty('tests.timeoutSuite') == null) {
// override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man
systemProperty 'tests.timeoutSuite', '1800000!'
}
}

processTestResources {
inputs.properties(project(':distribution').restTestExpansions)
MavenFilteringHack.filter(it, project(':distribution').restTestExpansions)
}
}

/*****************************************************************************
* Maven config *
*****************************************************************************/
configure(subprojects.findAll { it.name.contains('zip') }) {
// only zip distributions go to maven
BuildPlugin.configurePomGeneration(project)
apply plugin: 'nebula.info-scm'
apply plugin: 'nebula.maven-base-publish'
apply plugin: 'nebula.maven-scm'

// note: the group must be correct before applying the nexus plugin, or
// it will capture the wrong value...
project.group = "org.elasticsearch.distribution.${project.name}"

publishing {
publications {
nebula {
artifactId 'elasticsearch'
artifact buildDist
}
/*
* HUGE HACK: the underlying maven publication library refuses to
* deploy any attached artifacts when the packaging type is set to
* 'pom'. But Sonatype's OSS repositories require source files for
* artifacts that are of type 'zip'. We already publish the source
* and javadoc for Elasticsearch under the various other subprojects.
* So here we create another publication using the same name that
* has the "real" pom, and rely on the fact that gradle will execute
* the publish tasks in alphabetical order. This lets us publish the
* zip file and even though the pom says the type is 'pom' instead of
* 'zip'. We cannot setup a dependency between the tasks because the
* publishing tasks are created *extremely* late in the configuration
* phase, so that we cannot get ahold of the actual task. Furthermore,
* this entire hack only exists so we can make publishing to maven
* local work, since we publish to maven central externally.
*/
nebulaRealPom(MavenPublication) {
artifactId 'elasticsearch'
pom.packaging = 'pom'
pom.withXml { XmlProvider xml ->
Node root = xml.asNode()
root.appendNode('name', 'Elasticsearch')
root.appendNode('description', 'A Distributed RESTful Search Engine')
root.appendNode('url', PluginBuildPlugin.urlFromOrigin(project.scminfo.origin))
Node scmNode = root.appendNode('scm')
scmNode.appendNode('url', project.scminfo.origin)
}
}
}
}
}

74 changes: 2 additions & 72 deletions distribution/archives/integ-test-zip/build.gradle
Original file line number Diff line number Diff line change
@@ -1,72 +1,2 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import org.elasticsearch.gradle.plugin.PluginBuildPlugin
import org.apache.tools.ant.taskdefs.condition.Os

task buildZip(type: Zip) {
dependsOn createLogDir, createPluginsDir
baseName = 'elasticsearch'
with archivesFiles
}

artifacts {
'default' buildZip
archives buildZip
}

publishing {
publications {
nebula {
artifactId 'elasticsearch'
artifact buildZip
}
/* HUGE HACK: the underlying maven publication library refuses to deploy any attached artifacts
* when the packaging type is set to 'pom'. But Sonatype's OSS repositories require source files
* for artifacts that are of type 'zip'. We already publish the source and javadoc for Elasticsearch
* under the various other subprojects. So here we create another publication using the same
* name that has the "real" pom, and rely on the fact that gradle will execute the publish tasks
* in alphabetical order. This lets us publish the zip file and even though the pom says the
* type is 'pom' instead of 'zip'. We cannot setup a dependency between the tasks because the
* publishing tasks are created *extremely* late in the configuration phase, so that we cannot get
* ahold of the actual task. Furthermore, this entire hack only exists so we can make publishing to
* maven local work, since we publish to maven central externally. */
nebulaRealPom(MavenPublication) {
artifactId 'elasticsearch'
pom.packaging = 'pom'
pom.withXml { XmlProvider xml ->
Node root = xml.asNode()
root.appendNode('name', 'Elasticsearch')
root.appendNode('description', 'A Distributed RESTful Search Engine')
root.appendNode('url', PluginBuildPlugin.urlFromOrigin(project.scminfo.origin))
Node scmNode = root.appendNode('scm')
scmNode.appendNode('url', project.scminfo.origin)
}
}
}
}

integTestRunner {
if (Os.isFamily(Os.FAMILY_WINDOWS) && System.getProperty('tests.timeoutSuite') == null) {
// override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man
systemProperty 'tests.timeoutSuite', '1800000!'
}
}

integTest.dependsOn buildZip
// This file is intentionally blank. All configuration of the
// distribution is done in the parent project.
35 changes: 2 additions & 33 deletions distribution/archives/tar/build.gradle
Original file line number Diff line number Diff line change
@@ -1,33 +1,2 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

task buildTar(type: Tar) {
dependsOn createLogDir, createPluginsDir
baseName = 'elasticsearch'
extension = 'tar.gz'
with archivesFiles
compression = Compression.GZIP
dirMode 0755
fileMode 0644
}

artifacts {
'default' buildTar
}

// This file is intentionally blank. All configuration of the
// distribution is done in the parent project.
Loading

0 comments on commit 299f2b2

Please sign in to comment.