diff --git a/build.gradle b/build.gradle index b3e5e09e07a32..369bd30cc2f3e 100644 --- a/build.gradle +++ b/build.gradle @@ -142,6 +142,7 @@ subprojects { substitute module("org.elasticsearch:test-framework:${version}") with project("${projectsPrefix}:test-framework") } substitute module("org.elasticsearch.distribution.zip:elasticsearch:${version}") with project("${projectsPrefix}:distribution:zip") + substitute module("org.elasticsearch.distribution.tar:elasticsearch:${version}") with project("${projectsPrefix}:distribution:tar") } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 388259c7a9ca4..2e2062e175f95 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -24,6 +24,9 @@ import org.gradle.api.tasks.Input /** Configuration for an elasticsearch cluster, used for integration tests. */ class ClusterConfiguration { + @Input + String distribution = 'zip' + @Input int numNodes = 1 @@ -33,6 +36,9 @@ class ClusterConfiguration { @Input int transportPort = 9500 + @Input + String jvmArgs = System.getProperty('tests.jvm.argline', '') + Map systemProperties = new HashMap<>() @Input diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 3a06da929dd27..3a6c8865d2a28 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -22,6 +22,7 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.ElasticsearchProperties import org.gradle.api.DefaultTask import org.gradle.api.GradleException +import org.gradle.api.InvalidUserDataException import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.tasks.Copy @@ -42,7 +43,7 @@ class ClusterFormationTasks { // no need to cluster formation if the task won't run! return } - addZipConfiguration(project) + configureDistributionDependency(project, config.distribution) File clusterDir = new File(project.buildDir, 'cluster' + File.separator + task.name) if (config.numNodes == 1) { addNodeStartupTasks(project, task, config, clusterDir) @@ -57,22 +58,38 @@ class ClusterFormationTasks { } static void addNodeStartupTasks(Project project, Task task, ClusterConfiguration config, File baseDir) { + File pidFile = pidFile(baseDir) String clusterName = "${task.path.replace(':', '_').substring(1)}" - File home = new File(baseDir, "elasticsearch-${ElasticsearchProperties.version}") - List setupDependsOn = [project.configurations.elasticsearchZip] - setupDependsOn.addAll(task.dependsOn) - Task setup = project.tasks.create(name: task.name + '#setup', type: Copy, dependsOn: setupDependsOn) { - from { project.zipTree(project.configurations.elasticsearchZip.singleFile) } - into baseDir + File home = homeDir(baseDir, config.distribution) + Map esConfig = [ + 'cluster.name': clusterName, + 'http.port': config.httpPort, + 'transport.tcp.port': config.transportPort, + 'pidfile': pidFile, + // TODO: make this work for multi node! + 'discovery.zen.ping.unicast.hosts': "localhost:${config.transportPort}", + 'path.repo': "${home}/repo", + 'path.shared_data': "${home}/../", + // Define a node attribute so we can test that it exists + 'node.testattr': 'test', + 'repositories.url.allowed_urls': 'http://snapshot.test*' + ] + Map esEnv = [ + 'JAVA_HOME': System.getProperty('java.home'), + 'ES_GC_OPTS': config.jvmArgs + ] + + List setupDeps = [] // need to copy the deps, since start will later be added, which would create a circular task dep! + setupDeps.addAll(task.dependsOn) + Task setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setupDeps) { + delete baseDir } + setup = configureExtractTask(project, "${task.name}#extract", config.distribution, baseDir, setup) // chain setup tasks to maintain their order - setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setup) { - delete new File(home, 'plugins'), new File(home, 'data'), new File(home, 'logs') - } setup = project.tasks.create(name: "${task.name}#configure", type: DefaultTask, dependsOn: setup) << { - File configFile = new File(home, 'config' + File.separator + 'elasticsearch.yml') + File configFile = new File(home, 'config/elasticsearch.yml') logger.info("Configuring ${configFile}") - configFile.setText("cluster.name: ${clusterName}", 'UTF-8') + configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8') } for (Map.Entry command : config.setupCommands.entrySet()) { Task nextSetup = project.tasks.create(name: "${task.name}#${command.getKey()}", type: Exec, dependsOn: setup) { @@ -100,15 +117,7 @@ class ClusterFormationTasks { setup = nextSetup } - File pidFile = pidFile(baseDir) - List esArgs = [ - "-Des.http.port=${config.httpPort}", - "-Des.transport.tcp.port=${config.transportPort}", - "-Des.pidfile=${pidFile}", - "-Des.path.repo=${home}/repo", - "-Des.path.shared_data=${home}/../", - ] - esArgs.addAll(config.systemProperties.collect {key, value -> "-D${key}=${value}"}) + List esArgs = config.systemProperties.collect {key, value -> "-D${key}=${value}"} Closure esPostStartActions = { ant, logger -> ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${task.name}#start") { and { @@ -125,12 +134,13 @@ class ClusterFormationTasks { throw new GradleException('Failed to start elasticsearch') } } - Task start; + Task start if (Os.isFamily(Os.FAMILY_WINDOWS)) { // elasticsearch.bat is spawned as it has no daemon mode start = project.tasks.create(name: "${task.name}#start", type: DefaultTask, dependsOn: setup) << { // Fall back to Ant exec task as Gradle Exec task does not support spawning yet ant.exec(executable: 'cmd', spawn: true, dir: home) { + esEnv.each { env(key: key, value: value) } (['/C', 'call', 'bin/elasticsearch'] + esArgs).each { arg(value: it) } } esPostStartActions(ant, logger) @@ -141,6 +151,7 @@ class ClusterFormationTasks { executable 'sh' args 'bin/elasticsearch', '-d' // daemonize! args esArgs + environment esEnv errorOutput = new ByteArrayOutputStream() doLast { if (errorOutput.toString().isEmpty() == false) { @@ -157,9 +168,44 @@ class ClusterFormationTasks { task.dependsOn(start) } + static Task configureExtractTask(Project project, String name, String distro, File baseDir, Task setup) { + List extractDependsOn = [project.configurations.elasticsearchDistro, setup] + Task extract + switch (distro) { + case 'zip': + extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { + from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) } + into baseDir + } + break; + case 'tar': + extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { + from { project.tarTree(project.resources.gzip(project.configurations.elasticsearchDistro.singleFile)) } + into baseDir + } + break; + default: + throw new InvalidUserDataException("Unknown distribution: ${distro}") + } + return extract + } + + static File homeDir(File baseDir, String distro) { + String path + switch (distro) { + case 'zip': + case 'tar': + path = "elasticsearch-${ElasticsearchProperties.version}" + break; + default: + throw new InvalidUserDataException("Unknown distribution: ${distro}") + } + return new File(baseDir, path) + } + static void addNodeStopTask(Project project, Task task, File baseDir) { LazyPidReader pidFile = new LazyPidReader(pidFile: pidFile(baseDir)) - Task stop = project.tasks.create(name: task.name + '#stop', type: Exec) { + Task stop = project.tasks.create(name: "${task.name}#stop", type: Exec) { if (Os.isFamily(Os.FAMILY_WINDOWS)) { executable 'Taskkill' args '/PID', pidFile, '/F' @@ -187,13 +233,14 @@ class ClusterFormationTasks { return new File(dir, 'es.pid') } - static void addZipConfiguration(Project project) { + static void configureDistributionDependency(Project project, String distro) { String elasticsearchVersion = ElasticsearchProperties.version + String packaging = distro == 'tar' ? 'tgz' : distro project.configurations { - elasticsearchZip + elasticsearchDistro } project.dependencies { - elasticsearchZip "org.elasticsearch.distribution.zip:elasticsearch:${elasticsearchVersion}@zip" + elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}" } } } diff --git a/distribution/build.gradle b/distribution/build.gradle index 0ea94659d36ac..7770c0d98abd2 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -36,11 +36,19 @@ buildscript { allprojects { project.ext { // this is common configuration for distributions, but we also add it here for the license check to use - deps = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api') + dependencyFiles = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api') } } subprojects { + /***************************************************************************** + * Rest test config * + *****************************************************************************/ + apply plugin: 'elasticsearch.rest-test' + integTest { + includePackaged true + } + /***************************************************************************** * Maven config * *****************************************************************************/ @@ -50,8 +58,8 @@ subprojects { // we must create our own install task, because it is only added when the java plugin is added task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') { configuration = configurations.archives - MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention); - repositoriesHandler.mavenInstaller(); + MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention) + repositoriesHandler.mavenInstaller() } // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run... @@ -81,7 +89,7 @@ subprojects { libFiles = copySpec { into 'lib' from project("${projectsPrefix}:core").jar - from deps + from dependencyFiles } configFiles = copySpec { @@ -151,7 +159,7 @@ buildDeb.dependsOn createEmptyDir /***************************************************************************** * Deb and rpm configuration * *****************************************************************************/ -configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { +configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) { apply plugin: 'nebula.ospackage-base' ospackage { packageName = 'elasticsearch' @@ -173,29 +181,17 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { } directory('/etc/elasticsearch/scripts') } - if (project.name == 'deb') { - task buildDeb(type: Deb) { - dependsOn deps - } - artifacts { - archives buildDeb - } - } else if (project.name == 'rpm') { - task buildRpm(type: Rpm) { - dependsOn deps - } - artifacts { - archives buildRpm - } - } + + // TODO: re-enable tests when we have real rpm and deb distros! + integTest.enabled = false } // TODO: dependency checks should really be when building the jar itself, which would remove the need // for this hackery and instead we can do this inside the BuildPlugin task check(group: 'Verification', description: 'Runs all checks.') {} // dummy task! DependencyLicensesTask.configure(project) { - dependsOn = [deps] - dependencies = deps + dependsOn = [dependencyFiles] + dependencies = dependencyFiles mapping from: /lucene-.*/, to: 'lucene' mapping from: /jackson-.*/, to: 'jackson' } diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle index 66a6c3b5426f5..24eabe785983b 100644 --- a/distribution/deb/build.gradle +++ b/distribution/deb/build.gradle @@ -1,4 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ -/*task buildDeb(type: Deb) { - dependsOn deps -}*/ +task buildDeb(type: Deb) { + dependsOn dependencyFiles +} + +artifacts { + archives buildDeb +} + +integTest.dependsOn buildDeb diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle index fdf800ba2a4ba..04f0c0bda9b16 100644 --- a/distribution/rpm/build.gradle +++ b/distribution/rpm/build.gradle @@ -1,4 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ -/*task buildRpm(type: Rpm) { - dependsOn deps -}*/ +task buildRpm(type: Rpm) { + dependsOn dependencyFiles +} + +artifacts { + archives buildRpm +} + +integTest.dependsOn buildRpm diff --git a/distribution/tar/build.gradle b/distribution/tar/build.gradle index f5991f349f780..b49058b3e4275 100644 --- a/distribution/tar/build.gradle +++ b/distribution/tar/build.gradle @@ -1,10 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ -task buildTar(type: Tar, dependsOn: deps) { +task buildTar(type: Tar, dependsOn: dependencyFiles) { baseName = 'elasticsearch' with archivesFiles compression = Compression.GZIP } artifacts { + 'default' buildTar archives buildTar } + +integTest { + dependsOn buildTar + clusterConfig.distribution = 'tar' +} diff --git a/distribution/zip/build.gradle b/distribution/zip/build.gradle index 53ad5bc076267..d636e66f1529b 100644 --- a/distribution/zip/build.gradle +++ b/distribution/zip/build.gradle @@ -1,5 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ -task buildZip(type: Zip, dependsOn: deps) { +task buildZip(type: Zip, dependsOn: dependencyFiles) { baseName = 'elasticsearch' with archivesFiles } @@ -9,3 +27,5 @@ artifacts { archives buildZip } +integTest.dependsOn buildZip +