From 8a3a92f9d5c7ae3799238b56009cf1c679ff6aed Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 24 Jan 2019 17:26:42 +0200 Subject: [PATCH] Ground work to start up the docker image in the build (#37754) This change adds a docker compose configuration that's used with the `elasticsearch.test.fixtures` plugin to start up the image and check that the TCP ports are up. We can build on this to add other checks for culster health, run REST tests, etc. We can add multiple containers and configurations to the compose file (e.x. test different env vars) and form clusters. --- .../testfixtures/TestFixtureExtension.java | 1 - .../testfixtures/TestFixturesPlugin.java | 92 +++++++++---------- distribution/docker/build.gradle | 17 ++++ distribution/docker/docker-compose.yml | 21 +++++ x-pack/test/smb-fixture/build.gradle | 2 + 5 files changed, 85 insertions(+), 48 deletions(-) create mode 100644 distribution/docker/docker-compose.yml diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java index edc0fd09f1677..b4ddcf0bed183 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java @@ -44,5 +44,4 @@ public void useFixture(String path) { fixtures.add(fixtureProject); } - } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java index 32a50fb4b0750..35a7eacf1fde1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java @@ -20,6 +20,7 @@ import com.avast.gradle.dockercompose.ComposeExtension; import com.avast.gradle.dockercompose.DockerComposePlugin; +import com.avast.gradle.dockercompose.tasks.ComposeUp; import org.elasticsearch.gradle.precommit.JarHellTask; import org.elasticsearch.gradle.precommit.TestingConventionsTasks; import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask; @@ -54,7 +55,6 @@ public void apply(Project project) { // convenience boilerplate with build plugin // Can't reference tasks that are implemented in Groovy, use reflection instead disableTaskByType(tasks, getTaskClass("org.elasticsearch.gradle.precommit.LicenseHeadersTask")); - disableTaskByType(tasks, getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask")); disableTaskByType(tasks, ThirdPartyAuditTask.class); disableTaskByType(tasks, JarHellTask.class); @@ -62,7 +62,6 @@ public void apply(Project project) { Task preProcessFixture = project.getTasks().create("preProcessFixture"); buildFixture.dependsOn(preProcessFixture); Task postProcessFixture = project.getTasks().create("postProcessFixture"); - buildFixture.dependsOn(postProcessFixture); if (dockerComposeSupported(project) == false) { preProcessFixture.setEnabled(false); @@ -83,7 +82,7 @@ public void apply(Project project) { buildFixture.dependsOn(tasks.getByName("composeUp")); tasks.getByName("composeUp").mustRunAfter(preProcessFixture); - postProcessFixture.dependsOn("composeUp"); + postProcessFixture.dependsOn(buildFixture); configureServiceInfoForTask( postProcessFixture, @@ -91,38 +90,36 @@ public void apply(Project project) { (name, port) -> postProcessFixture.getExtensions() .getByType(ExtraPropertiesExtension.class).set(name, port) ); - } else { - extension.fixtures.all(fixtureProject -> project.evaluationDependsOn(fixtureProject.getPath())); - if (dockerComposeSupported(project) == false) { - project.getLogger().warn( - "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " + - "but none could be found so these will be skipped", project.getPath() + } + + extension.fixtures.all(fixtureProject -> project.evaluationDependsOn(fixtureProject.getPath())); + if (dockerComposeSupported(project) == false) { + project.getLogger().warn( + "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " + + "but none could be found so these will be skipped", project.getPath() + ); + disableTaskByType(tasks, getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask")); + // conventions are not honored when the tasks are disabled + disableTaskByType(tasks, TestingConventionsTasks.class); + disableTaskByType(tasks, ComposeUp.class); + return; + } + tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task -> + extension.fixtures.all(fixtureProject -> { + fixtureProject.getTasks().matching(it -> it.getName().equals("buildFixture")).all(buildFixture -> + task.dependsOn(buildFixture) ); - tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task -> - task.setEnabled(false) + fixtureProject.getTasks().matching(it -> it.getName().equals("composeDown")).all(composeDown -> + task.finalizedBy(composeDown) ); - // conventions are not honored when the tasks are disabled - tasks.withType(TestingConventionsTasks.class, task -> - task.setEnabled(false) + configureServiceInfoForTask( + task, + fixtureProject, + (name, port) -> setSystemProperty(task, name, port) ); - return; - } - tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task -> - extension.fixtures.all(fixtureProject -> { - fixtureProject.getTasks().matching(it->it.getName().equals("buildFixture")).all(buildFixture -> - task.dependsOn(buildFixture) - ); - fixtureProject.getTasks().matching(it->it.getName().equals("composeDown")).all(composeDown -> - task.finalizedBy(composeDown) - ); - configureServiceInfoForTask( - task, - fixtureProject, - (name, port) -> setSystemProperty(task, name, port) - ); - }) - ); - } + }) + ); + } private void configureServiceInfoForTask(Task task, Project fixtureProject, BiConsumer consumer) { @@ -131,23 +128,24 @@ private void configureServiceInfoForTask(Task task, Project fixtureProject, BiCo task.doFirst(theTask -> fixtureProject.getExtensions().getByType(ComposeExtension.class).getServicesInfos() .forEach((service, infos) -> { - theTask.getLogger().info( - "Port maps for {}\nTCP:{}\nUDP:{}\nexposed to {}", - fixtureProject.getPath(), - infos.getTcpPorts(), - infos.getUdpPorts(), - theTask.getPath() - ); infos.getTcpPorts() - .forEach((container, host) -> consumer.accept( - "test.fixtures." + service + ".tcp." + container, - host - )); + .forEach((container, host) -> { + String name = "test.fixtures." + service + ".tcp." + container; + theTask.getLogger().info("port mapping property: {}={}", name, host); + consumer.accept( + name, + host + ); + }); infos.getUdpPorts() - .forEach((container, host) -> consumer.accept( - "test.fixtures." + service + ".udp." + container, - host - )); + .forEach((container, host) -> { + String name = "test.fixtures." + service + ".udp." + container; + theTask.getLogger().info("port mapping property: {}={}", name, host); + consumer.accept( + name, + host + ); + }); }) ); } diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 219d81e1117ec..4d228e3c44fdc 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -4,6 +4,7 @@ import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.VersionProperties apply plugin: 'base' +apply plugin: 'elasticsearch.test.fixtures' configurations { dockerPlugins @@ -68,6 +69,22 @@ void addCopyDockerfileTask(final boolean oss) { } } + +preProcessFixture { + dependsOn taskName("copy", true, "DockerContext") + dependsOn taskName("copy", true, "Dockerfile") + dependsOn taskName("copy", false, "DockerContext") + dependsOn taskName("copy", false, "Dockerfile") +} + +postProcessFixture.doLast { + println "docker default distro is on port: ${ext."test.fixtures.elasticsearch-default.tcp.9200"}, " + + "oss is on: ${ext."test.fixtures.elasticsearch-oss.tcp.9200"}" +} + +// TODO: Add some actual tests, this will just check that the TPC port in the container is up +check.dependsOn postProcessFixture + void addBuildDockerImage(final boolean oss) { final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) { dependsOn taskName("copy", oss, "DockerContext") diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml new file mode 100644 index 0000000000000..3f220aa9e91c0 --- /dev/null +++ b/distribution/docker/docker-compose.yml @@ -0,0 +1,21 @@ +# Only used for testing the docker images +version: '3' +services: + elasticsearch-default: + build: + context: ./build/docker + dockerfile: Dockerfile + environment: + - cluster.name=elasticsearch-default + - discovery.type=single-node + ports: + - "9200" + elasticsearch-oss: + build: + context: ./build/oss-docker + dockerfile: Dockerfile + environment: + - cluster.name=elasticsearch-oss + - discovery.type=single-node + ports: + - "9200" diff --git a/x-pack/test/smb-fixture/build.gradle b/x-pack/test/smb-fixture/build.gradle index c361737e22c6d..846c38829870a 100644 --- a/x-pack/test/smb-fixture/build.gradle +++ b/x-pack/test/smb-fixture/build.gradle @@ -1,2 +1,4 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.test.fixtures' + +unitTest.enabled = false