From 6fb68474566a2d8f3c67cd15faea5aa3fdf03bb7 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 25 Jan 2019 12:56:40 +0100 Subject: [PATCH 1/5] Use TestFixturesPlugin to Run Minio in Tests (#37852) * Use TestFixturesPlugin to Run Minio in Tests * Closes #37680 * Closes #37783 --- plugins/repository-s3/build.gradle | 163 +++-------------------- plugins/repository-s3/docker-compose.yml | 9 ++ 2 files changed, 29 insertions(+), 143 deletions(-) create mode 100644 plugins/repository-s3/docker-compose.yml diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 96a90332e187b..7284b2af0204f 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -1,13 +1,10 @@ -import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.LoggedExec +import org.elasticsearch.gradle.BuildPlugin import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.test.AntFixture import org.elasticsearch.gradle.test.ClusterConfiguration import org.elasticsearch.gradle.test.RestIntegTestTask import com.carrotsearch.gradle.junit4.RandomizedTestingTask -import java.lang.reflect.Field - /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -139,25 +136,6 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) { throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present") } - -final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z' -final String minioBinDir = "${buildDir}/minio/bin" -final String minioDataDir = "${buildDir}/minio/data" -final String minioAddress = "127.0.0.1" - -String minioDistribution -String minioCheckSum -if (Os.isFamily(Os.FAMILY_MAC)) { - minioDistribution = 'darwin-amd64' - minioCheckSum = '96b0bcb2f590e8e65fb83d5c3e221f9bd1106b49fa6f22c6b726b80b845d7c60' -} else if (Os.isFamily(Os.FAMILY_UNIX)) { - minioDistribution = 'linux-amd64' - minioCheckSum = '713dac7c105285eab3b92649be92b5e793b29d3525c7929fa7aaed99374fad99' -} else { - minioDistribution = null - minioCheckSum = null -} - buildscript { repositories { maven { @@ -169,140 +147,39 @@ buildscript { } } -if (useFixture && minioDistribution) { - apply plugin: 'de.undercouch.download' - - final String minioFileName = "minio.${minioVersion}" - final String minioDownloadURL = "https://dl.minio.io/server/minio/release/${minioDistribution}/archive/${minioFileName}" - final String minioFilePath = "${gradle.gradleUserHomeDir}/downloads/minio/${minioDistribution}/${minioFileName}" - - task downloadMinio(type: Download) { - src minioDownloadURL - dest minioFilePath - onlyIfModified true - } +if (useFixture) { - task verifyMinioChecksum(type: Verify, dependsOn: downloadMinio) { - src minioFilePath - algorithm 'SHA-256' - checksum minioCheckSum - } + apply plugin: 'elasticsearch.test.fixtures' - task installMinio(type: Sync, dependsOn: verifyMinioChecksum) { - from minioFilePath - into minioBinDir - fileMode 0755 + RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) { + description = "Runs REST tests using the Minio repository." } - task startMinio { - dependsOn installMinio - - ext.minioPid = 0L - ext.minioPort = 0 - + Task writeDockerFile = project.tasks.create('writeDockerFile') { + File minioDockerfile = new File("${project.buildDir}/minio-docker/Dockerfile") + outputs.file(minioDockerfile) doLast { - // get free port - for (int port = 60920; port < 60940; port++) { - try { - javax.net.ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName(minioAddress)).close() - minioPort = port - break - } catch (BindException e) { - logger.info("Port " + port + " for Minio process is already taken", e) - } - } - if (minioPort == 0) { - throw new GradleException("Could not find a free port for Minio") - } - - new File("${minioDataDir}/${s3PermanentBucket}").mkdirs() - // we skip these tests on Windows so we do no need to worry about compatibility here - final ProcessBuilder minio = new ProcessBuilder( - "${minioBinDir}/${minioFileName}", - "server", - "--address", - minioAddress + ":" + minioPort, - minioDataDir) - minio.environment().put('MINIO_ACCESS_KEY', s3PermanentAccessKey) - minio.environment().put('MINIO_SECRET_KEY', s3PermanentSecretKey) - final Process process = minio.start() - if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { - try { - Class cProcessImpl = process.getClass() - Field fPid = cProcessImpl.getDeclaredField("pid") - if (!fPid.isAccessible()) { - fPid.setAccessible(true) - } - minioPid = fPid.getInt(process) - } catch (Exception e) { - logger.error("failed to read pid from minio process", e) - process.destroyForcibly() - throw e - } - } else { - minioPid = process.pid() - } - - new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br -> - String line - int httpPort = 0 - while ((line = br.readLine()) != null) { - logger.info(line) - if (line.matches('.*Endpoint.*:\\d+$')) { - assert httpPort == 0 - final int index = line.lastIndexOf(":") - assert index >= 0 - httpPort = Integer.parseInt(line.substring(index + 1)) - assert httpPort == minioPort : "Port mismatch, expected ${minioPort} but was ${httpPort}" - - final File script = new File(project.buildDir, "minio/minio.killer.sh") - script.setText( - ["function shutdown {", - " kill ${minioPid}", - "}", - "trap shutdown EXIT", - // will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies - "read line\n"].join('\n'), 'UTF-8') - final ProcessBuilder killer = new ProcessBuilder("bash", script.absolutePath) - killer.start() - break - } - } - - assert httpPort > 0 - } - } - } - - task stopMinio(type: LoggedExec) { - onlyIf { startMinio.minioPid > 0 } - - doFirst { - logger.info("Shutting down minio with pid ${startMinio.minioPid}") + minioDockerfile.parentFile.mkdirs() + minioDockerfile.text = "FROM minio/minio:RELEASE.2019-01-23T23-18-58Z\n" + + "RUN mkdir -p /minio/data/${s3PermanentBucket}\n" + + "ENV MINIO_ACCESS_KEY ${s3PermanentAccessKey}\n" + + "ENV MINIO_SECRET_KEY ${s3PermanentSecretKey}" } - - final Object pid = "${ -> startMinio.minioPid }" - - // we skip these tests on Windows so we do no need to worry about compatibility here - executable = 'kill' - args('-9', pid) - } - - RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) { - description = "Runs REST tests using the Minio repository." } + preProcessFixture.dependsOn(writeDockerFile) // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks: project.afterEvaluate { ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration cluster.dependsOn(project.bundlePlugin) - cluster.dependsOn(startMinio) // otherwise we don't know the Minio port + cluster.dependsOn(postProcessFixture) cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey Closure minioAddressAndPort = { - assert startMinio.minioPort > 0 - return 'http://' + minioAddress + ':' + startMinio.minioPort + int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000" + assert minioPort > 0 + return 'http://127.0.0.1:' + minioPort } cluster.setting 's3.client.integration_test_permanent.endpoint', "${ -> minioAddressAndPort.call()}" @@ -317,8 +194,7 @@ if (useFixture && minioDistribution) { restIntegTestTask.clusterConfig.jvmArgs = jvmArgs } - integTestMinioRunner.dependsOn(startMinio) - integTestMinioRunner.finalizedBy(stopMinio) + integTestMinioRunner.dependsOn(postProcessFixture) // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968 integTestMinioRunner.systemProperty 'tests.rest.blacklist', [ 'repository_s3/30_repository_temporary_credentials/*', @@ -327,6 +203,7 @@ if (useFixture && minioDistribution) { ].join(",") project.check.dependsOn(integTestMinio) + BuildPlugin.requireDocker(integTestMinio) } File parentFixtures = new File(project.buildDir, "fixtures") diff --git a/plugins/repository-s3/docker-compose.yml b/plugins/repository-s3/docker-compose.yml new file mode 100644 index 0000000000000..e44750550e271 --- /dev/null +++ b/plugins/repository-s3/docker-compose.yml @@ -0,0 +1,9 @@ +version: '3' +services: + minio-fixture: + build: + context: ./build/minio-docker + dockerfile: Dockerfile + ports: + - "9000" + command: ["server", "/minio/data"] \ No newline at end of file From 501293a009f45c51b196cf7f5e0b5861f61688d0 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 25 Jan 2019 22:55:29 +0100 Subject: [PATCH 2/5] Fix S3 Repository ITs When Docker is not Available (#37878) * Disable Minio fixture and tests that require it when fixtures are disabled or Docker is not available * Relates #37852 --- plugins/repository-s3/build.gradle | 48 ++++++++++++++++-------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 7284b2af0204f..037a28eb25cf8 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -170,28 +170,33 @@ if (useFixture) { preProcessFixture.dependsOn(writeDockerFile) // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks: project.afterEvaluate { - ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration - cluster.dependsOn(project.bundlePlugin) - cluster.dependsOn(postProcessFixture) - cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey - cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey - - Closure minioAddressAndPort = { - int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000" - assert minioPort > 0 - return 'http://127.0.0.1:' + minioPort + // Only configure the Minio tests if postProcessFixture is configured to skip them if Docker is not available + // or fixtures have been disabled + if (postProcessFixture.enabled) { + ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration + cluster.dependsOn(project.bundlePlugin) + cluster.dependsOn(postProcessFixture) + cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey + cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey + + Closure minioAddressAndPort = { + int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000" + assert minioPort > 0 + return 'http://127.0.0.1:' + minioPort + } + cluster.setting 's3.client.integration_test_permanent.endpoint', "${-> minioAddressAndPort.call()}" + + Task restIntegTestTask = project.tasks.getByName('integTestMinio') + restIntegTestTask.clusterConfig.plugin(project.path) + + // Default jvm arguments for all test clusters + String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') + + " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') + + " " + System.getProperty('tests.jvm.argline', '') + + restIntegTestTask.clusterConfig.jvmArgs = jvmArgs + project.check.dependsOn(integTestMinio) } - cluster.setting 's3.client.integration_test_permanent.endpoint', "${ -> minioAddressAndPort.call()}" - - Task restIntegTestTask = project.tasks.getByName('integTestMinio') - restIntegTestTask.clusterConfig.plugin(project.path) - - // Default jvm arguments for all test clusters - String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') + - " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') + - " " + System.getProperty('tests.jvm.argline', '') - - restIntegTestTask.clusterConfig.jvmArgs = jvmArgs } integTestMinioRunner.dependsOn(postProcessFixture) @@ -202,7 +207,6 @@ if (useFixture) { 'repository_s3/50_repository_ecs_credentials/*' ].join(",") - project.check.dependsOn(integTestMinio) BuildPlugin.requireDocker(integTestMinio) } From e9990468f37eef6963ad9612710b26d0a5eb37ac Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 7 Mar 2019 10:22:44 -0800 Subject: [PATCH 3/5] add explicit composeUp dependsOn --- plugins/repository-s3/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 037a28eb25cf8..3c47646d55e65 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -167,6 +167,7 @@ if (useFixture) { } } + preProcessFixture.dependsOn(composeUp) preProcessFixture.dependsOn(writeDockerFile) // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks: project.afterEvaluate { From d920a7aed9a339bad8a023418cadfa3dae3c3f02 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 8 Mar 2019 12:57:16 -0800 Subject: [PATCH 4/5] reverse roles --- plugins/repository-s3/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 3c47646d55e65..63bc39049e2be 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -167,8 +167,9 @@ if (useFixture) { } } - preProcessFixture.dependsOn(composeUp) preProcessFixture.dependsOn(writeDockerFile) + composeUp.dependsOn(preProcessFixture) + // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks: project.afterEvaluate { // Only configure the Minio tests if postProcessFixture is configured to skip them if Docker is not available From c8c322aec415ff5fe9dd72e06078683ef5e1dbf2 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 8 Mar 2019 12:58:21 -0800 Subject: [PATCH 5/5] reversereverse --- plugins/repository-s3/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 63bc39049e2be..14ee80d075a1e 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -168,7 +168,7 @@ if (useFixture) { } preProcessFixture.dependsOn(writeDockerFile) - composeUp.dependsOn(preProcessFixture) + composeUp.dependsOn(writeDockerFile) // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks: project.afterEvaluate {