From c76b2910625e42410f7951b4183528bd1b642297 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 13 Jan 2021 12:02:41 +0000 Subject: [PATCH] Extend the selective muting of memory tests on Debian 8 The selective muting implemented for autoscaling in #67159 is extended to the ML tests that also fail when machine memory is reported as 0. Most of the logic to determine when memory will not be accurately reported is now in a utility method in the base class. Relates #66885 --- .../elasticsearch/test/ESIntegTestCase.java | 20 +++++++++++++++++++ ...ansportGetAutoscalingCapacityActionIT.java | 10 +--------- .../integration/BasicDistributedJobsIT.java | 4 ++++ .../integration/MlDistributedFailureIT.java | 3 +++ .../xpack/ml/integration/TooManyJobsIT.java | 4 ++++ 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 6d1026ebbaa42..7228da77f36c5 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -56,6 +56,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.ClusterAdminClient; @@ -120,6 +121,7 @@ import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.store.IndicesStore; +import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.node.NodeMocksPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; @@ -2232,4 +2234,22 @@ public static String resolveCustomDataPath(String index) { public static boolean inFipsJvm() { return Boolean.parseBoolean(System.getProperty(FIPS_SYSPROP)); } + + /** + * On Debian 8 the "memory" subsystem is not mounted by default + * when cgroups are enabled, and this confuses many versions of + * Java prior to Java 15. Tests that rely on machine memory + * being accurately determined will not work on such setups, + * and can use this method for selective muting. + * See https://github.com/elastic/elasticsearch/issues/67089 + * and https://github.com/elastic/elasticsearch/issues/66885 + */ + protected boolean willSufferDebian8MemoryProblem() { + final NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet(); + final boolean anyDebian8Nodes = response.getNodes() + .stream() + .anyMatch(ni -> ni.getInfo(OsInfo.class).getPrettyName().equals("Debian GNU/Linux 8 (jessie)")); + boolean java15Plus = JavaVersion.current().compareTo(JavaVersion.parse("15")) >= 0; + return anyDebian8Nodes && java15Plus == false; + } } diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityActionIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityActionIT.java index d9421bca73cb0..4ab173297f457 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityActionIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityActionIT.java @@ -6,10 +6,7 @@ package org.elasticsearch.xpack.autoscaling.action; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.monitor.os.OsProbe; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.autoscaling.AutoscalingIntegTestCase; @@ -27,13 +24,8 @@ public class TransportGetAutoscalingCapacityActionIT extends AutoscalingIntegTestCase { public void testCurrentCapacity() throws Exception { - final NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet(); - final boolean anyDebian8Nodes = response.getNodes() - .stream() - .anyMatch(ni -> ni.getInfo(OsInfo.class).getPrettyName().equals("Debian GNU/Linux 8 (jessie)")); - boolean java15Plus = JavaVersion.current().compareTo(JavaVersion.parse("15")) >= 0; // see: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654 - assumeTrue("cannot run on debian 8 prior to java 15", java15Plus || anyDebian8Nodes == false); + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); assertThat(capacity().results().keySet(), Matchers.empty()); long memory = OsProbe.getInstance().getTotalPhysicalMemorySize(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index a2b3fa98e4599..fba3055cc7d7e 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -420,6 +420,10 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception { } public void testCloseUnassignedLazyJobAndDatafeed() throws Exception { + + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); + internalCluster().ensureAtLeastNumDataNodes(3); ensureStableCluster(3); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index 2645c29aaa553..2cbbc04bd8d01 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -397,6 +397,9 @@ public void testStopAndForceStopDatafeed() throws Exception { public void testJobRelocationIsMemoryAware() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); + internalCluster().ensureAtLeastNumDataNodes(1); ensureStableCluster(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index 8a44cbb0b13d8..ddd17350acc35 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -126,10 +126,14 @@ public void testLazyNodeValidation() throws Exception { } public void testSingleNode() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); verifyMaxNumberOfJobsLimit(1, randomIntBetween(1, 20), randomBoolean()); } public void testMultipleNodes() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); verifyMaxNumberOfJobsLimit(3, randomIntBetween(1, 20), randomBoolean()); }