[7.17] Migrate core rest tests with security to new testing framework (…

…#92575) (#92686)

* Migrate core rest tests with security to new testing framework (#92575)

# Conflicts:
#	x-pack/qa/core-rest-tests-with-security/build.gradle

* Fixes

* More fixes

* More fixes

* More more fixes

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -55,7 +55,8 @@ public void apply(Project project) {
         project.getTasks().withType(Test.class).configureEach(test -> {
             File testOutputDir = new File(test.getReports().getJunitXml().getOutputLocation().getAsFile().get(), "output");
 
-            ErrorReportingTestListener listener = new ErrorReportingTestListener(test.getTestLogging(), test.getLogger(), testOutputDir);
+            ErrorReportingTestListener listener = new ErrorReportingTestListener(test, testOutputDir);
+            test.getExtensions().getExtraProperties().set("dumpOutputOnFailure", true);
             test.getExtensions().add("errorReportingTestListener", listener);
             test.addTestOutputListener(listener);
             test.addTestListener(listener);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java

@@ -10,12 +10,12 @@
 import org.gradle.api.internal.tasks.testing.logging.FullExceptionFormatter;
 import org.gradle.api.internal.tasks.testing.logging.TestExceptionFormatter;
 import org.gradle.api.logging.Logger;
+import org.gradle.api.tasks.testing.Test;
 import org.gradle.api.tasks.testing.TestDescriptor;
 import org.gradle.api.tasks.testing.TestListener;
 import org.gradle.api.tasks.testing.TestOutputEvent;
 import org.gradle.api.tasks.testing.TestOutputListener;
 import org.gradle.api.tasks.testing.TestResult;
-import org.gradle.api.tasks.testing.logging.TestLogging;
 
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
@@ -39,16 +39,18 @@
 public class ErrorReportingTestListener implements TestOutputListener, TestListener {
     private static final String REPRODUCE_WITH_PREFIX = "REPRODUCE WITH";
 
+    private final Test testTask;
     private final TestExceptionFormatter formatter;
     private final File outputDirectory;
     private final Logger taskLogger;
     private Map<Descriptor, EventWriter> eventWriters = new ConcurrentHashMap<>();
     private Map<Descriptor, Deque<String>> reproductionLines = new ConcurrentHashMap<>();
     private Set<Descriptor> failedTests = new LinkedHashSet<>();
 
-    public ErrorReportingTestListener(TestLogging testLogging, Logger taskLogger, File outputDirectory) {
-        this.formatter = new FullExceptionFormatter(testLogging);
-        this.taskLogger = taskLogger;
+    public ErrorReportingTestListener(Test testTask, File outputDirectory) {
+        this.testTask = testTask;
+        this.formatter = new FullExceptionFormatter(testTask.getTestLogging());
+        this.taskLogger = testTask.getLogger();
         this.outputDirectory = outputDirectory;
     }
 
@@ -81,34 +83,37 @@ public void afterSuite(final TestDescriptor suite, TestResult result) {
         Descriptor descriptor = Descriptor.of(suite);
 
         try {
-            // if the test suite failed, report all captured output
-            if (result.getResultType().equals(TestResult.ResultType.FAILURE)) {
-                EventWriter eventWriter = eventWriters.get(descriptor);
-
-                if (eventWriter != null) {
-                    // It's not explicit what the threading guarantees are for TestListener method execution so we'll
-                    // be explicitly safe here to avoid interleaving output from multiple test suites
-                    synchronized (this) {
-                        // make sure we've flushed everything to disk before reading
-                        eventWriter.flush();
-
-                        System.err.println("\n\nSuite: " + suite);
-
-                        try (BufferedReader reader = eventWriter.reader()) {
-                            PrintStream out = System.out;
-                            for (String message = reader.readLine(); message != null; message = reader.readLine()) {
-                                if (message.startsWith("  1> ")) {
-                                    out = System.out;
-                                } else if (message.startsWith("  2> ")) {
-                                    out = System.err;
+            if (isDumpOutputEnabled()) {
+                // if the test suite failed, report all captured output
+                if (result.getResultType().equals(TestResult.ResultType.FAILURE)) {
+                    EventWriter eventWriter = eventWriters.get(descriptor);
+
+                    if (eventWriter != null) {
+                        // It's not explicit what the threading guarantees are for TestListener method execution so we'll
+                        // be explicitly safe here to avoid interleaving output from multiple test suites
+                        synchronized (this) {
+                            // make sure we've flushed everything to disk before reading
+                            eventWriter.flush();
+
+                            System.err.println("\n\nSuite: " + suite);
+
+                            try (BufferedReader reader = eventWriter.reader()) {
+                                PrintStream out = System.out;
+                                for (String message = reader.readLine(); message != null; message = reader.readLine()) {
+                                    if (message.startsWith("  1> ")) {
+                                        out = System.out;
+                                    } else if (message.startsWith("  2> ")) {
+                                        out = System.err;
+                                    }
+
+                                    out.println(message);
                                 }
-
-                                out.println(message);
                             }
                         }
                     }
                 }
             }
+
             if (suite.getParent() == null) {
                 // per test task top level gradle test run suite finished
                 if (getFailedTests().size() > 0) {
@@ -281,4 +286,9 @@ public void close() throws IOException {
             outputFile.delete();
         }
     }
+
+    private boolean isDumpOutputEnabled() {
+        Object errorReportingEnabled = testTask.getExtensions().getExtraProperties().get("dumpOutputOnFailure");
+        return errorReportingEnabled == null || (boolean) errorReportingEnabled;
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -112,8 +112,12 @@ public void apply(Project project) {
             task.dependsOn(integTestDistro, modulesConfiguration);
             registerDistributionInputs(task, integTestDistro);
 
+            // Enable parallel execution for these tests since each test gets its own cluster
             task.setMaxParallelForks(task.getProject().getGradle().getStartParameter().getMaxWorkerCount() / 2);
 
+            // Disable test failure reporting since this stuff is now captured in build scans
+            task.getExtensions().getExtraProperties().set("dumpOutputOnFailure", false);
+
             // Disable the security manager and syscall filter since the test framework needs to fork processes
             task.systemProperty("tests.security.manager", "false");
             task.systemProperty("tests.system_call_filter", "false");

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -332,6 +332,9 @@ private void commonNodeConfig() {
         }
         ElasticsearchNode firstNode = null;
         for (ElasticsearchNode node : nodes) {
+            if (node.getTestDistribution().equals(TestDistribution.INTEG_TEST)) {
+                node.defaultConfig.put("xpack.security.enabled", "false");
+            }
             if (node.getTestDistribution().equals(TestDistribution.DEFAULT)) {
                 if (node.getVersion().onOrAfter("7.16.0")) {
                     node.defaultConfig.put("cluster.deprecation_indexing.enabled", "false");

distribution/archives/build.gradle

@@ -65,7 +65,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
 distribution_archives {
   integTestZip {
     content {
-      archiveFiles(transportModulesFiles, 'zip', null, 'x64', true, false)
+      archiveFiles(integTestModulesFiles, 'zip', null, 'x64', true, false)
     }
   }
 

distribution/build.gradle

@@ -99,7 +99,7 @@ tasks.withType(NoticeTask).configureEach {
 String ossOutputs = 'build/outputs/oss'
 String defaultOutputs = 'build/outputs/default'
 String systemdOutputs = 'build/outputs/systemd'
-String transportOutputs = 'build/outputs/transport-only'
+String integTestOutputs = 'build/outputs/integ-test'
 String externalTestOutputs = 'build/outputs/external-test'
 
 def processDefaultOutputsTaskProvider = tasks.register("processDefaultOutputs", Sync) {
@@ -116,8 +116,9 @@ def processExternalTestOutputsTaskProvider = tasks.register("processExternalTest
 
 // Integ tests work over the rest http layer, so we need a transport included with the integ test zip.
 // All transport modules are included so that they may be randomized for testing
-def processTransportOutputsTaskProvider = tasks.register("processTransportOutputs", Sync) {
-  into transportOutputs
+// Additionally install security plugin so we can run tests with security enabled using this distribution
+def processIntegTestOutputsTaskProvider = tasks.register("processIntegTestOutputs", Sync) {
+  into integTestOutputs
 }
 
 def defaultModulesFiles = fileTree("${defaultOutputs}/modules") {
@@ -127,17 +128,23 @@ def defaultModulesFiles = fileTree("${defaultOutputs}/modules") {
 def defaultBinFiles = fileTree("${defaultOutputs}/bin") {
   builtBy processDefaultOutputsTaskProvider
 }
+def integTestBinFiles = fileTree("${integTestOutputs}/bin") {
+  builtBy processIntegTestOutputsTaskProvider
+}
 def defaultConfigFiles = fileTree("${defaultOutputs}/config") {
   builtBy processDefaultOutputsTaskProvider
 }
+def integTestConfigFiles = fileTree("${integTestOutputs}/config") {
+  builtBy processIntegTestOutputsTaskProvider
+}
 
 def systemdModuleFiles = fileTree("${systemdOutputs}/modules") {
   builtBy processSystemdOutputsTaskProvider
 }
 
-def buildTransportModulesTaskProvider = tasks.register("buildTransportModules") {
-  dependsOn processTransportOutputsTaskProvider
-  outputs.dir "${transportOutputs}/modules"
+def buildIntegTestModulesTaskProvider = tasks.register("buildIntegTestModules") {
+  dependsOn processIntegTestOutputsTaskProvider
+  outputs.dir "${integTestOutputs}/modules"
 }
 def buildExternalTestModulesTaskProvider = tasks.register("buildExternalTestModules") {
   dependsOn "processExternalTestOutputs"
@@ -226,7 +233,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {
 
   copyModule(processDefaultOutputsTaskProvider, module)
   if (module.name.startsWith('transport-')) {
-    copyModule(processTransportOutputsTaskProvider, module)
+    copyModule(processIntegTestOutputsTaskProvider, module)
   }
 
   restTestExpansions['expected.modules.count'] += 1
@@ -243,6 +250,9 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
     }
   }
   copyModule(processDefaultOutputsTaskProvider, xpackModule)
+  if (xpackModule.name.equals('core') || xpackModule.name.equals('security')) {
+    copyModule(processIntegTestOutputsTaskProvider, xpackModule)
+  }
 }
 
 copyModule(processSystemdOutputsTaskProvider, project(':modules:systemd'))
@@ -364,8 +374,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
-    transportModulesFiles = copySpec {
-      from buildTransportModulesTaskProvider
+    integTestModulesFiles = copySpec {
+      from buildIntegTestModulesTaskProvider
     }
 
     configFiles = { distributionType, testDistro, jdk ->
@@ -377,7 +387,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           filter("tokens" : expansionsForDistribution(distributionType, testDistro, jdk), ReplaceTokens.class)
         }
         from buildDefaultLog4jConfigTaskProvider
-        from defaultConfigFiles
+        from testDistro ? integTestConfigFiles : defaultConfigFiles
       }
     }
 
@@ -407,9 +417,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         // module provided bin files
         with copySpec {
           eachFile { it.setMode(0755) }
-          if(testDistro == false) {
-            from(defaultBinFiles)
-          }
+          from(testDistro ? integTestBinFiles : defaultBinFiles)
           if (distributionType != 'zip') {
             exclude '*.bat'
           }

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -186,6 +186,7 @@ public static boolean hasXPack() {
      */
     private static RestClient adminClient;
     private static Boolean hasXPack;
+    private static Boolean hasShutdown;
     private static TreeSet<Version> nodeVersions;
 
     @Before
@@ -194,6 +195,7 @@ public void initClient() throws IOException {
             assert adminClient == null;
             assert clusterHosts == null;
             assert hasXPack == null;
+            assert hasShutdown == null;
             assert nodeVersions == null;
             String cluster = getTestRestCluster();
             String[] stringUrls = cluster.split(",");
@@ -213,6 +215,7 @@ public void initClient() throws IOException {
             adminClient = buildClient(restAdminSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()]));
 
             hasXPack = false;
+            hasShutdown = false;
             nodeVersions = new TreeSet<>();
             Map<?, ?> response = entityAsMap(adminClient.performRequest(new Request("GET", "_nodes/plugins")));
             Map<?, ?> nodes = (Map<?, ?>) response.get("nodes");
@@ -221,16 +224,21 @@ public void initClient() throws IOException {
                 nodeVersions.add(Version.fromString(nodeInfo.get("version").toString()));
                 for (Object module : (List<?>) nodeInfo.get("modules")) {
                     Map<?, ?> moduleInfo = (Map<?, ?>) module;
-                    if (moduleInfo.get("name").toString().startsWith("x-pack-")) {
+                    String moduleName = moduleInfo.get("name").toString();
+                    if (moduleName.startsWith("x-pack-")) {
                         hasXPack = true;
                     }
+                    if (moduleName.equals("x-pack-shutdown")) {
+                        hasShutdown = true;
+                    }
                 }
             }
         }
         assert client != null;
         assert adminClient != null;
         assert clusterHosts != null;
         assert hasXPack != null;
+        assert hasShutdown != null;
         assert nodeVersions != null;
     }
 
@@ -397,6 +405,7 @@ public static void closeClients() throws IOException {
             client = null;
             adminClient = null;
             hasXPack = null;
+            hasShutdown = null;
             nodeVersions = null;
         }
     }
@@ -897,7 +906,7 @@ private Set<String> getAllUnexpectedTemplates() throws IOException {
      */
     @SuppressWarnings("unchecked")
     protected void deleteAllNodeShutdownMetadata() throws IOException {
-        if (hasXPack() == false || minimumNodeVersion().before(Version.V_7_15_0)) {
+        if (hasShutdown == false || minimumNodeVersion().before(Version.V_7_15_0)) {
             // Node shutdown APIs are only present in xpack
             return;
         }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSettingsProvider.java

@@ -28,7 +28,9 @@ public Map<String, String> get(LocalNodeSpec nodeSpec) {
         settings.put("transport.port", "0");
         settings.put("network.host", "_local_");
 
-        if (nodeSpec.getDistributionType() == DistributionType.DEFAULT) {
+        if (nodeSpec.getDistributionType() == DistributionType.INTEG_TEST) {
+            settings.put("xpack.security.enabled", "false");
+        } else {
             // Disable deprecation indexing which is enabled by default in 7.16
             if (nodeSpec.getVersion().onOrAfter("7.16.0")) {
                 settings.put("cluster.deprecation_indexing.enabled", "false");

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java

@@ -430,9 +430,7 @@ private void installModule(String moduleName, List<Path> modulePaths) {
             Path destination = distributionDir.resolve("modules").resolve(moduleName);
             if (Files.notExists(destination)) {
                 Path modulePath = modulePaths.stream()
-                    .map(path -> Pair.of(BUNDLE_ARTIFACT_PATTERN.matcher(path.getFileName().toString()), path))
-                    .filter(pair -> pair.left.matches())
-                    .map(p -> p.right.getParent().resolve(p.left.group(1)))
+                    .filter(path -> path.getFileName().toString().startsWith(moduleName))
                     .findFirst()
                     .orElseThrow(() -> {
                         String taskPath = System.getProperty("tests.task");

x-pack/qa/core-rest-tests-with-security/build.gradle

@@ -1,11 +1,13 @@
-import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.info.BuildParams
-
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.authenticated-testclusters'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 dependencies {
   testImplementation project(':x-pack:qa')
+  clusterModules project(':modules:mapper-extras')
+  clusterModules project(':modules:rank-eval')
+  clusterModules project(xpackModule('stack'))
+  clusterModules project(xpackModule('ilm'))
+  clusterModules project(xpackModule('mapper-constant-keyword'))
+  clusterModules project(xpackModule('data-streams'))
 }
 
 restResources {
@@ -21,11 +23,3 @@ tasks.named("yamlRestTest").configure {
         'indices.get_alias/10_basic/Get alias against closed indices'
       ].join(',')
 }
-
-testClusters.matching { it.name == "yamlRestTest" }.configureEach {
-  testDistribution = 'DEFAULT'
-  setting 'xpack.watcher.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'indices.lifecycle.history_index_enabled', 'false'
-}