[Profiling] Load integration test data from file (#101947)

With this commit we remove the hardcoded document construction from
profiling integration tests and instead load test data from ndjson
files.

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/CancellationIT.java

@@ -56,12 +56,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return plugins;
     }
 
-    @Override
-    protected boolean useOnlyAllEvents() {
-        // we assume that all indices have been created to simplify the testing logic.
-        return false;
-    }
-
     public void testAutomaticCancellation() throws Exception {
         Request restRequest = new Request("POST", "/_profiling/stacktraces");
         restRequest.setEntity(new StringEntity("""

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java

@@ -8,19 +8,14 @@
 package org.elasticsearch.xpack.profiling;
 
 public class GetFlameGraphActionIT extends ProfilingTestCase {
-    @Override
-    protected boolean useOnlyAllEvents() {
-        return true;
-    }
-
     public void testGetStackTracesUnfiltered() throws Exception {
-        GetStackTracesRequest request = new GetStackTracesRequest(1, null);
+        GetStackTracesRequest request = new GetStackTracesRequest(10, null);
         GetFlamegraphResponse response = client().execute(GetFlamegraphAction.INSTANCE, request).get();
         // only spot-check top level properties - detailed tests are done in unit tests
-        assertEquals(4, response.getSize());
+        assertEquals(231, response.getSize());
         assertEquals(1.0d, response.getSamplingRate(), 0.001d);
-        assertEquals(3, response.getSelfCPU());
-        assertEquals(4, response.getTotalCPU());
-        assertEquals(1, response.getTotalSamples());
+        assertEquals(60, response.getSelfCPU());
+        assertEquals(1204, response.getTotalCPU());
+        assertEquals(40, response.getTotalSamples());
     }
 }

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java

@@ -11,23 +11,28 @@
 
 public class GetStackTracesActionIT extends ProfilingTestCase {
     public void testGetStackTracesUnfiltered() throws Exception {
-        GetStackTracesRequest request = new GetStackTracesRequest(1, null);
+        GetStackTracesRequest request = new GetStackTracesRequest(10, null);
+        request.setAdjustSampleCount(true);
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
-        assertEquals(1, response.getTotalFrames());
+        assertEquals(40, response.getTotalSamples());
+        assertEquals(285, response.getTotalFrames());
+
+        assertNotNull(response.getStackTraceEvents());
+        assertEquals(4, (int) response.getStackTraceEvents().get("14cFLjgoe-BTQd17mhedeA"));
+
         assertNotNull(response.getStackTraces());
-        StackTrace stackTrace = response.getStackTraces().get("QjoLteG7HX3VUUXr-J4kHQ");
-        assertEquals(List.of(1083999), stackTrace.addressOrLines);
-        assertEquals(List.of("QCCDqjSg3bMK1C4YRK6Tiw"), stackTrace.fileIds);
-        assertEquals(List.of("QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf"), stackTrace.frameIds);
-        assertEquals(List.of(2), stackTrace.typeIds);
+        // just do a high-level spot check. Decoding is tested in unit-tests
+        StackTrace stackTrace = response.getStackTraces().get("JvISdnJ47BQ01489cwF9DA");
+        assertEquals(4, stackTrace.addressOrLines.size());
+        assertEquals(4, stackTrace.fileIds.size());
+        assertEquals(4, stackTrace.frameIds.size());
+        assertEquals(4, stackTrace.typeIds.size());
 
         assertNotNull(response.getStackFrames());
-        StackFrame stackFrame = response.getStackFrames().get("QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf");
-        assertEquals(List.of("_raw_spin_unlock_irqrestore", "inlined_frame_1", "inlined_frame_0"), stackFrame.functionName);
-        assertNotNull(response.getStackTraceEvents());
-        assertEquals(1, (int) response.getStackTraceEvents().get("QjoLteG7HX3VUUXr-J4kHQ"));
+        StackFrame stackFrame = response.getStackFrames().get("lHp5_WAgpLy2alrUVab6HAAAAAAATgeq");
+        assertEquals(List.of("blkdev_issue_flush"), stackFrame.functionName);
 
         assertNotNull(response.getExecutables());
-        assertNotNull("libc.so.6", response.getExecutables().get("QCCDqjSg3bMK1C4YRK6Tiw"));
+        assertNotNull("vmlinux", response.getExecutables().get("lHp5_WAgpLy2alrUVab6HA"));
     }
 }

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
@@ -19,6 +20,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.transport.netty4.Netty4Plugin;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.ilm.IndexLifecycle;
@@ -27,7 +29,7 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.time.Instant;
+import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -71,18 +73,6 @@ private void indexDoc(String index, String id, Map<String, Object> source) {
         assertEquals(RestStatus.CREATED, indexResponse.status());
     }
 
-    /**
-     * Only the index "profiling-events-all" is always present. All other indices (e.g. "profiling-events-5pow02") are created on demand
-     * at a later point when there are enough samples. With this flag we simulate that data should be retrieved briefly after cluster
-     * start when only profiling-events-all is present. We expect that also in this case, available data is returned but we rely only
-     * on the single existing index.
-     *
-     * @return <code>true</code> iff this test should rely on only "profiling-events-all" being present.
-     */
-    protected boolean useOnlyAllEvents() {
-        return randomBoolean();
-    }
-
     /**
      * @return <code>true</code> iff this test relies that data (and the corresponding indices / data streams) are present for this test.
      */
@@ -112,37 +102,30 @@ protected void updateProfilingTemplatesEnabled(boolean newValue) {
         assertTrue("Update of profiling templates enabled setting is not acknowledged", response.isAcknowledged());
     }
 
+    protected final byte[] read(String resource) throws IOException {
+        return ProfilingTestCase.class.getClassLoader().getResourceAsStream(resource).readAllBytes();
+    }
+
+    protected final void bulkIndex(String file) throws Exception {
+        byte[] bulkData = read(file);
+        BulkResponse response = client().prepareBulk().add(bulkData, 0, bulkData.length, XContentType.JSON).execute().actionGet();
+        assertFalse(response.hasFailures());
+    }
+
     @Before
     public void setupData() throws Exception {
         if (requiresDataSetup() == false) {
             return;
         }
         // only enable index management while setting up indices to avoid interfering with the rest of the test infrastructure
         updateProfilingTemplatesEnabled(true);
-        Collection<String> eventsIndices = useOnlyAllEvents() ? List.of(EventsIndex.FULL_INDEX.getName()) : EventsIndex.indexNames();
         waitForIndices();
         ensureGreen();
 
-        // ensure that we have this in every index, so we find an event
-        for (String idx : eventsIndices) {
-            indexDoc(
-                idx,
-                "QjoLteG7HX3VUUXr-J4kHQ",
-                Map.of("@timestamp", Instant.now().toEpochMilli(), "Stacktrace.id", "QjoLteG7HX3VUUXr-J4kHQ", "Stacktrace.count", 1)
-            );
-        }
-
-        indexDoc(
-            "profiling-stacktraces",
-            "QjoLteG7HX3VUUXr-J4kHQ",
-            Map.of("Stacktrace.frame.ids", "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf", "Stacktrace.frame.types", "AQI")
-        );
-        indexDoc(
-            "profiling-stackframes",
-            "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf",
-            Map.of("Stackframe.function.name", List.of("_raw_spin_unlock_irqrestore", "inlined_frame_1", "inlined_frame_0"))
-        );
-        indexDoc("profiling-executables", "QCCDqjSg3bMK1C4YRK6Tiw", Map.of("Executable.file.name", "libc.so.6"));
+        bulkIndex("data/profiling-events-all.ndjson");
+        bulkIndex("data/profiling-stacktraces.ndjson");
+        bulkIndex("data/profiling-stackframes.ndjson");
+        bulkIndex("data/profiling-executables.ndjson");
 
         refresh();
     }