address SmokeTestWatcherWithSecurityIT#testSearchInputWithInsufficien…

…tPrivileges (#42764)

This commit adds busy wait and increases the interval for
SmokeTestWatcherWithSecurityIT#testSearchInputWithInsufficientPrivileges.

Watcher will not allow the same watch to be executed concurrently. If it
finds that case, it will update the watch history with a "not_executed_already_queued"
status. Given a slow machine, and 1 second interval this is possible.

To address this, this commit increases the interval so the watch can fire at most 2
times with a greater interval between the executions and adds a busy wait for the
expected state.

While this does not gaurntee a fix, it should greatly reduce the chances of this
test erroring.

x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

@@ -164,7 +164,7 @@ public void testSearchInputWithInsufficientPrivileges() throws Exception {
         String indexName = "index_not_allowed_to_read";
         try (XContentBuilder builder = jsonBuilder()) {
             builder.startObject();
-            builder.startObject("trigger").startObject("schedule").field("interval", "1s").endObject().endObject();
+            builder.startObject("trigger").startObject("schedule").field("interval", "4s").endObject().endObject();
             builder.startObject("input").startObject("search").startObject("request")
                     .startArray("indices").value(indexName).endArray()
                     .startObject("body").startObject("query").startObject("match_all").endObject().endObject().endObject()
@@ -180,8 +180,10 @@ public void testSearchInputWithInsufficientPrivileges() throws Exception {
 
         // check history, after watch has fired
         ObjectPath objectPath = getWatchHistoryEntry(watchId);
-        String state = objectPath.evaluate("hits.hits.0._source.state");
-        assertThat(state, is("execution_not_needed"));
+        assertBusy(() -> {
+            String state = objectPath.evaluate("hits.hits.0._source.state");
+            assertThat(state, is("execution_not_needed"));
+        });
         boolean conditionMet = objectPath.evaluate("hits.hits.0._source.result.condition.met");
         assertThat(conditionMet, is(false));
     }