Do not remove legacy watcher templates in 7.x (#82167)

Some previous 7.x release validate that these legacy templates exist in
the cluster (the validation executes on non-master nodes too). This can
cause the rolling upgrade test (particulary the ones that do not upgrade
the entire cluster ie. mixed cluster scenarios) to hang waiting for
watcher to be started, however it will never start because of the
legacy templates validation failing.

This is not a problem per se for "real life" rolling upgrades as this does not
block the rolling upgrades, nor does it raise any errors, it will
however stop watcher on the nodes that detect it (until the said nodes are
upgraded, at which point watcher is started).

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java

@@ -720,11 +720,6 @@ public void onIndexModule(IndexModule module) {
     public UnaryOperator<Map<String, IndexTemplateMetadata>> getIndexTemplateMetadataUpgrader() {
         return map -> {
             map.keySet().removeIf(name -> name.startsWith("watch_history_"));
-            // watcher migrated to using system indices so these legacy templates are not needed anymore
-            map.remove(".watches");
-            map.remove(".triggered_watches");
-            // post 7.x we moved to typeless watch-history-10
-            map.remove(".watch-history-9");
             return map;
         };
     }

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java

@@ -11,7 +11,6 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -21,12 +20,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 
 public class WatcherRestartIT extends AbstractUpgradeTestCase {
@@ -135,31 +132,6 @@ private void assertIndexTemplateExists(String expectedTemplate) throws IOExcepti
         );
     }
 
-    public void testEnsureWatcherDeletesLegacyTemplates() throws Exception {
-        if (CLUSTER_TYPE.equals(ClusterType.UPGRADED)) {
-            // legacy index template created in previous releases should not be present anymore
-            assertBusy(() -> {
-                Request request = new Request("GET", "/_template/*watch*");
-                try {
-                    Response response = client().performRequest(request);
-                    Map<String, Object> responseLevel = entityAsMap(response);
-                    assertNotNull(responseLevel);
-
-                    assertThat(responseLevel.containsKey(".watches"), is(false));
-                    assertThat(responseLevel.containsKey(".triggered_watches"), is(false));
-                    assertThat(responseLevel.containsKey(".watch-history-9"), is(false));
-                } catch (ResponseException e) {
-                    // Not found is fine
-                    assertThat(
-                        "Unexpected failure getting templates: " + e.getResponse().getStatusLine(),
-                        e.getResponse().getStatusLine().getStatusCode(),
-                        is(404)
-                    );
-                }
-            }, 30, TimeUnit.SECONDS);
-        }
-    }
-
     private void ensureWatcherStopped() throws Exception {
         assertBusy(() -> {
             Response stats = client().performRequest(new Request("GET", "_watcher/stats"));