Increase package install max timeout + add concurrency control to rol…

…lover flow
elastic · Sep 19, 2023 · e2e30bf · e2e30bf
1 parent 6a37dcd
commit e2e30bf
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 19 deletions.
diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts
@@ -9,7 +9,7 @@ import { ElasticsearchAssetType, KibanaAssetType } from '../types/models';
 
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
-export const MAX_TIME_COMPLETE_INSTALL = 60000;
+export const MAX_TIME_COMPLETE_INSTALL = 30 * 60 * 1000; // 30 minutes
 
 export const FLEET_SYSTEM_PACKAGE = 'system';
 export const FLEET_ELASTIC_AGENT_PACKAGE = 'elastic_agent';

diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
@@ -11,6 +11,8 @@ import type {
   MappingTypeMapping,
 } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
+import pMap from 'p-map';
+
 import type { Field, Fields } from '../../fields/field';
 import type {
   RegistryDataStream,
@@ -723,15 +725,22 @@ const updateAllDataStreams = async (
   esClient: ElasticsearchClient,
   logger: Logger
 ): Promise<void> => {
-  const updatedataStreamPromises = indexNameWithTemplates.map((templateEntry) => {
-    return updateExistingDataStream({
-      esClient,
-      logger,
-      dataStreamName: templateEntry.dataStreamName,
-    });
-  });
-  await Promise.all(updatedataStreamPromises);
+  await pMap(
+    indexNameWithTemplates,
+    (templateEntry) => {
+      return updateExistingDataStream({
+        esClient,
+        logger,
+        dataStreamName: templateEntry.dataStreamName,
+      });
+    },
+    {
+      // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster
+      concurrency: 20,
+    }
+  );
 };
+
 const updateExistingDataStream = async ({
   dataStreamName,
   esClient,

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
@@ -99,18 +99,30 @@ export async function _installPackage({
   try {
     // if some installation already exists
     if (installedPkg) {
+      const isStatusInstalling = installedPkg.attributes.install_status === 'installing';
+      const hasExceededTimeout =
+        Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
+        MAX_TIME_COMPLETE_INSTALL;
+
       // if the installation is currently running, don't try to install
       // instead, only return already installed assets
-      if (
-        installedPkg.attributes.install_status === 'installing' &&
-        Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
-          MAX_TIME_COMPLETE_INSTALL
-      ) {
-        throw new ConcurrentInstallOperationError(
-          `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${
-            pkgVersion || 'unknown'
-          } detected, aborting.`
-        );
+      if (isStatusInstalling && hasExceededTimeout) {
+        // If this is a forced installation, ignore the timeout and restart the installation anyway
+        if (force) {
+          await restartInstallation({
+            savedObjectsClient,
+            pkgName,
+            pkgVersion,
+            installSource,
+            verificationResult,
+          });
+        } else {
+          throw new ConcurrentInstallOperationError(
+            `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${
+              pkgVersion || 'unknown'
+            } detected, aborting.`
+          );
+        }
       } else {
         // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL
         // (it might be stuck) update the saved object and proceed