[Defend Workflows][Main port] Unblock fleet setup when validating tho…

…usands of uninstall tokens (#174737)

`main` port of the following PR merged to `8.12`:
- #174535

the goal of this port is to sync between `8.12` and `main`, while other
changes are expected to the related uninstall token validation

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/fleet/common/types/index.ts

@@ -40,6 +40,7 @@ export interface FleetConfigType {
   };
   setup?: {
     agentPolicySchemaUpgradeBatchSize?: number;
+    uninstallTokenVerificationBatchSize?: number;
   };
   developer?: {
     maxAgentPoliciesWithInactivityTimeout?: number;

x-pack/plugins/fleet/server/config.ts

@@ -151,6 +151,7 @@ export const config: PluginConfigDescriptor = {
       setup: schema.maybe(
         schema.object({
           agentPolicySchemaUpgradeBatchSize: schema.maybe(schema.number()),
+          uninstallTokenVerificationBatchSize: schema.maybe(schema.number()),
         })
       ),
       developer: schema.object({

x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.test.ts

@@ -13,6 +13,8 @@ import type { SavedObjectsClientContract } from '@kbn/core/server';
 import type { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-plugin/server';
 import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks';
 
+import { errors } from '@elastic/elasticsearch';
+
 import { UninstallTokenError } from '../../../../common/errors';
 
 import { SO_SEARCH_LIMIT } from '../../../../common';
@@ -527,6 +529,48 @@ describe('UninstallTokenService', () => {
           ).resolves.toBeNull();
         });
 
+        describe('avoiding `too_many_nested_clauses` error', () => {
+          it('performs one query if number of policies is smaller than batch size', async () => {
+            mockCreatePointInTimeFinderAsInternalUser();
+            await uninstallTokenService.checkTokenValidityForAllPolicies();
+
+            expect(esoClientMock.createPointInTimeFinderDecryptedAsInternalUser).toBeCalledTimes(1);
+            expect(esoClientMock.createPointInTimeFinderDecryptedAsInternalUser).toBeCalledWith({
+              filter:
+                'fleet-uninstall-tokens.id: "test-so-id" or fleet-uninstall-tokens.id: "test-so-id-two"',
+              perPage: 10000,
+              type: 'fleet-uninstall-tokens',
+            });
+          });
+
+          it('performs multiple queries if number of policies is larger than batch size', async () => {
+            // @ts-ignore
+            appContextService.getConfig().setup = { uninstallTokenVerificationBatchSize: 1 };
+
+            mockCreatePointInTimeFinderAsInternalUser();
+
+            await uninstallTokenService.checkTokenValidityForAllPolicies();
+
+            expect(esoClientMock.createPointInTimeFinderDecryptedAsInternalUser).toBeCalledTimes(2);
+
+            expect(
+              esoClientMock.createPointInTimeFinderDecryptedAsInternalUser
+            ).toHaveBeenNthCalledWith(1, {
+              filter: 'fleet-uninstall-tokens.id: "test-so-id"',
+              perPage: 10000,
+              type: 'fleet-uninstall-tokens',
+            });
+
+            expect(
+              esoClientMock.createPointInTimeFinderDecryptedAsInternalUser
+            ).toHaveBeenNthCalledWith(2, {
+              filter: 'fleet-uninstall-tokens.id: "test-so-id-two"',
+              perPage: 10000,
+              type: 'fleet-uninstall-tokens',
+            });
+          });
+        });
+
         it('returns error if any of the tokens is missing', async () => {
           mockCreatePointInTimeFinderAsInternalUser([okaySO, missingTokenSO2]);
 
@@ -597,6 +641,26 @@ describe('UninstallTokenService', () => {
           });
         });
 
+        it('returns error on `too_many_nested_clauses` error', async () => {
+          // @ts-ignore
+          const responseError = new errors.ResponseError({});
+          responseError.message = 'this is a too_many_nested_clauses error';
+
+          esoClientMock.createPointInTimeFinderDecryptedAsInternalUser = jest
+            .fn()
+            .mockRejectedValueOnce(responseError);
+
+          await expect(
+            uninstallTokenService.checkTokenValidityForAllPolicies()
+          ).resolves.toStrictEqual({
+            error: new UninstallTokenError(
+              'Failed to validate uninstall tokens: `too_many_nested_clauses` error received. ' +
+                'Setting/decreasing the value of `xpack.fleet.setup.uninstallTokenVerificationBatchSize` in your kibana.yml should help. ' +
+                `Current value is 500.`
+            ),
+          });
+        });
+
         it('throws error in case of unknown error', async () => {
           esoClientMock.createPointInTimeFinderDecryptedAsInternalUser = jest
             .fn()

x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.ts

@@ -23,13 +23,15 @@ import type {
 import type { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-plugin/server';
 import type { KibanaRequest } from '@kbn/core-http-server';
 import { SECURITY_EXTENSION_ID } from '@kbn/core-saved-objects-server';
-import { asyncForEach } from '@kbn/std';
+import { asyncForEach, asyncMap } from '@kbn/std';
 
 import type {
   AggregationsTermsInclude,
   AggregationsTermsExclude,
 } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
+import { isResponseError } from '@kbn/es-errors';
+
 import { UninstallTokenError } from '../../../../common/errors';
 
 import type { GetUninstallTokensMetadataResponse } from '../../../../common/types/rest_spec/uninstall_token';
@@ -205,15 +207,31 @@ export class UninstallTokenService implements UninstallTokenServiceInterface {
       return [];
     }
 
-    const filter: string = tokenObjectHits
-      .map(({ _id }) => {
-        return `${UNINSTALL_TOKENS_SAVED_OBJECT_TYPE}.id: "${_id}"`;
-      })
-      .join(' or ');
+    const filterEntries: string[] = tokenObjectHits.map(
+      ({ _id }) => `${UNINSTALL_TOKENS_SAVED_OBJECT_TYPE}.id: "${_id}"`
+    );
+
+    const uninstallTokenChunks: UninstallToken[][] = await asyncMap(
+      chunk(filterEntries, this.getUninstallTokenVerificationBatchSize()),
+      (entries) => {
+        const filter = entries.join(' or ');
+        return this.getDecryptedTokens({ filter });
+      }
+    );
 
-    return this.getDecryptedTokens({ filter });
+    return uninstallTokenChunks.flat();
   }
 
+  private getUninstallTokenVerificationBatchSize = () => {
+    /** If `uninstallTokenVerificationBatchSize` is too large, we get an error of `too_many_nested_clauses`.
+     *  Assuming that `max_clause_count` >= 1024, and experiencing that batch size should be less than half
+     *  than `max_clause_count` with our current query, batch size below 512 should be okay on every env.
+     */
+    const config = appContextService.getConfig();
+
+    return config?.setup?.uninstallTokenVerificationBatchSize ?? 500;
+  };
+
   private getDecryptedTokens = async (
     options: Partial<SavedObjectsCreatePointInTimeFinderOptions>
   ): Promise<UninstallToken[]> => {
@@ -520,6 +538,16 @@ export class UninstallTokenService implements UninstallTokenServiceInterface {
       if (error instanceof UninstallTokenError) {
         // known errors are considered non-fatal
         return { error };
+      } else if (isResponseError(error) && error.message.includes('too_many_nested_clauses')) {
+        // `too_many_nested_clauses` is considered non-fatal
+        const errorMessage =
+          'Failed to validate uninstall tokens: `too_many_nested_clauses` error received. ' +
+          'Setting/decreasing the value of `xpack.fleet.setup.uninstallTokenVerificationBatchSize` in your kibana.yml should help. ' +
+          `Current value is ${this.getUninstallTokenVerificationBatchSize()}.`;
+
+        appContextService.getLogger().warn(`${errorMessage}: '${error}'`);
+
+        return { error: new UninstallTokenError(errorMessage) };
       } else {
         const errorMessage = 'Unknown error happened while checking Uninstall Tokens validity';
         appContextService.getLogger().error(`${errorMessage}: '${error}'`);