[Fleet] Changes to bulk upgrade api for allowing rolling upgrades (#1…

…31947)

* [Fleet] Changes to bulk upgrade api for allowing rolling upgrades

* Remove one query and some tests

* Skip version check if fleet server agents are being upgraded

* Fix test

* Fixing tests again

* Fix failing check

* Fix another test

* Fix another test

* Fix api integration tests

* Remove parameter

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Nicolas Chaulet <nicolas.chaulet@elastic.co>

x-pack/plugins/fleet/common/services/get_max_version.test.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getMaxVersion } from './get_max_version';
+
+describe('Fleet - getMaxVersion', () => {
+  it('returns the maximum version', () => {
+    const versions = ['8.1.0', '8.3.0', '8.2.1', '7.16.0', '8.2.0', '7.16.1', '8.3.1'];
+    expect(getMaxVersion(versions)).toEqual('8.3.1');
+  });
+
+  it('returns the maximum version when there are duplicates', () => {
+    const versions = ['8.1.0', '8.3.0', '8.2.1', '7.16.0', '8.2.0', '7.16.1', '8.2.0', '7.15.1'];
+    expect(getMaxVersion(versions)).toEqual('8.3.0');
+  });
+
+  it('returns the maximum version when there is a snapshot version', () => {
+    const versions = ['8.1.0', '8.2.0-SNAPSHOT', '7.16.0', '7.16.1'];
+    expect(getMaxVersion(versions)).toEqual('8.2.0-SNAPSHOT');
+  });
+
+  it('returns the maximum version and prefers the major version to the snapshot', () => {
+    const versions = ['8.1.0', '8.2.0-SNAPSHOT', '8.2.0', '7.16.0', '7.16.1'];
+    expect(getMaxVersion(versions)).toEqual('8.2.0');
+  });
+
+  it('when there is only a version returns it', () => {
+    const versions = ['8.1.0'];
+    expect(getMaxVersion(versions)).toEqual('8.1.0');
+  });
+
+  it('returns an empty string when the passed array is empty', () => {
+    const versions: string[] = [];
+    expect(getMaxVersion(versions)).toEqual('');
+  });
+});

x-pack/plugins/fleet/common/services/get_max_version.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { uniq } from 'lodash';
+import semverGt from 'semver/functions/gt';
+import semverCoerce from 'semver/functions/coerce';
+
+// Find max version from an array of string versions
+export function getMaxVersion(versions: string[]) {
+  const uniqVersions: string[] = uniq(versions);
+
+  if (uniqVersions.length === 1) {
+    const semverVersion = semverCoerce(uniqVersions[0])?.version;
+    return semverVersion ? semverVersion : '';
+  } else if (uniqVersions.length > 1) {
+    const sorted = uniqVersions.sort((a, b) => (semverGt(a, b) ? 1 : -1));
+    return sorted[sorted.length - 1];
+  }
+  return '';
+}

x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts

@@ -7,15 +7,24 @@
 
 import type { RequestHandler } from '@kbn/core/server';
 import type { TypeOf } from '@kbn/config-schema';
+import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server';
+
 import semverCoerce from 'semver/functions/coerce';
+import semverGt from 'semver/functions/gt';
 
 import type { PostAgentUpgradeResponse, PostBulkAgentUpgradeResponse } from '../../../common/types';
 import type { PostAgentUpgradeRequestSchema, PostBulkAgentUpgradeRequestSchema } from '../../types';
 import * as AgentService from '../../services/agents';
 import { appContextService } from '../../services';
 import { defaultIngestErrorHandler } from '../../errors';
+import { SO_SEARCH_LIMIT } from '../../../common';
 import { isAgentUpgradeable } from '../../../common/services';
-import { getAgentById } from '../../services/agents';
+import { getAgentById, getAgentsByKuery } from '../../services/agents';
+import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, AGENTS_PREFIX } from '../../constants';
+
+import { getMaxVersion } from '../../../common/services/get_max_version';
+
+import { packagePolicyService } from '../../services/package_policy';
 
 export const postAgentUpgradeHandler: RequestHandler<
   TypeOf<typeof PostAgentUpgradeRequestSchema.params>,
@@ -28,7 +37,7 @@ export const postAgentUpgradeHandler: RequestHandler<
   const { version, source_uri: sourceUri, force } = request.body;
   const kibanaVersion = appContextService.getKibanaVersion();
   try {
-    checkVersionIsSame(version, kibanaVersion);
+    checkKibanaVersion(version, kibanaVersion);
     checkSourceUriAllowed(sourceUri);
   } catch (err) {
     return response.customError({
@@ -90,8 +99,9 @@ export const postBulkAgentsUpgradeHandler: RequestHandler<
   } = request.body;
   const kibanaVersion = appContextService.getKibanaVersion();
   try {
-    checkVersionIsSame(version, kibanaVersion);
+    checkKibanaVersion(version, kibanaVersion);
     checkSourceUriAllowed(sourceUri);
+    await checkFleetServerVersion(version, agents, soClient, esClient);
   } catch (err) {
     return response.customError({
       statusCode: 400,
@@ -125,17 +135,17 @@ export const postBulkAgentsUpgradeHandler: RequestHandler<
   }
 };
 
-export const checkVersionIsSame = (version: string, kibanaVersion: string) => {
+export const checkKibanaVersion = (version: string, kibanaVersion: string) => {
   // get version number only in case "-SNAPSHOT" is in it
   const kibanaVersionNumber = semverCoerce(kibanaVersion)?.version;
   if (!kibanaVersionNumber) throw new Error(`kibanaVersion ${kibanaVersionNumber} is not valid`);
   const versionToUpgradeNumber = semverCoerce(version)?.version;
   if (!versionToUpgradeNumber)
     throw new Error(`version to upgrade ${versionToUpgradeNumber} is not valid`);
-  // temporarily only allow upgrading to the same version as the installed kibana version
-  if (kibanaVersionNumber !== versionToUpgradeNumber)
+
+  if (semverGt(version, kibanaVersion))
     throw new Error(
-      `cannot upgrade agent to ${versionToUpgradeNumber} because it is different than the installed kibana version ${kibanaVersionNumber}`
+      `cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the installed kibana version ${kibanaVersionNumber}`
     );
 };
 
@@ -146,3 +156,67 @@ const checkSourceUriAllowed = (sourceUri?: string) => {
     );
   }
 };
+
+// Check the installed fleet server versions
+// Allow upgrading if the agents to upgrade include fleet server agents
+const checkFleetServerVersion = async (
+  versionToUpgradeNumber: string,
+  agentsIds: string | string[],
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient
+) => {
+  let packagePolicyData;
+  try {
+    packagePolicyData = await packagePolicyService.list(soClient, {
+      perPage: SO_SEARCH_LIMIT,
+      kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name: fleet_server`,
+    });
+  } catch (error) {
+    throw new Error(error.message);
+  }
+  const agentPoliciesIds = packagePolicyData?.items.map((item) => item.policy_id);
+
+  if (agentPoliciesIds.length === 0) {
+    return;
+  }
+
+  let agentsResponse;
+  try {
+    agentsResponse = await getAgentsByKuery(esClient, {
+      showInactive: false,
+      perPage: SO_SEARCH_LIMIT,
+      kuery: `${AGENTS_PREFIX}.policy_id:${agentPoliciesIds.map((id) => `"${id}"`).join(' or ')}`,
+    });
+  } catch (error) {
+    throw new Error(error.message);
+  }
+
+  const { agents: fleetServerAgents } = agentsResponse;
+
+  if (fleetServerAgents.length === 0) {
+    return;
+  }
+  const fleetServerIds = fleetServerAgents.map((agent) => agent.id);
+
+  let hasFleetServerAgents: boolean;
+  if (Array.isArray(agentsIds)) {
+    hasFleetServerAgents = agentsIds.some((id) => fleetServerIds.includes(id));
+  } else {
+    hasFleetServerAgents = fleetServerIds.includes(agentsIds);
+  }
+  if (hasFleetServerAgents) {
+    return;
+  }
+
+  const fleetServerVersions = fleetServerAgents.map(
+    (agent) => agent.local_metadata.elastic.agent.version
+  ) as string[];
+
+  const maxFleetServerVersion = getMaxVersion(fleetServerVersions);
+
+  if (semverGt(versionToUpgradeNumber, maxFleetServerVersion)) {
+    throw new Error(
+      `cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the latest fleet server version ${maxFleetServerVersion}`
+    );
+  }
+};