[8.7][APM] Fleet migration support for bundled APM package (#153159) (#…

…155281) Backport of #153159. (cherry picked from commit d1dff0b) Closes #149342. It accomplishes this by returning the ArchivePackage, unzipped bundled package that includes most of the same fields as the RegistryPackage. These fields are used in APM to support the fleet migration workflow.
elastic · Apr 19, 2023 · 87e661c · 87e661c
1 parent 8b46647
commit 87e661c
Show file tree

Hide file tree

Showing 13 changed files with 273 additions and 12 deletions.
diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml
@@ -176,6 +176,7 @@ enabled:
   - x-pack/test/api_integration/apis/uptime/config.ts
   - x-pack/test/api_integration/apis/watcher/config.ts
   - x-pack/test/apm_api_integration/basic/config.ts
+  - x-pack/test/apm_api_integration/cloud/config.ts
   - x-pack/test/apm_api_integration/rules/config.ts
   - x-pack/test/apm_api_integration/trial/config.ts
   - x-pack/test/banners_functional/config.ts

diff --git a/x-pack/plugins/apm/public/components/app/settings/schema/schema_overview.tsx b/x-pack/plugins/apm/public/components/app/settings/schema/schema_overview.tsx
@@ -243,7 +243,7 @@ function getDisabledReason({
       )
     );
   }
-  if (!hasCloudAgentPolicy) {
+  if (hasRequiredRole && !hasCloudAgentPolicy) {
     reasons.push(
       i18n.translate(
         'xpack.apm.settings.schema.disabledReason.hasCloudAgentPolicy',

diff --git a/x-pack/plugins/apm/server/routes/fleet/get_latest_apm_package.ts b/x-pack/plugins/apm/server/routes/fleet/get_latest_apm_package.ts
@@ -17,12 +17,19 @@ export async function getLatestApmPackage({
   request: KibanaRequest;
 }) {
   const packageClient = fleetPluginStart.packageService.asScoped(request);
-  const { name, version } = await packageClient.fetchFindLatestPackage(
+  const latestPackage = await packageClient.fetchFindLatestPackage(
     APM_PACKAGE_NAME
   );
-  const registryPackage = await packageClient.getPackage(name, version);
-  const { title, policy_templates: policyTemplates } =
-    registryPackage.packageInfo;
+  const packageInfo =
+    'buffer' in latestPackage
+      ? (await packageClient.readBundledPackage(latestPackage)).packageInfo
+      : latestPackage;
+  const {
+    name,
+    version,
+    title,
+    policy_templates: policyTemplates,
+  } = packageInfo;
   const firstTemplate = policyTemplates?.[0];
   const policyTemplateInputVars =
     firstTemplate && 'inputs' in firstTemplate

diff --git a/x-pack/plugins/apm/server/routes/fleet/route.ts b/x-pack/plugins/apm/server/routes/fleet/route.ts
@@ -195,6 +195,17 @@ const getMigrationCheckRoute = createApmServerRoute({
       plugins.security.start(),
     ]);
     const hasRequiredRole = isSuperuser({ securityPluginStart, request });
+    if (!hasRequiredRole) {
+      return {
+        has_cloud_agent_policy: false,
+        has_cloud_apm_package_policy: false,
+        cloud_apm_migration_enabled: cloudApmMigrationEnabled,
+        has_required_role: false,
+        cloud_apm_package_policy: undefined,
+        has_apm_integrations: false,
+        latest_apm_package_version: '',
+      };
+    }
     const cloudAgentPolicy = hasRequiredRole
       ? await getCloudAgentPolicy({
           savedObjectsClient,
@@ -203,14 +214,14 @@ const getMigrationCheckRoute = createApmServerRoute({
       : undefined;
     const apmPackagePolicy = getApmPackagePolicy(cloudAgentPolicy);
     const coreStart = await core.start();
-    const packagePolicies = await getApmPackagePolicies({
-      coreStart,
-      fleetPluginStart,
-    });
     const latestApmPackage = await getLatestApmPackage({
       fleetPluginStart,
       request,
     });
+    const packagePolicies = await getApmPackagePolicies({
+      coreStart,
+      fleetPluginStart,
+    });
     return {
       has_cloud_agent_policy: !!cloudAgentPolicy,
       has_cloud_apm_package_policy: !!apmPackagePolicy,

diff --git a/x-pack/plugins/apm/server/test_helpers/create_apm_users/authentication.ts b/x-pack/plugins/apm/server/test_helpers/create_apm_users/authentication.ts
@@ -16,6 +16,7 @@ export enum ApmUsername {
   apmManageOwnAgentKeys = 'apm_manage_own_agent_keys',
   apmManageOwnAndCreateAgentKeys = 'apm_manage_own_and_create_agent_keys',
   apmMonitorClusterAndIndices = 'apm_monitor_cluster_and_indices',
+  apmManageServiceAccount = 'apm_manage_service_account',
 }
 
 export enum ApmCustomRolename {
@@ -24,6 +25,7 @@ export enum ApmCustomRolename {
   apmManageOwnAgentKeys = 'apm_manage_own_agent_keys',
   apmManageOwnAndCreateAgentKeys = 'apm_manage_own_and_create_agent_keys',
   apmMonitorClusterAndIndices = 'apm_monitor_cluster_and_indices',
+  apmManageServiceAccount = 'apm_manage_service_account',
 }
 
 export const customRoles = {
@@ -88,6 +90,11 @@ export const customRoles = {
       cluster: ['monitor'],
     },
   },
+  [ApmCustomRolename.apmManageServiceAccount]: {
+    elasticsearch: {
+      cluster: ['manage_service_account'],
+    },
+  },
 };
 
 export const users: Record<
@@ -123,6 +130,10 @@ export const users: Record<
     builtInRoleNames: ['viewer'],
     customRoleNames: [ApmCustomRolename.apmMonitorClusterAndIndices],
   },
+  [ApmUsername.apmManageServiceAccount]: {
+    builtInRoleNames: ['editor'],
+    customRoleNames: [ApmCustomRolename.apmManageServiceAccount],
+  },
 };
 
 export const APM_TEST_PASSWORD = 'changeme';
diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts
@@ -11,6 +11,7 @@ const createClientMock = (): jest.Mocked<PackageClient> => ({
   getInstallation: jest.fn(),
   ensureInstalledPackage: jest.fn(),
   fetchFindLatestPackage: jest.fn(),
+  readBundledPackage: jest.fn(),
   getPackage: jest.fn(),
   getPackages: jest.fn(),
   reinstallEsAssets: jest.fn(),

diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts
@@ -26,13 +26,15 @@ import * as epmPackagesGet from './packages/get';
 import * as epmPackagesInstall from './packages/install';
 import * as epmRegistry from './registry';
 import * as epmTransformsInstall from './elasticsearch/transform/install';
+import * as epmArchiveParse from './archive/parse';
 
 const testKeys = [
   'getInstallation',
   'ensureInstalledPackage',
   'fetchFindLatestPackage',
   'getPackage',
   'reinstallEsAssets',
+  'readBundledPackage',
 ];
 
 function getTest(
@@ -144,6 +146,23 @@ function getTest(
         ],
       };
       break;
+    case testKeys[5]:
+      const bundledPackage = { name: 'package name', version: '8.0.0', buffer: Buffer.from([]) };
+      test = {
+        method: mocks.packageClient.readBundledPackage.bind(mocks.packageClient),
+        args: [bundledPackage],
+        spy: jest.spyOn(epmArchiveParse, 'generatePackageInfoFromArchiveBuffer'),
+        spyArgs: [bundledPackage.buffer, 'application/zip'],
+        spyResponse: {
+          packageInfo: { name: 'readBundledPackage test' },
+          paths: ['/some/test/path'],
+        },
+        expectedReturnValue: {
+          packageInfo: { name: 'readBundledPackage test' },
+          paths: ['/some/test/path'],
+        },
+      };
+      break;
     default:
       throw new Error('invalid test key');
   }

diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts
@@ -32,6 +32,7 @@ import { installTransforms, isTransform } from './elasticsearch/transform/instal
 import type { FetchFindLatestPackageOptions } from './registry';
 import { fetchFindLatestPackageOrThrow, getPackage } from './registry';
 import { ensureInstalledPackage, getInstallation, getPackages } from './packages';
+import { generatePackageInfoFromArchiveBuffer } from './archive';
 
 export type InstalledAssetType = EsAssetReference;
 
@@ -54,6 +55,10 @@ export interface PackageClient {
     options?: FetchFindLatestPackageOptions
   ): Promise<RegistryPackage | BundledPackage>;
 
+  readBundledPackage(
+    bundledPackage: BundledPackage
+  ): Promise<{ packageInfo: ArchivePackage; paths: string[] }>;
+
   getPackage(
     packageName: string,
     packageVersion: string
@@ -137,6 +142,11 @@ class PackageClientImpl implements PackageClient {
     return fetchFindLatestPackageOrThrow(packageName, options);
   }
 
+  public async readBundledPackage(bundledPackage: BundledPackage) {
+    await this.#runPreflight();
+    return generatePackageInfoFromArchiveBuffer(bundledPackage.buffer, 'application/zip');
+  }
+
   public async getPackage(
     packageName: string,
     packageVersion: string,

diff --git a/x-pack/test/apm_api_integration/cloud/config.ts b/x-pack/test/apm_api_integration/cloud/config.ts
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { configs } from '../configs';
+
+export default configs.cloud;
diff --git a/x-pack/test/apm_api_integration/common/config.ts b/x-pack/test/apm_api_integration/common/config.ts
@@ -56,7 +56,8 @@ type ApmApiClientKey =
   | 'noMlAccessUser'
   | 'manageOwnAgentKeysUser'
   | 'createAndAllAgentKeysUser'
-  | 'monitorClusterAndIndicesUser';
+  | 'monitorClusterAndIndicesUser'
+  | 'manageServiceAccount';
 
 export type ApmApiClient = Record<ApmApiClientKey, Awaited<ReturnType<typeof getApmApiClient>>>;
 
@@ -146,6 +147,10 @@ export function createTestConfig(
               kibanaServer,
               username: ApmUsername.apmMonitorClusterAndIndices,
             }),
+            manageServiceAccount: await getApmApiClient({
+              kibanaServer,
+              username: ApmUsername.apmManageServiceAccount,
+            }),
           };
         },
         ml: MachineLearningAPIProvider,

diff --git a/x-pack/test/apm_api_integration/configs/index.ts b/x-pack/test/apm_api_integration/configs/index.ts
@@ -37,6 +37,14 @@ const apmFtrConfigs = {
       'logging.loggers': [apmDebugLogger],
     },
   },
+  cloud: {
+    license: 'basic' as const,
+    kibanaConfig: {
+      'xpack.apm.agent.migrations.enabled': 'true',
+      'xpack.apm.forceSyntheticSource': 'true',
+      'logging.loggers': [apmDebugLogger],
+    },
+  },
 };
 
 export type APMFtrConfigName = keyof typeof apmFtrConfigs;

diff --git a/x-pack/test/apm_api_integration/tests/fleet/apm_package_policy_setup.ts b/x-pack/test/apm_api_integration/tests/fleet/apm_package_policy_setup.ts
@@ -11,14 +11,15 @@ export function setupFleet(bettertest: BetterTest) {
   return bettertest({ pathname: '/api/fleet/setup', method: 'post' });
 }
 
-export async function createAgentPolicy(bettertest: BetterTest) {
+export async function createAgentPolicy(bettertest: BetterTest, id?: string) {
   const agentPolicyResponse = await bettertest<{ item: AgentPolicy }>({
     pathname: '/api/fleet/agent_policies',
     method: 'post',
     query: { sys_monitoring: true },
     body: {
       name: 'test_agent_policy',
       description: '',
+      id,
       namespace: 'default',
       monitoring_enabled: ['logs', 'metrics'],
     },
@@ -27,7 +28,11 @@ export async function createAgentPolicy(bettertest: BetterTest) {
   return agentPolicyResponse.body.item.id;
 }
 
-export async function createPackagePolicy(bettertest: BetterTest, agentPolicyId: string) {
+export async function createPackagePolicy(
+  bettertest: BetterTest,
+  agentPolicyId: string,
+  id?: string
+) {
   // Get version of available APM package
   const apmPackageResponse = await bettertest<{ item: any }>({
     pathname: `/api/fleet/epm/packages/apm`,
@@ -43,6 +48,7 @@ export async function createPackagePolicy(bettertest: BetterTest, agentPolicyId:
       description: '',
       namespace: 'default',
       policy_id: agentPolicyId,
+      id,
       enabled: true,
       inputs: [{ type: 'apm', policy_template: 'apmserver', enabled: true, streams: [], vars: {} }],
       package: { name: 'apm', title: 'Elastic APM', version: apmPackageVersion },