From 4d52ad2cd5d269c296af12a1fbf549ba8c1db95a Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 30 Nov 2023 08:52:37 -0500 Subject: [PATCH] [Fleet] Fix get file handler for bundled package (#172182) --- .../server/routes/epm/file_handler.test.ts | 241 ++++++++++++++++++ .../fleet/server/routes/epm/file_handler.ts | 138 ++++++++++ .../fleet/server/routes/epm/handlers.ts | 84 +----- .../plugins/fleet/server/routes/epm/index.ts | 2 +- x-pack/plugins/fleet/tsconfig.json | 1 + 5 files changed, 382 insertions(+), 84 deletions(-) create mode 100644 x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts create mode 100644 x-pack/plugins/fleet/server/routes/epm/file_handler.ts diff --git a/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts b/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts new file mode 100644 index 00000000000000..708d0e16a66fd6 --- /dev/null +++ b/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts @@ -0,0 +1,241 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { httpServerMock } from '@kbn/core-http-server-mocks'; +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; +import { Headers } from 'node-fetch'; + +import { getBundledPackageByPkgKey } from '../../services/epm/packages/bundled_packages'; +import { getFile, getInstallation } from '../../services/epm/packages/get'; +import type { FleetRequestHandlerContext } from '../..'; +import { appContextService } from '../../services'; +import { unpackBufferEntries, getArchiveEntry } from '../../services/epm/archive'; +import { getAsset } from '../../services/epm/archive/storage'; + +import { getFileHandler } from './file_handler'; + +jest.mock('../../services/app_context'); +jest.mock('../../services/epm/archive'); +jest.mock('../../services/epm/archive/storage'); +jest.mock('../../services/epm/packages/bundled_packages'); +jest.mock('../../services/epm/packages/get'); + +const mockedGetBundledPackageByPkgKey = jest.mocked(getBundledPackageByPkgKey); +const mockedGetInstallation = jest.mocked(getInstallation); +const mockedGetFile = jest.mocked(getFile); +const mockedGetArchiveEntry = jest.mocked(getArchiveEntry); +const mockedUnpackBufferEntries = jest.mocked(unpackBufferEntries); +const mockedGetAsset = jest.mocked(getAsset); + +function mockContext() { + const mockSavedObjectsClient = savedObjectsClientMock.create(); + const mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + return { + fleet: { + internalSOClient: async () => mockSavedObjectsClient, + }, + core: { + savedObjects: { + client: mockSavedObjectsClient, + }, + elasticsearch: { + client: { + asInternalUser: mockElasticsearchClient, + }, + }, + }, + } as unknown as FleetRequestHandlerContext; +} + +describe('getFileHandler', () => { + beforeEach(() => { + const logger = loggingSystemMock.createLogger(); + jest.mocked(appContextService).getLogger.mockReturnValue(logger); + mockedGetBundledPackageByPkgKey.mockReset(); + mockedUnpackBufferEntries.mockReset(); + mockedGetFile.mockReset(); + mockedGetInstallation.mockReset(); + mockedGetArchiveEntry.mockReset(); + mockedGetAsset.mockReset(); + }); + + it('should return the file for bundled package and an existing file', async () => { + mockedGetBundledPackageByPkgKey.mockResolvedValue({} as any); + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'README.md', + }, + }); + const buffer = Buffer.from(`TEST`); + mockedUnpackBufferEntries.mockResolvedValue([ + { + path: 'test-1.0.0/README.md', + buffer, + }, + ]); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 200, + body: buffer, + headers: expect.objectContaining({ + 'content-type': 'text/markdown; charset=utf-8', + }), + }) + ); + }); + + it('should a 404 for bundled package with a non existing file', async () => { + mockedGetBundledPackageByPkgKey.mockResolvedValue({} as any); + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'idonotexists.md', + }, + }); + mockedUnpackBufferEntries.mockResolvedValue([ + { + path: 'test-1.0.0/README.md', + buffer: Buffer.from(`TEST`), + }, + ]); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 404, + body: 'bundled package file not found: idonotexists.md', + }) + ); + }); + + it('should proxy registry 200 for non bundled and non installed package', async () => { + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'idonotexists.md', + }, + }); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + + mockedGetFile.mockResolvedValue({ + status: 200, + // @ts-expect-error + body: 'test', + headers: new Headers({ + raw: '', + 'content-type': 'text/markdown', + }), + }); + + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 200, + body: 'test', + headers: expect.objectContaining({ + 'content-type': 'text/markdown', + }), + }) + ); + }); + + it('should proxy registry 404 for non bundled and non installed package', async () => { + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'idonotexists.md', + }, + }); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + + mockedGetFile.mockResolvedValue({ + status: 404, + // @ts-expect-error + body: 'not found', + headers: new Headers({ + raw: '', + 'content-type': 'text', + }), + }); + + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 404, + body: 'not found', + headers: expect.objectContaining({ + 'content-type': 'text', + }), + }) + ); + }); + + it('should return the file from installation for installed package', async () => { + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'README.md', + }, + }); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + + mockedGetInstallation.mockResolvedValue({ version: '1.0.0' } as any); + mockedGetArchiveEntry.mockReturnValue(Buffer.from('test')); + + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 200, + headers: expect.objectContaining({ + 'content-type': 'text/markdown; charset=utf-8', + }), + }) + ); + }); + + it('should a 404 if the file from installation do not exists for installed package', async () => { + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'README.md', + }, + }); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + + mockedGetInstallation.mockResolvedValue({ version: '1.0.0' } as any); + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 404, + body: 'installed package file not found: README.md', + }) + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/routes/epm/file_handler.ts b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts new file mode 100644 index 00000000000000..b7572f1980cdc8 --- /dev/null +++ b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; + +import type { TypeOf } from '@kbn/config-schema'; +import mime from 'mime-types'; +import type { ResponseHeaders, KnownHeaders, HttpResponseOptions } from '@kbn/core/server'; + +import type { GetFileRequestSchema, FleetRequestHandler } from '../../types'; +import { getFile, getInstallation } from '../../services/epm/packages'; +import { defaultFleetErrorHandler } from '../../errors'; +import { getArchiveEntry } from '../../services/epm/archive'; +import { getAsset } from '../../services/epm/archive/storage'; +import { getBundledPackageByPkgKey } from '../../services/epm/packages/bundled_packages'; +import { pkgToPkgKey } from '../../services/epm/registry'; +import { unpackBufferEntries } from '../../services/epm/archive'; + +const CACHE_CONTROL_10_MINUTES_HEADER: HttpResponseOptions['headers'] = { + 'cache-control': 'max-age=600', +}; +export const getFileHandler: FleetRequestHandler< + TypeOf +> = async (context, request, response) => { + try { + const { pkgName, pkgVersion, filePath } = request.params; + const savedObjectsClient = (await context.fleet).internalSoClient; + + const installation = await getInstallation({ savedObjectsClient, pkgName }); + const useLocalFile = pkgVersion === installation?.version; + const assetPath = `${pkgName}-${pkgVersion}/${filePath}`; + + if (useLocalFile) { + const fileBuffer = getArchiveEntry(assetPath); + // only pull local installation if we don't have it cached + const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath })); + + // error, if neither is available + if (!fileBuffer && !storedAsset) { + return response.custom({ + body: `installed package file not found: ${filePath}`, + statusCode: 404, + }); + } + + // if storedAsset is not available, fileBuffer *must* be + // b/c we error if we don't have at least one, and storedAsset is the least likely + const { buffer, contentType } = storedAsset + ? { + contentType: storedAsset.media_type, + buffer: storedAsset.data_utf8 + ? Buffer.from(storedAsset.data_utf8, 'utf8') + : Buffer.from(storedAsset.data_base64, 'base64'), + } + : { + contentType: mime.contentType(path.extname(assetPath)), + buffer: fileBuffer, + }; + + if (!contentType) { + return response.custom({ + body: `unknown content type for file: ${filePath}`, + statusCode: 400, + }); + } + + return response.custom({ + body: buffer, + statusCode: 200, + headers: { + ...CACHE_CONTROL_10_MINUTES_HEADER, + 'content-type': contentType, + }, + }); + } + + const bundledPackage = await getBundledPackageByPkgKey( + pkgToPkgKey({ name: pkgName, version: pkgVersion }) + ); + if (bundledPackage) { + const bufferEntries = await unpackBufferEntries(bundledPackage.buffer, 'application/zip'); + + const fileBuffer = bufferEntries.find((entry) => entry.path === assetPath)?.buffer; + + if (!fileBuffer) { + return response.custom({ + body: `bundled package file not found: ${filePath}`, + statusCode: 404, + }); + } + + // if storedAsset is not available, fileBuffer *must* be + // b/c we error if we don't have at least one, and storedAsset is the least likely + const { buffer, contentType } = { + contentType: mime.contentType(path.extname(assetPath)), + buffer: fileBuffer, + }; + + if (!contentType) { + return response.custom({ + body: `unknown content type for file: ${filePath}`, + statusCode: 400, + }); + } + + return response.custom({ + body: buffer, + statusCode: 200, + headers: { + ...CACHE_CONTROL_10_MINUTES_HEADER, + 'content-type': contentType, + }, + }); + } else { + const registryResponse = await getFile(pkgName, pkgVersion, filePath); + const headersToProxy: KnownHeaders[] = ['content-type']; + const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => { + const value = registryResponse.headers.get(knownHeader); + if (value !== null) { + headers[knownHeader] = value; + } + return headers; + }, {} as ResponseHeaders); + + return response.custom({ + body: registryResponse.body, + statusCode: registryResponse.status, + headers: { ...CACHE_CONTROL_10_MINUTES_HEADER, ...proxiedHeaders }, + }); + } + } catch (error) { + return defaultFleetErrorHandler({ error, response }); + } +}; diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index c03272119dd16e..6fadeff5180c21 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -5,12 +5,9 @@ * 2.0. */ -import path from 'path'; - import type { TypeOf } from '@kbn/config-schema'; -import mime from 'mime-types'; import semverValid from 'semver/functions/valid'; -import type { ResponseHeaders, KnownHeaders, HttpResponseOptions } from '@kbn/core/server'; +import type { HttpResponseOptions } from '@kbn/core/server'; import { pick } from 'lodash'; @@ -41,7 +38,6 @@ import type { GetPackagesRequestSchema, GetInstalledPackagesRequestSchema, GetDataStreamsRequestSchema, - GetFileRequestSchema, GetInfoRequestSchema, InstallPackageFromRegistryRequestSchema, InstallPackageByUploadRequestSchema, @@ -60,21 +56,17 @@ import { getCategories, getPackages, getInstalledPackages, - getFile, getPackageInfo, isBulkInstallError, installPackage, removeInstallation, getLimitedPackages, - getInstallation, getBulkAssets, getTemplateInputs, } from '../../services/epm/packages'; import type { BulkInstallResponse } from '../../services/epm/packages'; import { defaultFleetErrorHandler, fleetErrorToResponseOptions, FleetError } from '../../errors'; import { appContextService, checkAllowedPackages } from '../../services'; -import { getArchiveEntry } from '../../services/epm/archive/cache'; -import { getAsset } from '../../services/epm/archive/storage'; import { getPackageUsageStats } from '../../services/epm/packages/get'; import { updatePackage } from '../../services/epm/packages/update'; import { getGpgKeyIdOrUndefined } from '../../services/epm/packages/package_verification'; @@ -206,80 +198,6 @@ export const getLimitedListHandler: FleetRequestHandler< } }; -export const getFileHandler: FleetRequestHandler< - TypeOf -> = async (context, request, response) => { - try { - const { pkgName, pkgVersion, filePath } = request.params; - const savedObjectsClient = (await context.fleet).internalSoClient; - const installation = await getInstallation({ savedObjectsClient, pkgName }); - const useLocalFile = pkgVersion === installation?.version; - - if (useLocalFile) { - const assetPath = `${pkgName}-${pkgVersion}/${filePath}`; - const fileBuffer = getArchiveEntry(assetPath); - // only pull local installation if we don't have it cached - const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath })); - - // error, if neither is available - if (!fileBuffer && !storedAsset) { - return response.custom({ - body: `installed package file not found: ${filePath}`, - statusCode: 404, - }); - } - - // if storedAsset is not available, fileBuffer *must* be - // b/c we error if we don't have at least one, and storedAsset is the least likely - const { buffer, contentType } = storedAsset - ? { - contentType: storedAsset.media_type, - buffer: storedAsset.data_utf8 - ? Buffer.from(storedAsset.data_utf8, 'utf8') - : Buffer.from(storedAsset.data_base64, 'base64'), - } - : { - contentType: mime.contentType(path.extname(assetPath)), - buffer: fileBuffer, - }; - - if (!contentType) { - return response.custom({ - body: `unknown content type for file: ${filePath}`, - statusCode: 400, - }); - } - - return response.custom({ - body: buffer, - statusCode: 200, - headers: { - ...CACHE_CONTROL_10_MINUTES_HEADER, - 'content-type': contentType, - }, - }); - } else { - const registryResponse = await getFile(pkgName, pkgVersion, filePath); - const headersToProxy: KnownHeaders[] = ['content-type']; - const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => { - const value = registryResponse.headers.get(knownHeader); - if (value !== null) { - headers[knownHeader] = value; - } - return headers; - }, {} as ResponseHeaders); - - return response.custom({ - body: registryResponse.body, - statusCode: registryResponse.status, - headers: { ...CACHE_CONTROL_10_MINUTES_HEADER, ...proxiedHeaders }, - }); - } - } catch (error) { - return defaultFleetErrorHandler({ error, response }); - } -}; - export const getInfoHandler: FleetRequestHandler< TypeOf, TypeOf diff --git a/x-pack/plugins/fleet/server/routes/epm/index.ts b/x-pack/plugins/fleet/server/routes/epm/index.ts index 6e0000bf4ccbf6..5245381a409dab 100644 --- a/x-pack/plugins/fleet/server/routes/epm/index.ts +++ b/x-pack/plugins/fleet/server/routes/epm/index.ts @@ -55,7 +55,6 @@ import { getListHandler, getInstalledListHandler, getLimitedListHandler, - getFileHandler, getInfoHandler, getBulkAssetsHandler, installPackageFromRegistryHandler, @@ -70,6 +69,7 @@ import { createCustomIntegrationHandler, getInputsHandler, } from './handlers'; +import { getFileHandler } from './file_handler'; const MAX_FILE_SIZE_BYTES = 104857600; // 100MB diff --git a/x-pack/plugins/fleet/tsconfig.json b/x-pack/plugins/fleet/tsconfig.json index 4d3c850b9e75d2..4503d328e15039 100644 --- a/x-pack/plugins/fleet/tsconfig.json +++ b/x-pack/plugins/fleet/tsconfig.json @@ -102,5 +102,6 @@ "@kbn/dashboard-plugin", "@kbn/cloud", "@kbn/config", + "@kbn/core-http-server-mocks", ] }