From 21f42fb27b7b617243cffc9292c6d3c99345b3b7 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 5 Jun 2023 13:46:27 +0200 Subject: [PATCH 1/6] [ML] Support pipelines deletion and force flag for delete action (#158671) --- x-pack/plugins/ml/common/types/common.ts | 5 ++ .../model_management/delete_models_modal.tsx | 89 +++++++++++++++++-- .../model_management/expanded_row.tsx | 6 +- .../model_management/force_stop_dialog.tsx | 4 +- .../model_management/model_actions.tsx | 49 +++++++--- .../model_management/models_list.tsx | 18 ++-- .../services/ml_api_service/trained_models.ts | 9 +- .../model_management/models_provider.ts | 15 +++- .../server/routes/schemas/inference_schema.ts | 5 ++ .../ml/server/routes/trained_models.ts | 14 ++- .../translations/translations/fr-FR.json | 1 - .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - .../model_management/model_list.ts | 58 +++++++----- .../services/ml/trained_models_table.ts | 42 ++++++++- 15 files changed, 244 insertions(+), 73 deletions(-) diff --git a/x-pack/plugins/ml/common/types/common.ts b/x-pack/plugins/ml/common/types/common.ts index 4559dd168d693c..77742635b52ff6 100644 --- a/x-pack/plugins/ml/common/types/common.ts +++ b/x-pack/plugins/ml/common/types/common.ts @@ -52,3 +52,8 @@ type Without = { [P in Exclude]?: never }; export type XOR = T | U extends object ? (Without & U) | (Without & T) : T | U; export type AwaitReturnType = T extends PromiseLike ? U : T; + +/** + * Removes an optional modifier from a property in a type. + */ +export type WithRequired = T & { [P in K]-?: Exclude }; diff --git a/x-pack/plugins/ml/public/application/model_management/delete_models_modal.tsx b/x-pack/plugins/ml/public/application/model_management/delete_models_modal.tsx index e730dad0c36aa7..8eb0fc8c03112d 100644 --- a/x-pack/plugins/ml/public/application/model_management/delete_models_modal.tsx +++ b/x-pack/plugins/ml/public/application/model_management/delete_models_modal.tsx @@ -5,36 +5,58 @@ * 2.0. */ -import React, { FC, useState, useCallback } from 'react'; +import React, { FC, useCallback, useState } from 'react'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; import { + EuiButton, + EuiButtonEmpty, + EuiCallOut, + EuiCheckbox, EuiModal, + EuiModalBody, + EuiModalFooter, EuiModalHeader, EuiModalHeaderTitle, - EuiModalFooter, - EuiButtonEmpty, - EuiButton, } from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n-react'; +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { type WithRequired } from '../../../common/types/common'; import { useTrainedModelsApiService } from '../services/ml_api_service/trained_models'; import { useToastNotificationService } from '../services/toast_notification_service'; import { DeleteSpaceAwareItemCheckModal } from '../components/delete_space_aware_item_check_modal'; +import { type ModelItem } from './models_list'; interface DeleteModelsModalProps { - modelIds: string[]; + models: ModelItem[]; onClose: (refreshList?: boolean) => void; } -export const DeleteModelsModal: FC = ({ modelIds, onClose }) => { +export const DeleteModelsModal: FC = ({ models, onClose }) => { const trainedModelsApiService = useTrainedModelsApiService(); const { displayErrorToast, displaySuccessToast } = useToastNotificationService(); const [canDeleteModel, setCanDeleteModel] = useState(false); + const [deletePipelines, setDeletePipelines] = useState(false); + + const modelIds = models.map((m) => m.model_id); + + const modelsWithPipelines = models.filter((m) => isPopulatedObject(m.pipelines)) as Array< + WithRequired + >; + + const pipelinesCount = modelsWithPipelines.reduce((acc, curr) => { + return acc + Object.keys(curr.pipelines).length; + }, 0); const deleteModels = useCallback(async () => { try { await Promise.all( - modelIds.map((modelId) => trainedModelsApiService.deleteTrainedModel(modelId)) + modelIds.map((modelId) => + trainedModelsApiService.deleteTrainedModel(modelId, { + with_pipelines: deletePipelines, + force: pipelinesCount > 0, + }) + ) ); displaySuccessToast( i18n.translate('xpack.ml.trainedModels.modelsList.successfullyDeletedMessage', { @@ -59,7 +81,7 @@ export const DeleteModelsModal: FC = ({ modelIds, onClos } onClose(true); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [modelIds, trainedModelsApiService]); + }, [modelIds, trainedModelsApiService, deletePipelines, pipelinesCount]); return canDeleteModel ? ( = ({ modelIds, onClos + {modelsWithPipelines.length > 0 ? ( + + + } + color="warning" + iconType="warning" + > +
+

+ +

+ + } + checked={deletePipelines} + onChange={setDeletePipelines.bind(null, (prev) => !prev)} + data-test-subj="mlModelsDeleteModalDeletePipelinesCheckbox" + /> +
+
    + {modelsWithPipelines.flatMap((model) => { + return Object.keys(model.pipelines).map((pipelineId) => ( +
  • {pipelineId}
    • + )); + })} +
+ + + ) : null} + = ({ item }) => { id="xpack.ml.trainedModels.modelsList.expandedRow.pipelinesTabLabel" defaultMessage="Pipelines" /> - - {isPopulatedObject(pipelines) ? Object.keys(pipelines!).length : 0} - + {isPopulatedObject(pipelines) ? ( + {Object.keys(pipelines).length} + ) : null} ), content: ( diff --git a/x-pack/plugins/ml/public/application/model_management/force_stop_dialog.tsx b/x-pack/plugins/ml/public/application/model_management/force_stop_dialog.tsx index 39b108127b127b..4721ddb898fa32 100644 --- a/x-pack/plugins/ml/public/application/model_management/force_stop_dialog.tsx +++ b/x-pack/plugins/ml/public/application/model_management/force_stop_dialog.tsx @@ -150,13 +150,13 @@ export const StopModelDeploymentsConfirmDialog: FC -

+

    {pipelineWarning.map((pipelineName) => { return
  • {pipelineName}
    • ; })}
-

+
) : null} diff --git a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx index 8d27068a4ab6ec..1868ae0b8d85bf 100644 --- a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx +++ b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx @@ -9,7 +9,7 @@ import { Action } from '@elastic/eui/src/components/basic_table/action_types'; import { i18n } from '@kbn/i18n'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { EuiToolTip } from '@elastic/eui'; -import React, { useCallback, useMemo } from 'react'; +import React, { useCallback, useMemo, useEffect, useState } from 'react'; import { BUILT_IN_MODEL_TAG, DEPLOYMENT_STATE, @@ -42,7 +42,7 @@ export function useModelActions({ }: { isLoading: boolean; onTestAction: (model: ModelItem) => void; - onModelsDeleteRequest: (modelsIds: string[]) => void; + onModelsDeleteRequest: (models: ModelItem[]) => void; onLoading: (isLoading: boolean) => void; fetchModels: () => Promise; modelAndDeploymentIds: string[]; @@ -53,9 +53,12 @@ export function useModelActions({ overlays, theme, docLinks, + mlServices: { mlApiServices }, }, } = useMlKibana(); + const [canManageIngestPipelines, setCanManageIngestPipelines] = useState(false); + const startModelDeploymentDocUrl = docLinks.links.ml.startTrainedModelsDeployment; const navigateToPath = useNavigateToPath(); @@ -70,6 +73,23 @@ export function useModelActions({ const canTestTrainedModels = capabilities.ml.canTestTrainedModels as boolean; const canDeleteTrainedModels = capabilities.ml.canDeleteTrainedModels as boolean; + useEffect(() => { + let isMounted = true; + mlApiServices + .hasPrivileges({ + cluster: ['manage_ingest_pipelines'], + }) + .then((result) => { + const canManagePipelines = result.cluster.manage_ingest_pipelines; + if (isMounted) { + setCanManageIngestPipelines(canManagePipelines); + } + }); + return () => { + isMounted = false; + }; + }, [mlApiServices]); + const getUserConfirmation = useMemo( () => getUserConfirmationProvider(overlays, theme), [overlays, theme] @@ -394,17 +414,12 @@ export function useModelActions({ }, { name: (model) => { - const hasPipelines = isPopulatedObject(model.pipelines); const hasDeployments = model.state === MODEL_STATE.STARTED; return ( { - onModelsDeleteRequest([model.model_id]); + onModelsDeleteRequest([model]); + }, + available: (item) => { + const hasZeroPipelines = Object.keys(item.pipelines ?? {}).length === 0; + return ( + canDeleteTrainedModels && + !isBuiltInModel(item) && + !item.putModelConfig && + (hasZeroPipelines || canManageIngestPipelines) + ); }, - available: (item) => - canDeleteTrainedModels && !isBuiltInModel(item) && !item.putModelConfig, enabled: (item) => { - // TODO check for permissions to delete ingest pipelines. - // ATM undefined means pipelines fetch failed server-side. - return item.state !== MODEL_STATE.STARTED && !isPopulatedObject(item.pipelines); + return item.state !== MODEL_STATE.STARTED; }, }, { @@ -476,6 +496,7 @@ export function useModelActions({ isBuiltInModel, onTestAction, canTestTrainedModels, + canManageIngestPipelines, ] ); } diff --git a/x-pack/plugins/ml/public/application/model_management/models_list.tsx b/x-pack/plugins/ml/public/application/model_management/models_list.tsx index 609b294931e149..b2032b43ebbc39 100644 --- a/x-pack/plugins/ml/public/application/model_management/models_list.tsx +++ b/x-pack/plugins/ml/public/application/model_management/models_list.tsx @@ -133,7 +133,7 @@ export const ModelsList: FC = ({ const [isLoading, setIsLoading] = useState(false); const [items, setItems] = useState([]); const [selectedModels, setSelectedModels] = useState([]); - const [modelIdsToDelete, setModelIdsToDelete] = useState([]); + const [modelsToDelete, setModelsToDelete] = useState([]); const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>( {} ); @@ -348,7 +348,7 @@ export const ModelsList: FC = ({ isLoading, fetchModels: fetchModelsData, onTestAction: setModelToTest, - onModelsDeleteRequest: setModelIdsToDelete, + onModelsDeleteRequest: setModelsToDelete, onLoading: setIsLoading, modelAndDeploymentIds, }); @@ -502,13 +502,7 @@ export const ModelsList: FC = ({ - m.model_id) - )} - > + = ({ data-test-subj={isLoading ? 'mlModelsTable loading' : 'mlModelsTable loaded'} /> - {modelIdsToDelete.length > 0 && ( + {modelsToDelete.length > 0 && ( { - setModelIdsToDelete([]); + setModelsToDelete([]); if (refreshList) { fetchModelsData(); } }} - modelIds={modelIdsToDelete} + models={modelsToDelete} /> )} {modelToTest === null ? null : ( diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts index 30abf6f4dd2ead..0ea4b1d1fde4bf 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts @@ -114,11 +114,18 @@ export function trainedModelsApiProvider(httpService: HttpService) { * * @param modelId - Model ID */ - deleteTrainedModel(modelId: string) { + deleteTrainedModel( + modelId: string, + options: { with_pipelines?: boolean; force?: boolean } = { + with_pipelines: false, + force: false, + } + ) { return httpService.http<{ acknowledge: boolean }>({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/${modelId}`, method: 'DELETE', version: '1', + query: options, }); }, diff --git a/x-pack/plugins/ml/server/models/model_management/models_provider.ts b/x-pack/plugins/ml/server/models/model_management/models_provider.ts index e4ba7e2df04a95..702e8454660f6c 100644 --- a/x-pack/plugins/ml/server/models/model_management/models_provider.ts +++ b/x-pack/plugins/ml/server/models/model_management/models_provider.ts @@ -6,7 +6,6 @@ */ import type { IScopedClusterClient } from '@kbn/core/server'; - import type { PipelineDefinition } from '../../../common/types/trained_models'; export type ModelService = ReturnType; @@ -51,5 +50,19 @@ export function modelsProvider(client: IScopedClusterClient) { return modelIdsMap; }, + + /** + * Deletes associated pipelines of the requested model + * @param modelIds + */ + async deleteModelPipelines(modelIds: string[]) { + const pipelines = await this.getModelsPipelines(modelIds); + const pipelinesIds: string[] = [ + ...new Set([...pipelines.values()].flatMap((v) => Object.keys(v!))), + ]; + await Promise.all( + pipelinesIds.map((id) => client.asCurrentUser.ingest.deletePipeline({ id })) + ); + }, }; } diff --git a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts index bc96a734cd54cb..e15fd108c5f5b6 100644 --- a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts @@ -72,3 +72,8 @@ export const stopDeploymentSchema = schema.object({ /** force stop */ force: schema.maybe(schema.boolean()), }); + +export const deleteTrainedModelQuerySchema = schema.object({ + with_pipelines: schema.maybe(schema.boolean({ defaultValue: false })), + force: schema.maybe(schema.boolean({ defaultValue: false })), +}); diff --git a/x-pack/plugins/ml/server/routes/trained_models.ts b/x-pack/plugins/ml/server/routes/trained_models.ts index dd1d4fa693a84f..cefb8a9dce0fdb 100644 --- a/x-pack/plugins/ml/server/routes/trained_models.ts +++ b/x-pack/plugins/ml/server/routes/trained_models.ts @@ -11,6 +11,7 @@ import { ML_INTERNAL_BASE_PATH } from '../../common/constants/app'; import { RouteInitialization } from '../types'; import { wrapError } from '../client/error_wrapper'; import { + deleteTrainedModelQuerySchema, getInferenceQuerySchema, inferTrainedModelBody, inferTrainedModelQuery, @@ -22,7 +23,6 @@ import { threadingParamsSchema, updateDeploymentParamsSchema, } from './schemas/inference_schema'; - import { TrainedModelConfigResponse } from '../../common/types/trained_models'; import { mlLog } from '../lib/log'; import { forceQuerySchema } from './schemas/anomaly_detectors_schema'; @@ -303,15 +303,25 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization) validate: { request: { params: modelIdSchema, + query: deleteTrainedModelQuerySchema, }, }, }, - routeGuard.fullLicenseAPIGuard(async ({ mlClient, request, response }) => { + routeGuard.fullLicenseAPIGuard(async ({ mlClient, request, response, client }) => { try { const { modelId } = request.params; + const { with_pipelines: withPipelines, force } = request.query; + + if (withPipelines) { + // first we need to delete pipelines, otherwise ml api return an error + await modelsProvider(client).deleteModelPipelines(modelId.split(',')); + } + const body = await mlClient.deleteTrainedModel({ model_id: modelId, + force, }); + return response.ok({ body, }); diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index c31c08adaf65b6..ffe6d5515bb0c6 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -24943,7 +24943,6 @@ "xpack.ml.trainedModels.modelsList.builtInModelMessage": "Modèle intégré", "xpack.ml.trainedModels.modelsList.collapseRow": "Réduire", "xpack.ml.trainedModels.modelsList.createdAtHeader": "Créé à", - "xpack.ml.trainedModels.modelsList.deleteDisabledTooltip": "Le modèle a des pipelines associés", "xpack.ml.trainedModels.modelsList.deleteModal.cancelButtonLabel": "Annuler", "xpack.ml.trainedModels.modelsList.deleteModal.deleteButtonLabel": "Supprimer", "xpack.ml.trainedModels.modelsList.deleteModelActionLabel": "Supprimer le modèle", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index fe7b22c4b506b8..81add29898d3b6 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -24929,7 +24929,6 @@ "xpack.ml.trainedModels.modelsList.builtInModelMessage": "ビルトインモデル", "xpack.ml.trainedModels.modelsList.collapseRow": "縮小", "xpack.ml.trainedModels.modelsList.createdAtHeader": "作成日時：", - "xpack.ml.trainedModels.modelsList.deleteDisabledTooltip": "モデルにはパイプラインが関連付けられています", "xpack.ml.trainedModels.modelsList.deleteModal.cancelButtonLabel": "キャンセル", "xpack.ml.trainedModels.modelsList.deleteModal.deleteButtonLabel": "削除", "xpack.ml.trainedModels.modelsList.deleteModelActionLabel": "モデルを削除", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 8db7d6a86c6992..940c0f9e108790 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -24928,7 +24928,6 @@ "xpack.ml.trainedModels.modelsList.builtInModelMessage": "内置模型", "xpack.ml.trainedModels.modelsList.collapseRow": "折叠", "xpack.ml.trainedModels.modelsList.createdAtHeader": "创建于", - "xpack.ml.trainedModels.modelsList.deleteDisabledTooltip": "模型有关联的管道", "xpack.ml.trainedModels.modelsList.deleteModal.cancelButtonLabel": "取消", "xpack.ml.trainedModels.modelsList.deleteModal.deleteButtonLabel": "删除", "xpack.ml.trainedModels.modelsList.deleteModelActionLabel": "删除模型", diff --git a/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts b/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts index bba29abe0b914a..e404eb3e451e6a 100644 --- a/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts +++ b/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts @@ -52,6 +52,28 @@ export default function ({ getService }: FtrProviderContext) { modelTypes: ['regression', 'tree_ensemble'], }; + describe('for ML user with read-only access', () => { + before(async () => { + await ml.securityUI.loginAsMlViewer(); + await ml.navigation.navigateToTrainedModels(); + await ml.commonUI.waitForRefreshButtonEnabled(); + }); + + after(async () => { + await ml.securityUI.logout(); + }); + + it('renders expanded row content correctly for model with pipelines', async () => { + await ml.trainedModelsTable.ensureRowIsExpanded(modelWithPipelineData.modelId); + await ml.trainedModelsTable.assertDetailsTabContent(); + await ml.trainedModelsTable.assertInferenceConfigTabContent(); + await ml.trainedModelsTable.assertStatsTabContent(); + await ml.trainedModelsTable.assertPipelinesTabContent(true, [ + { pipelineName: `pipeline_${modelWithPipelineData.modelId}`, expectDefinition: false }, + ]); + }); + }); + describe('for ML power user', () => { before(async () => { await ml.securityUI.loginAsMlPowerUser(); @@ -138,7 +160,7 @@ export default function ({ getService }: FtrProviderContext) { ); }); - it('displays a model with an ingest pipeline and delete action is disabled', async () => { + it('displays a model with an ingest pipeline and model can be deleted with associated ingest pipelines', async () => { await ml.testExecution.logTestStep('should display the model in the table'); await ml.trainedModelsTable.filterWithSearchString(modelWithPipelineData.modelId, 1); @@ -152,10 +174,20 @@ export default function ({ getService }: FtrProviderContext) { }); await ml.testExecution.logTestStep( - 'should show disabled delete action for the model in the table' + 'should show enabled delete action for the model in the table' ); await ml.trainedModelsTable.assertModelDeleteActionButtonEnabled( + modelWithPipelineData.modelId, + true + ); + + await ml.testExecution.logTestStep('should show the delete modal'); + await ml.trainedModelsTable.clickDeleteAction(modelWithPipelineData.modelId); + + await ml.testExecution.logTestStep('should delete the model with pipelines'); + await ml.trainedModelsTable.confirmDeleteModel(true); + await ml.trainedModelsTable.assertModelDisplayedInTable( modelWithPipelineData.modelId, false ); @@ -224,27 +256,5 @@ export default function ({ getService }: FtrProviderContext) { } }); }); - - describe('for ML user with read-only access', () => { - before(async () => { - await ml.securityUI.loginAsMlViewer(); - await ml.navigation.navigateToTrainedModels(); - await ml.commonUI.waitForRefreshButtonEnabled(); - }); - - after(async () => { - await ml.securityUI.logout(); - }); - - it('renders expanded row content correctly for model with pipelines', async () => { - await ml.trainedModelsTable.ensureRowIsExpanded(modelWithPipelineData.modelId); - await ml.trainedModelsTable.assertDetailsTabContent(); - await ml.trainedModelsTable.assertInferenceConfigTabContent(); - await ml.trainedModelsTable.assertStatsTabContent(); - await ml.trainedModelsTable.assertPipelinesTabContent(true, [ - { pipelineName: `pipeline_${modelWithPipelineData.modelId}`, expectDefinition: false }, - ]); - }); - }); }); } diff --git a/x-pack/test/functional/services/ml/trained_models_table.ts b/x-pack/test/functional/services/ml/trained_models_table.ts index 71ce07da7d6d65..e136d6553606f7 100644 --- a/x-pack/test/functional/services/ml/trained_models_table.ts +++ b/x-pack/test/functional/services/ml/trained_models_table.ts @@ -284,16 +284,54 @@ export function TrainedModelsTableProvider( await testSubjects.missingOrFail('mlModelsDeleteModal', { timeout: 60 * 1000 }); } - public async confirmDeleteModel() { + public async getCheckBoxState(testSubj: string): Promise { + return (await testSubjects.getAttribute(testSubj, 'checked')) === 'true'; + } + + public async assertDeletePipelinesCheckboxSelected(expectedValue: boolean) { + const actualCheckState = await this.getCheckBoxState( + 'mlModelsDeleteModalDeletePipelinesCheckbox' + ); + expect(actualCheckState).to.eql( + expectedValue, + `Delete model pipelines checkbox should be ${expectedValue} (got ${actualCheckState})` + ); + } + + public async setDeletePipelinesCheckbox() { + await this.assertDeletePipelinesCheckboxSelected(false); + + const checkboxLabel = await find.byCssSelector(`label[for="delete-model-pipelines"]`); + await checkboxLabel.click(); + + await this.assertDeletePipelinesCheckboxSelected(true); + } + + public async confirmDeleteModel(withPipelines: boolean = false) { await retry.tryForTime(30 * 1000, async () => { await this.assertDeleteModalExists(); + + if (withPipelines) { + await this.setDeletePipelinesCheckbox(); + } + await testSubjects.click('mlModelsDeleteModalConfirmButton'); await this.assertDeleteModalNotExists(); }); } public async clickDeleteAction(modelId: string) { - await testSubjects.click(this.rowSelector(modelId, 'mlModelsTableRowDeleteAction')); + const actionsButtonExists = await this.doesModelCollapsedActionsButtonExist(modelId); + + if (actionsButtonExists) { + await this.toggleActionsContextMenu(modelId, true); + const panelElement = await find.byCssSelector('.euiContextMenuPanel'); + const actionButton = await panelElement.findByTestSubject('mlModelsTableRowDeleteAction'); + await actionButton.click(); + } else { + await testSubjects.click(this.rowSelector(modelId, 'mlModelsTableRowDeleteAction')); + } + await this.assertDeleteModalExists(); } From 16b9614de268eedc35c418b602ce16b8019c0a8e Mon Sep 17 00:00:00 2001 From: Giorgos Bamparopoulos Date: Mon, 5 Jun 2023 13:16:16 +0100 Subject: [PATCH 2/6] [APM] Display size for hidden indices in storage explorer (#158746) - Adds `expand_wildcards: 'all'` to the Index stats API call to fix an issue with missing statistics for hidden data streams in storage explorer - Fixes an issue with number of replicas not being displayed --- .../storage_explorer/get_storage_details_per_service.ts | 2 +- .../server/routes/storage_explorer/indices_stats_helpers.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/apm/server/routes/storage_explorer/get_storage_details_per_service.ts b/x-pack/plugins/apm/server/routes/storage_explorer/get_storage_details_per_service.ts index e019c563601cb0..c029ab3253b8f5 100644 --- a/x-pack/plugins/apm/server/routes/storage_explorer/get_storage_details_per_service.ts +++ b/x-pack/plugins/apm/server/routes/storage_explorer/get_storage_details_per_service.ts @@ -257,7 +257,7 @@ export async function getStorageDetailsPerIndex({ ? indexInfo.settings?.index?.number_of_shards ?? 0 : undefined, replica: indexInfo - ? indexInfo.settings?.number_of_replicas ?? 0 + ? indexInfo.settings?.index?.number_of_replicas ?? 0 : undefined, size, dataStream: indexInfo?.data_stream, diff --git a/x-pack/plugins/apm/server/routes/storage_explorer/indices_stats_helpers.ts b/x-pack/plugins/apm/server/routes/storage_explorer/indices_stats_helpers.ts index 20584f86997902..7c4ce4390bf319 100644 --- a/x-pack/plugins/apm/server/routes/storage_explorer/indices_stats_helpers.ts +++ b/x-pack/plugins/apm/server/routes/storage_explorer/indices_stats_helpers.ts @@ -18,7 +18,10 @@ export async function getTotalIndicesStats({ }) { const index = getApmIndicesCombined(apmEventClient); const esClient = (await context.core).elasticsearch.client; - const totalStats = await esClient.asCurrentUser.indices.stats({ index }); + const totalStats = await esClient.asCurrentUser.indices.stats({ + index, + expand_wildcards: 'all', + }); return totalStats; } From 948f995bad06c97164482c69818874455c1e6946 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 5 Jun 2023 14:30:03 +0200 Subject: [PATCH 3/6] [Synthetics] Added inspect panel for monitor config (#157978) Co-authored-by: Abdul Zahid Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/fleet/server/mocks/index.ts | 1 + .../server/services/package_policy.test.ts | 48 ++++ .../fleet/server/services/package_policy.ts | 43 +++ .../server/services/package_policy_service.ts | 5 + .../synthetics/common/constants/rest_api.ts | 1 + .../common/components/monitor_inspect.tsx | 205 +++++++++++++ .../components/monitor_add_edit/portals.tsx | 2 + .../monitor_add_edit/steps/index.tsx | 2 + .../steps/inspect_monitor_portal.tsx | 19 ++ .../public/apps/synthetics/routes.tsx | 7 +- .../state/monitor_management/api.ts | 18 ++ .../public/utils/api_service/api_service.ts | 3 +- .../server/common/unzipt_project_code.ts | 35 +++ .../plugins/synthetics/server/routes/index.ts | 2 + .../routes/monitor_cruds/add_monitor.ts | 46 +-- .../routes/monitor_cruds/inspect_monitor.ts | 131 +++++++++ .../convert_to_data_stream.ts | 2 +- .../synthetics_private_location.test.ts | 4 +- .../synthetics_private_location.ts | 65 ++++- .../service_api_client.test.ts | 45 ++- .../synthetics_service/service_api_client.ts | 272 +++++++++++------- .../synthetics_monitor_client.ts | 104 +++++-- .../synthetics_service/synthetics_service.ts | 31 +- .../apis/synthetics/add_monitor_project.ts | 2 +- .../fixtures/inspect_browser_monitor.json | 84 ++++++ .../api_integration/apis/synthetics/index.ts | 1 + .../apis/synthetics/inspect_monitor.ts | 227 +++++++++++++++ .../services/private_location_test_service.ts | 31 +- .../synthetics_monitor_test_service.ts | 32 +++ x-pack/test/tsconfig.json | 1 + 30 files changed, 1270 insertions(+), 199 deletions(-) create mode 100644 x-pack/plugins/synthetics/public/apps/synthetics/components/common/components/monitor_inspect.tsx create mode 100644 x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/steps/inspect_monitor_portal.tsx create mode 100644 x-pack/plugins/synthetics/server/common/unzipt_project_code.ts create mode 100644 x-pack/plugins/synthetics/server/routes/monitor_cruds/inspect_monitor.ts create mode 100644 x-pack/test/api_integration/apis/synthetics/fixtures/inspect_browser_monitor.json create mode 100644 x-pack/test/api_integration/apis/synthetics/inspect_monitor.ts diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 528891aa6ab4ce..890f7372758c37 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -119,6 +119,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked { }); }); + describe('inspect', () => { + it('should return compiled inputs', async () => { + const soClient = savedObjectsClientMock.create(); + + soClient.create.mockResolvedValueOnce({ + id: 'test-package-policy', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + + mockAgentPolicyGet(); + + const policyResult = await packagePolicyService.inspect( + soClient, + { + id: 'b684f590-feeb-11ed-b202-b7f403f1dee9', + name: 'Test Package Policy', + namespace: 'test', + enabled: true, + policy_id: 'test', + inputs: [], + package: { + name: 'test', + title: 'Test', + version: '0.0.1', + }, + } + // Skipping unique name verification just means we have to less mocking/setup + ); + + expect(policyResult).toEqual({ + elasticsearch: undefined, + enabled: true, + inputs: [], + name: 'Test Package Policy', + namespace: 'test', + package: { + name: 'test', + title: 'Test', + version: '0.0.1', + }, + policy_id: 'test', + id: 'b684f590-feeb-11ed-b202-b7f403f1dee9', + }); + }); + }); + describe('bulkCreate', () => { it('should call audit logger', async () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index b4ad90bc350780..c17f8f4ec2ea3c 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -445,6 +445,49 @@ class PackagePolicyClientImpl implements PackagePolicyClient { }; } + /** Purpose of this function is to take a package policy and compile the inputs + This is primarily used by the Synthetics UI to display the inputs which are passed to agent + Purpose is to debug the inputs which are passed to the agent and also compared them to the config + which is passed to public service locations */ + public async inspect( + soClient: SavedObjectsClientContract, + packagePolicy: NewPackagePolicyWithId + ): Promise { + if (!packagePolicy.id) { + packagePolicy.id = SavedObjectsUtils.generateId(); + } + + const packageInfos = await getPackageInfoForPackagePolicies([packagePolicy], soClient); + const agentPolicyId = packagePolicy.policy_id; + + let inputs = getInputsWithStreamIds(packagePolicy, packagePolicy.id); + const { id, ...pkgPolicyWithoutId } = packagePolicy; + + let elasticsearch: PackagePolicy['elasticsearch']; + if (packagePolicy.package) { + const pkgInfo = packageInfos.get( + `${packagePolicy.package.name}-${packagePolicy.package.version}` + ); + + inputs = pkgInfo + ? await _compilePackagePolicyInputs(pkgInfo, packagePolicy.vars || {}, inputs) + : inputs; + + elasticsearch = pkgInfo?.elasticsearch; + } + + return { + id: packagePolicy.id, + ...pkgPolicyWithoutId, + ...(packagePolicy.package + ? { package: omit(packagePolicy.package, 'experimental_data_stream_features') } + : {}), + inputs, + elasticsearch, + policy_id: agentPolicyId, + }; + } + public async get( soClient: SavedObjectsClientContract, id: string diff --git a/x-pack/plugins/fleet/server/services/package_policy_service.ts b/x-pack/plugins/fleet/server/services/package_policy_service.ts index 816d0339a1277a..9519cafbc6a736 100644 --- a/x-pack/plugins/fleet/server/services/package_policy_service.ts +++ b/x-pack/plugins/fleet/server/services/package_policy_service.ts @@ -56,6 +56,11 @@ export interface PackagePolicyClient { request?: KibanaRequest ): Promise; + inspect( + soClient: SavedObjectsClientContract, + packagePolicy: NewPackagePolicyWithId + ): Promise; + bulkCreate( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, diff --git a/x-pack/plugins/synthetics/common/constants/rest_api.ts b/x-pack/plugins/synthetics/common/constants/rest_api.ts index b64d0af6d81cd2..f5b263b22f2178 100644 --- a/x-pack/plugins/synthetics/common/constants/rest_api.ts +++ b/x-pack/plugins/synthetics/common/constants/rest_api.ts @@ -40,6 +40,7 @@ export enum API_URLS { INDEX_TEMPLATES = '/internal/uptime/service/index_templates', SERVICE_LOCATIONS = '/internal/uptime/service/locations', SYNTHETICS_MONITORS = '/internal/uptime/service/monitors', + SYNTHETICS_MONITOR_INSPECT = '/internal/uptime/service/monitor/inspect', GET_SYNTHETICS_MONITOR = '/internal/uptime/service/monitor/{monitorId}', SYNTHETICS_ENABLEMENT = '/internal/uptime/service/enablement', RUN_ONCE_MONITOR = '/internal/uptime/service/monitors/run_once', diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/common/components/monitor_inspect.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/common/components/monitor_inspect.tsx new file mode 100644 index 00000000000000..9df8ecbfb4c53f --- /dev/null +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/common/components/monitor_inspect.tsx @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { useFormContext } from 'react-hook-form'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { enableInspectEsQueries } from '@kbn/observability-plugin/common'; +import { useFetcher } from '@kbn/observability-shared-plugin/public'; +import { i18n } from '@kbn/i18n'; +import { + EuiFlyout, + EuiButton, + EuiCodeBlock, + EuiFlyoutHeader, + EuiTitle, + EuiFlyoutFooter, + EuiHorizontalRule, + EuiSpacer, + EuiFlyoutBody, + EuiToolTip, + EuiSwitch, +} from '@elastic/eui'; + +import { ClientPluginsStart } from '../../../../../plugin'; +import { useSyntheticsSettingsContext } from '../../../contexts'; +import { LoadingState } from '../../monitors_page/overview/overview/monitor_detail_flyout'; +import { DataStream, SyntheticsMonitor } from '../../../../../../common/runtime_types'; +import { inspectMonitorAPI, MonitorInspectResponse } from '../../../state/monitor_management/api'; + +export const MonitorInspectWrapper = () => { + const { + services: { uiSettings }, + } = useKibana(); + + const { isDev } = useSyntheticsSettingsContext(); + + const isInspectorEnabled = uiSettings?.get(enableInspectEsQueries); + + return isDev || isInspectorEnabled ? : null; +}; + +const MonitorInspect = () => { + const { isDev } = useSyntheticsSettingsContext(); + + const [hideParams, setHideParams] = useState(() => !isDev); + const [isFlyoutVisible, setIsFlyoutVisible] = useState(false); + + const closeFlyout = () => { + setIsFlyoutVisible(false); + setIsInspecting(false); + }; + + const [isInspecting, setIsInspecting] = useState(false); + const onButtonClick = () => { + setIsInspecting(() => !isInspecting); + setIsFlyoutVisible(() => !isFlyoutVisible); + }; + + const { getValues, formState } = useFormContext(); + + const { data, loading, error } = useFetcher(() => { + if (isInspecting) { + return inspectMonitorAPI({ + hideParams, + monitor: getValues() as SyntheticsMonitor, + }); + } + }, [isInspecting, hideParams]); + + let flyout; + + if (isFlyoutVisible) { + flyout = ( + + + +

{CONFIG_LABEL}

+ + + + setHideParams(e.target.checked)} + /> + + {!loading && data ? ( + <> + + {formatContent(data.result)} + + {data.decodedCode && } + + ) : loading && !error ? ( + + ) : ( +

{error?.message}

+ )} + + + + {CLOSE_LABEL} + + + + ); + } + return ( + <> + + + {INSPECT_MONITOR_LABEL} + + + + {flyout} + + ); +}; + +// @ts-ignore: Unused variable +// tslint:disable-next-line: no-unused-variable +const MonitorCode = ({ code }: { code: string }) => ( + <> + + +

{SOURCE_CODE_LABEL}

+ + + + {code} + + +); + +const formatContent = (result: MonitorInspectResponse) => { + const firstResult = result.publicConfigs?.[0]?.monitors?.[0]; + + const currentInput = result.privateConfig?.inputs.find((input) => input.enabled); + const compiledConfig = currentInput?.streams.find((stream) => + Object.values(DataStream).includes(stream.data_stream.dataset as DataStream) + )?.compiled_stream; + + return JSON.stringify( + { publicConfig: firstResult ?? {}, privateConfig: compiledConfig ?? {} }, + null, + 2 + ); +}; + +const CONFIG_LABEL = i18n.translate('xpack.synthetics.monitorInspect.configLabel', { + defaultMessage: 'Configuration', +}); + +const VALID_CONFIG_LABEL = i18n.translate( + 'xpack.synthetics.monitorInspect.formattedConfigLabel.valid', + { + defaultMessage: 'Only valid form configurations can be inspected.', + } +); + +const FORMATTED_CONFIG_DESCRIPTION = i18n.translate( + 'xpack.synthetics.monitorInspect.formattedConfigLabel.description', + { + defaultMessage: 'View formatted configuration for this monitor.', + } +); +const CLOSE_LABEL = i18n.translate('xpack.synthetics.monitorInspect.closeLabel', { + defaultMessage: 'Close', +}); + +export const SOURCE_CODE_LABEL = i18n.translate('xpack.synthetics.monitorInspect.sourceCodeLabel', { + defaultMessage: 'Source code', +}); + +export const INSPECT_MONITOR_LABEL = i18n.translate( + 'xpack.synthetics.monitorInspect.inspectLabel', + { + defaultMessage: 'Inspect configuration', + } +); + +const HIDE_PARAMS = i18n.translate('xpack.synthetics.monitorInspect.hideParams', { + defaultMessage: 'Hide params values', +}); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/portals.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/portals.tsx index cd32ca9d914d47..8ebd3143eb1283 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/portals.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/portals.tsx @@ -9,4 +9,6 @@ import { createHtmlPortalNode } from 'react-reverse-portal'; export const MonitorTypePortalNode = createHtmlPortalNode(); +export const InspectMonitorPortalNode = createHtmlPortalNode(); + export const MonitorDetailsLinkPortalNode = createHtmlPortalNode(); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/steps/index.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/steps/index.tsx index 4f2433835fed83..f08452dc836074 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/steps/index.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/steps/index.tsx @@ -8,6 +8,7 @@ import React from 'react'; import { EuiSteps, EuiPanel, EuiText, EuiSpacer } from '@elastic/eui'; import { useFormContext } from 'react-hook-form'; +import { InspectMonitorPortal } from './inspect_monitor_portal'; import { ConfigKey, FormMonitorType, StepMap } from '../types'; import { AdvancedConfig } from '../advanced'; import { MonitorTypePortal } from './monitor_type_portal'; @@ -54,6 +55,7 @@ export const MonitorSteps = ({ )} + ); }; diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/steps/inspect_monitor_portal.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/steps/inspect_monitor_portal.tsx new file mode 100644 index 00000000000000..80dd5e0f6a16cd --- /dev/null +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/steps/inspect_monitor_portal.tsx @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { InPortal } from 'react-reverse-portal'; +import { MonitorInspectWrapper } from '../../common/components/monitor_inspect'; +import { InspectMonitorPortalNode } from '../portals'; + +export const InspectMonitorPortal = () => { + return ( + + + + ); +}; diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/routes.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/routes.tsx index b80420cdede50a..8f3ff6e74ffe99 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/routes.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/routes.tsx @@ -31,6 +31,7 @@ import { MonitorAddPageWithServiceAllowed } from './components/monitor_add_edit/ import { MonitorEditPageWithServiceAllowed } from './components/monitor_add_edit/monitor_edit_page'; import { GettingStartedPage } from './components/getting_started/getting_started_page'; import { + InspectMonitorPortalNode, MonitorDetailsLinkPortalNode, MonitorTypePortalNode, } from './components/monitor_add_edit/portals'; @@ -97,6 +98,7 @@ const getRoutes = ( defaultMessage="Create Monitor" /> ), + rightSideItems: [], children: ( ), - rightSideItems: [], + rightSideItems: [ + , + , + ], breadcrumbs: [ { text: , diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/state/monitor_management/api.ts b/x-pack/plugins/synthetics/public/apps/synthetics/state/monitor_management/api.ts index 2b68e1a104cc1f..27999495fd8712 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/state/monitor_management/api.ts +++ b/x-pack/plugins/synthetics/public/apps/synthetics/state/monitor_management/api.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { PackagePolicy } from '@kbn/fleet-plugin/common'; import { apiService } from '../../../../utils/api_service'; import { EncryptedSyntheticsMonitor, @@ -23,6 +24,23 @@ export const createMonitorAPI = async ({ return await apiService.post(API_URLS.SYNTHETICS_MONITORS, monitor); }; +export interface MonitorInspectResponse { + publicConfigs: any[]; + privateConfig: PackagePolicy | null; +} + +export const inspectMonitorAPI = async ({ + monitor, + hideParams, +}: { + hideParams?: boolean; + monitor: SyntheticsMonitor | EncryptedSyntheticsMonitor; +}): Promise<{ result: MonitorInspectResponse; decodedCode: string }> => { + return await apiService.post(API_URLS.SYNTHETICS_MONITOR_INSPECT, monitor, undefined, { + hideParams, + }); +}; + export const updateMonitorAPI = async ({ monitor, id, diff --git a/x-pack/plugins/synthetics/public/utils/api_service/api_service.ts b/x-pack/plugins/synthetics/public/utils/api_service/api_service.ts index 9df3339e1b7958..b73b353f708f2e 100644 --- a/x-pack/plugins/synthetics/public/utils/api_service/api_service.ts +++ b/x-pack/plugins/synthetics/public/utils/api_service/api_service.ts @@ -73,10 +73,11 @@ class ApiService { return response; } - public async post(apiUrl: string, data?: any, decodeType?: any) { + public async post(apiUrl: string, data?: any, decodeType?: any, params?: HttpFetchQuery) { const response = await this._http!.post(apiUrl, { method: 'POST', body: JSON.stringify(data), + query: params, }); this.addInspectorRequest?.({ data: response, status: FETCH_STATUS.SUCCESS, loading: false }); diff --git a/x-pack/plugins/synthetics/server/common/unzipt_project_code.ts b/x-pack/plugins/synthetics/server/common/unzipt_project_code.ts new file mode 100644 index 00000000000000..551b738944e2d2 --- /dev/null +++ b/x-pack/plugins/synthetics/server/common/unzipt_project_code.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { join } from 'path'; +import { writeFile } from 'fs/promises'; +import os from 'os'; +import AdmZip from 'adm-zip'; + +export function generateUniqueId() { + return `${Date.now() + Math.floor(Math.random() * 1e13)}`; +} + +export function generateTempPath() { + return join(os.tmpdir(), `synthetics-${generateUniqueId()}`); +} + +export async function unzipFile(content: string) { + const decoded = Buffer.from(content, 'base64'); + const pathToZip = generateTempPath(); + await writeFile(pathToZip, decoded); + const zip = new AdmZip(pathToZip); + const zipEntries = zip.getEntries(); + + let allData = ''; + + for (const entry of zipEntries) { + const entryData = entry.getData().toString(); + allData += entryData; + } + return allData; +} diff --git a/x-pack/plugins/synthetics/server/routes/index.ts b/x-pack/plugins/synthetics/server/routes/index.ts index 70e5145b7d8f0c..50069c95dc4434 100644 --- a/x-pack/plugins/synthetics/server/routes/index.ts +++ b/x-pack/plugins/synthetics/server/routes/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { inspectSyntheticsMonitorRoute } from './monitor_cruds/inspect_monitor'; import { deletePackagePolicyRoute } from './monitor_cruds/delete_integration'; import { createJourneyScreenshotRoute } from './pings/journey_screenshots'; import { createJourneyScreenshotBlocksRoute } from './pings/journey_screenshot_blocks'; @@ -98,6 +99,7 @@ export const syntheticsAppRestApiRoutes: SyntheticsRestApiRouteFactory[] = [ getLocationMonitors, getPrivateLocationsRoute, getSyntheticsFilters, + inspectSyntheticsMonitorRoute, ]; export const syntheticsAppStreamingApiRoutes: SyntheticsStreamingRouteFactory[] = [ diff --git a/x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts b/x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts index 55df65b7fa581f..63bafe787497d8 100644 --- a/x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts +++ b/x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts @@ -75,13 +75,11 @@ export const addSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () => ({ ); try { - const spaceId = server.spaces?.spacesService.getSpaceId(request) ?? DEFAULT_SPACE_ID; const { errors, newMonitor } = await syncNewMonitor({ normalizedMonitor, routeContext, id, privateLocations, - spaceId, }); if (errors && errors.length > 0) { @@ -149,28 +147,22 @@ export const createNewSavedObjectMonitor = async ({ ); }; -export const syncNewMonitor = async ({ - id, +export const hydrateMonitorFields = ({ + newMonitorId, normalizedMonitor, - privateLocations, - spaceId, routeContext, }: { - id?: string; + newMonitorId: string; normalizedMonitor: SyntheticsMonitor; routeContext: RouteContext; - privateLocations: PrivateLocation[]; - spaceId: string; }) => { - const { savedObjectsClient, server, syntheticsMonitorClient, request } = routeContext; - const newMonitorId = id ?? uuidV4(); + const { server, request } = routeContext; + const { preserve_namespace: preserveNamespace } = request.query as Record< string, { preserve_namespace?: boolean } >; - - let monitorSavedObject: SavedObject | null = null; - const monitorWithNamespace = { + return { ...normalizedMonitor, [ConfigKey.MONITOR_QUERY_ID]: normalizedMonitor[ConfigKey.CUSTOM_HEARTBEAT_ID] || newMonitorId, [ConfigKey.CONFIG_ID]: newMonitorId, @@ -178,6 +170,28 @@ export const syncNewMonitor = async ({ ? normalizedMonitor[ConfigKey.NAMESPACE] : getMonitorNamespace(server, request, normalizedMonitor[ConfigKey.NAMESPACE]), }; +}; + +export const syncNewMonitor = async ({ + id, + normalizedMonitor, + privateLocations, + routeContext, +}: { + id?: string; + normalizedMonitor: SyntheticsMonitor; + routeContext: RouteContext; + privateLocations: PrivateLocation[]; +}) => { + const { savedObjectsClient, server, syntheticsMonitorClient, request, spaceId } = routeContext; + const newMonitorId = id ?? uuidV4(); + + let monitorSavedObject: SavedObject | null = null; + const monitorWithNamespace = hydrateMonitorFields({ + normalizedMonitor, + routeContext, + newMonitorId, + }); try { const newMonitorPromise = createNewSavedObjectMonitor({ @@ -263,7 +277,7 @@ export const deleteMonitorIfCreated = async ({ } }; -const getPrivateLocations = async ( +export const getPrivateLocations = async ( soClient: SavedObjectsClientContract, normalizedMonitor: SyntheticsMonitor ) => { @@ -275,7 +289,7 @@ const getPrivateLocations = async ( return await getSyntheticsPrivateLocations(soClient); }; -const getMonitorNamespace = ( +export const getMonitorNamespace = ( server: UptimeServerSetup, request: KibanaRequest, configuredNamespace: string diff --git a/x-pack/plugins/synthetics/server/routes/monitor_cruds/inspect_monitor.ts b/x-pack/plugins/synthetics/server/routes/monitor_cruds/inspect_monitor.ts new file mode 100644 index 00000000000000..fbff0ba34fd14d --- /dev/null +++ b/x-pack/plugins/synthetics/server/routes/monitor_cruds/inspect_monitor.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { v4 as uuidV4 } from 'uuid'; +import { schema } from '@kbn/config-schema'; +import { unzipFile } from '../../common/unzipt_project_code'; +import { + ConfigKey, + MonitorFields, + SyntheticsMonitor, + PrivateLocation, +} from '../../../common/runtime_types'; +import { SyntheticsRestApiRouteFactory } from '../../legacy_uptime/routes/types'; +import { API_URLS } from '../../../common/constants'; +import { DEFAULT_FIELDS } from '../../../common/constants/monitor_defaults'; +import { validateMonitor } from './monitor_validation'; +import { getPrivateLocations, hydrateMonitorFields } from './add_monitor'; + +export const inspectSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () => ({ + method: 'POST', + path: API_URLS.SYNTHETICS_MONITOR_INSPECT, + validate: { + body: schema.any(), + query: schema.object({ + id: schema.maybe(schema.string()), + preserve_namespace: schema.maybe(schema.boolean()), + hideParams: schema.maybe(schema.boolean()), + }), + }, + writeAccess: true, + handler: async (routeContext): Promise => { + const { savedObjectsClient, server, syntheticsMonitorClient, request, spaceId, response } = + routeContext; + // usually id is auto generated, but this is useful for testing + const { id, hideParams = true } = request.query; + + const monitor: SyntheticsMonitor = request.body as SyntheticsMonitor; + const monitorType = monitor[ConfigKey.MONITOR_TYPE]; + const monitorWithDefaults = { + ...DEFAULT_FIELDS[monitorType], + ...monitor, + }; + + const validationResult = validateMonitor(monitorWithDefaults as MonitorFields); + + if (!validationResult.valid || !validationResult.decodedMonitor) { + const { reason: message, details, payload } = validationResult; + return response.badRequest({ body: { message, attributes: { details, ...payload } } }); + } + + const normalizedMonitor = validationResult.decodedMonitor; + + const privateLocations: PrivateLocation[] = await getPrivateLocations( + savedObjectsClient, + normalizedMonitor + ); + + const canSave = + Boolean((await server.coreStart?.capabilities.resolveCapabilities(request)).uptime.save) ?? + false; + + try { + const newMonitorId = id ?? uuidV4(); + + const monitorWithNamespace = hydrateMonitorFields({ + normalizedMonitor, + routeContext, + newMonitorId, + }); + + const result = await syntheticsMonitorClient.inspectMonitor( + { monitor: monitorWithNamespace as MonitorFields, id: newMonitorId }, + request, + savedObjectsClient, + privateLocations, + spaceId, + hideParams, + canSave + ); + + const publicConfigs = result.publicConfigs; + + const sampleMonitor = publicConfigs?.[0]?.monitors?.[0]; + + const hasSourceContent = sampleMonitor?.streams[0][ConfigKey.SOURCE_PROJECT_CONTENT]; + let decodedCode = ''; + if (hasSourceContent) { + decodedCode = await unzipFile(hasSourceContent); + } + + return response.ok({ body: { result, decodedCode: formatCode(decodedCode) } }); + } catch (getErr) { + server.logger.error( + `Unable to inspect Synthetics monitor ${monitorWithDefaults[ConfigKey.NAME]}` + ); + server.logger.error(getErr); + + return response.customError({ + body: { message: getErr.message }, + statusCode: 500, + }); + } + }, +}); + +const formatCode = (code: string) => { + const replacements = [ + { pattern: /\(\d*,\s*import_synthetics\d*\.step\)/g, replacement: 'step' }, + { pattern: /\(\d*,\s*import_synthetics\d*\.journey\)/g, replacement: 'journey' }, + { pattern: /import_synthetics\d*\.monitor/g, replacement: 'monitor' }, + { pattern: /\(\d*,\s*import_synthetics\d*\.expect\)/g, replacement: 'expect' }, + ]; + + let updated = code; + replacements.forEach(({ pattern, replacement }) => { + updated = updated.replace(pattern, replacement); + }); + + return updated + .replace( + /var import_synthetics\d* = require\("@elastic\/synthetics"\);/, + "import { step, journey, monitor, expect } from '@elastic/synthetics';" + ) + .replace( + /var import_synthetics = require\("@elastic\/synthetics"\);/, + "import { step, journey, monitor, expect } from '@elastic/synthetics';" + ); +}; diff --git a/x-pack/plugins/synthetics/server/synthetics_service/formatters/public_formatters/convert_to_data_stream.ts b/x-pack/plugins/synthetics/server/synthetics_service/formatters/public_formatters/convert_to_data_stream.ts index 52ae921c78643c..78842274d02e0e 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/formatters/public_formatters/convert_to_data_stream.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/formatters/public_formatters/convert_to_data_stream.ts @@ -8,7 +8,7 @@ import { DEFAULT_NAMESPACE_STRING } from '../../../../common/constants/monitor_defaults'; import { DataStream, MonitorFields } from '../../../../common/runtime_types'; -interface DataStreamConfig { +export interface DataStreamConfig { type: DataStream; id: string; schedule: string; diff --git a/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts b/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts index 9c6b16ba89e8c6..7fd986ed12ca63 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts @@ -90,7 +90,7 @@ describe('SyntheticsPrivateLocation', () => { } as unknown as UptimeServerSetup; it.each([ - [true, 'Unable to create Synthetics package policy for private location'], + [true, 'Unable to create Synthetics package policy template for private location'], [ false, 'Unable to create Synthetics package policy for monitor. Fleet write permissions are needed to use Synthetics private locations.', @@ -120,7 +120,7 @@ describe('SyntheticsPrivateLocation', () => { }); it.each([ - [true, 'Unable to create Synthetics package policy for private location'], + [true, 'Unable to create Synthetics package policy template for private location'], [ false, 'Unable to update Synthetics package policy for monitor. Fleet write permissions are needed to use Synthetics private locations.', diff --git a/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts b/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts index c2a878b15cd1d1..5f06839d4bc03d 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts @@ -38,14 +38,18 @@ export class SyntheticsPrivateLocation { this.server = _server; } - async buildNewPolicy( - savedObjectsClient: SavedObjectsClientContract - ): Promise { - return await this.server.fleet.packagePolicyService.buildPackagePolicyFromPackage( + async buildNewPolicy(savedObjectsClient: SavedObjectsClientContract): Promise { + const newPolicy = await this.server.fleet.packagePolicyService.buildPackagePolicyFromPackage( savedObjectsClient, 'synthetics', this.server.logger ); + + if (!newPolicy) { + throw new Error(`Unable to create Synthetics package policy template for private location`); + } + + return newPolicy; } getPolicyId(config: HeartbeatConfig, locId: string, spaceId: string) { @@ -131,10 +135,6 @@ export class SyntheticsPrivateLocation { const newPolicyTemplate = await this.buildNewPolicy(savedObjectsClient); - if (!newPolicyTemplate) { - throw new Error(`Unable to create Synthetics package policy for private location`); - } - for (const { config, globalParams } of configs) { try { const { locations } = config; @@ -188,6 +188,51 @@ export class SyntheticsPrivateLocation { } } + async inspectPackagePolicy({ + privateConfig, + savedObjectsClient, + spaceId, + allPrivateLocations, + }: { + privateConfig?: PrivateConfig; + savedObjectsClient: SavedObjectsClientContract; + allPrivateLocations: PrivateLocation[]; + spaceId: string; + }) { + if (!privateConfig) { + return null; + } + const newPolicyTemplate = await this.buildNewPolicy(savedObjectsClient); + + const { config, globalParams } = privateConfig; + try { + const { locations } = config; + + const privateLocation = locations.find((loc) => !loc.isServiceManaged); + + const location = allPrivateLocations?.find((loc) => loc.id === privateLocation?.id)!; + + const newPolicy = this.generateNewPolicy( + config, + location, + savedObjectsClient, + newPolicyTemplate, + spaceId, + globalParams + ); + + const pkgPolicy = { + ...newPolicy, + id: this.getPolicyId(config, location.id, spaceId), + } as NewPackagePolicyWithId; + + return await this.server.fleet.packagePolicyService.inspect(savedObjectsClient, pkgPolicy); + } catch (e) { + this.server.logger.error(e); + return null; + } + } + async editMonitors( configs: Array<{ config: HeartbeatConfig; globalParams: Record }>, request: KibanaRequest, @@ -206,10 +251,6 @@ export class SyntheticsPrivateLocation { const newPolicyTemplate = await this.buildNewPolicy(savedObjectsClient); - if (!newPolicyTemplate) { - throw new Error(`Unable to create Synthetics package policy for private location`); - } - const policiesToUpdate: NewPackagePolicyWithId[] = []; const policiesToCreate: NewPackagePolicyWithId[] = []; const policiesToDelete: string[] = []; diff --git a/x-pack/plugins/synthetics/server/synthetics_service/service_api_client.test.ts b/x-pack/plugins/synthetics/server/synthetics_service/service_api_client.test.ts index 700f339516215b..15fc1f50ab6c2f 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/service_api_client.test.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/service_api_client.test.ts @@ -182,55 +182,42 @@ describe('callAPI', () => { const output = { hosts: ['https://localhost:9200'], api_key: '12345' }; - await apiClient.callAPI('POST', { + const serviceData = { monitors: testMonitors, output, license: licenseMock.license, - }); + endpoint: 'monitors' as const, + }; + + await apiClient.callAPI('POST', serviceData); expect(spy).toHaveBeenCalledTimes(3); const devUrl = 'https://service.dev'; + const monitorsByLocation = apiClient.processServiceData(serviceData); + expect(spy).toHaveBeenNthCalledWith( 1, - { - isEdit: undefined, - monitors: testMonitors.filter((monitor: any) => - monitor.locations.some((loc: any) => loc.id === 'us_central') - ), - output, - license: licenseMock.license, - }, + monitorsByLocation.find(({ location: { id } }) => id === 'us_central')?.data, 'POST', - devUrl + devUrl, + 'monitors' ); expect(spy).toHaveBeenNthCalledWith( 2, - { - isEdit: undefined, - monitors: testMonitors.filter((monitor: any) => - monitor.locations.some((loc: any) => loc.id === 'us_central_qa') - ), - output, - license: licenseMock.license, - }, + monitorsByLocation.find(({ location: { id } }) => id === 'us_central_qa')?.data, 'POST', - 'https://qa.service.elstc.co' + 'https://qa.service.elstc.co', + 'monitors' ); expect(spy).toHaveBeenNthCalledWith( 3, - { - isEdit: undefined, - monitors: testMonitors.filter((monitor: any) => - monitor.locations.some((loc: any) => loc.id === 'us_central_staging') - ), - output, - license: licenseMock.license, - }, + monitorsByLocation.find(({ location: { id } }) => id === 'us_central_staging')?.data, 'POST', - 'https://qa.service.stg.co' + 'https://qa.service.stg.co', + 'monitors' ); expect(axiosSpy).toHaveBeenCalledTimes(3); diff --git a/x-pack/plugins/synthetics/server/synthetics_service/service_api_client.ts b/x-pack/plugins/synthetics/server/synthetics_service/service_api_client.ts index 02a4b1ea51e93c..0e709c89589768 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/service_api_client.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/service_api_client.ts @@ -5,14 +5,17 @@ * 2.0. */ -import axios, { AxiosError, AxiosRequestConfig } from 'axios'; +import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'; import { forkJoin, from as rxjsFrom, Observable, of } from 'rxjs'; import { catchError, tap } from 'rxjs/operators'; import * as https from 'https'; import { SslConfig } from '@kbn/server-http-tools'; import { Logger } from '@kbn/core/server'; import { LicenseGetLicenseInformation } from '@elastic/elasticsearch/lib/api/types'; -import { convertToDataStreamFormat } from './formatters/public_formatters/convert_to_data_stream'; +import { + convertToDataStreamFormat, + DataStreamConfig, +} from './formatters/public_formatters/convert_to_data_stream'; import { UptimeServerSetup } from '../legacy_uptime/lib/adapters'; import { sendErrorTelemetryEvents } from '../routes/telemetry/monitor_upgrade_sender'; import { MonitorFields, PublicLocations, ServiceLocationErrors } from '../../common/runtime_types'; @@ -31,6 +34,20 @@ export interface ServiceData { license: LicenseGetLicenseInformation; } +export interface ServicePayload { + monitors: DataStreamConfig[]; + output: { + hosts: string[]; + api_key: string; + }; + stack_version: string; + is_edit?: boolean; + license_level: string; + license_issued_to: string; + deployment_id?: string; + cloud_id?: string; +} + export class ServiceAPIClient { private readonly username?: string; private readonly authorization: string; @@ -57,6 +74,46 @@ export class ServiceAPIClient { this.server = server; } + addVersionHeader(req: AxiosRequestConfig) { + req.headers = { ...req.headers, 'x-kibana-version': this.stackVersion }; + return req; + } + + async checkAccountAccessStatus() { + if (this.authorization) { + // in case username/password is provided, we assume it's always allowed + return { allowed: true, signupUrl: null }; + } + + if (this.locations.length > 0) { + // get a url from a random location + const url = this.locations[Math.floor(Math.random() * this.locations.length)].url; + + /* url is required for service locations, but omitted for private locations. + /* this.locations is only service locations */ + const httpsAgent = this.getHttpsAgent(url); + + if (httpsAgent) { + try { + const { data } = await axios( + this.addVersionHeader({ + method: 'GET', + url: url + '/allowed', + httpsAgent, + }) + ); + + const { allowed, signupUrl } = data; + return { allowed, signupUrl }; + } catch (e) { + this.logger.error(e); + } + } + } + + return { allowed: false, signupUrl: null }; + } + getHttpsAgent(targetUrl: string) { const parsedTargetUrl = new URL(targetUrl); @@ -81,135 +138,92 @@ export class ServiceAPIClient { return baseHttpsAgent; } + async inspect(data: ServiceData) { + const monitorsByLocation = this.processServiceData(data); + + return monitorsByLocation.map(({ data: payload }) => payload); + } + async post(data: ServiceData) { - return this.callAPI('POST', data); + return (await this.callAPI('POST', data)).pushErrors; } async put(data: ServiceData) { - return this.callAPI('PUT', data); + return (await this.callAPI('PUT', data)).pushErrors; } async delete(data: ServiceData) { - return this.callAPI('DELETE', data); + return (await this.callAPI('DELETE', data)).pushErrors; } async runOnce(data: ServiceData) { - return this.callAPI('POST', { ...data, endpoint: 'runOnce' }); + return (await this.callAPI('POST', { ...data, endpoint: 'runOnce' })).pushErrors; } async syncMonitors(data: ServiceData) { - return this.callAPI('PUT', { ...data, endpoint: 'sync' }); + return (await this.callAPI('PUT', { ...data, endpoint: 'sync' })).pushErrors; } - addVersionHeader(req: AxiosRequestConfig) { - req.headers = { ...req.headers, 'x-kibana-version': this.stackVersion }; - return req; - } - - async checkAccountAccessStatus() { - if (this.authorization) { - // in case username/password is provided, we assume it's always allowed - return { allowed: true, signupUrl: null }; - } - - if (this.locations.length > 0) { - // get a url from a random location - const url = this.locations[Math.floor(Math.random() * this.locations.length)].url; - - /* url is required for service locations, but omitted for private locations. - /* this.locations is only service locations */ - const httpsAgent = this.getHttpsAgent(url); - - if (httpsAgent) { - try { - const { data } = await axios( - this.addVersionHeader({ - method: 'GET', - url: url + '/allowed', - httpsAgent, - }) - ); - - const { allowed, signupUrl } = data; - return { allowed, signupUrl }; - } catch (e) { - this.logger.error(e); - } + processServiceData({ monitors, ...restOfData }: ServiceData) { + // group monitors by location + const monitorsByLocation: Array<{ + location: { id: string; url: string }; + monitors: ServiceData['monitors']; + data: ServicePayload; + }> = []; + this.locations.forEach(({ id, url }) => { + const locMonitors = monitors.filter(({ locations }) => + locations?.find((loc) => loc.id === id && loc.isServiceManaged) + ); + if (locMonitors.length > 0) { + const data = this.getRequestData({ ...restOfData, monitors: locMonitors }); + monitorsByLocation.push({ location: { id, url }, monitors: locMonitors, data }); } - } - - return { allowed: false, signupUrl: null }; + }); + return monitorsByLocation; } - async callAPI( - method: 'POST' | 'PUT' | 'DELETE', - { monitors: allMonitors, output, endpoint, isEdit, license }: ServiceData - ) { + async callAPI(method: 'POST' | 'PUT' | 'DELETE', serviceData: ServiceData) { + const { endpoint } = serviceData; if (this.username === TEST_SERVICE_USERNAME) { // we don't want to call service while local integration tests are running - return; + return { result: [] as ServicePayload[], pushErrors: [] }; } const pushErrors: ServiceLocationErrors = []; - const promises: Array> = []; - this.locations.forEach(({ id, url }) => { - const locMonitors = allMonitors.filter(({ locations }) => - locations?.find((loc) => loc.id === id && loc.isServiceManaged) + const monitorsByLocation = this.processServiceData(serviceData); + + monitorsByLocation.forEach(({ location: { url, id }, monitors, data }) => { + const promise = this.callServiceEndpoint(data, method, url, endpoint); + promises.push( + rxjsFrom(promise).pipe( + tap((result) => { + this.logSuccessMessage(url, method, monitors.length, result); + }), + catchError((err: AxiosError<{ reason: string; status: number }>) => { + pushErrors.push({ locationId: id, error: err.response?.data! }); + this.logServiceError(err, url, method, monitors.length); + // we don't want to throw an unhandled exception here + return of(true); + }) + ) ); - if (locMonitors.length > 0) { - const promise = this.callServiceEndpoint( - { monitors: locMonitors, isEdit, endpoint, output, license }, - method, - url - ); - promises.push( - rxjsFrom(promise).pipe( - tap((result) => { - this.logger.debug(result.data); - this.logger.debug( - `Successfully called service location ${url}${result.request?.path} with method ${method} with ${locMonitors.length} monitors` - ); - }), - catchError((err: AxiosError<{ reason: string; status: number }>) => { - pushErrors.push({ locationId: id, error: err.response?.data! }); - const reason = err.response?.data?.reason ?? ''; - - err.message = `Failed to call service location ${url}${err.request?.path} with method ${method} with ${locMonitors.length} monitors: ${err.message}, ${reason}`; - this.logger.error(err); - sendErrorTelemetryEvents(this.logger, this.server.telemetry, { - reason: err.response?.data?.reason, - message: err.message, - type: 'syncError', - code: err.code, - status: err.response?.data?.status, - url, - stackVersion: this.server.stackVersion, - }); - // we don't want to throw an unhandled exception here - return of(true); - }) - ) - ); - } }); - await forkJoin(promises).toPromise(); + const result = await forkJoin(promises).toPromise(); - return pushErrors; + return { pushErrors, result }; } async callServiceEndpoint( - { monitors, output, endpoint = 'monitors', isEdit, license }: ServiceData, + data: ServicePayload, + // INSPECT is a special case where we don't want to call the service, but just return the data method: 'POST' | 'PUT' | 'DELETE', - baseUrl: string + baseUrl: string, + endpoint: string = 'monitors' ) { - // don't need to pass locations to heartbeat - const monitorsStreams = monitors.map(({ locations, ...rest }) => - convertToDataStreamFormat(rest) - ); - let url = baseUrl; switch (endpoint) { case 'monitors': @@ -229,19 +243,63 @@ export class ServiceAPIClient { this.addVersionHeader({ method, url, - data: { - monitors: monitorsStreams, - output, - stack_version: this.stackVersion, - is_edit: isEdit, - license_level: license.type, - license_issued_to: license.issued_to, - deployment_id: this.server.cloud?.deploymentId, - cloud_id: this.server.cloud?.cloudId, - }, + data, headers: authHeader, httpsAgent: this.getHttpsAgent(baseUrl), }) ); } + + getRequestData({ monitors, output, isEdit, license }: ServiceData) { + // don't need to pass locations to heartbeat + const monitorsStreams = monitors.map(({ locations, ...rest }) => + convertToDataStreamFormat(rest) + ); + + return { + monitors: monitorsStreams, + output, + stack_version: this.stackVersion, + is_edit: isEdit, + license_level: license.type, + license_issued_to: license.issued_to, + deployment_id: this.server.cloud?.deploymentId, + cloud_id: this.server.cloud?.cloudId, + }; + } + + logSuccessMessage( + url: string, + method: string, + numMonitors: number, + result: AxiosResponse | ServicePayload + ) { + if ('status' in result || 'request' in result) { + this.logger.debug(result.data); + this.logger.debug( + `Successfully called service location ${url}${result.request?.path} with method ${method} with ${numMonitors} monitors` + ); + } + } + + logServiceError( + err: AxiosError<{ reason: string; status: number }>, + url: string, + method: string, + numMonitors: number + ) { + const reason = err.response?.data?.reason ?? ''; + + err.message = `Failed to call service location ${url}${err.request?.path} with method ${method} with ${numMonitors} monitors: ${err.message}, ${reason}`; + this.logger.error(err); + sendErrorTelemetryEvents(this.logger, this.server.telemetry, { + reason: err.response?.data?.reason, + message: err.message, + type: 'syncError', + code: err.code, + status: err.response?.data?.status, + url, + stackVersion: this.server.stackVersion, + }); + } } diff --git a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts index b828668a50868e..95169f39d49434 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts @@ -22,13 +22,13 @@ import { import { SyntheticsService } from '../synthetics_service'; import { - MonitorFields, - SyntheticsMonitorWithId, + EncryptedSyntheticsMonitor, HeartbeatConfig, + MonitorFields, + MonitorServiceLocation, PrivateLocation, - EncryptedSyntheticsMonitor, + SyntheticsMonitorWithId, SyntheticsMonitorWithSecrets, - MonitorServiceLocation, } from '../../../common/runtime_types'; import { ConfigData, @@ -58,20 +58,12 @@ export class SyntheticsMonitorClient { const paramsBySpace = await this.syntheticsService.getSyntheticsParams({ spaceId }); for (const monitorObj of monitors) { - const { monitor, id } = monitorObj; - const config = { - monitor, - configId: id, - params: paramsBySpace[spaceId], - }; - - const { str: paramsString, params } = mixParamsWithGlobalParams( - paramsBySpace[spaceId], - monitor + const { formattedConfig, params, config } = await this.formatConfigWithParams( + monitorObj, + spaceId, + paramsBySpace ); - const formattedConfig = formatHeartbeatRequest(config, paramsString); - const { privateLocations, publicLocations } = this.parseLocations(formattedConfig); if (privateLocations.length > 0) { privateConfigs.push({ config: formattedConfig, globalParams: params }); @@ -350,4 +342,84 @@ export class SyntheticsMonitorClient { return heartbeatConfigs; } + + async formatConfigWithParams( + monitorObj: { monitor: MonitorFields; id: string }, + spaceId: string, + paramsBySpace: Record> + ) { + const { monitor, id } = monitorObj; + const config = { + monitor, + configId: id, + params: paramsBySpace[spaceId], + }; + + const { str: paramsString, params } = mixParamsWithGlobalParams( + paramsBySpace[spaceId], + monitor + ); + + const formattedConfig = formatHeartbeatRequest(config, paramsString); + return { formattedConfig, params, config }; + } + + async inspectMonitor( + monitorObj: { monitor: MonitorFields; id: string }, + request: KibanaRequest, + savedObjectsClient: SavedObjectsClientContract, + allPrivateLocations: PrivateLocation[], + spaceId: string, + hideParams: boolean, + canSave: boolean + ) { + const privateConfigs: PrivateConfig[] = []; + const paramsBySpace = await this.syntheticsService.getSyntheticsParams({ + spaceId, + canSave, + hideParams, + }); + + const { formattedConfig, params, config } = await this.formatConfigWithParams( + monitorObj, + spaceId, + paramsBySpace + ); + + if (hideParams) { + formattedConfig.params = hideParamsHelper(formattedConfig.params); + config.monitor.params = hideParamsHelper(config.monitor.params); + } + + const { privateLocations, publicLocations } = this.parseLocations(formattedConfig); + if (privateLocations.length > 0) { + privateConfigs.push({ config: formattedConfig, globalParams: params }); + } + + const publicPromise = this.syntheticsService.inspectConfig( + publicLocations.length > 0 ? config : undefined + ); + const privatePromise = this.privateLocationAPI.inspectPackagePolicy({ + privateConfig: privateConfigs?.[0], + savedObjectsClient, + allPrivateLocations, + spaceId, + }); + + const [publicConfigs, privateConfig] = await Promise.all([publicPromise, privatePromise]); + return { publicConfigs, privateConfig }; + } } + +const hideParamsHelper = (params?: string) => { + if (!params) return params; + + const parsedParams = JSON.parse(params); + // replace all values with '***' + const newParams = Object.create(null); + Object.keys(parsedParams).forEach((key) => { + newParams[key] = '"********"'; + }); + + return JSON.stringify(newParams); +}; diff --git a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts index fa4c9165c51895..470ba51a445770 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts @@ -299,6 +299,24 @@ export class SyntheticsService { }; } + async inspectConfig(config?: ConfigData) { + if (!config) { + return null; + } + const monitors = this.formatConfigs(config); + const license = await this.getLicense(); + + const output = await this.getOutput(); + if (output) { + return await this.apiClient.inspect({ + monitors, + output, + license, + }); + } + return null; + } + async addConfigs(configs: ConfigData[]) { try { if (configs.length === 0) { @@ -568,7 +586,14 @@ export class SyntheticsService { >; } - async getSyntheticsParams({ spaceId }: { spaceId?: string } = {}) { + async getSyntheticsParams({ + spaceId, + hideParams = false, + canSave = true, + }: { spaceId?: string; canSave?: boolean; hideParams?: boolean } = {}) { + if (!canSave) { + return Object.create(null); + } const encryptedClient = this.server.encryptedSavedObjects.getClient(); const paramsBySpace: Record> = Object.create(null); @@ -586,7 +611,9 @@ export class SyntheticsService { if (!paramsBySpace[namespace]) { paramsBySpace[namespace] = Object.create(null); } - paramsBySpace[namespace][param.attributes.key] = param.attributes.value; + paramsBySpace[namespace][param.attributes.key] = hideParams + ? '"*******"' + : param.attributes.value; }); }); } diff --git a/x-pack/test/api_integration/apis/synthetics/add_monitor_project.ts b/x-pack/test/api_integration/apis/synthetics/add_monitor_project.ts index abd8b7a6c687a4..74313585252db6 100644 --- a/x-pack/test/api_integration/apis/synthetics/add_monitor_project.ts +++ b/x-pack/test/api_integration/apis/synthetics/add_monitor_project.ts @@ -224,7 +224,7 @@ export default function ({ getService }: FtrProviderContext) { } finally { await Promise.all([ successfulMonitors.map((monitor) => { - return deleteMonitor(monitor.id, project); + // return deleteMonitor(monitor.id, project); }), ]); } diff --git a/x-pack/test/api_integration/apis/synthetics/fixtures/inspect_browser_monitor.json b/x-pack/test/api_integration/apis/synthetics/fixtures/inspect_browser_monitor.json new file mode 100644 index 00000000000000..3a6bad21b45f4e --- /dev/null +++ b/x-pack/test/api_integration/apis/synthetics/fixtures/inspect_browser_monitor.json @@ -0,0 +1,84 @@ +{ + "type": "browser", + "form_monitor_type": "multistep", + "enabled": true, + "alert": { + "status": { + "enabled": true + } + }, + "schedule": { + "number": "10", + "unit": "m" + }, + "service.name": "", + "config_id": "0088b13c-9bb0-4fc6-a0b5-63b9b024eabb", + "tags": [], + "timeout": null, + "name": "check if title is present", + "locations": [ + { + "id": "localhost", + "label": "Local Synthetics Service", + "geo": { + "lat": 0, + "lon": 0 + }, + "isServiceManaged": true + } + ], + "namespace": "default", + "origin": "project", + "journey_id": "bb82f7de-d832-4b14-8097-38a464d5fe49", + "hash": "ekrjelkjrelkjre", + "id": "bb82f7de-d832-4b14-8097-38a464d5fe49-test-project-cb47c83a-45e7-416a-9301-cb476b5bff01-default", + "params": "", + "project_id": "test-project-cb47c83a-45e7-416a-9301-cb476b5bff01", + "playwright_options": "{\"headless\":true,\"chromiumSandbox\":false}", + "__ui": { + "script_source": { + "is_generated_script": false, + "file_name": "" + } + }, + "url.port": null, + "source.inline.script": "", + "source.project.content": "UEsDBBQACAAIAON5qVQAAAAAAAAAAAAAAAAfAAAAZXhhbXBsZXMvdG9kb3MvYmFzaWMuam91cm5leS50c22Q0WrDMAxF3/sVF7MHB0LMXlc6RvcN+wDPVWNviW0sdUsp/fe5SSiD7UFCWFfHujIGlpnkybwxFTZfoY/E3hsaLEtwhs9RPNWKDU12zAOxkXRIbN4tB9d9pFOJdO6EN2HMqQguWN9asFBuQVMmJ7jiWNII9fIXrbabdUYr58l9IhwhQQZCYORCTFFUC31Btj21NRc7Mq4Nds+4bDD/pNVgT9F52Jyr2Fa+g75LAPttg8yErk+S9ELpTmVotlVwnfNCuh2lepl3+JflUmSBJ3uggt1v9INW/lHNLKze9dJe1J3QJK8pSvWkm6aTtCet5puq+x63+AFQSwcIAPQ3VfcAAACcAQAAUEsBAi0DFAAIAAgA43mpVAD0N1X3AAAAnAEAAB8AAAAAAAAAAAAgAKSBAAAAAGV4YW1wbGVzL3RvZG9zL2Jhc2ljLmpvdXJuZXkudHNQSwUGAAAAAAEAAQBNAAAARAEAAAAA", + "playwright_text_assertion": "", + "urls": "", + "screenshots": "on", + "synthetics_args": [], + "filter_journeys.match": "check if title is present", + "filter_journeys.tags": [], + "ignore_https_errors": false, + "throttling": { + "value": { + "download": "5", + "upload": "3", + "latency": "20" + }, + "id": "default", + "label": "Default" + }, + "ssl.certificate_authorities": "", + "ssl.certificate": "", + "ssl.key": "", + "ssl.key_passphrase": "", + "ssl.verification_mode": "full", + "ssl.supported_protocols": [ + "TLSv1.1", + "TLSv1.2", + "TLSv1.3" + ], + "original_space": "default", + "custom_heartbeat_id": "bb82f7de-d832-4b14-8097-38a464d5fe49-test-project-cb47c83a-45e7-416a-9301-cb476b5bff01-default", + "revision": 1, + "source.inline": { + "type": "inline", + "script": "", + "fileName": "" + }, + "service": { + "name": "" + } +} diff --git a/x-pack/test/api_integration/apis/synthetics/index.ts b/x-pack/test/api_integration/apis/synthetics/index.ts index a2d4afacc7fb36..13c581bf168b00 100644 --- a/x-pack/test/api_integration/apis/synthetics/index.ts +++ b/x-pack/test/api_integration/apis/synthetics/index.ts @@ -31,5 +31,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./sync_global_params')); loadTestFile(require.resolve('./add_edit_params')); loadTestFile(require.resolve('./add_monitor_project_private_location')); + loadTestFile(require.resolve('./inspect_monitor')); }); } diff --git a/x-pack/test/api_integration/apis/synthetics/inspect_monitor.ts b/x-pack/test/api_integration/apis/synthetics/inspect_monitor.ts new file mode 100644 index 00000000000000..a55c8b8f0d6c25 --- /dev/null +++ b/x-pack/test/api_integration/apis/synthetics/inspect_monitor.ts @@ -0,0 +1,227 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MonitorFields } from '@kbn/synthetics-plugin/common/runtime_types'; +import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import expect from '@kbn/expect'; +import { syntheticsParamType } from '@kbn/synthetics-plugin/common/types/saved_objects'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { getFixtureJson } from './helper/get_fixture_json'; +import { SyntheticsMonitorTestService } from './services/synthetics_monitor_test_service'; +import { PrivateLocationTestService } from './services/private_location_test_service'; + +export default function ({ getService }: FtrProviderContext) { + describe('inspectSyntheticsMonitor', function () { + this.tags('skipCloud'); + + const supertest = getService('supertest'); + + const monitorTestService = new SyntheticsMonitorTestService(getService); + const testPrivateLocations = new PrivateLocationTestService(getService); + const kibanaServer = getService('kibanaServer'); + + let _monitors: MonitorFields[]; + + before(async () => { + await kibanaServer.savedObjects.clean({ types: [syntheticsParamType] }); + await supertest.put(API_URLS.SYNTHETICS_ENABLEMENT).set('kbn-xsrf', 'true').expect(200); + _monitors = [getFixtureJson('http_monitor'), getFixtureJson('inspect_browser_monitor')]; + }); + + it('inspect http monitor', async () => { + const apiResponse = await monitorTestService.inspectMonitor({ + ..._monitors[0], + locations: [ + { + id: 'localhost', + label: 'Local Synthetics Service', + isServiceManaged: true, + }, + ], + }); + + expect(apiResponse).eql({ + result: { + publicConfigs: [ + { + monitors: [ + { + type: 'http', + schedule: '@every 5m', + enabled: true, + data_stream: { namespace: 'testnamespace' }, + streams: [ + { + data_stream: { dataset: 'http', type: 'synthetics' }, + type: 'http', + enabled: true, + schedule: '@every 5m', + tags: ['tag1', 'tag2'], + timeout: '3ms', + name: 'test-monitor-name', + namespace: 'testnamespace', + origin: 'ui', + urls: 'https://nextjs-test-synthetics.vercel.app/api/users', + max_redirects: '3', + password: 'test', + proxy_url: 'http://proxy.com', + 'response.include_body': 'never', + 'response.include_headers': true, + 'check.response.status': ['200', '201'], + 'check.request.body': 'testValue', + 'check.request.headers': { sampleHeader: 'sampleHeaderValue' }, + username: 'test-username', + mode: 'any', + 'response.include_body_max_bytes': '1024', + ipv4: true, + ipv6: true, + fields: {}, + fields_under_root: true, + }, + ], + }, + ], + output: { hosts: [] }, + license_level: 'trial', + }, + ], + privateConfig: null, + }, + decodedCode: '', + }); + }); + + it('inspect project browser monitor', async () => { + const apiResponse = await monitorTestService.inspectMonitor({ + ..._monitors[1], + params: JSON.stringify({ + username: 'elastic', + password: 'changeme', + }), + locations: [ + { + id: 'localhost', + label: 'Local Synthetics Service', + isServiceManaged: true, + }, + ], + }); + expect(apiResponse).eql({ + result: { + publicConfigs: [ + { + monitors: [ + { + type: 'browser', + schedule: '@every 10m', + enabled: true, + data_stream: { namespace: 'default' }, + streams: [ + { + data_stream: { dataset: 'browser', type: 'synthetics' }, + type: 'browser', + enabled: true, + schedule: '@every 10m', + name: 'check if title is present', + namespace: 'default', + origin: 'project', + params: { + username: '"********"', + password: '"********"', + }, + playwright_options: { headless: true, chromiumSandbox: false }, + 'source.project.content': + 'UEsDBBQACAAIAON5qVQAAAAAAAAAAAAAAAAfAAAAZXhhbXBsZXMvdG9kb3MvYmFzaWMuam91cm5leS50c22Q0WrDMAxF3/sVF7MHB0LMXlc6RvcN+wDPVWNviW0sdUsp/fe5SSiD7UFCWFfHujIGlpnkybwxFTZfoY/E3hsaLEtwhs9RPNWKDU12zAOxkXRIbN4tB9d9pFOJdO6EN2HMqQguWN9asFBuQVMmJ7jiWNII9fIXrbabdUYr58l9IhwhQQZCYORCTFFUC31Btj21NRc7Mq4Nds+4bDD/pNVgT9F52Jyr2Fa+g75LAPttg8yErk+S9ELpTmVotlVwnfNCuh2lepl3+JflUmSBJ3uggt1v9INW/lHNLKze9dJe1J3QJK8pSvWkm6aTtCet5puq+x63+AFQSwcIAPQ3VfcAAACcAQAAUEsBAi0DFAAIAAgA43mpVAD0N1X3AAAAnAEAAB8AAAAAAAAAAAAgAKSBAAAAAGV4YW1wbGVzL3RvZG9zL2Jhc2ljLmpvdXJuZXkudHNQSwUGAAAAAAEAAQBNAAAARAEAAAAA', + screenshots: 'on', + 'filter_journeys.match': 'check if title is present', + ignore_https_errors: false, + throttling: { download: 5, upload: 3, latency: 20 }, + original_space: 'default', + fields: { + 'monitor.project.name': 'test-project-cb47c83a-45e7-416a-9301-cb476b5bff01', + 'monitor.project.id': 'test-project-cb47c83a-45e7-416a-9301-cb476b5bff01', + }, + fields_under_root: true, + }, + ], + }, + ], + license_level: 'trial', + output: { hosts: [] }, + }, + ], + privateConfig: null, + }, + decodedCode: + '// asset:/Users/vigneshh/elastic/synthetics/examples/todos/basic.journey.ts\nimport { journey, step, expect } from "@elastic/synthetics";\njourney("check if title is present", ({ page, params }) => {\n step("launch app", async () => {\n await page.goto(params.url);\n });\n step("assert title", async () => {\n const header = await page.$("h1");\n expect(await header.textContent()).toBe("todos");\n });\n});\n', + }); + }); + + it('inspect http monitor in private location', async () => { + const location = await testPrivateLocations.addTestPrivateLocation(); + const apiResponse = await monitorTestService.inspectMonitor({ + ..._monitors[0], + locations: [ + { + id: location.id, + label: location.label, + isServiceManaged: false, + }, + ], + }); + + const privateConfig = apiResponse.result.privateConfig!; + + const enabledStream = privateConfig.inputs + .find((input) => input.enabled) + ?.streams.find((stream) => stream.enabled); + + const compiledStream = enabledStream?.compiled_stream; + + delete compiledStream.id; + delete compiledStream.processors[0].add_fields.fields.config_id; + + expect(enabledStream?.compiled_stream).eql({ + __ui: { is_tls_enabled: false }, + type: 'http', + name: 'test-monitor-name', + origin: 'ui', + 'run_from.id': location.id, + 'run_from.geo.name': 'Test private location 0', + enabled: true, + urls: 'https://nextjs-test-synthetics.vercel.app/api/users', + schedule: '@every 5m', + timeout: '3ms', + max_redirects: 3, + proxy_url: 'http://proxy.com', + tags: ['tag1', 'tag2'], + username: 'test-username', + password: 'test', + 'response.include_headers': true, + 'response.include_body': 'never', + 'response.include_body_max_bytes': 1024, + 'check.request.method': null, + 'check.request.headers': { sampleHeader: 'sampleHeaderValue' }, + 'check.request.body': 'testValue', + 'check.response.status': ['200', '201'], + mode: 'any', + ipv4: true, + ipv6: true, + processors: [ + { + add_fields: { + target: '', + fields: { + 'monitor.fleet_managed': true, + }, + }, + }, + ], + }); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/synthetics/services/private_location_test_service.ts b/x-pack/test/api_integration/apis/synthetics/services/private_location_test_service.ts index 8b29c7f7ab0450..7436f5da5df360 100644 --- a/x-pack/test/api_integration/apis/synthetics/services/private_location_test_service.ts +++ b/x-pack/test/api_integration/apis/synthetics/services/private_location_test_service.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { v4 as uuidv4 } from 'uuid'; import { privateLocationsSavedObjectName } from '@kbn/synthetics-plugin/common/saved_objects/private_locations'; import { privateLocationsSavedObjectId } from '@kbn/synthetics-plugin/server/legacy_uptime/lib/saved_objects/private_locations'; import { FtrProviderContext } from '../../../ftr_provider_context'; @@ -34,6 +34,12 @@ export class PrivateLocationTestService { } } + async addTestPrivateLocation() { + const apiResponse = await this.addFleetPolicy(uuidv4()); + const testPolicyId = apiResponse.body.item.id; + return (await this.setTestLocations([testPolicyId]))[0]; + } + async addFleetPolicy(name: string) { return this.supertest .post('/api/fleet/agent_policies?sys_monitoring=true') @@ -50,22 +56,25 @@ export class PrivateLocationTestService { async setTestLocations(testFleetPolicyIds: string[]) { const server = this.getService('kibanaServer'); + const locations = testFleetPolicyIds.map((id, index) => ({ + label: 'Test private location ' + index, + agentPolicyId: id, + id, + geo: { + lat: '', + lon: '', + }, + concurrentMonitors: 1, + })); + await server.savedObjects.create({ type: privateLocationsSavedObjectName, id: privateLocationsSavedObjectId, attributes: { - locations: testFleetPolicyIds.map((id, index) => ({ - label: 'Test private location ' + index, - agentPolicyId: id, - id, - geo: { - lat: '', - lon: '', - }, - concurrentMonitors: 1, - })), + locations, }, overwrite: true, }); + return locations; } } diff --git a/x-pack/test/api_integration/apis/synthetics/services/synthetics_monitor_test_service.ts b/x-pack/test/api_integration/apis/synthetics/services/synthetics_monitor_test_service.ts index 0d45c902dbc009..003c590cbc10ff 100644 --- a/x-pack/test/api_integration/apis/synthetics/services/synthetics_monitor_test_service.ts +++ b/x-pack/test/api_integration/apis/synthetics/services/synthetics_monitor_test_service.ts @@ -7,6 +7,9 @@ import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; import { syntheticsMonitorType } from '@kbn/synthetics-plugin/common/types/saved_objects'; +import { SavedObject } from '@kbn/core-saved-objects-common/src/server_types'; +import { MonitorFields } from '@kbn/synthetics-plugin/common/runtime_types'; +import { MonitorInspectResponse } from '@kbn/synthetics-plugin/public/apps/synthetics/state/monitor_management/api'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { KibanaSupertestProvider } from '../../../../../../test/api_integration/services/supertest'; @@ -27,6 +30,35 @@ export class SyntheticsMonitorTestService { return this.supertest.get(url).set('kbn-xsrf', 'true').expect(200); } + async addMonitor(monitor: any) { + const res = await this.supertest + .post(API_URLS.SYNTHETICS_MONITORS) + .set('kbn-xsrf', 'true') + .send(monitor) + .expect(200); + + return res.body as SavedObject; + } + + async inspectMonitor(monitor: any, hideParams: boolean = true) { + const res = await this.supertest + .post(API_URLS.SYNTHETICS_MONITOR_INSPECT) + .set('kbn-xsrf', 'true') + .send(monitor) + .expect(200); + + // remove the id and config_id from the response + delete res.body.result?.publicConfigs?.[0].monitors[0].id; + delete res.body.result?.publicConfigs?.[0].monitors[0].streams[0].id; + delete res.body.result?.publicConfigs?.[0].monitors[0].streams[0].config_id; + delete res.body.result?.publicConfigs?.[0].monitors[0].streams[0].fields.config_id; + delete res.body.result?.publicConfigs?.[0].output.api_key; + delete res.body.result?.publicConfigs?.[0].license_issued_to; + delete res.body.result?.publicConfigs?.[0].stack_version; + + return res.body as { result: MonitorInspectResponse; decodedCode: string }; + } + async addProjectMonitors(project: string, monitors: any) { const { body } = await this.supertest .put(API_URLS.SYNTHETICS_MONITORS_PROJECT_UPDATE.replace('{projectName}', project)) diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index cb60103314031f..1f08d0884d81e3 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -125,6 +125,7 @@ "@kbn/observability-shared-plugin", "@kbn/maps-vector-tile-utils", "@kbn/server-route-repository", + "@kbn/core-saved-objects-common", "@kbn/core-http-common", "@kbn/slo-schema" ] From 6ea3f39b61a37104f06d8e412a1369793d431588 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Mon, 5 Jun 2023 14:55:49 +0200 Subject: [PATCH 4/6] [Defend Workflows] Adjust osquery editor to use monaco (#158192) --- .../osquery/cypress/e2e/all/live_query.cy.ts | 17 +- .../osquery/cypress/screens/live_query.ts | 2 +- .../osquery/cypress/tasks/saved_queries.ts | 12 +- .../osquery/public/editor/ace_types.ts | 19 -- .../plugins/osquery/public/editor/index.tsx | 110 +++---- .../editor/osquery_highlight_rules.test.ts | 73 +++++ .../public/editor/osquery_highlight_rules.ts | 293 ++++++++++++------ .../osquery/public/editor/osquery_mode.ts | 35 --- .../osquery/public/editor/osquery_tables.ts | 13 +- .../form/live_query_query_field.tsx | 1 - .../osquery/public/results/results_table.tsx | 2 +- .../saved_queries/form/playground_flyout.tsx | 2 +- .../saved_queries/saved_query_flyout.tsx | 1 + x-pack/plugins/osquery/tsconfig.json | 3 +- 14 files changed, 357 insertions(+), 226 deletions(-) delete mode 100644 x-pack/plugins/osquery/public/editor/ace_types.ts create mode 100644 x-pack/plugins/osquery/public/editor/osquery_highlight_rules.test.ts delete mode 100644 x-pack/plugins/osquery/public/editor/osquery_mode.ts diff --git a/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts index a350aae9d05cb3..865033208b15de 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts @@ -172,7 +172,7 @@ describe('ALL - Live Query', () => { .should('be.visible') .click(); - cy.react('ReactAce', { props: { value: 'select * from users;' } }).should('exist'); + cy.get(LIVE_QUERY_EDITOR).contains('select * from users;'); }); it('should open query details by clicking the details icon', () => { @@ -239,25 +239,22 @@ describe('ALL - Live Query', () => { "where pos.remote_port !='0' {shift+enter}" + 'limit 1000;'; cy.contains('New live query').click(); - cy.react('ReactAce').invoke('height').and('be.gt', 99).and('be.lt', 110); + cy.get(LIVE_QUERY_EDITOR).invoke('height').and('be.gt', 99).and('be.lt', 110); cy.get(LIVE_QUERY_EDITOR).click().invoke('val', multilineQuery); inputQuery(multilineQuery); - cy.wait(2000); - cy.react('ReactAce').invoke('height').should('be.gt', 220).and('be.lt', 300); + cy.get(LIVE_QUERY_EDITOR).invoke('height').should('be.gt', 220).and('be.lt', 300); selectAllAgents(); submitQuery(); - checkResults(); + cy.getBySel('osqueryResultsPanel'); // check if it get's bigger when we add more lines - cy.react('ReactAce').invoke('height').should('be.gt', 220).and('be.lt', 300); + cy.get(LIVE_QUERY_EDITOR).invoke('height').should('be.gt', 220).and('be.lt', 300); inputQuery(multilineQuery); - cy.wait(2000); - cy.react('ReactAce').invoke('height').should('be.gt', 350).and('be.lt', 500); + cy.get(LIVE_QUERY_EDITOR).invoke('height').should('be.gt', 350).and('be.lt', 550); inputQuery('{selectall}{backspace}{selectall}{backspace}'); - cy.wait(2000); // not sure if this is how it used to work when I implemented the functionality, but let's leave it like this for now - cy.react('ReactAce').invoke('height').should('be.gt', 350).and('be.lt', 500); + cy.get(LIVE_QUERY_EDITOR).invoke('height').should('be.gt', 200).and('be.lt', 350); }); }); diff --git a/x-pack/plugins/osquery/cypress/screens/live_query.ts b/x-pack/plugins/osquery/cypress/screens/live_query.ts index 32d3a54881b1d3..b44764331fb3ac 100644 --- a/x-pack/plugins/osquery/cypress/screens/live_query.ts +++ b/x-pack/plugins/osquery/cypress/screens/live_query.ts @@ -7,7 +7,7 @@ export const AGENT_FIELD = '[data-test-subj="comboBoxInput"]'; export const ALL_AGENTS_OPTION = '[title="All agents"]'; -export const LIVE_QUERY_EDITOR = '#osquery_editor'; +export const LIVE_QUERY_EDITOR = '.kibanaCodeEditor'; export const SUBMIT_BUTTON = '#submit-button'; export const RESULTS_TABLE = 'osqueryResultsTable'; diff --git a/x-pack/plugins/osquery/cypress/tasks/saved_queries.ts b/x-pack/plugins/osquery/cypress/tasks/saved_queries.ts index a7c267e510cf73..978a025c8d9401 100644 --- a/x-pack/plugins/osquery/cypress/tasks/saved_queries.ts +++ b/x-pack/plugins/osquery/cypress/tasks/saved_queries.ts @@ -114,11 +114,13 @@ export const getSavedQueriesComplexTest = () => // Disabled submit button in test configuration cy.contains('Submit').should('not.be.disabled'); - // this clears the input - inputQuery('{selectall}{backspace}{selectall}{backspace}'); - cy.contains('Submit').should('be.disabled'); - inputQuery(BIG_QUERY); - cy.contains('Submit').should('not.be.disabled'); + cy.getBySel('osquery-save-query-flyout').within(() => { + cy.contains('Query is a required field').should('not.exist'); + // this clears the input + inputQuery('{selectall}{backspace}{selectall}{backspace}'); + cy.contains('Query is a required field'); + inputQuery(BIG_QUERY); + }); // Save edited cy.react('EuiButton').contains('Update query').click(); diff --git a/x-pack/plugins/osquery/public/editor/ace_types.ts b/x-pack/plugins/osquery/public/editor/ace_types.ts deleted file mode 100644 index 93d8b4c5e2d2bb..00000000000000 --- a/x-pack/plugins/osquery/public/editor/ace_types.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * Ace#define is not defined in the published types, so we define our own - * interface. - */ -export interface AceInterface { - define: ( - name: string, - deps: string[], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - cb: (acequire: (name: string) => any, exports: any) => void - ) => void; -} diff --git a/x-pack/plugins/osquery/public/editor/index.tsx b/x-pack/plugins/osquery/public/editor/index.tsx index 971c19190df1f2..42494426e62099 100644 --- a/x-pack/plugins/osquery/public/editor/index.tsx +++ b/x-pack/plugins/osquery/public/editor/index.tsx @@ -5,38 +5,26 @@ * 2.0. */ -import React, { useEffect, useState, useCallback, useRef } from 'react'; +import React, { useEffect, useState, useCallback } from 'react'; import useDebounce from 'react-use/lib/useDebounce'; -import styled from 'styled-components'; -import { EuiResizeObserver } from '@elastic/eui'; +import { CodeEditor } from '@kbn/kibana-react-plugin/public'; -import type { EuiCodeEditorProps } from '../shared_imports'; -import { EuiCodeEditor } from '../shared_imports'; +import { monaco } from '@kbn/monaco'; -import './osquery_mode'; -import 'brace/theme/tomorrow'; - -const EDITOR_SET_OPTIONS = { - enableBasicAutocompletion: true, - enableLiveAutocompletion: true, -}; - -const EDITOR_PROPS = { - $blockScrolling: true, -}; +import { initializeOsqueryEditor } from './osquery_highlight_rules'; interface OsqueryEditorProps { defaultValue: string; onChange: (newValue: string) => void; - commands?: EuiCodeEditorProps['commands']; + commands?: Array<{ + name: string; + exec: () => void; + }>; } -const ResizeWrapper = styled.div` - overflow: auto; - resize: vertical; - min-height: 100px; -`; - +const editorOptions = { + theme: 'osquery', +}; const MIN_HEIGHT = 100; const OsqueryEditorComponent: React.FC = ({ defaultValue, @@ -45,20 +33,10 @@ const OsqueryEditorComponent: React.FC = ({ }) => { const [editorValue, setEditorValue] = useState(defaultValue ?? ''); const [height, setHeight] = useState(MIN_HEIGHT); - const editorRef = useRef<{ renderer: { layerConfig: { maxHeight: number; minHeight: number } } }>( - { - renderer: { layerConfig: { maxHeight: 100, minHeight: 100 } }, - } - ); useDebounce( () => { onChange(editorValue); - const config = editorRef.current?.renderer.layerConfig; - - if (config.maxHeight > config.minHeight) { - setHeight(config.maxHeight); - } }, 500, [editorValue] @@ -66,41 +44,47 @@ const OsqueryEditorComponent: React.FC = ({ useEffect(() => setEditorValue(defaultValue), [defaultValue]); - const resizeEditor = useCallback((editorInstance) => { - editorRef.current.renderer = editorInstance.renderer; + useEffect(() => { + const disposable = initializeOsqueryEditor(); - setTimeout(() => { - const { maxHeight } = editorInstance.renderer.layerConfig; - if (maxHeight > MIN_HEIGHT) { - setHeight(maxHeight); - } - }, 0); + return () => { + disposable?.dispose(); + }; }, []); - const onResize = useCallback((dimensions) => { - setHeight(dimensions.height); - }, []); + const editorDidMount = useCallback( + (editor: monaco.editor.IStandaloneCodeEditor) => { + const minHeight = 100; + const maxHeight = 1000; + + commands?.map((command) => { + if (command.name === 'submitOnCmdEnter') { + // on CMD/CTRL + Enter submit the query + // eslint-disable-next-line no-bitwise + editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter, command.exec); + } + }); + + const updateHeight = () => { + const contentHeight = Math.min(maxHeight, Math.max(minHeight, editor.getContentHeight())); + setHeight(contentHeight); + }; + + editor.onDidContentSizeChange(updateHeight); + }, + [commands] + ); return ( - - {(resizeRef) => ( - - - - )} - + ); }; diff --git a/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.test.ts b/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.test.ts new file mode 100644 index 00000000000000..e086bfe13837e1 --- /dev/null +++ b/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + builtinConstants, + builtinFunctions, + dataTypes, + getEditorAutoCompleteSuggestion, + keywords, + osqueryTableNames, +} from './osquery_highlight_rules'; +import { flatMap, uniq } from 'lodash'; + +describe('Osquery Editor', () => { + const regex = /\s*[\s,]\s*/; + + test('should split properly by regex', () => { + const value = 'Select description, user_account from services; /n Where des'; + const split = value.split(regex); + const result = [ + 'Select', + 'description', + 'user_account', + 'from', + 'services;', + '/n', + 'Where', + 'des', + ]; + + expect(split).toEqual(result); + }); + test('should provide proper suggestions', () => { + const value = 'Select description, user_account from services; /n Where des'; + + const range = { startLineNumber: 2, endLineNumber: 2, startColumn: 2, endColumn: 2 }; + + // @ts-expect-error TS2339: Property 'suggestions' does not exist on type 'ProviderResult '. + const { suggestions } = getEditorAutoCompleteSuggestion(range, value, false); + + const flatSuggestionLabels = flatMap(suggestions, (obj) => obj.label); + expect(flatSuggestionLabels).toEqual(suggestionLabels); + }); + test('should provide just keywords if column is 1', () => { + const value = 'Select description, user_account from services; /n Where des'; + + const range = { startLineNumber: 1, endLineNumber: 1, startColumn: 1, endColumn: 1 }; + + // @ts-expect-error TS2339: Property 'suggestions' does not exist on type 'ProviderResult '. + const { suggestions } = getEditorAutoCompleteSuggestion(range, value, false); + + const flatSuggestionLabels = flatMap(suggestions, (obj) => obj.label); + expect(flatSuggestionLabels).toEqual(keywordsSuggestionLabels); + }); +}); + +const keywordsSuggestionLabels = keywords.map((kw) => kw.toUpperCase()); + +const suggestionLabels = uniq([ + ...keywordsSuggestionLabels, + ...osqueryTableNames, + ...builtinConstants, + ...builtinFunctions, + ...dataTypes, + 'description', + 'user_account', + 'services;', + '/n', +]); diff --git a/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.ts b/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.ts index 7ec61e6a2d80ad..b9cdcff73793fc 100644 --- a/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.ts +++ b/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.ts @@ -5,14 +5,13 @@ * 2.0. */ -import ace from 'brace'; -import 'brace/ext/language_tools'; -import type { AceInterface } from './ace_types'; -import { getOsqueryTableNames } from './osquery_tables'; +import { monaco } from '@kbn/monaco'; +import { findLast, map, uniqBy } from 'lodash'; +import { getOsqueryTableNames, osqueryTablesRecord } from './osquery_tables'; -const osqueryTables = getOsqueryTableNames().join('|'); +export const osqueryTableNames = getOsqueryTableNames(); -const keywords = [ +export const keywords = [ 'select', 'insert', 'update', @@ -57,11 +56,11 @@ const keywords = [ 'database', 'drop', 'grant', -].join('|'); +]; -const builtinConstants = ['true', 'false'].join('|'); +export const builtinConstants = ['true', 'false']; -const builtinFunctions = [ +export const builtinFunctions = [ 'avg', 'count', 'first', @@ -81,9 +80,9 @@ const builtinFunctions = [ 'ifnull', 'isnull', 'nvl', -].join('|'); +]; -const dataTypes = [ +export const dataTypes = [ 'int', 'numeric', 'decimal', @@ -102,84 +101,202 @@ const dataTypes = [ 'real', 'number', 'integer', -].join('|'); - -// This is gross, but the types exported by brace are lagging and incorrect: https://github.com/thlorenz/brace/issues/182 -(ace as unknown as AceInterface).define( - 'ace/mode/osquery_highlight_rules', - ['require', 'exports', 'ace/mode/sql_highlight_rules'], - // eslint-disable-next-line prefer-arrow-callback - function (acequire, exports) { - 'use strict'; - - const SqlHighlightRules = acequire('./sql_highlight_rules').SqlHighlightRules; - - class OsqueryHighlightRules extends SqlHighlightRules { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - constructor(...args: any) { - super(...args); - const keywordMapper = this.createKeywordMapper( - { - 'osquery-token': osqueryTables, - 'support.function': builtinFunctions, - keyword: keywords, - 'constant.language': builtinConstants, - 'storage.type': dataTypes, - }, - 'identifier', - true - ); - - this.$rules = { - start: [ - { - token: 'comment', - regex: '--.*$', - }, - { - token: 'comment', - start: '/\\*', - end: '\\*/', - }, - { - token: 'string', // " string - regex: '".*?"', - }, - { - token: 'string', // ' string - regex: "'.*?'", - }, - { - token: 'constant.numeric', // float - regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b', - }, - { - token: keywordMapper, - regex: '[a-zA-Z_$][a-zA-Z0-9_$]*\\b', - }, - { - token: 'keyword.operator', - regex: '\\+|\\-|\\/|\\/\\/|%|<@>|@>|<@|&|\\^|~|<|>|<=|=>|==|!=|<>|=', - }, - { - token: 'paren.lparen', - regex: '[\\(]', - }, - { - token: 'paren.rparen', - regex: '[\\)]', - }, - { - token: 'text', - regex: '\\s+', - }, +]; + +interface Range { + startLineNumber: number; + endLineNumber: number; + startColumn: number; + endColumn: number; +} + +interface IDisposable { + dispose: () => void; +} + +const theme = { + base: 'vs' as const, + inherit: false, + rules: [ + { token: 'osquery' }, + { token: 'support.function', foreground: '4271AE' }, + { token: 'keyword', foreground: '8959A8' }, + { token: 'storage.type', foreground: '8959A8' }, + { token: 'constant.language', foreground: 'F5871F' }, + { token: 'comment', foreground: '8E908C' }, + { token: 'string', foreground: '718C00' }, + { token: 'constant.numeric', foreground: 'F5871F' }, + { token: 'keyword.operator', foreground: '3E999F' }, + ], + colors: { + 'editorGutter.background': '#F6F6F6', + }, +}; + +export const initializeOsqueryEditor = () => { + let disposable: IDisposable | null = null; + if (monaco) { + disposable = monaco.languages.onLanguage('sql', () => { + monaco.languages.setMonarchTokensProvider('sql', { + ignoreCase: true, + osqueryTableNames, + builtinFunctions, + keywords, + builtinConstants, + dataTypes, + brackets: [ + { open: '[', close: ']', token: 'delimiter.square' }, + { open: '(', close: ')', token: 'delimiter.parenthesis' }, + ], + tokenizer: { + root: [ + [ + '[a-zA-Z_$][a-zA-Z0-9_$]*\\b', + { + cases: { + '@osqueryTableNames': 'osquery', + '@builtinFunctions': 'support.function', + '@keywords': 'keyword', + '@builtinConstants': 'constant.language', + '@dataTypes': 'storage.type', + }, + }, + ], + ['--.*$', 'comment'], + ['/\\*.*\\*/', 'comment'], + ['".*?"', 'string'], + ["'.*?'", 'string'], + [/[ \t\r\n]+/, { token: 'whitespace' }], + ['[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b', 'constant.numeric'], + ['\\+|\\-|\\/|\\/\\/|%|<@>|@>|<@|&|\\^|~|<|>|<=|=>|==|!=|<>|=', 'keyword.operator'], + ['[\\(]', 'paren.lparen'], + ['[\\)]', 'paren.rparen'], + ['\\s+', 'text'], ], - }; + }, + }); + monaco?.editor.defineTheme('osquery', theme); + monaco?.languages.registerCompletionItemProvider('sql', { + triggerCharacters: ['.'], + provideCompletionItems: (model: monaco.editor.ITextModel, position: monaco.Position) => { + const value = model.getValue(); + const tokens = monaco.editor.tokenize(value, 'sql'); + const findOsqueryToken = findLast( + tokens[position.lineNumber - 1], + (token) => token.type === 'osquery.sql' + ); + + const osqueryTable = model.getWordAtPosition({ + lineNumber: position.lineNumber, + column: (findOsqueryToken?.offset || 0) + 1, + }); + + const lineContent = model.getLineContent(position.lineNumber); + + const word = model.getWordUntilPosition(position); - this.normalizeRules(); - } - } + const isDot = + lineContent.charAt(lineContent.length - 1) === '.' || + lineContent.charAt(lineContent.length - 2) === '.'; - exports.OsqueryHighlightRules = OsqueryHighlightRules; + const range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn, + }; + + return getEditorAutoCompleteSuggestion(range, value, isDot, osqueryTable?.word); + }, + }); + }); + + return disposable; } -); +}; + +const regex = /\s*[\s,]\s*/; +export const getEditorAutoCompleteSuggestion = ( + range: Range, + value: string, + isDot: boolean, + name?: string +): monaco.languages.ProviderResult => { + // we do not want to suggest the last word (currently being typed) + const localValue = value.split(regex).slice(0, -1); + const localKeywords = localValue.map((kw) => ({ + label: kw, + kind: monaco.languages.CompletionItemKind.Snippet, + detail: 'Local', + insertText: kw, + range, + })); + + const suggestionsFromDefaultKeywords = keywords.map((kw) => ({ + label: `${kw.toUpperCase()}`, + kind: monaco.languages.CompletionItemKind.Keyword, + detail: 'Keyword', + insertText: `${kw.toUpperCase()} `, + range, + })); + + const osqueryColumns = name + ? map(osqueryTablesRecord[name]?.columns, ({ name: columnName }) => ({ + label: columnName, + kind: monaco.languages.CompletionItemKind.Folder, + detail: `${name} column`, + insertText: columnName, + range, + })) + : []; + + const tableNameKeywords = osqueryTableNames.map((tableName: string) => ({ + label: tableName, + kind: monaco.languages.CompletionItemKind.Folder, + detail: 'Osquery', + insertText: tableName, + range, + })); + const builtinConstantsKeywords = builtinConstants.map((constant: string) => ({ + label: constant, + kind: monaco.languages.CompletionItemKind.Constant, + detail: 'Constant', + insertText: constant, + range, + })); + const builtinFunctionsKeywords = builtinFunctions.map((builtinFunction: string) => ({ + label: builtinFunction, + kind: monaco.languages.CompletionItemKind.Function, + detail: 'Function', + insertText: builtinFunction, + range, + })); + const dataTypesKeywords = dataTypes.map((dataType: string) => ({ + label: dataType, + kind: monaco.languages.CompletionItemKind.TypeParameter, + detail: 'Type', + insertText: dataType, + range, + })); + + return { + suggestions: + // first word has to be an SQL keyword + range.startColumn === 1 + ? suggestionsFromDefaultKeywords + : // if last char is === '.' it means we are joining so we want to present just specific osquery table suggestions + isDot + ? osqueryColumns + : uniqBy( + [ + ...suggestionsFromDefaultKeywords, + ...tableNameKeywords, + ...builtinConstantsKeywords, + ...builtinFunctionsKeywords, + ...dataTypesKeywords, + ...localKeywords, + ], + (word) => word.label.toLowerCase() + ), + }; +}; diff --git a/x-pack/plugins/osquery/public/editor/osquery_mode.ts b/x-pack/plugins/osquery/public/editor/osquery_mode.ts deleted file mode 100644 index 25751c3c04a125..00000000000000 --- a/x-pack/plugins/osquery/public/editor/osquery_mode.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import ace from 'brace'; -import 'brace/mode/sql'; -import 'brace/ext/language_tools'; -import type { AceInterface } from './ace_types'; -import './osquery_highlight_rules'; - -(ace as unknown as AceInterface).define( - 'ace/mode/osquery', - ['require', 'exports', 'ace/mode/sql', 'ace/mode/osquery_highlight_rules'], - // eslint-disable-next-line prefer-arrow-callback - function (acequire, exports) { - const TextMode = acequire('./sql').Mode; - const OsqueryHighlightRules = acequire('./osquery_highlight_rules').OsqueryHighlightRules; - - class Mode extends TextMode { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - constructor(...args: any[]) { - super(...args); - this.HighlightRules = OsqueryHighlightRules; - } - } - - Mode.prototype.lineCommentStart = '--'; - Mode.prototype.$id = 'ace/mode/osquery'; - - exports.Mode = Mode; - } -); diff --git a/x-pack/plugins/osquery/public/editor/osquery_tables.ts b/x-pack/plugins/osquery/public/editor/osquery_tables.ts index fbfa20dc39ede0..5b2e990fcccca2 100644 --- a/x-pack/plugins/osquery/public/editor/osquery_tables.ts +++ b/x-pack/plugins/osquery/public/editor/osquery_tables.ts @@ -23,4 +23,15 @@ export const getOsqueryTables = () => { return osqueryTables; }; -export const getOsqueryTableNames = () => flatMap(getOsqueryTables(), 'name'); +const normalizedOsqueryTables = getOsqueryTables(); + +export const osqueryTablesRecord: Record }> = + normalizedOsqueryTables.reduce( + (acc, table) => ({ + ...acc, + [table.name]: table, + }), + {} + ); + +export const getOsqueryTableNames = () => flatMap(normalizedOsqueryTables, 'name'); diff --git a/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx b/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx index 7a707d1a4f4c8a..72458cdb327180 100644 --- a/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx +++ b/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx @@ -112,7 +112,6 @@ const LiveQueryQueryFieldComponent: React.FC = ({ ? [ { name: 'submitOnCmdEnter', - bindKey: { win: 'ctrl+enter', mac: 'cmd+enter' }, exec: handleSubmitForm, }, ] diff --git a/x-pack/plugins/osquery/public/results/results_table.tsx b/x-pack/plugins/osquery/public/results/results_table.tsx index 2aa4a781c44eed..e7fae8d6c78bd7 100644 --- a/x-pack/plugins/osquery/public/results/results_table.tsx +++ b/x-pack/plugins/osquery/public/results/results_table.tsx @@ -436,7 +436,7 @@ const ResultsTableComponent: React.FC = ({ )} {!allResultsData?.edges.length ? ( - + ) : ( diff --git a/x-pack/plugins/osquery/public/saved_queries/form/playground_flyout.tsx b/x-pack/plugins/osquery/public/saved_queries/form/playground_flyout.tsx index a38c2f7728969e..da5cb715f4a0fa 100644 --- a/x-pack/plugins/osquery/public/saved_queries/form/playground_flyout.tsx +++ b/x-pack/plugins/osquery/public/saved_queries/form/playground_flyout.tsx @@ -35,7 +35,7 @@ const PlaygroundFlyoutComponent: React.FC = ({ enabled, o const serializedFormData = useMemo(() => serializer(watchedValues), [ecsMapping]); return ( - +
diff --git a/x-pack/plugins/osquery/public/saved_queries/saved_query_flyout.tsx b/x-pack/plugins/osquery/public/saved_queries/saved_query_flyout.tsx index 130f4591ef18f5..49186b81b42680 100644 --- a/x-pack/plugins/osquery/public/saved_queries/saved_query_flyout.tsx +++ b/x-pack/plugins/osquery/public/saved_queries/saved_query_flyout.tsx @@ -62,6 +62,7 @@ const SavedQueryFlyoutComponent: React.FC = ({ return ( Date: Mon, 5 Jun 2023 06:11:38 -0700 Subject: [PATCH 5/6] [SOR] Adds support for validation schema with models (#158527) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../src/lib/apis/helpers/validation.test.ts | 95 ++++++++++++++ .../src/lib/apis/helpers/validation.ts | 29 ++++- .../lib/apis/helpers/validation_fixtures.ts | 123 ++++++++++++++++++ .../src/validation/validator.test.ts | 71 +++++----- .../src/validation/validator.ts | 20 +-- .../src/model_version/schemas.ts | 6 + 6 files changed, 301 insertions(+), 43 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation_fixtures.ts diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation.test.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation.test.ts new file mode 100644 index 00000000000000..323aa18c8f6ef9 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { type SavedObjectSanitizedDoc } from '@kbn/core-saved-objects-server'; +import { ValidationHelper } from './validation'; +import { typedef, typedef1, typedef2 } from './validation_fixtures'; +import { SavedObjectTypeRegistry } from '@kbn/core-saved-objects-base-server-internal'; + +const defaultVersion = '8.10.0'; +const modelVirtualVersion = '10.1.0'; +const typeA = 'my-typeA'; +const typeB = 'my-typeB'; +const typeC = 'my-typeC'; + +describe('Saved Objects type validation helper', () => { + let helper: ValidationHelper; + let logger: MockedLogger; + let typeRegistry: SavedObjectTypeRegistry; + + const createMockObject = ( + type: string, + attr: Partial + ): SavedObjectSanitizedDoc => ({ + type, + id: 'test-id', + references: [], + attributes: {}, + ...attr, + }); + const registerType = (name: string, parts: Partial) => { + typeRegistry.registerType({ + name, + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, + }); + }; + beforeEach(() => { + logger = loggerMock.create(); + typeRegistry = new SavedObjectTypeRegistry(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('validation helper', () => { + beforeEach(() => { + registerType(typeA, typedef); + registerType(typeB, typedef1); + registerType(typeC, typedef2); + }); + + it('should validate objects against stack versions', () => { + helper = new ValidationHelper({ + logger, + registry: typeRegistry, + kibanaVersion: defaultVersion, + }); + const data = createMockObject(typeA, { attributes: { foo: 'hi', count: 1 } }); + expect(() => helper.validateObjectForCreate(typeA, data)).not.toThrowError(); + }); + + it('should validate objects against model versions', () => { + helper = new ValidationHelper({ + logger, + registry: typeRegistry, + kibanaVersion: modelVirtualVersion, + }); + const data = createMockObject(typeB, { attributes: { foo: 'hi', count: 1 } }); + expect(() => helper.validateObjectForCreate(typeB, data)).not.toThrowError(); + }); + + it('should fail validation against invalid objects when version requested does not support a field', () => { + helper = new ValidationHelper({ + logger, + registry: typeRegistry, + kibanaVersion: defaultVersion, + }); + const validationError = new Error( + '[attributes.count]: definition for this key is missing: Bad Request' + ); + const data = createMockObject(typeC, { attributes: { foo: 'hi', count: 1 } }); + expect(() => helper.validateObjectForCreate(typeC, data)).toThrowError(validationError); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation.ts index 96224953ba459b..e9c8c6bcf50cda 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation.ts @@ -9,7 +9,10 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import type { Logger } from '@kbn/logging'; import type { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server'; -import { SavedObjectsTypeValidator } from '@kbn/core-saved-objects-base-server-internal'; +import { + SavedObjectsTypeValidator, + modelVersionToVirtualVersion, +} from '@kbn/core-saved-objects-base-server-internal'; import { SavedObjectsErrorHelpers, type SavedObjectSanitizedDoc, @@ -91,7 +94,7 @@ export class ValidationHelper { } const validator = this.getTypeValidator(type); try { - validator.validate(doc, this.kibanaVersion); + validator.validate(doc); } catch (error) { throw SavedObjectsErrorHelpers.createBadRequestError(error.message); } @@ -100,10 +103,30 @@ export class ValidationHelper { private getTypeValidator(type: string): SavedObjectsTypeValidator { if (!this.typeValidatorMap[type]) { const savedObjectType = this.registry.getType(type); + + const stackVersionSchemas = + typeof savedObjectType?.schemas === 'function' + ? savedObjectType.schemas() + : savedObjectType?.schemas ?? {}; + + const modelVersionCreateSchemas = + typeof savedObjectType?.modelVersions === 'function' + ? savedObjectType.modelVersions() + : savedObjectType?.modelVersions ?? {}; + + const combinedSchemas = { ...stackVersionSchemas }; + Object.entries(modelVersionCreateSchemas).reduce((map, [key, modelVersion]) => { + if (modelVersion.schemas?.create) { + const virtualVersion = modelVersionToVirtualVersion(key); + combinedSchemas[virtualVersion] = modelVersion.schemas!.create!; + } + return map; + }, {}); + this.typeValidatorMap[type] = new SavedObjectsTypeValidator({ logger: this.logger.get('type-validator'), type, - validationMap: savedObjectType!.schemas ?? {}, + validationMap: combinedSchemas, defaultVersion: this.kibanaVersion, }); } diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation_fixtures.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation_fixtures.ts new file mode 100644 index 00000000000000..539c5f13c840d7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/validation_fixtures.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema } from '@kbn/config-schema'; +import { SavedObjectsType } from '@kbn/core-saved-objects-server'; + +export const typedef: Partial = { + mappings: { + properties: { + foo: { + type: 'keyword', + }, + count: { + type: 'integer', + }, + }, + }, + schemas: { + '8.9.0': schema.object({ + foo: schema.string(), + }), + '8.10.0': schema.object({ + foo: schema.string(), + count: schema.number(), + }), + }, + switchToModelVersionAt: '8.10.0', +}; + +export const typedef1: Partial = { + mappings: { + properties: { + foo: { + type: 'keyword', + }, + count: { + type: 'integer', + }, + }, + }, + schemas: { + '8.9.0': schema.object({ + foo: schema.string(), + }), + '8.10.0': schema.object({ + foo: schema.string(), + count: schema.number(), + }), + }, + switchToModelVersionAt: '8.10.0', + modelVersions: { + '1': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + count: { + properties: { + count: { + type: 'integer', + }, + }, + }, + }, + }, + ], + schemas: { + create: schema.object({ + foo: schema.string(), + count: schema.number(), + }), + }, + }, + }, +}; + +export const typedef2: Partial = { + mappings: { + properties: { + foo: { + type: 'keyword', + }, + count: { + type: 'integer', + }, + }, + }, + schemas: { + '8.9.0': schema.object({ + foo: schema.string(), + }), + }, + switchToModelVersionAt: '8.10.0', + modelVersions: { + '1': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + count: { + properties: { + count: { + type: 'integer', + }, + }, + }, + }, + }, + ], + schemas: { + create: schema.object({ + foo: schema.string(), + count: schema.number(), + }), + }, + }, + }, +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/validation/validator.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/validation/validator.test.ts index e552aee1c89764..37ced142364daf 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/validation/validator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/validation/validator.test.ts @@ -51,18 +51,18 @@ describe('Saved Objects type validator', () => { it('should log when a validation fails', () => { const data = createMockObject({ attributes: { foo: false } }); - expect(() => validator.validate(data, '1.0.0')).toThrowError(); + expect(() => validator.validate(data)).toThrowError(); expect(logger.warn).toHaveBeenCalledTimes(1); }); it('should work when given valid values', () => { const data = createMockObject({ attributes: { foo: 'hi' } }); - expect(() => validator.validate(data, '1.0.0')).not.toThrowError(); + expect(() => validator.validate(data)).not.toThrowError(); }); it('should throw an error when given invalid values', () => { const data = createMockObject({ attributes: { foo: false } }); - expect(() => validator.validate(data, '1.0.0')).toThrowErrorMatchingInlineSnapshot( + expect(() => validator.validate(data)).toThrowErrorMatchingInlineSnapshot( `"[attributes.foo]: expected value of type [string] but got [boolean]"` ); }); @@ -71,7 +71,7 @@ describe('Saved Objects type validator', () => { const data = createMockObject({ attributes: { foo: 'hi' } }); // @ts-expect-error Intentionally malformed object data.updated_at = false; - expect(() => validator.validate(data, '1.0.0')).toThrowErrorMatchingInlineSnapshot( + expect(() => validator.validate(data)).toThrowErrorMatchingInlineSnapshot( `"[updated_at]: expected value of type [string] but got [boolean]"` ); }); @@ -86,7 +86,7 @@ describe('Saved Objects type validator', () => { }); const data = createMockObject({ attributes: { foo: 'hi' } }); - expect(() => validator.validate(data, '1.0.0')).not.toThrowError(); + expect(() => validator.validate(data)).not.toThrowError(); }); }); @@ -97,8 +97,8 @@ describe('Saved Objects type validator', () => { '2.7.0': createStubSpec(), '3.0.0': createStubSpec(), '3.5.0': createStubSpec(), - '4.0.0': createStubSpec(), - '4.3.0': createStubSpec(), + // we're intentionally leaving out 10.1.0 to test model version selection + '10.2.0': createStubSpec(), }; validator = new SavedObjectsTypeValidator({ logger, type, validationMap, defaultVersion }); }); @@ -118,51 +118,58 @@ describe('Saved Objects type validator', () => { return undefined; }; - it('should use the correct schema when specifying the version', () => { - let data = createMockObject({ typeMigrationVersion: '2.2.0' }); - validator.validate(data, '3.2.0'); + it('should use the correct schema for documents with typeMigrationVersion', () => { + const data = createMockObject({ typeMigrationVersion: '3.0.0' }); + validator.validate(data); expect(getCalledVersion()).toEqual('3.0.0'); - - jest.clearAllMocks(); - - data = createMockObject({ typeMigrationVersion: '3.5.0' }); - validator.validate(data, '4.5.0'); - expect(getCalledVersion()).toEqual('4.3.0'); }); - it('should use the correct schema for documents with typeMigrationVersion', () => { - let data = createMockObject({ typeMigrationVersion: '3.2.0' }); + it('should use the correct schema for documents with typeMigrationVersion greater than default version', () => { + const data = createMockObject({ typeMigrationVersion: '3.5.0' }); validator.validate(data); expect(getCalledVersion()).toEqual('3.0.0'); + }); - jest.clearAllMocks(); - - data = createMockObject({ typeMigrationVersion: '3.5.0' }); + it('should use the correct schema for documents with migrationVersion', () => { + const data = createMockObject({ + migrationVersion: { + [type]: '3.0.0', + }, + }); validator.validate(data); - expect(getCalledVersion()).toEqual('3.5.0'); + expect(getCalledVersion()).toEqual('3.0.0'); }); - it('should use the correct schema for documents with migrationVersion', () => { - let data = createMockObject({ + it('should use the correct schema for documents with migrationVersion higher than default', () => { + const data = createMockObject({ migrationVersion: { [type]: '4.6.0', }, }); validator.validate(data); - expect(getCalledVersion()).toEqual('4.3.0'); + // 4.6.0 > 3.3.0 (default), is not a valid virtual model and there aren't migrations for the type in the default version + expect(getCalledVersion()).toEqual('3.0.0'); + }); + + it("should use the correct schema for documents with virtualModelVersion that isn't registered", () => { + let data = createMockObject({ typeMigrationVersion: '10.1.0' }); + validator.validate(data); + expect(getCalledVersion()).toEqual('3.5.0'); jest.clearAllMocks(); - data = createMockObject({ - migrationVersion: { - [type]: '4.0.0', - }, - }); + data = createMockObject({ typeMigrationVersion: '10.3.0' }); + validator.validate(data); + expect(getCalledVersion()).toEqual('10.2.0'); + }); + + it('should use the correct schema for documents with virtualModelVersion that is registered', () => { + const data = createMockObject({ typeMigrationVersion: '10.2.0' }); validator.validate(data); - expect(getCalledVersion()).toEqual('4.0.0'); + expect(getCalledVersion()).toEqual('10.2.0'); }); - it('should use the correct schema for documents without a version specified', () => { + it('should use the correct schema for documents without a version', () => { const data = createMockObject({}); validator.validate(data); expect(getCalledVersion()).toEqual('3.0.0'); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/validation/validator.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/validation/validator.ts index e028c16f9bd2ff..e8344de7bc2f07 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/validation/validator.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/validation/validator.ts @@ -13,6 +13,7 @@ import type { SavedObjectSanitizedDoc, } from '@kbn/core-saved-objects-server'; import { createSavedObjectSanitizedDocSchema } from './schema'; +import { isVirtualModelVersion } from '../model_version'; /** * Helper class that takes a {@link SavedObjectsValidationMap} and runs validations for a @@ -45,13 +46,16 @@ export class SavedObjectsTypeValidator { this.orderedVersions = Object.keys(this.validationMap).sort(Semver.compare); } - public validate(document: SavedObjectSanitizedDoc, version?: string): void { - const docVersion = - version ?? - document.typeMigrationVersion ?? - document.migrationVersion?.[document.type] ?? - this.defaultVersion; - const schemaVersion = previousVersionWithSchema(this.orderedVersions, docVersion); + public validate(document: SavedObjectSanitizedDoc): void { + let usedVersion: string; + const docVersion = document.typeMigrationVersion ?? document.migrationVersion?.[document.type]; + if (docVersion) { + usedVersion = isVirtualModelVersion(docVersion) ? docVersion : this.defaultVersion; + } else { + usedVersion = this.defaultVersion; + } + const schemaVersion = previousVersionWithSchema(this.orderedVersions, usedVersion); + if (!schemaVersion || !this.validationMap[schemaVersion]) { return; } @@ -62,7 +66,7 @@ export class SavedObjectsTypeValidator { validationSchema.validate(document); } catch (e) { this.log.warn( - `Error validating object of type [${this.type}] against version [${docVersion}]` + `Error validating object of type [${this.type}] against version [${usedVersion}]` ); throw e; } diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/schemas.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/schemas.ts index bfcc76de5d4fe1..61bb3c342d9d63 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/schemas.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/schemas.ts @@ -7,6 +7,7 @@ */ import type { ObjectType } from '@kbn/config-schema'; +import type { SavedObjectsValidationSpec } from '../validation'; /** * The validation and conversion schemas associated with this model version. @@ -29,6 +30,11 @@ export interface SavedObjectsModelVersionSchemaDefinitions { * See {@link SavedObjectModelVersionForwardCompatibilitySchema} for more info. */ forwardCompatibility?: SavedObjectModelVersionForwardCompatibilitySchema; + /** + * The schema applied when creating a document of the current version + * Allows for validating properties using @kbn/config-schema validations + */ + create?: SavedObjectsValidationSpec; } /** From c57589ec57e5e8265a66cd9c8c2102005736f6d8 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 5 Jun 2023 15:15:07 +0200 Subject: [PATCH 6/6] Fix config stacking order (#158827) ## Summary Fixes: #155154 (introduced in #149878), builds on #155436 . - Adds tests to ensure the configuration merging order, check those for reference. - Updates the README to explain the intention For the tests, I needed to output something to the logs. I hope it's not a big issue to log it. If needed, I might hide that behind a verbose- or feature flag. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- config/README.md | 9 +- .../src/bootstrap.test.mocks.ts | 1 + .../src/bootstrap.ts | 4 +- .../integration_tests/config_ordering.test.ts | 242 ++++++++++++++++++ src/cli/serve/serve.js | 9 +- 5 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 src/cli/serve/integration_tests/config_ordering.test.ts diff --git a/config/README.md b/config/README.md index b5cad71cb0813e..83ef1b1c661205 100644 --- a/config/README.md +++ b/config/README.md @@ -5,9 +5,12 @@ this configuration, pass `--serverless={mode}` or run `yarn serverless-{mode}` valid modes are currently: `es`, `oblt`, and `security` configuration is applied in the following order, later values override - 1. kibana.yml - 2. serverless.yml - 3. serverless.{mode}.yml + 1. serverless.yml (serverless configs go first) + 2. serverless.{mode}.yml (serverless configs go first) + 3. base config, in this preference order: + - my-config.yml(s) (set by --config) + - env-config.yml (described by `env.KBN_CONFIG_PATHS`) + - kibana.yml (default @ `env.KBN_PATH_CONF`/kibana.yml) 4. kibana.dev.yml 5. serverless.dev.yml 6. serverless.{mode}.dev.yml diff --git a/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts b/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts index 07277d565f694c..be54edf7f2124e 100644 --- a/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts +++ b/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts @@ -21,5 +21,6 @@ jest.doMock('@kbn/config', () => ({ jest.doMock('./root', () => ({ Root: jest.fn(() => ({ shutdown: jest.fn(), + logger: { get: () => ({ info: jest.fn(), debug: jest.fn() }) }, })), })); diff --git a/packages/core/root/core-root-server-internal/src/bootstrap.ts b/packages/core/root/core-root-server-internal/src/bootstrap.ts index 3f55b6493a6bda..bb0e3ddc8c7011 100644 --- a/packages/core/root/core-root-server-internal/src/bootstrap.ts +++ b/packages/core/root/core-root-server-internal/src/bootstrap.ts @@ -78,6 +78,9 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot } const root = new Root(rawConfigService, env, onRootShutdown); + const cliLogger = root.logger.get('cli'); + + cliLogger.debug('Kibana configurations evaluated in this order: ' + env.configs.join(', ')); process.on('SIGHUP', () => reloadConfiguration()); @@ -93,7 +96,6 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot }); function reloadConfiguration(reason = 'SIGHUP signal received') { - const cliLogger = root.logger.get('cli'); cliLogger.info(`Reloading Kibana configuration (reason: ${reason}).`, { tags: ['config'] }); try { diff --git a/src/cli/serve/integration_tests/config_ordering.test.ts b/src/cli/serve/integration_tests/config_ordering.test.ts new file mode 100644 index 00000000000000..7245083aa5141b --- /dev/null +++ b/src/cli/serve/integration_tests/config_ordering.test.ts @@ -0,0 +1,242 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Fs from 'fs'; +import * as Path from 'path'; +import * as Os from 'os'; +import * as Child from 'child_process'; +import Del from 'del'; +import * as Rx from 'rxjs'; +import { filter, map, take, timeout } from 'rxjs/operators'; + +const tempDir = Path.join(Os.tmpdir(), 'kbn-config-test'); + +const kibanaPath = follow('../../../../scripts/kibana.js'); + +const TIMEOUT_MS = 20000; + +const envForTempDir = { + env: { KBN_PATH_CONF: tempDir }, +}; + +const TestFiles = { + fileList: [] as string[], + + createEmptyConfigFiles(fileNames: string[], root: string = tempDir): string[] { + const configFiles = []; + for (const fileName of fileNames) { + const filePath = Path.resolve(root, fileName); + + if (!Fs.existsSync(filePath)) { + Fs.writeFileSync(filePath, 'dummy'); + + TestFiles.fileList.push(filePath); + } + + configFiles.push(filePath); + } + + return configFiles; + }, + cleanUpEmptyConfigFiles() { + for (const filePath of TestFiles.fileList) { + Del.sync(filePath); + } + TestFiles.fileList.length = 0; + }, +}; + +describe('Server configuration ordering', () => { + let kibanaProcess: Child.ChildProcessWithoutNullStreams; + + beforeEach(() => { + Fs.mkdirSync(tempDir, { recursive: true }); + }); + + afterEach(async () => { + if (kibanaProcess !== undefined) { + const exitPromise = new Promise((resolve) => kibanaProcess?.once('exit', resolve)); + kibanaProcess.kill('SIGKILL'); + await exitPromise; + } + + Del.sync(tempDir, { force: true }); + TestFiles.cleanUpEmptyConfigFiles(); + }); + + it('loads default config set without any options', async function () { + TestFiles.createEmptyConfigFiles(['kibana.yml']); + + kibanaProcess = Child.spawn(process.execPath, [kibanaPath, '--verbose'], envForTempDir); + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual(['kibana.yml']); + }); + + it('loads serverless configs when --serverless is set', async () => { + TestFiles.createEmptyConfigFiles([ + 'serverless.yml', + 'serverless.oblt.yml', + 'kibana.yml', + 'serverless.recent.yml', + ]); + + kibanaProcess = Child.spawn( + process.execPath, + [kibanaPath, '--verbose', '--serverless', 'oblt'], + envForTempDir + ); + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual([ + 'serverless.yml', + 'serverless.oblt.yml', + 'kibana.yml', + 'serverless.recent.yml', + ]); + }); + + it('prefers --config options over default', async () => { + const [configPath] = TestFiles.createEmptyConfigFiles([ + 'potato.yml', + 'serverless.yml', + 'serverless.oblt.yml', + 'kibana.yml', + 'serverless.recent.yml', + ]); + + kibanaProcess = Child.spawn( + process.execPath, + [kibanaPath, '--verbose', '--serverless', 'oblt', '--config', configPath], + envForTempDir + ); + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual([ + 'serverless.yml', + 'serverless.oblt.yml', + 'potato.yml', + 'serverless.recent.yml', + ]); + }); + + it('defaults to "es" if --serverless and --dev are there', async () => { + TestFiles.createEmptyConfigFiles([ + 'serverless.yml', + 'serverless.es.yml', + 'kibana.yml', + 'kibana.dev.yml', + 'serverless.dev.yml', + ]); + + kibanaProcess = Child.spawn( + process.execPath, + [kibanaPath, '--verbose', '--serverless', '--dev'], + envForTempDir + ); + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual([ + 'serverless.yml', + 'serverless.es.yml', + 'kibana.yml', + 'serverless.recent.yml', + 'kibana.dev.yml', + 'serverless.dev.yml', + ]); + }); + + it('adds dev configs to the stack', async () => { + TestFiles.createEmptyConfigFiles([ + 'serverless.yml', + 'serverless.security.yml', + 'kibana.yml', + 'kibana.dev.yml', + 'serverless.dev.yml', + ]); + + kibanaProcess = Child.spawn( + process.execPath, + [kibanaPath, '--verbose', '--serverless', 'security', '--dev'], + envForTempDir + ); + + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual([ + 'serverless.yml', + 'serverless.security.yml', + 'kibana.yml', + 'serverless.recent.yml', + 'kibana.dev.yml', + 'serverless.dev.yml', + ]); + }); +}); + +async function extractConfigurationOrder( + proc: Child.ChildProcessWithoutNullStreams +): Promise { + const configMessage = await waitForMessage(proc, /[Cc]onfig.*order:/, TIMEOUT_MS); + + const configList = configMessage + .match(/order: (.*)$/) + ?.at(1) + ?.split(', ') + ?.map((path) => Path.basename(path)); + + return configList; +} + +async function waitForMessage( + proc: Child.ChildProcessWithoutNullStreams, + expression: string | RegExp, + timeoutMs: number +): Promise { + const message$ = Rx.fromEvent(proc.stdout!, 'data').pipe( + map((messages) => String(messages).split('\n').filter(Boolean)) + ); + + const trackedExpression$ = message$.pipe( + // We know the sighup handler will be registered before this message logged + filter((messages: string[]) => messages.some((m) => m.match(expression))), + take(1) + ); + + const error$ = message$.pipe( + filter((messages: string[]) => messages.some((line) => line.match(/fatal/i))), + take(1), + map((line) => new Error(line.join('\n'))) + ); + + const value = await Rx.firstValueFrom( + Rx.race(trackedExpression$, error$).pipe( + timeout({ + first: timeoutMs, + with: () => + Rx.throwError( + () => new Error(`Config options didn't appear in logs for ${timeoutMs / 1000}s...`) + ), + }) + ) + ); + + if (value instanceof Error) { + throw value; + } + + if (Array.isArray(value)) { + return value[0]; + } else { + return value; + } +} + +function follow(file: string) { + return Path.relative(process.cwd(), Path.resolve(__dirname, file)); +} diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index 45c9e04c7e5587..3d4b51be36fa54 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -20,6 +20,8 @@ import { readKeystore } from '../keystore/read_keystore'; /** @type {ServerlessProjectMode[]} */ const VALID_SERVERLESS_PROJECT_MODE = ['es', 'oblt', 'security']; +const isNotEmpty = _.negate(_.isEmpty); + /** * @param {Record} opts * @returns {ServerlessProjectMode | true | null} @@ -305,8 +307,13 @@ export default function (program) { } command.action(async function (opts) { + const cliConfigs = opts.config || []; + const envConfigs = getEnvConfigs(); + const defaultConfig = getConfigPath(); + + const configs = [cliConfigs, envConfigs, [defaultConfig]].find(isNotEmpty); + const unknownOptions = this.getUnknownOptions(); - const configs = [getConfigPath(), ...getEnvConfigs(), ...(opts.config || [])]; const serverlessMode = getServerlessProjectMode(opts); if (serverlessMode) {