From c8c3e51e2bd238ea8e527285185c23b5b474c7d6 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 5 Dec 2019 22:10:16 -0700 Subject: [PATCH 1/4] skip flaky suite (#52246) --- .../legacy/plugins/graph/public/components/search_bar.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/graph/public/components/search_bar.test.tsx b/x-pack/legacy/plugins/graph/public/components/search_bar.test.tsx index a0514576877d13..8d5e27f1e9aec9 100644 --- a/x-pack/legacy/plugins/graph/public/components/search_bar.test.tsx +++ b/x-pack/legacy/plugins/graph/public/components/search_bar.test.tsx @@ -63,7 +63,8 @@ function wrapSearchBarInContext(testProps: OuterSearchBarProps) { ); } -describe('search_bar', () => { +// FLAKY: https://github.com/elastic/kibana/issues/52246 +describe.skip('search_bar', () => { const defaultProps = { isLoading: false, onQuerySubmit: jest.fn(), From 68cc4de804ba8ab020b887503eaaf918ac94414b Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Thu, 5 Dec 2019 23:36:10 -0700 Subject: [PATCH 2/4] [SIEM][Detection Engine] Adds signal data index per spaces through index naming conventions (#52237) ## Summary Changes the signals output index to be based on the user's space * Adds the ability to create a space based index through `POST /api/detection_engine/index` * Adds the existence API for the index through `HEAD /api/detection_engine/index` * Adds an index check during the creation of a rule, `POST api/detection_engine/rules` that will return a status of 400 with an error message if the index does not exist * Adds a new optional key in kibana.dev.yml of `xpack.siem.signalsIndex` for developers working together who need to segregate signals indexes. * Splits apart the ECS mappings and the signal mappings into separate files for easier maintenance. * Deprecates the defaultSignalsIndex (will remove it once the UI is updated) * Updates the README.md to remove the SIGNALS_INDEX environment variable * Updates the existing unit tests * Adds more unit tests unit tests For people writing the UI: --- How do I check for the existence of a signals index? See [scripts/signal_index_exists.sh](https://github.com/elastic/kibana/blob/28937ebe00bfc90129cf7e3ca1a04755c6029331/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signal_index_exists.sh) ```sh HEAD /api/detection_engine/index ``` How do I create a new signals index if my user has correct privileges? See [scripts/post_signal_index.sh](https://github.com/elastic/kibana/blob/28937ebe00bfc90129cf7e3ca1a04755c6029331/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal_index.sh) ```sh POST /api/detection_engine/index ``` How do I delete _everything_ of all signal indexes, policies, and templates for a particular space? See [scripts/delete_signal_index.sh](https://github.com/elastic/kibana/blob/28937ebe00bfc90129cf7e3ca1a04755c6029331/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh) ```sh DELETE /api/detection_engine/index ``` FAQ for people testing --- What is the name of the index, policy, etc... per space? If you're using the default space the index, policies, etc... will be: ```sh .siem-signals-default ``` If you're using a custom space such as `test-space` they will be: ```sh .siem-signals-test-space ``` If you set your `xpack.siem.signalsIndex` in your `kibana.dev.yml` to something such as: ```yml xpack.siem.signalsIndex: .siem-signals-frank-hassanabad ``` And use the default space it will be: ```sh .siem-signals-frank-hassanabad-default ``` And for a custom space such as `test-space` they will be: ```sh .siem-signals-frank-hassanabad-test-space ``` What is the policy that is being set? See: [signals_policy.json](https://github.com/elastic/kibana/blob/28937ebe00bfc90129cf7e3ca1a04755c6029331/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_policy.json) ```json { "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "rollover": { "max_size": "10gb", "max_age": "7d" } } } } } } ``` What is the boot strap index that is being set look like? See: [create_bootstrap_index.ts](https://github.com/elastic/kibana/blob/28937ebe00bfc90129cf7e3ca1a04755c6029331/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/create_bootstrap_index.ts) You should see this when running: ```sh ./get_signal_index.sh | less ``` ```json ".siem-signals-default-000001": { "aliases": { ".siem-signals-default": { "is_write_index": true } }, ``` What is the template that is being set look like? See: [get_signals_template.ts](https://github.com/elastic/kibana/blob/28937ebe00bfc90129cf7e3ca1a04755c6029331/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.ts) You should see this at the bottom when running: ```sh ./get_signal_index.sh ``` ```json "settings": { "index": { "lifecycle": { "name": ".siem-signals-default", "rollover_alias": ".siem-signals-default" }, "number_of_shards": "1", "provided_name": ".siem-signals-default-000001", "creation_date": "1575502837772", "number_of_replicas": "1", "uuid": "GB0h3AYRQD6AWl8OfNonJA", "version": { "created": "8000099" } } } ``` For more in-depth of testing of spaces using dev tools of Kibana --- Different testing scenarios involving having spaces set in the URL, vs not having spaces set. Also different testing scenarios involving having a developer based `xpack.siem.signalsIndex` being set vs not having one set and gettin the default of `.siem-signals` With a default space and kibana.dev.yml setting of: * xpack.siem.signalsIndex: .siem-signals-frank-hassanabad You can use dev tools to check the results after doing a `./post_signal_index.sh` ``` sh GET /_template/.siem-signals-frank-hassanabad-default GET /.siem-signals-frank-hassanabad-default-000001 GET /_ilm/policy/.siem-signals-frank-hassanabad-default GET /_alias/.siem-signals-frank-hassanabad-default ``` With a default space and no `kibana.dev.yml` setting, you can use dev tools to check the results after doing a `./post_signal_index.sh` ```sh GET /.siem-signals-default GET /_template/.siem-signals-default GET /.siem-signals-default-000001 GET /_ilm/policy/.siem-signals-default GET /_alias/.siem-signals-default ``` Setting a space through: ```sh export SPACE_URL=/s/test-space ``` With a default space and `kibana.dev.yml` setting using a user name such as mine: * xpack.siem.signalsIndex: .siem-signals-frank-hassanabad You can use dev tools to check the results after doing a `./post_signal_index.sh` ``` GET /.siem-signals-frank-hassanabad-test-space GET /_template/.siem-signals-frank-hassanabad-test-space GET /.siem-signals-frank-hassanabad-test-space-000001 GET /_ilm/policy/.siem-signals-frank-hassanabad-test-space GET /_alias/.siem-signals-frank-hassanabad-test-space ``` With a default space and no `kibana.dev.yml` setting, you can use dev tools to check the results after doing a `./post_signal_index.sh` ``` GET /.siem-signals-test-space GET /_template/.siem-signals-test-space GET /.siem-signals-default-test-space-000001 GET /_ilm/policy/.siem-signals-test-space GET /_alias/.siem-signals-test-space ``` ### Checklist Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR. ~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~ ~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~ ~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~ - [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios ~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~ ### For maintainers ~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ - [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process) --- .../legacy/plugins/siem/common/constants.ts | 14 +- x-pack/legacy/plugins/siem/index.ts | 18 +- .../scripts/convert_saved_search_to_rules.js | 5 +- .../plugins/siem/server/kibana.index.ts | 14 +- .../server/lib/detection_engine/README.md | 31 ++- .../alerts/get_input_output_index.test.ts | 215 ++---------------- .../alerts/get_input_output_index.ts | 60 +---- .../alerts/rules_alert_type.ts | 13 +- .../index/create_bootstrap_index.ts | 32 +++ .../index/delete_all_index.ts | 18 ++ .../detection_engine/index/delete_policy.ts | 17 ++ .../detection_engine/index/delete_template.ts | 18 ++ .../index/get_index_exists.ts | 18 ++ .../index/get_policy_exists.ts | 29 +++ .../index/get_template_exists.ts | 18 ++ .../lib/detection_engine/index/read_index.ts | 18 ++ .../lib/detection_engine/index/set_policy.ts | 19 ++ .../detection_engine/index/set_template.ts | 20 ++ .../lib/detection_engine/index/types.ts | 7 + .../routes/__mocks__/_mock_server.ts | 9 +- .../routes/create_rules_route.test.ts | 8 +- .../routes/create_rules_route.ts | 155 +++++++------ .../routes/index/create_index_route.ts | 63 +++++ .../routes/index/delete_index_route.ts | 70 ++++++ .../index/ecs_mapping.json} | 180 --------------- .../routes/index/get_signals_template.test.ts | 31 +++ .../routes/index/get_signals_template.ts | 25 ++ .../routes/index/read_index_route.ts | 57 +++++ .../routes/index/signals_mapping.json | 186 +++++++++++++++ .../routes/index/signals_policy.json | 15 ++ .../lib/detection_engine/routes/utils.ts | 15 ++ .../scripts/check_env_variables.sh | 5 - .../scripts/delete_signal_index.sh | 6 +- ...ut_signal_index.sh => get_signal_index.sh} | 9 +- .../detection_engine/scripts/hard_reset.sh | 2 +- ...signal_mapping.sh => post_signal_index.sh} | 10 +- .../rules/filter_with_empty_query.json | 1 - .../scripts/rules/filter_without_query.json | 1 - .../scripts/signal_index_exists.sh | 16 ++ x-pack/legacy/plugins/siem/server/types.ts | 1 + 40 files changed, 888 insertions(+), 561 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/create_bootstrap_index.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_all_index.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_policy.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_template.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_index_exists.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_policy_exists.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_template_exists.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/read_index.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/set_policy.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/set_template.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/index/types.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/create_index_route.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/delete_index_route.ts rename x-pack/legacy/plugins/siem/server/lib/detection_engine/{signals_mapping.json => routes/index/ecs_mapping.json} (91%) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.test.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/read_index_route.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_mapping.json create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_policy.json rename x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/{put_signal_index.sh => get_signal_index.sh} (56%) rename x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/{get_signal_mapping.sh => post_signal_index.sh} (53%) create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signal_index_exists.sh diff --git a/x-pack/legacy/plugins/siem/common/constants.ts b/x-pack/legacy/plugins/siem/common/constants.ts index e5d1fc83dac264..11b97738fcf52c 100644 --- a/x-pack/legacy/plugins/siem/common/constants.ts +++ b/x-pack/legacy/plugins/siem/common/constants.ts @@ -15,7 +15,11 @@ export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults'; export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults'; export const DEFAULT_SIEM_TIME_RANGE = 'siem:timeDefaults'; export const DEFAULT_SIEM_REFRESH_INTERVAL = 'siem:refreshIntervalDefaults'; + +// DEPRECATED: THIS WILL BE REMOVED VERY SOON AND IS NO LONGER USED ON THE BACKEND +// TODO: Remove this as soon as no code is left that is pulling data from it. export const DEFAULT_SIGNALS_INDEX_KEY = 'siem:defaultSignalsIndex'; + export const DEFAULT_SIGNALS_INDEX = '.siem-signals'; export const DEFAULT_MAX_SIGNALS = 100; export const DEFAULT_SEARCH_AFTER_PAGE_SIZE = 100; @@ -32,12 +36,18 @@ export const DEFAULT_INTERVAL_VALUE = 300000; // ms export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges'; /** - * Id for the SIGNALS alerting type + * Id for the signals alerting type */ export const SIGNALS_ID = `${APP_ID}.signals`; /** - * Detection engine route + * Detection engine routes */ export const DETECTION_ENGINE_URL = '/api/detection_engine'; export const DETECTION_ENGINE_RULES_URL = `${DETECTION_ENGINE_URL}/rules`; +export const DETECTION_ENGINE_INDEX_URL = `${DETECTION_ENGINE_URL}/index`; + +/** + * Default signals index key for kibana.dev.yml + */ +export const SIGNALS_INDEX_KEY = 'signalsIndex'; diff --git a/x-pack/legacy/plugins/siem/index.ts b/x-pack/legacy/plugins/siem/index.ts index 72b4ec588a5a4d..fca4a13db8cb5f 100644 --- a/x-pack/legacy/plugins/siem/index.ts +++ b/x-pack/legacy/plugins/siem/index.ts @@ -7,6 +7,7 @@ import { i18n } from '@kbn/i18n'; import { resolve } from 'path'; import { Server } from 'hapi'; +import { Root } from 'joi'; import { PluginInitializerContext } from 'src/core/server'; import { plugin } from './server'; @@ -24,6 +25,7 @@ import { DEFAULT_FROM, DEFAULT_TO, DEFAULT_SIGNALS_INDEX, + SIGNALS_INDEX_KEY, DEFAULT_SIGNALS_INDEX_KEY, } from './common/constants'; import { defaultIndexPattern } from './default_index_pattern'; @@ -103,6 +105,8 @@ export const siem = (kibana: any) => { category: ['siem'], requiresPageReload: true, }, + // DEPRECATED: This should be removed once the front end is no longer using any parts of it. + // TODO: Remove this as soon as no code is left that is pulling data from it. [DEFAULT_SIGNALS_INDEX_KEY]: { name: i18n.translate('xpack.siem.uiSettings.defaultSignalsIndexLabel', { defaultMessage: 'Elasticsearch signals index', @@ -155,7 +159,11 @@ export const siem = (kibana: any) => { getInjectedUiAppVars, indexPatternsServiceFactory, injectUiAppVars, - plugins: { alerting: plugins.alerting, xpack_main: plugins.xpack_main }, + plugins: { + alerting: plugins.alerting, + xpack_main: plugins.xpack_main, + spaces: plugins.spaces, + }, route: route.bind(server), savedObjects, }; @@ -166,5 +174,13 @@ export const siem = (kibana: any) => { serverFacade ); }, + config(Joi: Root) { + return Joi.object() + .keys({ + enabled: Joi.boolean().default(true), + [SIGNALS_INDEX_KEY]: Joi.string().default(DEFAULT_SIGNALS_INDEX), + }) + .default(); + }, }); }; diff --git a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_rules.js b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_rules.js index 3e1c5f51ebb5c8..b282a8bf1e861a 100644 --- a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_rules.js +++ b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_rules.js @@ -42,9 +42,10 @@ const allRulesNdJson = 'all_rules.ndjson'; // For converting, if you want to use these instead of rely on the defaults then // comment these in and use them for the script. Otherwise this is commented out // so we can utilize the defaults of input and output which are based on saved objects -// of siem:defaultIndex and siem:defaultSignalsIndex +// of siem:defaultIndex and your kibana.dev.yml setting of xpack.siem.signalsIndex. If +// the setting of xpack.siem.signalsIndex is not set it defaults to .siem-signals // const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*']; -// const OUTPUT_INDEX = process.env.SIGNALS_INDEX || '.siem-signals'; +// const OUTPUT_INDEX = '.siem-signals-some-other-index'; const walk = dir => { const list = fs.readdirSync(dir); diff --git a/x-pack/legacy/plugins/siem/server/kibana.index.ts b/x-pack/legacy/plugins/siem/server/kibana.index.ts index 3d73b9f4d90b00..66ec436ea418fb 100644 --- a/x-pack/legacy/plugins/siem/server/kibana.index.ts +++ b/x-pack/legacy/plugins/siem/server/kibana.index.ts @@ -18,11 +18,14 @@ import { import { rulesAlertType } from './lib/detection_engine/alerts/rules_alert_type'; import { isAlertExecutor } from './lib/detection_engine/alerts/types'; import { createRulesRoute } from './lib/detection_engine/routes/create_rules_route'; +import { createIndexRoute } from './lib/detection_engine/routes/index/create_index_route'; +import { readIndexRoute } from './lib/detection_engine/routes/index/read_index_route'; import { readRulesRoute } from './lib/detection_engine/routes/read_rules_route'; import { findRulesRoute } from './lib/detection_engine/routes/find_rules_route'; import { deleteRulesRoute } from './lib/detection_engine/routes/delete_rules_route'; import { updateRulesRoute } from './lib/detection_engine/routes/update_rules_route'; import { ServerFacade } from './types'; +import { deleteIndexRoute } from './lib/detection_engine/routes/index/delete_index_route'; const APP_ID = 'siem'; @@ -43,15 +46,20 @@ export const initServerWithKibana = ( const libs = compose(kbnServer, mode); initServer(libs); - // Signals/Alerting Rules routes for - // routes such as ${DETECTION_ENGINE_RULES_URL} - // that have the REST endpoints of /api/detection_engine/rules + // Detection Engine Rule routes that have the REST endpoints of /api/detection_engine/rules + // All REST rule creation, deletion, updating, etc... createRulesRoute(kbnServer); readRulesRoute(kbnServer); updateRulesRoute(kbnServer); deleteRulesRoute(kbnServer); findRulesRoute(kbnServer); + // Detection Engine index routes that have the REST endpoints of /api/detection_engine/index + // All REST index creation, policy management for spaces + createIndexRoute(kbnServer); + readIndexRoute(kbnServer); + deleteIndexRoute(kbnServer); + const xpackMainPlugin = kbnServer.plugins.xpack_main; xpackMainPlugin.registerFeature({ id: APP_ID, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md index 75757bbaa0c1f0..755727f9870f60 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md @@ -2,11 +2,12 @@ README.md for developers working on the backend detection engine on how to get s using the CURL scripts in the scripts folder. The scripts rely on CURL and jq: -* [CURL](https://curl.haxx.se) -* [jq](https://stedolan.github.io/jq/) +- [CURL](https://curl.haxx.se) +- [jq](https://stedolan.github.io/jq/) Install curl and jq + ```sh brew update brew install curl @@ -21,7 +22,6 @@ export ELASTICSEARCH_USERNAME=${user} export ELASTICSEARCH_PASSWORD=${password} export ELASTICSEARCH_URL=https://${ip}:9200 export KIBANA_URL=http://localhost:5601 -export SIGNALS_INDEX=.siem-signals-${your user id} export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id} export KIBANA_INDEX=.kibana-${your user id} ``` @@ -32,6 +32,12 @@ source `$HOME/.zshrc` or `${HOME}.bashrc` to ensure variables are set: source ~/.zshrc ``` +Open your `kibana.dev.yml` file and add these lines: + +```sh +xpack.siem.signalsIndex: .siem-signals-${your user id} +``` + Restart Kibana and ensure that you are using `--no-base-path` as changing the base path is a feature but will get in the way of the CURL scripts written as is. You should see alerting and actions starting up like so afterwards @@ -40,18 +46,11 @@ server log [22:05:22.277] [info][status][plugin:alerting@8.0.0] Status changed f server log [22:05:22.270] [info][status][plugin:actions@8.0.0] Status changed from uninitialized to green - Ready ``` -Go into your SIEM Advanced settings and underneath the setting of `siem:defaultSignalsIndex`, set that to the same -value as you did with the environment variable of `${SIGNALS_INDEX}`, which should be `.siem-signals-${your user id}` - -``` -.siem-signals-${your user id} -``` - Go to the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run: ```sh ./hard_reset.sh -./post_signal.sh +./post_rule.sh ``` which will: @@ -59,9 +58,9 @@ which will: - Delete any existing actions you have - Delete any existing alerts you have - Delete any existing alert tasks you have -- Delete any existing signal mapping you might have had. -- Add the latest signal index and its mappings using your settings from `${SIGNALS_INDEX}` environment variable. -- Posts the sample rule from `rules/root_or_admin_1.json` by replacing its `output_index` with your `SIGNALS_INDEX` environment variable +- Delete any existing signal mapping, policies, and template, you might have previously had. +- Add the latest signal index and its mappings using your settings from `kibana.dev.yml` environment variable of `xpack.siem.signalsIndex`. +- Posts the sample rule from `rules/root_or_admin_1.json` - The sample rule checks for root or admin every 5 minutes and reports that as a signal if it is a positive hit Now you can run @@ -128,9 +127,9 @@ post rules to `test-space` you set `SPACE_URL` to be: export SPACE_URL=/s/test-space ``` -The `${SPACE_URL}` is in front of all the APIs to correctly create, modify, delete, and update +The `${SPACE_URL}` is in front of all the APIs to correctly create, modify, delete, and update them from within the defined space. If this variable is not defined the default which is the url of an -empty string will be used. +empty string will be used. Add the `.siem-signals-${your user id}` to your advanced SIEM settings to see any signals created which should update once every 5 minutes at this point. diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.test.ts index 07eb7c885b4435..bd7ba915af9b00 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.test.ts @@ -5,13 +5,9 @@ */ import { savedObjectsClientMock } from 'src/core/server/mocks'; -import { - DEFAULT_SIGNALS_INDEX_KEY, - DEFAULT_INDEX_KEY, - DEFAULT_SIGNALS_INDEX, -} from '../../../../common/constants'; +import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; import { AlertServices } from '../../../../../alerting/server/types'; -import { getInputOutputIndex, getOutputIndex, getInputIndex } from './get_input_output_index'; +import { getInputIndex } from './get_input_output_index'; import { defaultIndexPattern } from '../../../../default_index_pattern'; describe('get_input_output_index', () => { @@ -46,240 +42,61 @@ describe('get_input_output_index', () => { }); describe('getInputOutputIndex', () => { - test('Returns inputIndex as is if inputIndex and outputIndex are both passed in', async () => { + test('Returns inputIndex if inputIndex is passed in', async () => { savedObjectsClient.get = jest.fn().mockImplementation(() => ({ attributes: {}, })); - const { inputIndex } = await getInputOutputIndex( - servicesMock, - '8.0.0', - ['test-input-index-1'], - 'test-output-index' - ); + const inputIndex = await getInputIndex(servicesMock, '8.0.0', ['test-input-index-1']); expect(inputIndex).toEqual(['test-input-index-1']); }); - test('Returns outputIndex as is if inputIndex and outputIndex are both passed in', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: {}, - })); - const { outputIndex } = await getInputOutputIndex( - servicesMock, - '8.0.0', - ['test-input-index-1'], - 'test-output-index' - ); - expect(outputIndex).toEqual('test-output-index'); - }); - - test('Returns inputIndex as is if inputIndex is defined but outputIndex is null', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: {}, - })); - const { inputIndex } = await getInputOutputIndex( - servicesMock, - '8.0.0', - ['test-input-index-1'], - null - ); - expect(inputIndex).toEqual(['test-input-index-1']); - }); - - test('Returns outputIndex as is if inputIndex is null but outputIndex is defined', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: {}, - })); - const { outputIndex } = await getInputOutputIndex( - servicesMock, - '8.0.0', - null, - 'test-output-index' - ); - expect(outputIndex).toEqual('test-output-index'); - }); - - test('Returns a saved object outputIndex if both passed in are undefined', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: { - [DEFAULT_SIGNALS_INDEX_KEY]: '.signals-test-index', - }, - })); - const { outputIndex } = await getInputOutputIndex( - servicesMock, - '8.0.0', - undefined, - undefined - ); - expect(outputIndex).toEqual('.signals-test-index'); - }); - - test('Returns a saved object outputIndex if passed in outputIndex is undefined', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: { - [DEFAULT_SIGNALS_INDEX_KEY]: '.signals-test-index', - }, - })); - const { outputIndex } = await getInputOutputIndex( - servicesMock, - '8.0.0', - ['test-input-index-1'], - undefined - ); - expect(outputIndex).toEqual('.signals-test-index'); - }); - - test('Returns a saved object outputIndex default from constants if both passed in input and configuration are null', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: { - [DEFAULT_SIGNALS_INDEX_KEY]: null, - }, - })); - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: {}, - })); - const { outputIndex } = await getInputOutputIndex(servicesMock, '8.0.0', null, null); - expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX); - }); - - test('Returns a saved object outputIndex default from constants if both passed in input and configuration are missing', async () => { - const { outputIndex } = await getInputOutputIndex( - servicesMock, - '8.0.0', - undefined, - undefined - ); - expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX); - }); - - test('Returns a saved object inputIndex if passed in inputIndex and outputIndex are undefined', async () => { + test('Returns a saved object inputIndex if passed in inputIndex is undefined', async () => { savedObjectsClient.get = jest.fn().mockImplementation(() => ({ attributes: { [DEFAULT_INDEX_KEY]: ['configured-index-1', 'configured-index-2'], }, })); - const { inputIndex } = await getInputOutputIndex(servicesMock, '8.0.0', undefined, undefined); + const inputIndex = await getInputIndex(servicesMock, '8.0.0', undefined); expect(inputIndex).toEqual(['configured-index-1', 'configured-index-2']); }); - test('Returns a saved object inputIndex if passed in inputIndex is undefined', async () => { + test('Returns a saved object inputIndex if passed in inputIndex is null', async () => { savedObjectsClient.get = jest.fn().mockImplementation(() => ({ attributes: { [DEFAULT_INDEX_KEY]: ['configured-index-1', 'configured-index-2'], }, })); - const { inputIndex } = await getInputOutputIndex( - servicesMock, - '8.0.0', - undefined, - 'output-index-1' - ); + const inputIndex = await getInputIndex(servicesMock, '8.0.0', null); expect(inputIndex).toEqual(['configured-index-1', 'configured-index-2']); }); - test('Returns a saved object inputIndex default from constants if both passed in inputIndex and configuration is null', async () => { + test('Returns a saved object inputIndex default from constants if inputIndex passed in is null and the key is also null', async () => { savedObjectsClient.get = jest.fn().mockImplementation(() => ({ attributes: { [DEFAULT_INDEX_KEY]: null, }, })); - const { inputIndex } = await getInputOutputIndex(servicesMock, '8.0.0', null, null); - expect(inputIndex).toEqual(defaultIndexPattern); - }); - - test('Returns a saved object inputIndex default from constants if both passed in inputIndex and configuration attributes is missing', async () => { - const { inputIndex } = await getInputOutputIndex(servicesMock, '8.0.0', undefined, undefined); + const inputIndex = await getInputIndex(servicesMock, '8.0.0', null); expect(inputIndex).toEqual(defaultIndexPattern); }); - }); - - describe('getOutputIndex', () => { - test('test output index is returned when passed in as is', async () => { - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const outputIndex = getOutputIndex('output-index-1', mockConfiguration); - expect(outputIndex).toEqual('output-index-1'); - }); - - test('configured output index is returned when output index is null', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: { - [DEFAULT_SIGNALS_INDEX_KEY]: '.siem-test-signals', - }, - })); - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const outputIndex = getOutputIndex(null, mockConfiguration); - expect(outputIndex).toEqual('.siem-test-signals'); - }); - test('output index from constants is returned when output index is null and so is the configuration', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: { - [DEFAULT_SIGNALS_INDEX_KEY]: null, - }, - })); - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const outputIndex = getOutputIndex(null, mockConfiguration); - expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX); - }); - - test('output index from constants is returned when output index is null and configuration is missing', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: {}, - })); - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const outputIndex = getOutputIndex(null, mockConfiguration); - expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX); - }); - - test('output index from constants is returned when output index is null and attributes is missing', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({})); - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const outputIndex = getOutputIndex(null, mockConfiguration); - expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX); - }); - }); - - describe('getInputIndex', () => { - test('test input index is returned when passed in as is', async () => { - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const inputIndex = getInputIndex(['input-index-1'], mockConfiguration); - expect(inputIndex).toEqual(['input-index-1']); - }); - - test('configured input index is returned when input index is null', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: { - [DEFAULT_INDEX_KEY]: ['input-index-1', 'input-index-2'], - }, - })); - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const inputIndex = getInputIndex(null, mockConfiguration); - expect(inputIndex).toEqual(['input-index-1', 'input-index-2']); - }); - - test('input index from constants is returned when input index is null and so is the configuration', async () => { + test('Returns a saved object inputIndex default from constants if inputIndex passed in is undefined and the key is also null', async () => { savedObjectsClient.get = jest.fn().mockImplementation(() => ({ attributes: { [DEFAULT_INDEX_KEY]: null, }, })); - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const inputIndex = getInputIndex(null, mockConfiguration); + const inputIndex = await getInputIndex(servicesMock, '8.0.0', undefined); expect(inputIndex).toEqual(defaultIndexPattern); }); - test('input index from constants is returned when input index is null and configuration is missing', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({ - attributes: {}, - })); - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const inputIndex = getInputIndex(null, mockConfiguration); + test('Returns a saved object inputIndex default from constants if both passed in inputIndex and configuration attributes are missing and the index is undefined', async () => { + const inputIndex = await getInputIndex(servicesMock, '8.0.0', undefined); expect(inputIndex).toEqual(defaultIndexPattern); }); - test('input index from constants is returned when input index is null and attributes is missing', async () => { - savedObjectsClient.get = jest.fn().mockImplementation(() => ({})); - const mockConfiguration = await savedObjectsClient.get('config', '8.0.0'); - const inputIndex = getInputIndex(null, mockConfiguration); + test('Returns a saved object inputIndex default from constants if both passed in inputIndex and configuration attributes are missing and the index is null', async () => { + const inputIndex = await getInputIndex(servicesMock, '8.0.0', null); expect(inputIndex).toEqual(defaultIndexPattern); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.ts index 567ab27976d8d2..624e012717820d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.ts @@ -4,27 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SavedObject, SavedObjectAttributes } from 'src/core/server'; import { defaultIndexPattern } from '../../../../default_index_pattern'; import { AlertServices } from '../../../../../alerting/server/types'; -import { - DEFAULT_INDEX_KEY, - DEFAULT_SIGNALS_INDEX_KEY, - DEFAULT_SIGNALS_INDEX, -} from '../../../../common/constants'; +import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; -interface IndexObjectAttributes extends SavedObjectAttributes { - [DEFAULT_INDEX_KEY]: string[]; - [DEFAULT_SIGNALS_INDEX_KEY]: string; -} - -export const getInputIndex = ( - inputIndex: string[] | undefined | null, - configuration: SavedObject -): string[] => { +export const getInputIndex = async ( + services: AlertServices, + version: string, + inputIndex: string[] | null | undefined +): Promise => { if (inputIndex != null) { return inputIndex; } else { + const configuration = await services.savedObjectsClient.get('config', version); if (configuration.attributes != null && configuration.attributes[DEFAULT_INDEX_KEY] != null) { return configuration.attributes[DEFAULT_INDEX_KEY]; } else { @@ -32,41 +24,3 @@ export const getInputIndex = ( } } }; - -export const getOutputIndex = ( - outputIndex: string | undefined | null, - configuration: SavedObject -): string => { - if (outputIndex != null) { - return outputIndex; - } else { - if ( - configuration.attributes != null && - configuration.attributes[DEFAULT_SIGNALS_INDEX_KEY] != null - ) { - return configuration.attributes[DEFAULT_SIGNALS_INDEX_KEY]; - } else { - return DEFAULT_SIGNALS_INDEX; - } - } -}; - -export const getInputOutputIndex = async ( - services: AlertServices, - version: string, - inputIndex: string[] | null | undefined, - outputIndex: string | null | undefined -): Promise<{ - inputIndex: string[]; - outputIndex: string; -}> => { - if (inputIndex != null && outputIndex != null) { - return { inputIndex, outputIndex }; - } else { - const configuration = await services.savedObjectsClient.get('config', version); - return { - inputIndex: getInputIndex(inputIndex, configuration), - outputIndex: getOutputIndex(outputIndex, configuration), - }; - } -}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts index 61fe9c7c226395..577de2ce0d5323 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts @@ -16,7 +16,7 @@ import { buildEventsSearchQuery } from './build_events_query'; import { searchAfterAndBulkCreate } from './utils'; import { RuleAlertTypeDefinition } from './types'; import { getFilter } from './get_filter'; -import { getInputOutputIndex } from './get_input_output_index'; +import { getInputIndex } from './get_input_output_index'; export const rulesAlertType = ({ logger, @@ -84,12 +84,7 @@ export const rulesAlertType = ({ ? DEFAULT_SEARCH_AFTER_PAGE_SIZE : params.maxSignals; - const { inputIndex, outputIndex: signalsIndex } = await getInputOutputIndex( - services, - version, - index, - outputIndex - ); + const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ type, filter, @@ -122,7 +117,7 @@ export const rulesAlertType = ({ noReIndexResult.hits.total.value } signals from the indexes of "${inputIndex.join( ', ' - )}" using signal rule "id: ${alertId}", "ruleId: ${ruleId}", pushing signals to index ${signalsIndex}` + )}" using signal rule "id: ${alertId}", "ruleId: ${ruleId}", pushing signals to index ${outputIndex}` ); } @@ -132,7 +127,7 @@ export const rulesAlertType = ({ services, logger, id: alertId, - signalsIndex, + signalsIndex: outputIndex, name, createdBy, updatedBy, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/create_bootstrap_index.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/create_bootstrap_index.ts new file mode 100644 index 00000000000000..9c8dca0cb370f4 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/create_bootstrap_index.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; +import { CallWithRequest } from './types'; + +// See the reference(s) below on explanations about why -000001 was chosen and +// why the is_write_index is true as well as the bootstrapping step which is needed. +// Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/applying-policy-to-template.html +export const createBootstrapIndex = async ( + callWithRequest: CallWithRequest< + { path: string; method: 'PUT'; body: unknown }, + CallClusterOptions, + boolean + >, + index: string +): Promise => { + return callWithRequest('transport.request', { + path: `${index}-000001`, + method: 'PUT', + body: { + aliases: { + [index]: { + is_write_index: true, + }, + }, + }, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_all_index.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_all_index.ts new file mode 100644 index 00000000000000..6f16eb8fbdeb18 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_all_index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndicesDeleteParams } from 'elasticsearch'; +import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; +import { CallWithRequest } from './types'; + +export const deleteAllIndex = async ( + callWithRequest: CallWithRequest, + index: string +): Promise => { + return callWithRequest('indices.delete', { + index, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_policy.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_policy.ts new file mode 100644 index 00000000000000..153b9ae4e4136c --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_policy.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CallWithRequest } from './types'; + +export const deletePolicy = async ( + callWithRequest: CallWithRequest<{ path: string; method: 'DELETE' }, {}, unknown>, + policy: string +): Promise => { + return callWithRequest('transport.request', { + path: `_ilm/policy/${policy}`, + method: 'DELETE', + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_template.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_template.ts new file mode 100644 index 00000000000000..b048dd27efb83b --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/delete_template.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndicesDeleteTemplateParams } from 'elasticsearch'; +import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; +import { CallWithRequest } from './types'; + +export const deleteTemplate = async ( + callWithRequest: CallWithRequest, + name: string +): Promise => { + return callWithRequest('indices.deleteTemplate', { + name, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_index_exists.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_index_exists.ts new file mode 100644 index 00000000000000..24164e894788a4 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_index_exists.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndicesExistsParams } from 'elasticsearch'; +import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; +import { CallWithRequest } from './types'; + +export const getIndexExists = async ( + callWithRequest: CallWithRequest, + index: string +): Promise => { + return callWithRequest('indices.exists', { + index, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_policy_exists.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_policy_exists.ts new file mode 100644 index 00000000000000..847c32d9d61fb3 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_policy_exists.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CallWithRequest } from './types'; + +export const getPolicyExists = async ( + callWithRequest: CallWithRequest<{ path: string; method: 'GET' }, {}, unknown>, + policy: string +): Promise => { + try { + await callWithRequest('transport.request', { + path: `_ilm/policy/${policy}`, + method: 'GET', + }); + // Return true that there exists a policy which is not 404 or some error + // Since there is not a policy exists API, this is how we create one by calling + // into the API to get it if it exists or rely on it to throw a 404 + return true; + } catch (err) { + if (err.statusCode === 404) { + return false; + } else { + throw err; + } + } +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_template_exists.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_template_exists.ts new file mode 100644 index 00000000000000..482fc8d855828c --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_template_exists.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndicesExistsTemplateParams } from 'elasticsearch'; +import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; +import { CallWithRequest } from './types'; + +export const getTemplateExists = async ( + callWithRequest: CallWithRequest, + template: string +): Promise => { + return callWithRequest('indices.existsTemplate', { + name: template, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/read_index.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/read_index.ts new file mode 100644 index 00000000000000..6c9d529078a77d --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/read_index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndicesGetSettingsParams } from 'elasticsearch'; +import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; +import { CallWithRequest } from './types'; + +export const readIndex = async ( + callWithRequest: CallWithRequest, + index: string +): Promise => { + return callWithRequest('indices.get', { + index, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/set_policy.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/set_policy.ts new file mode 100644 index 00000000000000..2511984b412f34 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/set_policy.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CallWithRequest } from './types'; + +export const setPolicy = async ( + callWithRequest: CallWithRequest<{ path: string; method: 'PUT'; body: unknown }, {}, unknown>, + policy: string, + body: unknown +): Promise => { + return callWithRequest('transport.request', { + path: `_ilm/policy/${policy}`, + method: 'PUT', + body, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/set_template.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/set_template.ts new file mode 100644 index 00000000000000..a679a61e10c001 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/set_template.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndicesPutTemplateParams } from 'elasticsearch'; +import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; +import { CallWithRequest } from './types'; + +export const setTemplate = async ( + callWithRequest: CallWithRequest, + name: string, + body: unknown +): Promise => { + return callWithRequest('indices.putTemplate', { + name, + body, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/types.ts new file mode 100644 index 00000000000000..70b33ad274ee1f --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/types.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export type CallWithRequest = (endpoint: string, params: T, options?: U) => Promise; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/_mock_server.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/_mock_server.ts index c02af2c841a307..3c19ff6f1bb652 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/_mock_server.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/_mock_server.ts @@ -8,6 +8,7 @@ import Hapi from 'hapi'; import { KibanaConfig } from 'src/legacy/server/kbn_server'; import { alertsClientMock } from '../../../../../../alerting/server/alerts_client.mock'; import { actionsClientMock } from '../../../../../../actions/server/actions_client.mock'; +import { ElasticsearchPlugin } from 'src/legacy/core_plugins/elasticsearch'; const defaultConfig = { 'kibana.index': '.kibana', @@ -46,11 +47,17 @@ export const createMockServer = (config: Record = defaultConfig) const actionsClient = actionsClientMock.create(); const alertsClient = alertsClientMock.create(); + const elasticsearch = { + getCluster: jest.fn().mockImplementation(() => ({ + callWithRequest: jest.fn(), + })), + }; server.decorate('request', 'getAlertsClient', () => alertsClient); server.decorate('request', 'getBasePath', () => '/s/default'); server.decorate('request', 'getActionsClient', () => actionsClient); + server.plugins.elasticsearch = (elasticsearch as unknown) as ElasticsearchPlugin; - return { server, alertsClient, actionsClient }; + return { server, alertsClient, actionsClient, elasticsearch }; }; export const createMockServerWithoutAlertClientDecoration = ( diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.test.ts index 4c222c196300ce..4aa57f005445b2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.test.ts @@ -22,11 +22,15 @@ import { import { DETECTION_ENGINE_RULES_URL } from '../../../../common/constants'; describe('create_rules', () => { - let { server, alertsClient, actionsClient } = createMockServer(); + let { server, alertsClient, actionsClient, elasticsearch } = createMockServer(); beforeEach(() => { jest.resetAllMocks(); - ({ server, alertsClient, actionsClient } = createMockServer()); + ({ server, alertsClient, actionsClient, elasticsearch } = createMockServer()); + elasticsearch.getCluster = jest.fn().mockImplementation(() => ({ + callWithRequest: jest.fn().mockImplementation(() => true), + })); + createRulesRoute(server); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.ts index 2b69e57f2c2eea..31068dac5d23a6 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.ts @@ -14,97 +14,112 @@ import { RulesRequest } from '../alerts/types'; import { createRulesSchema } from './schemas'; import { ServerFacade } from '../../../types'; import { readRules } from '../alerts/read_rules'; -import { transformOrError, transformError } from './utils'; +import { transformOrError, transformError, getIndex, callWithRequestFactory } from './utils'; +import { getIndexExists } from '../index/get_index_exists'; -export const createCreateRulesRoute: Hapi.ServerRoute = { - method: 'POST', - path: DETECTION_ENGINE_RULES_URL, - options: { - tags: ['access:siem'], - validate: { - options: { - abortEarly: false, +export const createCreateRulesRoute = (server: ServerFacade): Hapi.ServerRoute => { + return { + method: 'POST', + path: DETECTION_ENGINE_RULES_URL, + options: { + tags: ['access:siem'], + validate: { + options: { + abortEarly: false, + }, + payload: createRulesSchema, }, - payload: createRulesSchema, }, - }, - async handler(request: RulesRequest, headers) { - const { - description, - enabled, - false_positives: falsePositives, - filter, - from, - immutable, - query, - language, - output_index: outputIndex, - saved_id: savedId, - meta, - filters, - rule_id: ruleId, - index, - interval, - max_signals: maxSignals, - risk_score: riskScore, - name, - severity, - tags, - threats, - to, - type, - references, - } = request.payload; - const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null; - const actionsClient = isFunction(request.getActionsClient) ? request.getActionsClient() : null; - - if (!alertsClient || !actionsClient) { - return headers.response().code(404); - } - - try { - if (ruleId != null) { - const rule = await readRules({ alertsClient, ruleId }); - if (rule != null) { - return new Boom(`rule_id ${ruleId} already exists`, { statusCode: 409 }); - } - } - - const createdRule = await createRules({ - alertsClient, - actionsClient, + async handler(request: RulesRequest, headers) { + const { description, enabled, - falsePositives, + false_positives: falsePositives, filter, from, immutable, query, language, - outputIndex, - savedId, + output_index: outputIndex, + saved_id: savedId, meta, filters, - ruleId: ruleId != null ? ruleId : uuid.v4(), + rule_id: ruleId, index, interval, - maxSignals, - riskScore, + max_signals: maxSignals, + risk_score: riskScore, name, severity, tags, + threats, to, type, - threats, references, - }); - return transformOrError(createdRule); - } catch (err) { - return transformError(err); - } - }, + } = request.payload; + const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null; + const actionsClient = isFunction(request.getActionsClient) + ? request.getActionsClient() + : null; + + if (!alertsClient || !actionsClient) { + return headers.response().code(404); + } + + try { + const finalIndex = outputIndex != null ? outputIndex : getIndex(request, server); + const callWithRequest = callWithRequestFactory(request); + const indexExists = await getIndexExists(callWithRequest, finalIndex); + if (!indexExists) { + return new Boom( + `To create a rule, the index must exist first. Index ${finalIndex} does not exist`, + { + statusCode: 400, + } + ); + } + if (ruleId != null) { + const rule = await readRules({ alertsClient, ruleId }); + if (rule != null) { + return new Boom(`rule_id ${ruleId} already exists`, { statusCode: 409 }); + } + } + const createdRule = await createRules({ + alertsClient, + actionsClient, + description, + enabled, + falsePositives, + filter, + from, + immutable, + query, + language, + outputIndex: finalIndex, + savedId, + meta, + filters, + ruleId: ruleId != null ? ruleId : uuid.v4(), + index, + interval, + maxSignals, + riskScore, + name, + severity, + tags, + to, + type, + threats, + references, + }); + return transformOrError(createdRule); + } catch (err) { + return transformError(err); + } + }, + }; }; export const createRulesRoute = (server: ServerFacade) => { - server.route(createCreateRulesRoute); + server.route(createCreateRulesRoute(server)); }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/create_index_route.ts new file mode 100644 index 00000000000000..14a061fd4ccb79 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/create_index_route.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import Boom from 'boom'; + +import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; +import signalsPolicy from './signals_policy.json'; +import { ServerFacade, RequestFacade } from '../../../../types'; +import { transformError, getIndex, callWithRequestFactory } from '../utils'; +import { getIndexExists } from '../../index/get_index_exists'; +import { getPolicyExists } from '../../index/get_policy_exists'; +import { setPolicy } from '../../index/set_policy'; +import { setTemplate } from '../../index/set_template'; +import { getSignalsTemplate } from './get_signals_template'; +import { getTemplateExists } from '../../index/get_template_exists'; +import { createBootstrapIndex } from '../../index/create_bootstrap_index'; + +export const createCreateIndexRoute = (server: ServerFacade): Hapi.ServerRoute => { + return { + method: 'POST', + path: DETECTION_ENGINE_INDEX_URL, + options: { + tags: ['access:siem'], + validate: { + options: { + abortEarly: false, + }, + }, + }, + async handler(request: RequestFacade) { + try { + const index = getIndex(request, server); + const callWithRequest = callWithRequestFactory(request); + const indexExists = await getIndexExists(callWithRequest, index); + if (indexExists) { + return new Boom(`index ${index} already exists`, { statusCode: 409 }); + } else { + const policyExists = await getPolicyExists(callWithRequest, index); + if (!policyExists) { + await setPolicy(callWithRequest, index, signalsPolicy); + } + const templateExists = await getTemplateExists(callWithRequest, index); + if (!templateExists) { + const template = getSignalsTemplate(index); + await setTemplate(callWithRequest, index, template); + } + createBootstrapIndex(callWithRequest, index); + return { acknowledged: true }; + } + } catch (err) { + return transformError(err); + } + }, + }; +}; + +export const createIndexRoute = (server: ServerFacade) => { + server.route(createCreateIndexRoute(server)); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/delete_index_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/delete_index_route.ts new file mode 100644 index 00000000000000..89f21bbada939f --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/delete_index_route.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import Boom from 'boom'; + +import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; +import { ServerFacade, RequestFacade } from '../../../../types'; +import { transformError, getIndex, callWithRequestFactory } from '../utils'; +import { getIndexExists } from '../../index/get_index_exists'; +import { getPolicyExists } from '../../index/get_policy_exists'; +import { deletePolicy } from '../../index/delete_policy'; +import { getTemplateExists } from '../../index/get_template_exists'; +import { deleteAllIndex } from '../../index/delete_all_index'; +import { deleteTemplate } from '../../index/delete_template'; + +/** + * Deletes all of the indexes, template, ilm policies, and aliases. You can check + * this by looking at each of these settings from ES after a deletion: + * GET /_template/.siem-signals-default + * GET /.siem-signals-default-000001/ + * GET /_ilm/policy/.signals-default + * GET /_alias/.siem-signals-default + * + * And ensuring they're all gone + */ +export const createDeleteIndexRoute = (server: ServerFacade): Hapi.ServerRoute => { + return { + method: 'DELETE', + path: DETECTION_ENGINE_INDEX_URL, + options: { + tags: ['access:siem'], + validate: { + options: { + abortEarly: false, + }, + }, + }, + async handler(request: RequestFacade) { + try { + const index = getIndex(request, server); + const callWithRequest = callWithRequestFactory(request); + const indexExists = await getIndexExists(callWithRequest, index); + if (!indexExists) { + return new Boom(`index ${index} does not exist`, { statusCode: 404 }); + } else { + await deleteAllIndex(callWithRequest, `${index}-*`); + const policyExists = await getPolicyExists(callWithRequest, index); + if (policyExists) { + await deletePolicy(callWithRequest, index); + } + const templateExists = await getTemplateExists(callWithRequest, index); + if (templateExists) { + await deleteTemplate(callWithRequest, index); + } + return { acknowledged: true }; + } + } catch (err) { + return transformError(err); + } + }, + }; +}; + +export const deleteIndexRoute = (server: ServerFacade) => { + server.route(createDeleteIndexRoute(server)); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/ecs_mapping.json similarity index 91% rename from x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json rename to x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/ecs_mapping.json index 94f251645d3fd6..06edf94484af3b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/ecs_mapping.json @@ -5,186 +5,6 @@ "@timestamp": { "type": "date" }, - "signal": { - "properties": { - "parent": { - "properties": { - "index": { - "type": "keyword" - }, - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "depth": { - "type": "long" - } - } - }, - "rule": { - "properties": { - "id": { - "type": "keyword" - }, - "rule_id": { - "type": "keyword" - }, - "false_positives": { - "type": "keyword" - }, - "saved_id": { - "type": "keyword" - }, - "max_signals": { - "type": "keyword" - }, - "risk_score": { - "type": "keyword" - }, - "output_index": { - "type": "keyword" - }, - "description": { - "type": "keyword" - }, - "filter": { - "type": "object" - }, - "from": { - "type": "keyword" - }, - "immutable": { - "type": "keyword" - }, - "index": { - "type": "keyword" - }, - "interval": { - "type": "keyword" - }, - "language": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "query": { - "type": "keyword" - }, - "references": { - "type": "keyword" - }, - "severity": { - "type": "keyword" - }, - "tags": { - "type": "keyword" - }, - "threats": { - "type": "object" - }, - "type": { - "type": "keyword" - }, - "size": { - "type": "keyword" - }, - "to": { - "type": "keyword" - }, - "enabled": { - "type": "keyword" - }, - "filters": { - "type": "object" - }, - "created_by": { - "type": "keyword" - }, - "updated_by": { - "type": "keyword" - } - } - }, - "original_time": { - "type": "date" - }, - "original_event": { - "properties": { - "action": { - "type": "keyword" - }, - "category": { - "type": "keyword" - }, - "code": { - "type": "keyword" - }, - "created": { - "type": "date" - }, - "dataset": { - "type": "keyword" - }, - "duration": { - "type": "long" - }, - "end": { - "type": "date" - }, - "hash": { - "type": "keyword" - }, - "id": { - "type": "keyword" - }, - "kind": { - "type": "keyword" - }, - "module": { - "type": "keyword" - }, - "original": { - "doc_values": false, - "index": false, - "type": "keyword" - }, - "outcome": { - "type": "keyword" - }, - "provider": { - "type": "keyword" - }, - "risk_score": { - "type": "float" - }, - "risk_score_norm": { - "type": "float" - }, - "sequence": { - "type": "long" - }, - "severity": { - "type": "long" - }, - "start": { - "type": "date" - }, - "timezone": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "status": { - "type": "keyword" - } - } - }, "agent": { "properties": { "ephemeral_id": { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.test.ts new file mode 100644 index 00000000000000..80594ca74a3535 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getSignalsTemplate } from './get_signals_template'; + +describe('get_signals_template', () => { + test('it should set the lifecycle name and the rollover alias to be the name of the index passed in', () => { + const template = getSignalsTemplate('test-index'); + expect(template.settings).toEqual({ + index: { lifecycle: { name: 'test-index', rollover_alias: 'test-index' } }, + }); + }); + + test('it should set have the index patterns with an ending glob in it', () => { + const template = getSignalsTemplate('test-index'); + expect(template.index_patterns).toEqual(['test-index-*']); + }); + + test('it should have a mappings section which is an object type', () => { + const template = getSignalsTemplate('test-index'); + expect(typeof template.mappings).toEqual('object'); + }); + + test('it should have a signals section which is an object type', () => { + const template = getSignalsTemplate('test-index'); + expect(typeof template.mappings.properties.signal).toEqual('object'); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.ts new file mode 100644 index 00000000000000..c6580f0bdda427 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import signalsMapping from './signals_mapping.json'; +import ecsMapping from './ecs_mapping.json'; + +export const getSignalsTemplate = (index: string) => { + ecsMapping.mappings.properties.signal = signalsMapping.mappings.properties.signal; + const template = { + settings: { + index: { + lifecycle: { + name: index, + rollover_alias: index, + }, + }, + }, + index_patterns: [`${index}-*`], + mappings: ecsMapping.mappings, + }; + return template; +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/read_index_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/read_index_route.ts new file mode 100644 index 00000000000000..87b6ffb985c371 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/read_index_route.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import Boom from 'boom'; + +import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; +import { ServerFacade, RequestFacade } from '../../../../types'; +import { transformError, getIndex, callWithRequestFactory } from '../utils'; +import { getIndexExists } from '../../index/get_index_exists'; + +export const createReadIndexRoute = (server: ServerFacade): Hapi.ServerRoute => { + return { + method: 'GET', + path: DETECTION_ENGINE_INDEX_URL, + options: { + tags: ['access:siem'], + validate: { + options: { + abortEarly: false, + }, + }, + }, + async handler(request: RequestFacade, headers) { + try { + const index = getIndex(request, server); + const callWithRequest = callWithRequestFactory(request); + const indexExists = await getIndexExists(callWithRequest, index); + if (indexExists) { + // head request is used for if you want to get if the index exists + // or not and it will return a content-length: 0 along with either a 200 or 404 + // depending on if the index exists or not. + if (request.method.toLowerCase() === 'head') { + return headers.response().code(200); + } else { + return headers.response({ name: index }).code(200); + } + } else { + if (request.method.toLowerCase() === 'head') { + return headers.response().code(404); + } else { + return new Boom('An index for this space does not exist', { statusCode: 404 }); + } + } + } catch (err) { + return transformError(err); + } + }, + }; +}; + +export const readIndexRoute = (server: ServerFacade) => { + server.route(createReadIndexRoute(server)); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_mapping.json new file mode 100644 index 00000000000000..501522105bdbc3 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_mapping.json @@ -0,0 +1,186 @@ +{ + "mappings": { + "properties": { + "signal": { + "properties": { + "parent": { + "properties": { + "index": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "depth": { + "type": "long" + } + } + }, + "rule": { + "properties": { + "id": { + "type": "keyword" + }, + "rule_id": { + "type": "keyword" + }, + "false_positives": { + "type": "keyword" + }, + "saved_id": { + "type": "keyword" + }, + "max_signals": { + "type": "keyword" + }, + "risk_score": { + "type": "keyword" + }, + "output_index": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "filter": { + "type": "object" + }, + "from": { + "type": "keyword" + }, + "immutable": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "interval": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "query": { + "type": "keyword" + }, + "references": { + "type": "keyword" + }, + "severity": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "threats": { + "type": "object" + }, + "type": { + "type": "keyword" + }, + "size": { + "type": "keyword" + }, + "to": { + "type": "keyword" + }, + "enabled": { + "type": "keyword" + }, + "filters": { + "type": "object" + }, + "created_by": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + } + } + }, + "original_time": { + "type": "date" + }, + "original_event": { + "properties": { + "action": { + "type": "keyword" + }, + "category": { + "type": "keyword" + }, + "code": { + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "kind": { + "type": "keyword" + }, + "module": { + "type": "keyword" + }, + "original": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "outcome": { + "type": "keyword" + }, + "provider": { + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "status": { + "type": "keyword" + } + } + } + } + } +} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_policy.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_policy.json new file mode 100644 index 00000000000000..640d8e14190cd0 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_policy.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "10gb", + "max_age": "7d" + } + } + } + } + } +} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts index 40a33e9d97a182..4c5a5a6af93de3 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts @@ -6,7 +6,9 @@ import Boom from 'boom'; import { pickBy } from 'lodash/fp'; +import { APP_ID, SIGNALS_INDEX_KEY } from '../../../../common/constants'; import { RuleAlertType, isAlertType, OutputRuleAlertRest, isAlertTypes } from '../alerts/types'; +import { ServerFacade, RequestFacade } from '../../../types'; export const getIdError = ({ id, @@ -88,3 +90,16 @@ export const transformError = (err: Error & { statusCode?: number }) => { } } }; + +export const getIndex = (request: RequestFacade, server: ServerFacade): string => { + const spaceId = server.plugins.spaces.getSpaceId(request); + const signalsIndex = server.config().get(`xpack.${APP_ID}.${SIGNALS_INDEX_KEY}`); + return `${signalsIndex}-${spaceId}`; +}; + +export const callWithRequestFactory = (request: RequestFacade) => { + const { callWithRequest } = request.server.plugins.elasticsearch.getCluster('data'); + return (endpoint: string, params: T, options?: U) => { + return callWithRequest(request, endpoint, params, options); + }; +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh index c2406dc7f62317..fb3bbbe0fad186 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh @@ -30,11 +30,6 @@ if [ -z "${KIBANA_URL}" ]; then exit 1 fi -if [ -z "${SIGNALS_INDEX}" ]; then - echo "Set SIGNALS_INDEX in your environment" - exit 1 -fi - if [ -z "${TASK_MANAGER_INDEX}" ]; then echo "Set TASK_MANAGER_INDEX in your environment" exit 1 diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh index 8d5deec1ba3a17..aa8f90f27d6d86 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh @@ -10,9 +10,7 @@ set -e ./check_env_variables.sh # Example: ./delete_signal_index.sh -# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-delete-index.html curl -s -k \ - -H "Content-Type: application/json" \ + -H 'kbn-xsrf: 123' \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ - -X DELETE ${ELASTICSEARCH_URL}/${SIGNALS_INDEX} \ - | jq . + -X DELETE ${KIBANA_URL}${SPACE_URL}/api/detection_engine/index | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_index.sh similarity index 56% rename from x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh rename to x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_index.sh index 1b3b148a991614..882451631c9ebb 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_index.sh @@ -9,10 +9,7 @@ set -e ./check_env_variables.sh -# Example: ./put_signal_index.sh +# Example: ./get_signal_index.sh curl -s -k \ - -H "Content-Type: application/json" \ - -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ - -d @../signals_mapping.json \ - -X PUT ${ELASTICSEARCH_URL}/${SIGNALS_INDEX} \ - | jq . + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}${SPACE_URL}/api/detection_engine/index | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh index ee8fa18e1234d3..3cd0bff7ee8ab7 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh @@ -13,4 +13,4 @@ set -e ./delete_all_alerts.sh ./delete_all_alert_tasks.sh ./delete_signal_index.sh -./put_signal_index.sh +./post_signal_index.sh diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal_index.sh similarity index 53% rename from x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh rename to x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal_index.sh index 8b384fcc76f722..e408f21888c5fb 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal_index.sh @@ -9,9 +9,9 @@ set -e ./check_env_variables.sh -# Example: ./get_signal_mapping.sh -# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-mapping.html +# Example: ./post_signal_index.sh curl -s -k \ - -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ - -X GET ${ELASTICSEARCH_URL}/${SIGNALS_INDEX}/_mapping \ - | jq . + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/index | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_with_empty_query.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_with_empty_query.json index c136c9b0fe808d..75a04b54265506 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_with_empty_query.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_with_empty_query.json @@ -9,7 +9,6 @@ "type": "query", "from": "now-24h", "to": "now", - "output_index": ".siem-signals", "language": "lucene", "query": "", "filters": [ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_without_query.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_without_query.json index 5b69fced90dafa..2417a000f9dce2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_without_query.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_without_query.json @@ -9,7 +9,6 @@ "type": "query", "from": "now-24h", "to": "now", - "output_index": ".siem-signals", "language": "lucene", "filters": [ { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signal_index_exists.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signal_index_exists.sh new file mode 100755 index 00000000000000..b4a494a102b542 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signal_index_exists.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./signal_index_exists.sh +curl -s -k --head \ + -H 'Content-Type: application/json' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + ${KIBANA_URL}${SPACE_URL}/api/detection_engine/index diff --git a/x-pack/legacy/plugins/siem/server/types.ts b/x-pack/legacy/plugins/siem/server/types.ts index d6fea820b26988..ad19872b7a75dc 100644 --- a/x-pack/legacy/plugins/siem/server/types.ts +++ b/x-pack/legacy/plugins/siem/server/types.ts @@ -13,6 +13,7 @@ export interface ServerFacade { injectUiAppVars: Legacy.Server['injectUiAppVars']; plugins: { alerting?: Legacy.Server['plugins']['alerting']; + spaces: Legacy.Server['plugins']['spaces']; xpack_main: Legacy.Server['plugins']['xpack_main']; }; route: Legacy.Server['route']; From 6db76a7e6df874dbb904126652a472f3612532e6 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Fri, 6 Dec 2019 08:23:48 +0100 Subject: [PATCH 3/4] add codeowners for legacy server folder (#52158) --- .github/CODEOWNERS | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 610681e83798ee..c5e6768c17d464 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,6 +5,7 @@ # App /x-pack/legacy/plugins/lens/ @elastic/kibana-app /x-pack/legacy/plugins/graph/ @elastic/kibana-app +/src/legacy/server/sample_data/ @elastic/kibana-app # App Architecture /src/plugins/data/ @elastic/kibana-app-arch @@ -66,14 +67,25 @@ /packages/kbn-es/ @elastic/kibana-operations /packages/kbn-pm/ @elastic/kibana-operations /packages/kbn-test/ @elastic/kibana-operations +/src/legacy/server/keystore/ @elastic/kibana-operations +/src/legacy/server/pid/ @elastic/kibana-operations +/src/legacy/server/sass/ @elastic/kibana-operations +/src/legacy/server/utils/ @elastic/kibana-operations +/src/legacy/server/warnings/ @elastic/kibana-operations # Platform /src/core/ @elastic/kibana-platform -/src/legacy/server/saved_objects/ @elastic/kibana-platform /config/kibana.yml @elastic/kibana-platform /x-pack/plugins/features/ @elastic/kibana-platform /x-pack/plugins/licensing/ @elastic/kibana-platform /packages/kbn-config-schema/ @elastic/kibana-platform +/src/legacy/server/config/ @elastic/kibana-platform +/src/legacy/server/csp/ @elastic/kibana-platform +/src/legacy/server/http/ @elastic/kibana-platform +/src/legacy/server/i18n/ @elastic/kibana-platform +/src/legacy/server/logging/ @elastic/kibana-platform +/src/legacy/server/saved_objects/ @elastic/kibana-platform +/src/legacy/server/status/ @elastic/kibana-platform # Security /x-pack/legacy/plugins/security/ @elastic/kibana-security From ca554024966ecca9a461fe53a61042317816dee0 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Fri, 6 Dec 2019 10:28:29 +0100 Subject: [PATCH 4/4] make defaultRoute accessible in NP Config (#52308) * defaultRoute was not provided to the NP * improve defaultRoute validation * add test that defaultRoute is read from config * update tests --- src/core/server/http/http_config.ts | 10 +++- ...gacy_object_to_config_adapter.test.ts.snap | 2 + .../config/legacy_object_to_config_adapter.ts | 1 + .../default_route_provider_config.test.ts | 53 +++++++++++++++++++ .../saved_objects/saved_objects_mixin.js | 2 +- 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 src/legacy/server/http/integration_tests/default_route_provider_config.test.ts diff --git a/src/core/server/http/http_config.ts b/src/core/server/http/http_config.ts index 89676380610a94..cb7726de4da5a6 100644 --- a/src/core/server/http/http_config.ts +++ b/src/core/server/http/http_config.ts @@ -38,7 +38,15 @@ export const config = { validate: match(validBasePathRegex, "must start with a slash, don't end with one"), }) ), - defaultRoute: schema.maybe(schema.string()), + defaultRoute: schema.maybe( + schema.string({ + validate(value) { + if (!value.startsWith('/')) { + return 'must start with a slash'; + } + }, + }) + ), cors: schema.conditional( schema.contextRef('dev'), true, diff --git a/src/core/server/legacy/config/__snapshots__/legacy_object_to_config_adapter.test.ts.snap b/src/core/server/legacy/config/__snapshots__/legacy_object_to_config_adapter.test.ts.snap index 172feec6746773..d327860052eb97 100644 --- a/src/core/server/legacy/config/__snapshots__/legacy_object_to_config_adapter.test.ts.snap +++ b/src/core/server/legacy/config/__snapshots__/legacy_object_to_config_adapter.test.ts.snap @@ -8,6 +8,7 @@ Object { "enabled": true, }, "cors": false, + "defaultRoute": undefined, "host": "host", "keepaliveTimeout": 5000, "maxPayload": 1000, @@ -30,6 +31,7 @@ Object { "enabled": true, }, "cors": false, + "defaultRoute": undefined, "host": "host", "keepaliveTimeout": 5000, "maxPayload": 1000, diff --git a/src/core/server/legacy/config/legacy_object_to_config_adapter.ts b/src/core/server/legacy/config/legacy_object_to_config_adapter.ts index 6f0757dece165b..8035596bb6072b 100644 --- a/src/core/server/legacy/config/legacy_object_to_config_adapter.ts +++ b/src/core/server/legacy/config/legacy_object_to_config_adapter.ts @@ -62,6 +62,7 @@ export class LegacyObjectToConfigAdapter extends ObjectToConfigAdapter { return { autoListen: configValue.autoListen, basePath: configValue.basePath, + defaultRoute: configValue.defaultRoute, cors: configValue.cors, host: configValue.host, maxPayload: configValue.maxPayloadBytes, diff --git a/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts b/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts new file mode 100644 index 00000000000000..da785a59893ab6 --- /dev/null +++ b/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import * as kbnTestServer from '../../../../test_utils/kbn_server'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { Root } from '../../../../core/server/root'; + +describe('default route provider', () => { + let root: Root; + + afterEach(async () => await root.shutdown()); + + it('redirects to the configured default route', async function() { + root = kbnTestServer.createRoot({ + server: { + defaultRoute: '/app/some/default/route', + }, + }); + + await root.setup(); + await root.start(); + + const kbnServer = kbnTestServer.getKbnServer(root); + + kbnServer.server.decorate('request', 'getSavedObjectsClient', function() { + return { + get: (type: string, id: string) => ({ attributes: {} }), + }; + }); + + const { status, header } = await kbnTestServer.request.get(root, '/'); + + expect(status).toEqual(302); + expect(header).toMatchObject({ + location: '/app/some/default/route', + }); + }); +}); diff --git a/src/legacy/server/saved_objects/saved_objects_mixin.js b/src/legacy/server/saved_objects/saved_objects_mixin.js index 7324a02095c678..e0d8b895af8387 100644 --- a/src/legacy/server/saved_objects/saved_objects_mixin.js +++ b/src/legacy/server/saved_objects/saved_objects_mixin.js @@ -55,7 +55,7 @@ function getImportableAndExportableTypes({ kbnServer, visibleTypes }) { ); } -export async function savedObjectsMixin(kbnServer, server) { +export function savedObjectsMixin(kbnServer, server) { const migrator = kbnServer.newPlatform.__internals.kibanaMigrator; const mappings = migrator.getActiveMappings(); const allTypes = Object.keys(getRootPropertiesObjects(mappings));