Skip to content

Commit

Permalink
Merge branch 'main' into unskipping-detection-page-filters
Browse files Browse the repository at this point in the history
  • Loading branch information
MadameSheema committed Nov 6, 2023
2 parents db7574a + 820cfc0 commit e43300c
Show file tree
Hide file tree
Showing 2,678 changed files with 45,540 additions and 26,094 deletions.
8 changes: 6 additions & 2 deletions .buildkite/ftr_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ disabled:
- x-pack/test/fleet_api_integration/config.base.ts
- x-pack/test/security_solution_api_integration/config/ess/config.base.ts
- x-pack/test/security_solution_api_integration/config/serverless/config.base.ts

- x-pack/test/security_solution_endpoint/config.base.ts

# QA suites that are run out-of-band
- x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
Expand Down Expand Up @@ -387,7 +387,9 @@ enabled:
- x-pack/test/security_functional/expired_session.config.ts
- x-pack/test/security_solution_endpoint_api_int/config.ts
- x-pack/test/security_solution_endpoint/endpoint.config.ts
- x-pack/test/security_solution_endpoint/serverless.endpoint.config.ts
- x-pack/test/security_solution_endpoint/integrations.config.ts
- x-pack/test/security_solution_endpoint/serverless.integrations.config.ts
- x-pack/test/session_view/basic/config.ts
- x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
- x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
Expand Down Expand Up @@ -457,7 +459,9 @@ enabled:
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts

- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts




Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,12 @@ export async function pickTestGroupRunOrder() {
]
: []),
// if we are running on a external job, like kibana-code-coverage-main, try finding times that are specific to that job
...(!prNumber && pipelineSlug !== 'kibana-on-merge'
// kibana-elasticsearch-serverless-verify-and-promote is not necessarily run in commit order -
// using kibana-on-merge groups will provide a closer approximation, with a failure mode -
// of too many ftr groups instead of potential timeouts.
...(!prNumber &&
pipelineSlug !== 'kibana-on-merge' &&
pipelineSlug !== 'kibana-elasticsearch-serverless-verify-and-promote'
? [
{
branch: ownBranch,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 4
parallelism: 6
retry:
automatic:
- exit_status: '*'
Expand Down
11 changes: 11 additions & 0 deletions .buildkite/pipelines/pull_request/deploy_project.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
steps:
- command: .buildkite/scripts/steps/serverless/build_and_deploy.sh
label: 'Build and Deploy Project'
agents:
queue: n2-16-spot
timeout_in_minutes: 60
soft_fail: true
retry:
automatic:
- exit_status: '-1'
limit: 3
23 changes: 18 additions & 5 deletions .buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,21 @@ steps:

- wait: ~

- label: ":judge::seedling: Trigger Manual Tests Phase"
command: "make -C /agent trigger-manual-verification-phase"
if: build.branch == "main"
agents:
image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
- group: "Kibana Release Manager"
steps:
- label: ":judge::seedling: Trigger Manual Tests Phase - Kibana Release Manager"
command: "make -C /agent trigger-manual-verification-phase"
env:
USE_GROUP_LABEL: true
agents:
image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

- group: "Observability"
steps:
- label: ":judge::seedling: Trigger Manual Tests Phase - Observability"
command: "make -C /agent trigger-manual-verification-phase"
env:
NOTIFICATION_APPENDIX: "<!subteam^S060GJFKGGP> please execute your manual testing plan."
USE_GROUP_LABEL: true
agents:
image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
11 changes: 6 additions & 5 deletions .buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ steps:

- wait: ~

- label: ":judge::seedling: Trigger Manual Tests Phase"
command: "make -C /agent trigger-manual-verification-phase"
if: build.branch == "main"
agents:
image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
- group: "Kibana Release Manager"
steps:
- label: ":judge::seedling: Trigger Manual Tests Phase"
command: "make -C /agent trigger-manual-verification-phase"
agents:
image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
1 change: 1 addition & 0 deletions .buildkite/scripts/lifecycle/post_command.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
buildkite-agent artifact upload 'target/kibana-*'
buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
buildkite-agent artifact upload 'target/kibana-osquery/**/*.png'
buildkite-agent artifact upload 'target/kibana-osquery/**/*.mp4'
buildkite-agent artifact upload 'target/kibana-fleet/**/*.png'
buildkite-agent artifact upload 'target/test-metrics/*'
buildkite-agent artifact upload 'target/test-suites-ci-plan.json'
Expand Down
6 changes: 6 additions & 0 deletions .buildkite/scripts/lifecycle/pre_command.sh
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,12 @@ export KIBANA_DOCKER_PASSWORD
EC_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-cloud-deploy)"
export EC_API_KEY

PROJECT_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-project-deploy)"
export PROJECT_API_KEY

PROJECT_API_DOMAIN="$(retry 5 5 vault read -field=pr_deploy_domain secret/kibana-issues/dev/kibana-ci-project-deploy)"
export PROJECT_API_DOMAIN

SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
export SYNTHETICS_SERVICE_USERNAME

Expand Down
8 changes: 8 additions & 0 deletions .buildkite/scripts/pipelines/pull_request/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,14 @@ const uploadPipeline = (pipelineContent: string | object) => {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
}

if (
GITHUB_PR_LABELS.includes('ci:project-deploy-es') ||
GITHUB_PR_LABELS.includes('ci:project-deploy-oblt') ||
GITHUB_PR_LABELS.includes('ci:project-deploy-security')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_project.yml'));
}

if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
pipeline.push(getPipeline('.buildkite/pipelines/artifacts_container_image.yml'));
}
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/scripts/steps/artifacts/docker_image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ else
fi

KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-serverless"
KIBANA_IMAGE="$KIBANA_BASE_IMAGE:$KIBANA_IMAGE_TAG"
export KIBANA_IMAGE="$KIBANA_BASE_IMAGE:$KIBANA_IMAGE_TAG"

echo "--- Verify manifest does not already exist"
echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
Expand Down
1 change: 1 addition & 0 deletions .buildkite/scripts/steps/checks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,5 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
.buildkite/scripts/steps/checks/saved_objects_compat_changes.sh
.buildkite/scripts/steps/checks/saved_objects_definition_change.sh
.buildkite/scripts/steps/code_generation/security_solution_codegen.sh
.buildkite/scripts/steps/code_generation/osquery_codegen.sh
.buildkite/scripts/steps/checks/yarn_deduplicate.sh
10 changes: 10 additions & 0 deletions .buildkite/scripts/steps/code_generation/osquery_codegen.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/scripts/common/util.sh

echo --- Osquery OpenAPI Code Generation

(cd x-pack/plugins/osquery && yarn openapi:generate)
check_for_changed_files "yarn openapi:generate" true
96 changes: 96 additions & 0 deletions .buildkite/scripts/steps/serverless/build_and_deploy.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
#!/bin/bash

set -euo pipefail


source .buildkite/scripts/common/util.sh
source .buildkite/scripts/steps/artifacts/docker_image.sh

PROJECT_TYPE=""
is_pr_with_label "ci:project-deploy-es" && PROJECT_TYPE="elasticsearch"
is_pr_with_label "ci:project-deploy-oblt" && PROJECT_TYPE="observability"
is_pr_with_label "ci:project-deploy-security" && PROJECT_TYPE="security"
if [ -z "${PROJECT_TYPE}" ]; then
echo "Mising project type"
exit 10
fi

PROJECT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST-$PROJECT_TYPE"
PROJECT_CREATE_CONFIGURATION='{
"name": "'"$PROJECT_NAME"'",
"region_id": "aws-eu-west-1",
"overrides": {
"kibana": {
"docker_image": "'"$KIBANA_IMAGE"'"
}
}
}'
PROJECT_UPDATE_CONFIGURATION='{
"name": "'"$PROJECT_NAME"'",
"overrides": {
"kibana": {
"docker_image": "'"$KIBANA_IMAGE"'"
}
}
}'

echo "--- Create project"
DEPLOY_LOGS=$(mktemp --suffix ".json")

echo "Checking if project already exists..."
curl -s \
-H "Authorization: ApiKey $PROJECT_API_KEY" \
"${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}" \
-XGET &>> $DEPLOY_LOGS

PROJECT_ID=$(jq -r --slurp '[.[0].items[] | select(.name == "'$PROJECT_NAME'")] | .[0].id' $DEPLOY_LOGS)
if [ -z "${PROJECT_ID}" ] || [ "$PROJECT_ID" = 'null' ]; then
echo "Creating project..."
curl -s \
-H "Authorization: ApiKey $PROJECT_API_KEY" \
-H "Content-Type: application/json" \
"${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}" \
-XPOST -d "$PROJECT_CREATE_CONFIGURATION" &>> $DEPLOY_LOGS

PROJECT_ID=$(jq -r --slurp '.[1].id' $DEPLOY_LOGS)

echo "Get credentials..."
curl -s -XPOST -H "Authorization: ApiKey $PROJECT_API_KEY" \
"${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}/${PROJECT_ID}/_reset-credentials" &>> $DEPLOY_LOGS

PROJECT_USERNAME=$(jq -r --slurp '.[2].username' $DEPLOY_LOGS)
PROJECT_PASSWORD=$(jq -r --slurp '.[2].password' $DEPLOY_LOGS)

echo "Write to vault..."
VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
retry 5 30 vault login -no-print "$VAULT_TOKEN"
retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
else
echo "Updating project..."
curl -s \
-H "Authorization: ApiKey $PROJECT_API_KEY" \
-H "Content-Type: application/json" \
"${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}/${PROJECT_ID}" \
-XPUT -d "$PROJECT_UPDATE_CONFIGURATION" &>> $DEPLOY_LOGS
fi

PROJECT_KIBANA_URL=$(jq -r --slurp '.[1].endpoints.kibana' $DEPLOY_LOGS)
PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
PROJECT_ELASTICSEARCH_URL=$(jq -r --slurp '.[1].endpoints.elasticsearch' $DEPLOY_LOGS)

cat << EOF | buildkite-agent annotate --style "info" --context project
### Project Deployment
Kibana: $PROJECT_KIBANA_LOGIN_URL
Elasticsearch: $PROJECT_ELASTICSEARCH_URL
Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME\`
Kibana image: \`$KIBANA_IMAGE\`
EOF

buildkite-agent meta-data set pr_comment:deploy_project:head "* [Project Deployment](${PROJECT_KIBANA_LOGIN_URL})"
buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
8 changes: 2 additions & 6 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,7 @@ x-pack/test/cloud_integration/plugins/saml_provider @elastic/kibana-core
x-pack/plugins/cloud_integrations/cloud_links @elastic/kibana-core
x-pack/plugins/cloud @elastic/kibana-core
x-pack/plugins/cloud_security_posture @elastic/kibana-cloud-security-posture
packages/shared-ux/code_editor/impl @elastic/appex-sharedux
packages/shared-ux/code_editor/mocks @elastic/appex-sharedux
packages/shared-ux/code_editor/types @elastic/appex-sharedux
packages/shared-ux/code_editor @elastic/appex-sharedux
packages/kbn-coloring @elastic/kibana-visualizations
packages/kbn-config @elastic/kibana-core
packages/kbn-config-mocks @elastic/kibana-core
Expand Down Expand Up @@ -676,9 +674,7 @@ examples/share_examples @elastic/appex-sharedux
src/plugins/share @elastic/appex-sharedux
packages/kbn-shared-svg @elastic/apm-ui
packages/shared-ux/avatar/solution @elastic/appex-sharedux
packages/shared-ux/button/exit_full_screen/impl @elastic/appex-sharedux
packages/shared-ux/button/exit_full_screen/mocks @elastic/appex-sharedux
packages/shared-ux/button/exit_full_screen/types @elastic/appex-sharedux
packages/shared-ux/button/exit_full_screen @elastic/appex-sharedux
packages/shared-ux/button_toolbar @elastic/appex-sharedux
packages/shared-ux/card/no_data/impl @elastic/appex-sharedux
packages/shared-ux/card/no_data/mocks @elastic/appex-sharedux
Expand Down
2 changes: 1 addition & 1 deletion api_docs/actions.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/actions
title: "actions"
image: https://source.unsplash.com/400x175/?github
description: API docs for the actions plugin
date: 2023-10-30
date: 2023-11-06
tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
---
import actionsObj from './actions.devdocs.json';
Expand Down
2 changes: 1 addition & 1 deletion api_docs/advanced_settings.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/advancedSettings
title: "advancedSettings"
image: https://source.unsplash.com/400x175/?github
description: API docs for the advancedSettings plugin
date: 2023-10-30
date: 2023-11-06
tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
---
import advancedSettingsObj from './advanced_settings.devdocs.json';
Expand Down
2 changes: 1 addition & 1 deletion api_docs/aiops.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiops
title: "aiops"
image: https://source.unsplash.com/400x175/?github
description: API docs for the aiops plugin
date: 2023-10-30
date: 2023-11-06
tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops']
---
import aiopsObj from './aiops.devdocs.json';
Expand Down
16 changes: 8 additions & 8 deletions api_docs/alerting.devdocs.json
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@
"trackAdoption": false,
"references": [
{
"plugin": "stackAlerts",
"path": "x-pack/plugins/stack_alerts/public/rule_types/es_query/index.ts"
"plugin": "ml",
"path": "x-pack/plugins/ml/public/alerting/register_ml_alerts.ts"
},
{
"plugin": "stackAlerts",
Expand All @@ -125,8 +125,8 @@
"path": "x-pack/plugins/stack_alerts/public/rule_types/es_query/index.ts"
},
{
"plugin": "ml",
"path": "x-pack/plugins/ml/public/alerting/register_ml_alerts.ts"
"plugin": "stackAlerts",
"path": "x-pack/plugins/stack_alerts/public/rule_types/es_query/index.ts"
}
],
"children": [
Expand Down Expand Up @@ -2949,7 +2949,7 @@
"label": "snoozeSchedule",
"description": [],
"signature": [
"Readonly<{ id?: string | undefined; skipRecurrences?: string[] | undefined; } & { duration: number; rRule: Readonly<{ count?: number | undefined; interval?: number | undefined; freq?: 0 | 2 | 6 | 5 | 4 | 3 | 1 | undefined; until?: string | undefined; byweekday?: (string | number)[] | undefined; bymonth?: number[] | undefined; wkst?: \"MO\" | \"TU\" | \"WE\" | \"TH\" | \"FR\" | \"SA\" | \"SU\" | undefined; bysetpos?: number[] | undefined; } & { dtstart: string; tzid: string; bymonthday: number[]; byyearday: number[]; byweekno: number[]; byhour: number[]; byminute: number[]; bysecond: number[]; }>; }>[] | undefined"
"Readonly<{ id?: string | undefined; skipRecurrences?: string[] | undefined; } & { duration: number; rRule: Readonly<{ count?: number | undefined; interval?: number | undefined; freq?: 0 | 2 | 6 | 5 | 4 | 3 | 1 | undefined; until?: string | undefined; byweekday?: (string | number)[] | undefined; bymonthday?: number[] | undefined; bymonth?: number[] | undefined; wkst?: \"MO\" | \"TU\" | \"WE\" | \"TH\" | \"FR\" | \"SA\" | \"SU\" | undefined; bysetpos?: number[] | undefined; byyearday?: number[] | undefined; byweekno?: number[] | undefined; byhour?: number[] | undefined; byminute?: number[] | undefined; bysecond?: number[] | undefined; } & { dtstart: string; tzid: string; }>; }>[] | undefined"
],
"path": "x-pack/plugins/alerting/server/application/rule/types/rule.ts",
"deprecated": false,
Expand Down Expand Up @@ -4492,7 +4492,7 @@
"label": "AlertingRulesConfig",
"description": [],
"signature": [
"Pick<Readonly<{} & { minimumScheduleInterval: Readonly<{} & { value: string; enforce: boolean; }>; maxScheduledPerMinute: number; run: Readonly<{ timeout?: string | undefined; ruleTypeOverrides?: Readonly<{ timeout?: string | undefined; } & { id: string; }>[] | undefined; } & { actions: Readonly<{ connectorTypeOverrides?: Readonly<{ max?: number | undefined; } & { id: string; }>[] | undefined; } & { max: number; }>; alerts: Readonly<{} & { max: number; }>; }>; }>, \"minimumScheduleInterval\" | \"maxScheduledPerMinute\"> & { isUsingSecurity: boolean; }"
"Pick<Readonly<{ overwriteProducer?: \"siem\" | \"observability\" | \"stackAlerts\" | undefined; } & { minimumScheduleInterval: Readonly<{} & { value: string; enforce: boolean; }>; maxScheduledPerMinute: number; run: Readonly<{ timeout?: string | undefined; ruleTypeOverrides?: Readonly<{ timeout?: string | undefined; } & { id: string; }>[] | undefined; } & { actions: Readonly<{ connectorTypeOverrides?: Readonly<{ max?: number | undefined; } & { id: string; }>[] | undefined; } & { max: number; }>; alerts: Readonly<{} & { max: number; }>; }>; }>, \"minimumScheduleInterval\" | \"maxScheduledPerMinute\"> & { isUsingSecurity: boolean; }"
],
"path": "x-pack/plugins/alerting/server/config.ts",
"deprecated": false,
Expand Down Expand Up @@ -4950,7 +4950,7 @@
"<never>)[]; total: number; taskIdsFailedToBeEnabled: string[]; }>; bulkDisableRules: (options: Readonly<{ filter?: string | undefined; ids?: string[] | undefined; } & {}>) => Promise<",
"BulkDisableRulesResult",
"<Record<string, any>>>; updateApiKey: (options: { id: string; }) => Promise<void>; enable: (options: { id: string; }) => Promise<void>; disable: (options: { id: string; }) => Promise<void>; snooze: (options: ",
"SnoozeParams",
"SnoozeRuleOptions",
") => Promise<void>; unsnooze: (options: ",
"UnsnoozeParams",
") => Promise<void>; clearExpiredSnoozes: (options: { rule: Pick<",
Expand Down Expand Up @@ -10491,7 +10491,7 @@
"section": "def-common.SavedObjectsResolveResponse",
"text": "SavedObjectsResolveResponse"
},
"<unknown>, \"saved_object\">"
"<unknown>, \"saved_object\"> & { outcome: string; alias_target_id?: string | undefined; }"
],
"path": "x-pack/plugins/alerting/common/rule.ts",
"deprecated": false,
Expand Down
2 changes: 1 addition & 1 deletion api_docs/alerting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/alerting
title: "alerting"
image: https://source.unsplash.com/400x175/?github
description: API docs for the alerting plugin
date: 2023-10-30
date: 2023-11-06
tags: ['contributor', 'dev', 'apidocs', 'kibana', 'alerting']
---
import alertingObj from './alerting.devdocs.json';
Expand Down
Loading

0 comments on commit e43300c

Please sign in to comment.