Merge branch 'main' into goal-convert-to-lens-gauge

elastic · Apr 4, 2024 · 7ca637b · 7ca637b
2 parents f2aca31 + c12f02c
commit 7ca637b
Show file tree

Hide file tree

Showing 115 changed files with 5,448 additions and 365 deletions.
diff --git a/.buildkite/scripts/steps/esql_grammar_sync.sh b/.buildkite/scripts/steps/esql_grammar_sync.sh
@@ -121,7 +121,7 @@ main () {
   git push origin "$BRANCH_NAME"
 
   # Create a PR
-  gh pr create --draft --title "$PR_TITLE" --body "$PR_BODY" --base main --head "${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:ESQL' 
+  gh pr create --title "$PR_TITLE" --body "$PR_BODY" --base main --head "${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:ESQL' 
 }
 
 main
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -692,6 +692,7 @@ x-pack/plugins/search_connectors @elastic/enterprise-search-frontend
 packages/kbn-search-errors @elastic/kibana-data-discovery
 examples/search_examples @elastic/kibana-data-discovery
 packages/kbn-search-index-documents @elastic/enterprise-search-frontend
+x-pack/plugins/search_notebooks @elastic/enterprise-search-frontend
 x-pack/plugins/search_playground @elastic/enterprise-search-frontend
 packages/kbn-search-response-warnings @elastic/kibana-data-discovery
 x-pack/plugins/searchprofiler @elastic/kibana-management

diff --git a/.node-version b/.node-version
@@ -1 +1 @@
-20.11.1
+20.12.1
diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-20.11.1
+20.12.1
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
@@ -22,13 +22,13 @@ load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install
 # Setup the Node.js toolchain for the architectures we want to support
 node_repositories(
   node_repositories = {
-    "20.11.1-darwin_amd64": ("node-v20.11.1-darwin-x64.tar.gz", "node-v20.11.1-darwin-x64", "c52e7fb0709dbe63a4cbe08ac8af3479188692937a7bd8e776e0eedfa33bb848"),
-    "20.11.1-darwin_arm64": ("node-v20.11.1-darwin-arm64.tar.gz", "node-v20.11.1-darwin-arm64", "e0065c61f340e85106a99c4b54746c5cee09d59b08c5712f67f99e92aa44995d"),
-    "20.11.1-linux_arm64": ("node-v20.11.1-linux-arm64.tar.xz", "node-v20.11.1-linux-arm64", "36bac185164aa11940715425da1db2ec46e3354325bda9ee0b98b4a607aa0d8b"),
-    "20.11.1-linux_amd64": ("node-v20.11.1-linux-x64.tar.xz", "node-v20.11.1-linux-x64", "e2c39cb70b9ff79575a02747dd1e89917817cce05da21bef6b94eb9e92442024"),
-    "20.11.1-windows_amd64": ("node-v20.11.1-win-x64.zip", "node-v20.11.1-win-x64", "bc032628d77d206ffa7f133518a6225a9c5d6d9210ead30d67e294ff37044bda"),
+    "20.12.1-darwin_amd64": ("node-v20.12.1-darwin-x64.tar.gz", "node-v20.12.1-darwin-x64", "f5dc3c71c87c58c9b019d9f85302db3a6a6c47167c5a0480b697f153d02ac316"),
+    "20.12.1-darwin_arm64": ("node-v20.12.1-darwin-arm64.tar.gz", "node-v20.12.1-darwin-arm64", "65df8cb0724e3a58c7757b75a70cc1057e1f67ffc5e852bfe6241de0b37c70a0"),
+    "20.12.1-linux_arm64": ("node-v20.12.1-linux-arm64.tar.xz", "node-v20.12.1-linux-arm64", "903a0f94312ba819f16a133e9dc378db128804ff45ea69e11af7152c303e8a85"),
+    "20.12.1-linux_amd64": ("node-v20.12.1-linux-x64.tar.xz", "node-v20.12.1-linux-x64", "e43b54ecea97b4419a5526af57bcf4f22a3d5583e1b2cacf461da71b9ba2befe"),
+    "20.12.1-windows_amd64": ("node-v20.12.1-win-x64.zip", "node-v20.12.1-win-x64", "629e2619ef88c5a8ce9944201f00ca3124f079c43ceef7ab0826c6fd19e09d75"),
   },
-  node_version = "20.11.1",
+  node_version = "20.12.1",
   node_urls = [
     "https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v{version}/{filename}",
   ],

diff --git a/docs/developer/advanced/upgrading-nodejs.asciidoc b/docs/developer/advanced/upgrading-nodejs.asciidoc
@@ -19,7 +19,7 @@ These files must be updated when upgrading Node.js:
   - {kib-repo}blob/{branch}/WORKSPACE.bazel[`WORKSPACE.bazel`] - The version is specified in the `node_version` property.
     Besides this property, the list of files under `node_repositories` must be updated along with their respective SHA256 hashes.
     These can be found in the `SHASUMS256.txt` file inside the public `kibana-custom-node-artifacts` GCP bucket.
-    Example for Node.js v20.11.1: https://storage.googleapis.com/kibana-custom-node-artifacts/node-glibc-217/dist/v20.11.1/SHASUMS256.txt[kibana-custom-node-artifacts/node-glibc-217/dist/v20.11.1/SHASUMS256.txt]
+    Example for Node.js v20.12.1: https://storage.googleapis.com/kibana-custom-node-artifacts/node-glibc-217/dist/v20.12.1/SHASUMS256.txt[kibana-custom-node-artifacts/node-glibc-217/dist/v20.12.1/SHASUMS256.txt]
 
 See PR {kib-repo}pull/128123[#128123] for an example of how the Node.js version has been upgraded previously.
 
@@ -43,7 +43,7 @@ The only difference between the offical Node.js build and our custom build, is t
 ==== How to start a new build
 
 To generate a new custom Node.js build, https://buildkite.com/elastic/kibana-custom-node-dot-js-builds#new[start a new build] on our dedicated Buildkite pipeline (requires Elastic employee permissions).
-Give it a clear name (e.g. `Node 20.11.1`) and remember so set the custom `OVERRIDE_TARGET_VERSION` environment variable to the desired Node.js version - e.g. `OVERRIDE_TARGET_VERSION=20.11.1`.
+Give it a clear name (e.g. `Node 20.12.1`) and remember so set the custom `OVERRIDE_TARGET_VERSION` environment variable to the desired Node.js version - e.g. `OVERRIDE_TARGET_VERSION=20.12.1`.
 You find the "Environment Variables" field by expanding "Options >" in the "New Build" dialog.
 
 === Backporting

diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc
@@ -757,6 +757,10 @@ It uses Chromium and Puppeteer underneath to run the browser in headless mode.
 |This plugin contains common assets and endpoints for the use of connectors in Kibana. Primarily used by the enterprise_search and serverless_search plugins.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/search_notebooks/README.mdx[searchNotebooks]
+|This plugin contains endpoints and components for rendering search python notebooks in the persistent dev console.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/search_playground/README.md[searchPlayground]
 |The Search Playground is a tool for developers to experiment with their own data using LLMs.
 

diff --git a/package.json b/package.json
@@ -74,7 +74,7 @@
     "url": "https://github.com/elastic/kibana.git"
   },
   "engines": {
-    "node": "20.11.1",
+    "node": "20.12.1",
     "yarn": "^1.22.19"
   },
   "resolutions": {
@@ -696,6 +696,7 @@
     "@kbn/search-errors": "link:packages/kbn-search-errors",
     "@kbn/search-examples-plugin": "link:examples/search_examples",
     "@kbn/search-index-documents": "link:packages/kbn-search-index-documents",
+    "@kbn/search-notebooks": "link:x-pack/plugins/search_notebooks",
     "@kbn/search-playground": "link:x-pack/plugins/search_playground",
     "@kbn/search-response-warnings": "link:packages/kbn-search-response-warnings",
     "@kbn/searchprofiler-plugin": "link:x-pack/plugins/searchprofiler",

diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts
@@ -176,7 +176,7 @@ export const HASH_TO_VERSION_MAP = {
   'guided-onboarding-guide-state|a3db59c45a3fd2730816d4f53c35c7d9': '10.0.0',
   'guided-onboarding-plugin-state|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
   'index-pattern|83c02d842fe2a94d14dfa13f7dcd6e87': '10.0.0',
-  'infra-custom-dashboards|6eed22cbe14594bad8c076fa864930de': '10.0.0',
+  'infra-custom-dashboards|1eb3c9e1888b8daea8495769e8d3ba2d': '10.2.0',
   'infrastructure-monitoring-log-view|c50526fc6040c5355ed027d34d05b35c': '10.0.0',
   'infrastructure-ui-source|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
   'ingest_manager_settings|b91ffb075799c78ffd7dbd51a279c8c9': '10.1.0',

diff --git a/packages/kbn-apm-config-loader/src/config.test.ts b/packages/kbn-apm-config-loader/src/config.test.ts
@@ -7,9 +7,9 @@
  */
 import type { AgentConfigOptions, Labels } from 'elastic-apm-node';
 import {
-  gitRevExecMock,
-  mockedRootDir,
   packageMock,
+  mockedRootDir,
+  gitRevExecMock,
   readUuidFileMock,
   resetAllMocks,
 } from './config.test.mocks';
@@ -153,7 +153,6 @@ describe('ApmConfiguration', () => {
       delete process.env.ELASTIC_APM_API_KEY;
       delete process.env.ELASTIC_APM_KIBANA_FRONTEND_ACTIVE;
       delete process.env.ELASTIC_APM_SERVER_URL;
-      delete process.env.ELASTIC_APM_GLOBAL_LABELS;
       delete process.env.NODE_ENV;
     });
 
@@ -186,21 +185,6 @@ describe('ApmConfiguration', () => {
           })
         );
       });
-
-      it('ELASTIC_APM_GLOBAL_LABELS', () => {
-        process.env.ELASTIC_APM_GLOBAL_LABELS = 'test1=1,test2=2';
-        const config = new ApmConfiguration(mockedRootDir, {}, true);
-
-        expect(config.getConfig('serviceName')).toEqual(
-          expect.objectContaining({
-            globalLabels: {
-              git_rev: 'sha',
-              test1: '1',
-              test2: '2',
-            },
-          })
-        );
-      });
     });
 
     it('ELASTIC_APM_KIBANA_FRONTEND_ACTIVE', () => {

diff --git a/packages/kbn-apm-config-loader/src/config.ts b/packages/kbn-apm-config-loader/src/config.ts
@@ -7,7 +7,6 @@
  */
 
 import { join } from 'path';
-import deepmerge from 'deepmerge';
 import { merge, isEmpty } from 'lodash';
 import { execSync } from 'child_process';
 import { getDataPath } from '@kbn/utils';
@@ -310,10 +309,7 @@ export class ApmConfiguration {
     const { servicesOverrides, redactUsers, ...configFromKibanaConfig } =
       this.getConfigFromKibanaConfig();
     const configFromEnv = this.getConfigFromEnv(configFromKibanaConfig);
-    const config = [configFromKibanaConfig, configFromEnv].reduce<AgentConfigOptions>(
-      (acc, conf) => deepmerge(acc, conf),
-      {}
-    );
+    const config = merge({}, configFromKibanaConfig, configFromEnv);
 
     if (config.active === false && config.contextPropagationOnly !== false) {
       throw new Error(

diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json
@@ -462,8 +462,8 @@
   ],
   "infra-custom-dashboards": [
     "assetType",
-    "dashboardIdList",
-    "kuery"
+    "dashboardFilterAssetIdEnabled",
+    "dashboardSavedObjectId"
   ],
   "infrastructure-monitoring-log-view": [
     "name"

diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -1559,15 +1559,16 @@
     }
   },
   "infra-custom-dashboards": {
+    "dynamic": false,
     "properties": {
       "assetType": {
         "type": "keyword"
       },
-      "dashboardIdList": {
+      "dashboardSavedObjectId": {
         "type": "keyword"
       },
-      "kuery": {
-        "type": "text"
+      "dashboardFilterAssetIdEnabled": {
+        "type": "boolean"
       }
     }
   },

diff --git a/packages/kbn-es-types/src/search.test.ts b/packages/kbn-es-types/src/search.test.ts
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { AggregateOfMap } from './search';
+
+xdescribe('AggregateOfMap', () => {
+  test('aggregations should assume buckets are there if type is explicit', () => {
+    type MyAggregation = {} & {
+      group_by: {
+        terms: unknown;
+      };
+    };
+
+    const aggregation = {} as unknown as AggregateOfMap<MyAggregation, unknown>;
+    aggregation.group_by.buckets.length.toFixed(); // using a number-specific method
+  });
+
+  test('aggregations should not assume buckets are there if the aggregation may be undefined', () => {
+    type MyAggregation =
+      | undefined
+      | ({} & {
+          group_by: {
+            terms: unknown;
+          };
+        });
+
+    const aggregation = {} as unknown as AggregateOfMap<MyAggregation, unknown>;
+    aggregation?.group_by.buckets.length.toFixed(); // using a number-specific method
+    // @ts-expect-error "aggregation" may be undefined
+    aggregation.group_by.buckets.length.toFixed(); // using a number-specific method
+  });
+
+  test('aggregations should not assume buckets are there if the bucket name may be undefined', () => {
+    type MyAggregation =
+      | {} & {
+          group_by?: {
+            terms: unknown;
+          };
+        };
+
+    const aggregation = {} as unknown as AggregateOfMap<MyAggregation, unknown>;
+    aggregation.group_by?.buckets.length.toFixed(); // using a number-specific method
+    // @ts-expect-error "group_by" may be undefined
+    aggregation.group_by.buckets.length.toFixed(); // using a number-specific method
+  });
+});
diff --git a/packages/kbn-es-types/src/search.ts b/packages/kbn-es-types/src/search.ts
@@ -604,8 +604,7 @@ export type AggregateOf<
 
 export type AggregateOfMap<TAggregationMap extends AggregationMap | undefined, TDocument> = {
   [TAggregationName in keyof TAggregationMap]: Required<TAggregationMap>[TAggregationName] extends AggregationsAggregationContainer
-    ? // @ts-expect-error not sure how to fix this, anything I've tried causes errors upstream - Dario
-      AggregateOf<TAggregationMap[TAggregationName], TDocument>
+    ? AggregateOf<Required<TAggregationMap>[TAggregationName], TDocument>
     : never; // using never means we effectively ignore optional keys, using {} creates a union type of { ... } | {}
 };
 

diff --git a/...es/kbn-search-connectors/components/sync_jobs/__snapshots__/documents_panel.test.tsx.snap b/...es/kbn-search-connectors/components/sync_jobs/__snapshots__/documents_panel.test.tsx.snap
diff --git a/packages/kbn-search-connectors/components/sync_jobs/documents_panel.tsx b/packages/kbn-search-connectors/components/sync_jobs/documents_panel.tsx
@@ -17,7 +17,6 @@ import { FlyoutPanel } from './flyout_panel';
 interface SyncJobDocumentsPanelProps {
   added: number;
   removed: number;
-  total: number;
   volume: number;
 }
 
@@ -26,19 +25,13 @@ export const SyncJobDocumentsPanel: React.FC<SyncJobDocumentsPanelProps> = (sync
     {
       field: 'added',
       name: i18n.translate('searchConnectors.index.syncJobs.documents.added', {
-        defaultMessage: 'Added',
+        defaultMessage: 'Upserted',
       }),
     },
     {
       field: 'removed',
       name: i18n.translate('searchConnectors.index.syncJobs.documents.removed', {
-        defaultMessage: 'Removed',
-      }),
-    },
-    {
-      field: 'total',
-      name: i18n.translate('searchConnectors.index.syncJobs.documents.total', {
-        defaultMessage: 'Total',
+        defaultMessage: 'Deleted',
       }),
     },
     {

diff --git a/packages/kbn-search-connectors/components/sync_jobs/sync_job_flyout.tsx b/packages/kbn-search-connectors/components/sync_jobs/sync_job_flyout.tsx
@@ -76,7 +76,6 @@ export const SyncJobFlyout: React.FC<SyncJobFlyoutProps> = ({ onClose, syncJob }
           <EuiFlexItem>
             <SyncJobDocumentsPanel
               added={syncJob.indexed_document_count}
-              total={syncJob.total_document_count ?? 0}
               removed={syncJob.deleted_document_count}
               volume={syncJob.indexed_document_volume ?? 0}
             />

diff --git a/packages/kbn-search-connectors/components/sync_jobs/sync_jobs_table.tsx b/packages/kbn-search-connectors/components/sync_jobs/sync_jobs_table.tsx
@@ -63,7 +63,7 @@ export const SyncJobsTable: React.FC<SyncJobHistoryTableProps> = ({
           {
             field: 'indexed_document_count',
             name: i18n.translate('searchConnectors.searchIndices.addedDocs.columnTitle', {
-              defaultMessage: 'Docs added',
+              defaultMessage: 'Docs upserted',
             }),
             sortable: true,
             truncateText: true,

diff --git a/packages/shared-ux/modal/tabbed/src/context/index.tsx b/packages/shared-ux/modal/tabbed/src/context/index.tsx
@@ -144,7 +144,7 @@ export function ModalContextProvider<T extends Array<ITabDeclaration<Record<stri
   }, []);
 
   const [state, dispatch] = useReducer(
-    combineReducers(reducersMap!),
+    combineReducers(reducersMap),
     initialModalState.current,
     createInitialState
   );

diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -103,7 +103,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "guided-onboarding-guide-state": "d338972ed887ac480c09a1a7fbf582d6a3827c91",
         "guided-onboarding-plugin-state": "bc109e5ef46ca594fdc179eda15f3095ca0a37a4",
         "index-pattern": "997108a9ea1e8076e22231e1c95517cdb192b9c5",
-        "infra-custom-dashboards": "b92b6db1c1f8998af6e2951a17b76cf886c6bee5",
+        "infra-custom-dashboards": "1a5994f2e05bb8a1609825ddbf5012f77c5c67f3",
         "infrastructure-monitoring-log-view": "5f86709d3c27aed7a8379153b08ee5d3d90d77f5",
         "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
         "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",

diff --git a/src/plugins/console/public/application/containers/editor/monaco/use_set_initial_value.ts b/src/plugins/console/public/application/containers/editor/monaco/use_set_initial_value.ts
@@ -41,11 +41,32 @@ const readLoadFromParam = () => {
  *
  * @param params The {@link SetInitialValueParams} to use.
  */
-export const useSetInitialValue = (params: SetInitialValueParams) => {
+export const useSetInitialValue = async (params: SetInitialValueParams) => {
   const { initialTextValue, setValue, toasts } = params;
 
-  const loadBufferFromRemote = (url: string) => {
-    // TODO: Add support for fetching from HTTP
+  const loadBufferFromRemote = async (url: string) => {
+    if (/^https?:\/\//.test(url)) {
+      // Check if this is a valid URL
+      try {
+        new URL(url);
+      } catch (e) {
+        return;
+      }
+      // Parse the URL to avoid issues with spaces and other special characters.
+      const parsedURL = new URL(url);
+      if (parsedURL.origin === 'https://www.elastic.co') {
+        const resp = await fetch(parsedURL);
+        const data = await resp.text();
+        setValue(`${initialTextValue}\n\n${data}`);
+      } else {
+        toasts.addWarning(
+          i18n.translate('console.loadFromDataUnrecognizedUrlErrorMessage', {
+            defaultMessage:
+              'Only URLs with the Elastic domain (www.elastic.co) can be loaded in Console.',
+          })
+        );
+      }
+    }
 
     // If we have a data URI instead of HTTP, LZ-decode it. This enables
     // opening requests in Console from anywhere in Kibana.
@@ -67,20 +88,20 @@ export const useSetInitialValue = (params: SetInitialValueParams) => {
   };
 
   // Support for loading a console snippet from a remote source, like support docs.
-  const onHashChange = debounce(() => {
+  const onHashChange = debounce(async () => {
     const url = readLoadFromParam();
     if (!url) {
       return;
     }
-    loadBufferFromRemote(url);
+    await loadBufferFromRemote(url);
   }, 200);
 
   window.addEventListener('hashchange', onHashChange);
 
   const loadFromParam = readLoadFromParam();
 
   if (loadFromParam) {
-    loadBufferFromRemote(loadFromParam);
+    await loadBufferFromRemote(loadFromParam);
   } else {
     setValue(initialTextValue || DEFAULT_INPUT_VALUE);
   }