Merge pull request #5 from Arpit-Bandejiya/pit-poc-1

Pit poc 1

.github/workflows/build_and_test_workflow.yml

@@ -309,8 +309,78 @@ jobs:
           path: ./artifacts/target/${{ env.ARTIFACT_BUILD_NAME }}
           retention-days: 1
 
+  build-min-artifact-tests-windows:
+    runs-on: windows-latest
+    name: Build min release artifacts on Windows
+    defaults:
+      run:
+        working-directory: artifacts
+    strategy:
+      matrix:
+        include:
+          - name: Windows x64
+            ext: zip
+            suffix: windows-x64
+            script: build-platform --windows --skip-os-packages
+    steps:
+      - name: Configure git's autocrlf
+        run: |
+          git config --global core.autocrlf false
+        working-directory: .
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          path: artifacts
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: 'artifacts/.nvmrc'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Setup Yarn
+        run: |
+          npm uninstall -g yarn
+          npm i -g yarn@1.22.10
+          yarn config set network-timeout 1000000 -g
+
+      - name: Configure Yarn Cache
+        run: |
+          echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $env:GITHUB_ENV
+          echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%"
+
+      - name: Initialize Yarn Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.YARN_CACHE_LOCATION }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
+
+      - name: Get package version
+        run: |
+          echo "VERSION=$(yarn --silent pkg-version)" >> $env:GITHUB_ENV
+
+      - name: Get artifact build name
+        run: |
+          echo "ARTIFACT_BUILD_NAME=opensearch-dashboards-${{ env.VERSION }}-${{ matrix.suffix }}.${{ matrix.ext }}" >> $env:GITHUB_ENV
+
+      - name: Run bootstrap
+        run: yarn osd bootstrap
+
+      - name: Build `${{ matrix.name }}`
+        run: yarn ${{ matrix.script }} --release
+
+      - uses: actions/upload-artifact@v3
+        if: success()
+        with:
+          name: ${{ matrix.suffix }}-${{ env.VERSION }}
+          path: artifacts/target/${{ env.ARTIFACT_BUILD_NAME }}
+          retention-days: 1
+
   bwc-tests:
-    needs: [build-min-artifact-tests]
+    needs: [build-min-artifact-tests-linux]
     runs-on: ubuntu-latest
     container:
       image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2

CHANGELOG.md

@@ -20,7 +20,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Security] Bumps hapi/statehood to 7.0.4 ([#3411](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3411))
 - [CVE-2023-25166] Bump formula to 3.0.1 ([#3416](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3416))
 - [CVE-2023-25653] Bump node-jose to 2.2.0 ([#3445](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3445))
-- [CVE-2023-26486][cve-2023-26487] Bump vega from 5.22.1 to 5.23.0 ([#3533](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3533))
+- [CVE-2023-26486][CVE-2023-26487] Bump vega from 5.22.1 to 5.23.0 ([#3533](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3533))
 
 ### 📈 Features/Enhancements
 
@@ -63,6 +63,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Optimizer] Increase timeout waiting for the exiting of an optimizer worker ([#3193](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3193))
 - [Data] Update `createAggConfig` so that newly created configs can be added to beginning of `aggConfig` array ([#3160](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3160))
 - Add disablePrototypePoisoningProtection configuration to prevent JS client from erroring when cluster utilizes JS reserved words ([#2992](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2992))
+- [Point in TIme] Add management plugin shell and empty state ([#2813](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2813))
 - [Multiple DataSource] Add support for SigV4 authentication ([#3058](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3058))
 - Make build scripts find and use the latest version of Node.js that satisfies `engines.node` ([#3467](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3467))
 - [Multiple DataSource] Refactor test connection to support SigV4 auth type ([#3456](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3456))
@@ -71,6 +72,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Use mirrors to download Node.js binaries to escape sporadic 404 errors ([#3619](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3619))
 - [Multiple DataSource] Refactor dev tool console to use opensearch-js client to send requests ([#3544](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3544))
 - [Data] Add geo shape filter field ([#3605](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3605))
+- [Multiple DataSource] Allow create and distinguish index pattern with same name but from different datasources ([#3571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3571))
+- [Multiple DataSource] Integrate multiple datasource with dev tool console ([#3754](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3754))
 
 ### 🐛 Bug Fixes
 
@@ -143,6 +146,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Doc] Add docker dev set up instruction ([#3444](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3444))
 - [Doc] UI actions explorer ([#3614](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3614))
 - [Doc] Update SECURITY.md with instructions for nested dependencies and backporting ([#3497](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3497))
+- [Doc] Update DEVELOPER_GUIDE.md with added manual bootstrap timeout solution and max virtual memory error solution with docker ([#3764](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3764))
 
 ### 🛠 Maintenance
 
@@ -167,6 +171,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Vis Builder] Removed Hard Coded Strings and Used i18n to transalte([#2867](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2867))
 - [Console] Replace jQuery.ajax with core.http when calling OSD APIs in console ([#3080](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3080))
 - [I18n] Fix Listr type errors and error handlers ([#3629](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3629))
+- [Multiple DataSource] Present the authentication type choices in a drop-down ([#3693](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3693))
 
 ### 🔩 Tests
 
@@ -219,6 +224,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixes management app breadcrumb error ([#2344](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2344))
 - [BUG] Fix suggestion list cutoff issue ([#2607](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2607))
 - [TSVB] Fixes undefined serial diff aggregation documentation link ([#3503](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3503))
+- [Console] Fix dev tool console autocomplete not loading issue ([#3775](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3775))
 
 ### 🚞 Infrastructure
 

DEVELOPER_GUIDE.md

@@ -98,7 +98,19 @@ The `osd bootstrap` command will install the project's dependencies and build al
 $ yarn osd bootstrap
 ```
 
-Note: If you experience a network timeout while bootstrapping, you can update the timeout by configuring it in the [`.yarnrc`](https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/.yarnrc). For example:
+Note: If you experience a network timeout while bootstrapping:
+
+```
+| There appears to be trouble with your network connection. Retrying...
+```
+
+You can run command with —network-timeout flag:
+
+```
+$ yarn osd bootstrap —network-timeout 1000000
+```
+
+Or use the timeout by configuring it in the [`.yarnrc`](https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/.yarnrc). For example:
 
 ```
 network-timeout 1000000
@@ -146,6 +158,24 @@ Note - it may take a couple minutes to generate all the necessary bundles. If th
 [success][@osd/optimizer] 28 bundles compiled successfully after 145.9 sec, watching for changes
 ```
 
+Note: If you run a docker image, an error may occur:
+
+```
+Error: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
+```
+
+This error is because there is not enough memory so more memory must be allowed to be used:
+
+```
+$ sudo sysctl -w vm.max_map_count=262144
+```
+
+For windows:
+
+```
+$ wsl -d docker-desktop
+$ sysctl -w vm.max_map_count=262144
+```
 ### Next Steps
 
 Now that you have a development environment to play with, there are a number of different paths you may take next.
@@ -190,7 +220,6 @@ By default, the snapshot command will run [a minimal distribution of OpenSearch]
 If you would like to run OpenSearch with a particular plugin installed on the cluster snapshot, pass the `--P` flag after `yarn opensearch snapshot`. You can use the flag multiple times to install multiple plugins. The argument value can be a URL to the plugin's zip file, maven coordinates of the plugin, or a local zip file path (use `file://` followed by the absolute or relative path, in that case). For example:
 
 _(Linux, Windows, Darwin (MacOS) only - for others, you'll need to [run OpenSearch from a tarball](#alternative---run-opensearch-from-tarball) instead)_
-
 ```bash
 $ yarn opensearch snapshot --P https://repo1.maven.org/maven2/org/opensearch/plugin/opensearch-test-plugin/2.4.0.0/opensearch-test-plugin-2.4.0.0.zip
 ```

config/opensearch_dashboards.yml

@@ -231,9 +231,14 @@
 
 # Set the value of this setting to true to enable the experimental multiple data source
 # support feature. Use with caution.
-#data_source.enabled: false
+data_source.enabled: true
 # Set the value of these settings to customize crypto materials to encryption saved credentials
 # in data sources.
-#data_source.encryption.wrappingKeyName: 'changeme'
-#data_source.encryption.wrappingKeyNamespace: 'changeme'
-#data_source.encryption.wrappingKey: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+data_source.encryption.wrappingKeyName: 'changeme'
+data_source.encryption.wrappingKeyNamespace: 'changeme'
+data_source.encryption.wrappingKey: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+pit.enabled: true
+opensearch.hosts: ["http://localhost:9200"]
+opensearch.username: "admin" # Default username on the docker image
+opensearch.password: "admin" # Default password on the docker image
+opensearch.ssl.verificationMode: none

package.json

@@ -134,7 +134,7 @@
     "@hapi/podium": "^4.1.3",
     "@hapi/vision": "^6.1.0",
     "@hapi/wreck": "^17.1.0",
-    "@opensearch-project/opensearch": "^2.1.0",
+    "@opensearch-project/opensearch": "^2.2.0",
     "@osd/ace": "1.0.0",
     "@osd/analytics": "1.0.0",
     "@osd/apm-config-loader": "1.0.0",

packages/osd-opensearch/package.json

@@ -12,7 +12,7 @@
     "osd:watch": "../../scripts/use_node scripts/build --watch"
   },
   "dependencies": {
-    "@opensearch-project/opensearch": "^2.1.0",
+    "@opensearch-project/opensearch": "^2.2.0",
     "@osd/dev-utils": "1.0.0",
     "abort-controller": "^3.0.0",
     "chalk": "^4.1.0",

src/core/public/application/types.ts

@@ -503,6 +503,10 @@ export interface AppMountParameters<HistoryLocationState = unknown> {
    * ```
    */
   setHeaderActionMenu: (menuMount: MountPoint | undefined) => void;
+  /**
+   * Optional datasource id to pass while mounting app
+   */
+  dataSourceId?: string;
 }
 
 /**

src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker

@@ -161,6 +161,7 @@ opensearch_dashboards_vars=(
     data_source.audit.appender.layout.kind
     data_source.audit.appender.layout.highlight
     data_source.audit.appender.layout.pattern
+    pit.enabled
 )
 
 longopts=''

src/plugins/console/opensearch_dashboards.json

@@ -4,6 +4,11 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["devTools"],
-  "optionalPlugins": ["usageCollection", "home"],
-  "requiredBundles": ["opensearchUiShared", "opensearchDashboardsReact", "opensearchDashboardsUtils", "home"]
+  "optionalPlugins": ["usageCollection", "home", "dataSource"],
+  "requiredBundles": [
+    "opensearchUiShared",
+    "opensearchDashboardsReact",
+    "opensearchDashboardsUtils",
+    "home"
+  ]
 }

src/plugins/console/public/application/containers/editor/editor.tsx

@@ -28,36 +28,50 @@
  * under the License.
  */
 
-import React, { useCallback, memo } from 'react';
+import React, { useCallback, memo, useEffect } from 'react';
 import { debounce } from 'lodash';
 import { EuiProgress } from '@elastic/eui';
 
 import { EditorContentSpinner } from '../../components';
 import { Panel, PanelsContainer } from '../../../../../opensearch_dashboards_react/public';
 import { Editor as EditorUI, EditorOutput } from './legacy/console_editor';
 import { StorageKeys } from '../../../services';
-import { useEditorReadContext, useServicesContext, useRequestReadContext } from '../../contexts';
+import {
+  useEditorReadContext,
+  useServicesContext,
+  useRequestReadContext,
+  useRequestActionContext,
+} from '../../contexts';
 
 const INITIAL_PANEL_WIDTH = 50;
 const PANEL_MIN_WIDTH = '100px';
 
 interface Props {
   loading: boolean;
+  dataSourceId?: string;
 }
 
-export const Editor = memo(({ loading }: Props) => {
+export const Editor = memo(({ loading, dataSourceId }: Props) => {
   const {
     services: { storage },
   } = useServicesContext();
 
   const { currentTextObject } = useEditorReadContext();
   const { requestInFlight } = useRequestReadContext();
+  const dispatch = useRequestActionContext();
 
   const [firstPanelWidth, secondPanelWidth] = storage.get(StorageKeys.WIDTH, [
     INITIAL_PANEL_WIDTH,
     INITIAL_PANEL_WIDTH,
   ]);
 
+  useEffect(() => {
+    dispatch({
+      type: 'resetLastResult',
+      payload: undefined,
+    });
+  }, [dispatch, dataSourceId]);
+
   /* eslint-disable-next-line react-hooks/exhaustive-deps */
   const onPanelWidthChange = useCallback(
     debounce((widths: number[]) => {
@@ -83,7 +97,7 @@ export const Editor = memo(({ loading }: Props) => {
           {loading ? (
             <EditorContentSpinner />
           ) : (
-            <EditorUI initialTextValue={currentTextObject.text} />
+            <EditorUI initialTextValue={currentTextObject.text} dataSourceId={dataSourceId} />
           )}
         </Panel>
         <Panel

src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.tsx

@@ -53,6 +53,7 @@ const { useUIAceKeyboardMode } = ace;
 
 export interface EditorProps {
   initialTextValue: string;
+  dataSourceId?: string;
 }
 
 interface QueryParams {
@@ -76,15 +77,15 @@ const DEFAULT_INPUT_VALUE = `GET _search
 
 const inputId = 'ConAppInputTextarea';
 
-function EditorUI({ initialTextValue }: EditorProps) {
+function EditorUI({ initialTextValue, dataSourceId }: EditorProps) {
   const {
-    services: { history, notifications, settings: settingsService, opensearchHostService },
+    services: { history, notifications, settings: settingsService, opensearchHostService, http },
     docLinkVersion,
   } = useServicesContext();
 
   const { settings } = useEditorReadContext();
   const setInputEditor = useSetInputEditor();
-  const sendCurrentRequestToOpenSearch = useSendCurrentRequestToOpenSearch();
+  const sendCurrentRequestToOpenSearch = useSendCurrentRequestToOpenSearch(dataSourceId);
   const saveCurrentTextObject = useSaveCurrentTextObject();
 
   const editorRef = useRef<HTMLDivElement | null>(null);
@@ -184,7 +185,12 @@ function EditorUI({ initialTextValue }: EditorProps) {
     setInputEditor(editor);
     setTextArea(editorRef.current!.querySelector('textarea'));
 
-    retrieveAutoCompleteInfo(settingsService, settingsService.getAutocomplete());
+    retrieveAutoCompleteInfo(
+      http,
+      settingsService,
+      settingsService.getAutocomplete(),
+      dataSourceId
+    );
 
     const unsubscribeResizer = subscribeResizeChecker(editorRef.current!, editor);
     setupAutosave();
@@ -197,7 +203,15 @@ function EditorUI({ initialTextValue }: EditorProps) {
         editorInstanceRef.current.getCoreEditor().destroy();
       }
     };
-  }, [saveCurrentTextObject, initialTextValue, history, setInputEditor, settingsService]);
+  }, [
+    saveCurrentTextObject,
+    initialTextValue,
+    history,
+    setInputEditor,
+    settingsService,
+    http,
+    dataSourceId,
+  ]);
 
   useEffect(() => {
     const { current: editor } = editorInstanceRef;