Enable data client with sample data server side (#4268) (#4340)

* Enable data client with sample data server side * Add dataSourceId into savedObject Signed-off-by: Kristen Tian <tyarong@amazon.com> * Functional list, install uninstall Signed-off-by: Kristen Tian <tyarong@amazon.com> * add change log Signed-off-by: Kristen Tian <tyarong@amazon.com> * address comments Signed-off-by: Kristen Tian <tyarong@amazon.com> * add ut Signed-off-by: Kristen Tian <tyarong@amazon.com> --------- Signed-off-by: Kristen Tian <tyarong@amazon.com> (cherry picked from commit b337bea) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> # Conflicts: # CHANGELOG.md Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Jun 22, 2023 · 4fbf897 · 4fbf897
1 parent b00edec
commit 4fbf897
Show file tree

Hide file tree

Showing 17 changed files with 647 additions and 106 deletions.
diff --git a/.lycheeexclude b/.lycheeexclude
@@ -121,3 +121,4 @@ https://yarnpkg.com/latest.msi
 https://forum.opensearch.org/
 https://facebook.github.io/jest/
 https://facebook.github.io/jest/docs/cli.html
+http://helpmenow.com/problem2
diff --git a/src/plugins/data_source/server/client/configure_client.ts b/src/plugins/data_source/server/client/configure_client.ts
@@ -74,7 +74,7 @@ export const configureClient = async (
       requireDecryption
     );
   } catch (error: any) {
-    logger.error(
+    logger.debug(
       `Failed to get data source client for dataSourceId: [${dataSourceId}]. ${error}: ${error.stack}`
     );
     // Re-throw as DataSourceError

diff --git a/src/plugins/data_source/server/legacy/configure_legacy_client.ts b/src/plugins/data_source/server/legacy/configure_legacy_client.ts
@@ -60,7 +60,7 @@ export const configureLegacyClient = async (
       dataSourceId
     );
   } catch (error: any) {
-    logger.error(
+    logger.debug(
       `Failed to get data source client for dataSourceId: [${dataSourceId}]. ${error}: ${error.stack}`
     );
     // Re-throw as DataSourceError

diff --git a/src/plugins/home/opensearch_dashboards.json b/src/plugins/home/opensearch_dashboards.json
@@ -4,7 +4,7 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["data", "urlForwarding"],
-  "optionalPlugins": ["usageCollection", "telemetry"],
+  "optionalPlugins": ["usageCollection", "telemetry", "dataSource"],
   "requiredBundles": [
     "opensearchDashboardsReact"
   ]

diff --git a/src/plugins/home/public/application/sample_data_client.js b/src/plugins/home/public/application/sample_data_client.js
@@ -36,12 +36,14 @@ function clearIndexPatternsCache() {
   getServices().indexPatternService.clearCache();
 }
 
-export async function listSampleDataSets() {
-  return await getServices().http.get(sampleDataUrl);
+export async function listSampleDataSets(dataSourceId) {
+  const query = buildQuery(dataSourceId);
+  return await getServices().http.get(sampleDataUrl, { query });
 }
 
-export async function installSampleDataSet(id, sampleDataDefaultIndex) {
-  await getServices().http.post(`${sampleDataUrl}/${id}`);
+export async function installSampleDataSet(id, sampleDataDefaultIndex, dataSourceId) {
+  const query = buildQuery(dataSourceId);
+  await getServices().http.post(`${sampleDataUrl}/${id}`, { query });
 
   if (getServices().uiSettings.isDefault('defaultIndex')) {
     getServices().uiSettings.set('defaultIndex', sampleDataDefaultIndex);
@@ -50,8 +52,9 @@ export async function installSampleDataSet(id, sampleDataDefaultIndex) {
   clearIndexPatternsCache();
 }
 
-export async function uninstallSampleDataSet(id, sampleDataDefaultIndex) {
-  await getServices().http.delete(`${sampleDataUrl}/${id}`);
+export async function uninstallSampleDataSet(id, sampleDataDefaultIndex, dataSourceId) {
+  const query = buildQuery(dataSourceId);
+  await getServices().http.delete(`${sampleDataUrl}/${id}`, { query });
 
   const uiSettings = getServices().uiSettings;
 
@@ -64,3 +67,13 @@ export async function uninstallSampleDataSet(id, sampleDataDefaultIndex) {
 
   clearIndexPatternsCache();
 }
+
+function buildQuery(dataSourceId) {
+  const query = {};
+
+  if (dataSourceId) {
+    query.data_source_id = dataSourceId;
+  }
+
+  return query;
+}
diff --git a/src/plugins/home/server/services/sample_data/data_sets/ecommerce/index.ts b/src/plugins/home/server/services/sample_data/data_sets/ecommerce/index.ts
@@ -33,6 +33,7 @@ import { i18n } from '@osd/i18n';
 import { getSavedObjects } from './saved_objects';
 import { fieldMappings } from './field_mappings';
 import { SampleDatasetSchema, AppLinkSchema } from '../../lib/sample_dataset_registry_types';
+import { getSavedObjectsWithDataSource, appendDataSourceId } from '../util';
 
 const ecommerceName = i18n.translate('home.sampleData.ecommerceSpecTitle', {
   defaultMessage: 'Sample eCommerce orders',
@@ -42,17 +43,21 @@ const ecommerceDescription = i18n.translate('home.sampleData.ecommerceSpecDescri
 });
 const initialAppLinks = [] as AppLinkSchema[];
 
+const DEFAULT_INDEX = 'ff959d40-b880-11e8-a6d9-e546fe2bba5f';
+const DASHBOARD_ID = '722b74f0-b882-11e8-a6d9-e546fe2bba5f';
+
 export const ecommerceSpecProvider = function (): SampleDatasetSchema {
   return {
     id: 'ecommerce',
     name: ecommerceName,
     description: ecommerceDescription,
     previewImagePath: '/plugins/home/assets/sample_data_resources/ecommerce/dashboard.png',
     darkPreviewImagePath: '/plugins/home/assets/sample_data_resources/ecommerce/dashboard_dark.png',
-    overviewDashboard: '722b74f0-b882-11e8-a6d9-e546fe2bba5f',
+    overviewDashboard: appendDataSourceId(DASHBOARD_ID),
     appLinks: initialAppLinks,
-    defaultIndex: 'ff959d40-b880-11e8-a6d9-e546fe2bba5f',
-    savedObjects: getSavedObjects(),
+    defaultIndex: appendDataSourceId(DEFAULT_INDEX),
+    savedObjects: (dataSourceId?: string, dataSourceTitle?: string) =>
+      getSavedObjectsWithDataSource(getSavedObjects(), dataSourceId, dataSourceTitle),
     dataIndices: [
       {
         id: 'ecommerce',

diff --git a/src/plugins/home/server/services/sample_data/data_sets/flights/index.ts b/src/plugins/home/server/services/sample_data/data_sets/flights/index.ts
@@ -33,6 +33,7 @@ import { i18n } from '@osd/i18n';
 import { getSavedObjects } from './saved_objects';
 import { fieldMappings } from './field_mappings';
 import { SampleDatasetSchema, AppLinkSchema } from '../../lib/sample_dataset_registry_types';
+import { getSavedObjectsWithDataSource, appendDataSourceId } from '../util';
 
 const flightsName = i18n.translate('home.sampleData.flightsSpecTitle', {
   defaultMessage: 'Sample flight data',
@@ -42,17 +43,21 @@ const flightsDescription = i18n.translate('home.sampleData.flightsSpecDescriptio
 });
 const initialAppLinks = [] as AppLinkSchema[];
 
+const DEFAULT_INDEX = 'd3d7af60-4c81-11e8-b3d7-01146121b73d';
+const DASHBOARD_ID = '7adfa750-4c81-11e8-b3d7-01146121b73d';
+
 export const flightsSpecProvider = function (): SampleDatasetSchema {
   return {
     id: 'flights',
     name: flightsName,
     description: flightsDescription,
     previewImagePath: '/plugins/home/assets/sample_data_resources/flights/dashboard.png',
     darkPreviewImagePath: '/plugins/home/assets/sample_data_resources/flights/dashboard_dark.png',
-    overviewDashboard: '7adfa750-4c81-11e8-b3d7-01146121b73d',
+    overviewDashboard: appendDataSourceId(DASHBOARD_ID),
     appLinks: initialAppLinks,
-    defaultIndex: 'd3d7af60-4c81-11e8-b3d7-01146121b73d',
-    savedObjects: getSavedObjects(),
+    defaultIndex: appendDataSourceId(DEFAULT_INDEX),
+    savedObjects: (dataSourceId?: string, dataSourceTitle?: string) =>
+      getSavedObjectsWithDataSource(getSavedObjects(), dataSourceId, dataSourceTitle),
     dataIndices: [
       {
         id: 'flights',

diff --git a/src/plugins/home/server/services/sample_data/data_sets/logs/index.ts b/src/plugins/home/server/services/sample_data/data_sets/logs/index.ts
@@ -33,6 +33,7 @@ import { i18n } from '@osd/i18n';
 import { getSavedObjects } from './saved_objects';
 import { fieldMappings } from './field_mappings';
 import { SampleDatasetSchema, AppLinkSchema } from '../../lib/sample_dataset_registry_types';
+import { appendDataSourceId, getSavedObjectsWithDataSource } from '../util';
 
 const logsName = i18n.translate('home.sampleData.logsSpecTitle', {
   defaultMessage: 'Sample web logs',
@@ -42,17 +43,21 @@ const logsDescription = i18n.translate('home.sampleData.logsSpecDescription', {
 });
 const initialAppLinks = [] as AppLinkSchema[];
 
+const DEFAULT_INDEX = '90943e30-9a47-11e8-b64d-95841ca0b247';
+const DASHBOARD_ID = 'edf84fe0-e1a0-11e7-b6d5-4dc382ef7f5b';
+
 export const logsSpecProvider = function (): SampleDatasetSchema {
   return {
     id: 'logs',
     name: logsName,
     description: logsDescription,
     previewImagePath: '/plugins/home/assets/sample_data_resources/logs/dashboard.png',
     darkPreviewImagePath: '/plugins/home/assets/sample_data_resources/logs/dashboard_dark.png',
-    overviewDashboard: 'edf84fe0-e1a0-11e7-b6d5-4dc382ef7f5b',
+    overviewDashboard: appendDataSourceId(DASHBOARD_ID),
     appLinks: initialAppLinks,
-    defaultIndex: '90943e30-9a47-11e8-b64d-95841ca0b247',
-    savedObjects: getSavedObjects(),
+    defaultIndex: appendDataSourceId(DEFAULT_INDEX),
+    savedObjects: (dataSourceId?: string, dataSourceTitle?: string) =>
+      getSavedObjectsWithDataSource(getSavedObjects(), dataSourceId, dataSourceTitle),
     dataIndices: [
       {
         id: 'logs',

diff --git a/src/plugins/home/server/services/sample_data/data_sets/util.ts b/src/plugins/home/server/services/sample_data/data_sets/util.ts
@@ -0,0 +1,84 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { SavedObject } from 'opensearch-dashboards/server';
+
+export const appendDataSourceId = (id: string) => {
+  return (dataSourceId?: string) => (dataSourceId ? `${dataSourceId}_` + id : id);
+};
+
+export const getSavedObjectsWithDataSource = (
+  saveObjectList: SavedObject[],
+  dataSourceId?: string,
+  dataSourceTitle?: string
+): SavedObject[] => {
+  if (dataSourceId) {
+    return saveObjectList.map((saveObject) => {
+      saveObject.id = `${dataSourceId}_` + saveObject.id;
+      // update reference
+      if (saveObject.type === 'dashboard') {
+        saveObject.references.map((reference) => {
+          if (reference.id) {
+            reference.id = `${dataSourceId}_` + reference.id;
+          }
+        });
+      }
+
+      // update reference
+      if (saveObject.type === 'visualization' || saveObject.type === 'search') {
+        const searchSourceString = saveObject.attributes?.kibanaSavedObjectMeta?.searchSourceJSON;
+        const visStateString = saveObject.attributes?.visState;
+
+        if (searchSourceString) {
+          const searchSource = JSON.parse(searchSourceString);
+          if (searchSource.index) {
+            searchSource.index = `${dataSourceId}_` + searchSource.index;
+            saveObject.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.stringify(
+              searchSource
+            );
+          }
+        }
+
+        if (visStateString) {
+          const visState = JSON.parse(visStateString);
+          const controlList = visState.params?.controls;
+          if (controlList) {
+            controlList.map((control) => {
+              if (control.indexPattern) {
+                control.indexPattern = `${dataSourceId}_` + control.indexPattern;
+              }
+            });
+          }
+          saveObject.attributes.visState = JSON.stringify(visState);
+        }
+      }
+
+      // update reference
+      if (saveObject.type === 'index-pattern') {
+        saveObject.references = [
+          {
+            id: `${dataSourceId}`,
+            type: 'data-source',
+            name: 'dataSource',
+          },
+        ];
+      }
+
+      if (dataSourceTitle) {
+        if (
+          saveObject.type === 'dashboard' ||
+          saveObject.type === 'visualization' ||
+          saveObject.type === 'search'
+        ) {
+          saveObject.attributes.title = saveObject.attributes.title + `_${dataSourceTitle}`;
+        }
+      }
+
+      return saveObject;
+    });
+  }
+
+  return saveObjectList;
+};
diff --git a/src/plugins/home/server/services/sample_data/lib/sample_dataset_registry_types.ts b/src/plugins/home/server/services/sample_data/lib/sample_dataset_registry_types.ts
@@ -87,15 +87,15 @@ export interface SampleDatasetSchema<T = unknown> {
   darkPreviewImagePath: string;
 
   // saved object id of main dashboard for sample data set
-  overviewDashboard: string;
+  overviewDashboard: (dataSourceId?: string) => string;
   appLinks: AppLinkSchema[];
 
   // saved object id of default index-pattern for sample data set
-  defaultIndex: string;
+  defaultIndex: (dataSourceId?: string) => string;
 
   // OpenSearch Dashboards saved objects (index patter, visualizations, dashboard, ...)
   // Should provide a nice demo of OpenSearch Dashboards's functionality with the sample data set
-  savedObjects: Array<SavedObject<T>>;
+  savedObjects: (dataSourceId?: string, dataSourceTitle?: string) => Array<SavedObject<T>>;
   dataIndices: DataIndexSchema[];
   status?: string | undefined;
   statusMsg?: unknown;