fix: logs page histogram optimisation

.github/workflows/playwright.yml

@@ -33,7 +33,7 @@ jobs:
       matrix:
         browser: [chrome]
         testfilename: 
-         ["logsquickmode.spec.js","logspage.spec.js", "sanity.spec.js","alerts.spec.js", "logsqueries.spec.js", "pipeline.spec.js","region.spec.js", "schema.spec.js","dashboardtype.spec.js"]
+         ["logsquickmode.spec.js","logspage.spec.js", "sanity.spec.js","alerts.spec.js", "logsqueries.spec.js", "pipeline.spec.js","schema.spec.js","dashboardtype.spec.js"]
 
     steps:
       - name: Remove unused tools

tests/ui-testing/playwright-tests/dashboardtype.spec.js

@@ -52,6 +52,7 @@ test.describe("dashboard testcases", () => {
         await page.locator("[data-test='logs-search-bar-refresh-btn']").click({
             force: true,
         });
+        await page.waitForTimeout(3000);
         // get the data from the search variable
         await expect.poll(async () => (await search).status()).toBe(200);
         // await search.hits.FIXME_should("be.an", "array");

web/src/composables/useLogs.ts

@@ -520,7 +520,7 @@ const useLogs = () => {
           histogram:
             "select histogram(" +
             store.state.zoConfig.timestamp_column +
-            ", '[INTERVAL]') AS zo_sql_key, count(*) AS zo_sql_num from query GROUP BY zo_sql_key ORDER BY zo_sql_key",
+            ", '[INTERVAL]') AS zo_sql_key, count(*) AS zo_sql_num from \"[INDEX_NAME]\" [WHERE_CLAUSE] GROUP BY zo_sql_key ORDER BY zo_sql_key",
         },
       };
 
@@ -723,8 +723,14 @@ const useLogs = () => {
             "[WHERE_CLAUSE]",
             " WHERE " + whereClause
           );
+
+          req.aggs.histogram = req.aggs.histogram.replace(
+            "[WHERE_CLAUSE]",
+            " WHERE " + whereClause
+          );
         } else {
           req.query.sql = req.query.sql.replace("[WHERE_CLAUSE]", "");
+          req.aggs.histogram = req.aggs.histogram.replace("[WHERE_CLAUSE]", "");
         }
 
         req.query.sql = req.query.sql.replace(
@@ -736,6 +742,11 @@ const useLogs = () => {
           "[INDEX_NAME]",
           searchObj.data.stream.selectedStream.value
         );
+
+        req.aggs.histogram = req.aggs.histogram.replace(
+          "[INDEX_NAME]",
+          searchObj.data.stream.selectedStream.value
+        );
         // const parsedSQL = parser.astify(req.query.sql);
         // const unparsedSQL = parser.sqlify(parsedSQL);
         // console.log(unparsedSQL);
@@ -838,11 +849,11 @@ const useLogs = () => {
         config.isEnterprise == "true" &&
         store.state.zoConfig.super_cluster_enabled
       ) {
-        if(queryReq.query.hasOwnProperty("regions")) {
-        partitionQueryReq["regions"] = queryReq.query.regions;
+        if (queryReq.query.hasOwnProperty("regions")) {
+          partitionQueryReq["regions"] = queryReq.query.regions;
         }
 
-        if(queryReq.query.hasOwnProperty("clusters")) {
+        if (queryReq.query.hasOwnProperty("clusters")) {
           partitionQueryReq["clusters"] = queryReq.query.clusters;
         }
       }
@@ -963,12 +974,38 @@ const useLogs = () => {
       let recordSize = 0;
       let from = 0;
       let lastPage = 0;
-      searchObj.data.queryResults.total = partitionDetail.partitionTotal.reduce(
-        (accumulator: number, currentValue: number) =>
-          accumulator + Math.max(currentValue, 0),
-        0
-      );
 
+      const parsedSQL: any = fnParsedSQL();
+
+      if (
+        (searchObj.data.queryResults.aggs == undefined &&
+          searchObj.data.resultGrid.currentPage == 1 &&
+          searchObj.loadingHistogram == false &&
+          searchObj.meta.showHistogram == true &&
+          (!searchObj.meta.sqlMode ||
+            (searchObj.meta.sqlMode && isNonAggregatedQuery(parsedSQL)))) ||
+        (searchObj.loadingHistogram == false &&
+          searchObj.meta.showHistogram == true &&
+          searchObj.meta.sqlMode == false &&
+          searchObj.data.resultGrid.currentPage == 1)
+      ) {
+        console.log(searchObj.data.queryResults)
+        if(searchObj.data.queryResults.hasOwnProperty("aggs") && searchObj.data.queryResults.aggs != null) {
+        searchObj.data.queryResults.total = searchObj.data.queryResults.aggs.reduce(
+            (accumulator: number, currentValue: any) =>
+              accumulator + Math.max(parseInt(currentValue.zo_sql_num, 10), 0),
+            0
+          );
+          partitionDetail.partitionTotal[0] = searchObj.data.queryResults.total;
+        }
+      } else {
+        searchObj.data.queryResults.total =
+          partitionDetail.partitionTotal.reduce(
+            (accumulator: number, currentValue: number) =>
+              accumulator + Math.max(currentValue, 0),
+            0
+          );
+      }
       // partitionDetail.partitions.forEach((item: any, index: number) => {
       for (const [index, item] of partitionDetail.partitions.entries()) {
         total = partitionDetail.partitionTotal[index];
@@ -1169,9 +1206,20 @@ const useLogs = () => {
         // copy query request for histogram query and same for customDownload
         searchObj.data.histogramQuery = JSON.parse(JSON.stringify(queryReq));
 
-        // Removing sql_mode from histogram query, as it is not required
-        delete searchObj.data.histogramQuery.query.sql_mode;
+        searchObj.data.histogramQuery.query.sql =
+          searchObj.data.histogramQuery.aggs.histogram;
+        searchObj.data.histogramQuery.query.sql_mode = "full";
+
+        searchObj.data.histogramQuery.query.start_time =
+          searchObj.data.datetime.startTime;
+        searchObj.data.histogramQuery.query.end_time =
+          searchObj.data.datetime.endTime;
+        delete searchObj.data.histogramQuery.query.quick_mode;
+        delete searchObj.data.histogramQuery.query.from;
 
+        // Removing sql_mode from histogram query, as it is not required
+        //delete searchObj.data.histogramQuery.query.sql_mode;
+        delete searchObj.data.histogramQuery.aggs;
         delete queryReq.aggs;
         searchObj.data.customDownloadQueryObj = queryReq;
         // get the current page detail and set it into query request
@@ -1245,7 +1293,8 @@ const useLogs = () => {
             searchObj.meta.sqlMode == false &&
             searchObj.data.resultGrid.currentPage == 1)
         ) {
-          getHistogramQueryData(searchObj.data.histogramQuery);
+          await getHistogramQueryData(searchObj.data.histogramQuery);
+          refreshPartitionPagination(true);
         } else if (searchObj.meta.sqlMode && !isNonAggregatedQuery(parsedSQL)) {
           searchObj.data.histogram = {
             xData: [],
@@ -1341,6 +1390,10 @@ const useLogs = () => {
   const getPageCount = async (queryReq: any) => {
     return new Promise((resolve, reject) => {
       queryReq.query.size = 0;
+      delete queryReq.query.from;
+      delete queryReq.query.quick_mode;
+      queryReq.query["sql_mode"] = "full";
+
       queryReq.query.track_total_hits = true;
       searchService
         .search({
@@ -1650,40 +1703,21 @@ const useLogs = () => {
         searchObj.data.histogram.errorDetail = "";
         searchObj.loadingHistogram = true;
         queryReq.query.size = 0;
-        queryReq.query.track_total_hits = true;
+        //queryReq.query.track_total_hits = true;
         searchService
           .search({
             org_identifier: searchObj.organizationIdetifier,
             query: queryReq,
             page_type: searchObj.data.stream.streamType,
           })
-          .then((res) => {
+          .then(async (res) => {
             searchObjDebug["histogramProcessingStartTime"] = performance.now();
             searchObj.loading = false;
-            searchObj.data.queryResults.aggs = res.data.aggs;
-            searchObj.data.queryResults.total = res.data.total;
+            searchObj.data.queryResults.aggs = res.data.hits;
             searchObj.data.queryResults.scan_size = res.data.scan_size;
-            searchObj.data.queryResults.took = res.data.took;
-            generateHistogramData();
+            searchObj.data.queryResults.took += res.data.took;
+            await generateHistogramData();
 
-            for (const [
-              index,
-              item,
-            ] of searchObj.data.queryResults.partitionDetail.partitions.entries()) {
-              if (
-                (searchObj.data.queryResults.partitionDetail.partitionTotal[
-                  index
-                ] == -1 ||
-                  searchObj.data.queryResults.partitionDetail.partitionTotal[
-                    index
-                  ] < res.data.total) &&
-                queryReq.query.start_time == item[0]
-              ) {
-                searchObj.data.queryResults.partitionDetail.partitionTotal[
-                  index
-                ] = res.data.total;
-              }
-            }
             let regeratePaginationFlag = false;
             if (res.data.hits.length != searchObj.meta.resultGrid.rowsPerPage) {
               regeratePaginationFlag = true;
@@ -1692,7 +1726,7 @@ const useLogs = () => {
             // setting up forceFlag to true to update pagination as we have check for pagination already created more than currentPage + 3 pages.
             refreshPartitionPagination(regeratePaginationFlag);
 
-            searchObj.data.histogram.chartParams.title = getHistogramTitle();
+            //searchObj.data.histogram.chartParams.title = getHistogramTitle();
             searchObj.loadingHistogram = false;
 
             searchObjDebug["histogramProcessingEndTime"] = performance.now();
@@ -1787,13 +1821,17 @@ const useLogs = () => {
       const schemaFields: any = [];
       if (searchObj.data.streamResults.list.length > 0) {
         const timestampField = store.state.zoConfig.timestamp_column;
+        const allField = store.state.zoConfig?.all_fields_name;
         const schemaInterestingFields: string[] = [];
+        let userDefineSchemaSettings: any = [];
         const schemaMaps: any = [];
         let fieldObj: any = {};
         const localInterestingFields: any = useLocalInterestingFields();
 
         for (const stream of searchObj.data.streamResults.list) {
           if (searchObj.data.stream.selectedStream.value == stream.name) {
+            userDefineSchemaSettings =
+              stream.settings?.defined_schema_fields?.slice() || [];
             // check for schema exist in the object or not
             // if not pull the schema from server.
             if (!stream.hasOwnProperty("schema")) {
@@ -1819,9 +1857,20 @@ const useLogs = () => {
 
             if (
               stream.settings.hasOwnProperty("defined_schema_fields") &&
-              stream.settings.defined_schema_fields.length > 0
+              userDefineSchemaSettings.length > 0
             ) {
               searchObj.meta.hasUserDefinedSchemas = true;
+              if (store.state.zoConfig.hasOwnProperty("timestamp_column")) {
+                userDefineSchemaSettings.push(
+                  store.state.zoConfig?.timestamp_column
+                );
+              }
+
+              if (store.state.zoConfig.hasOwnProperty("all_fields_name")) {
+                userDefineSchemaSettings.push(
+                  store.state.zoConfig?.all_fields_name
+                );
+              }
             } else {
               searchObj.meta.hasUserDefinedSchemas = false;
             }
@@ -1857,7 +1906,8 @@ const useLogs = () => {
                     ? true
                     : false,
                 isSchemaField: true,
-                showValues: field.name !== timestampField,
+                showValues:
+                  field.name !== timestampField && field.name !== allField,
                 isInterestingField:
                   searchObj.data.stream.interestingFieldList.includes(
                     field.name
@@ -1869,19 +1919,14 @@ const useLogs = () => {
                 store.state.zoConfig.user_defined_schemas_enabled &&
                 searchObj.meta.useUserDefinedSchemas == "user_defined_schema" &&
                 stream.settings.hasOwnProperty("defined_schema_fields") &&
-                stream.settings.defined_schema_fields.length > 0
+                userDefineSchemaSettings.length > 0
               ) {
-                if (
-                  stream.settings.defined_schema_fields.includes(field.name)
-                ) {
+                if (userDefineSchemaSettings.includes(field.name)) {
                   schemaMaps.push(fieldObj);
                   schemaFields.push(field.name);
                 }
 
-                if (
-                  schemaMaps.length ==
-                  stream.settings.defined_schema_fields.length
-                ) {
+                if (schemaMaps.length == userDefineSchemaSettings.length) {
                   break;
                 }
               } else {
@@ -1962,7 +2007,7 @@ const useLogs = () => {
             ] = searchObj.data.stream.interestingFieldList;
             useLocalInterestingFields(localFields);
             searchObj.data.stream.userDefinedSchema =
-              stream.settings.defined_schema_fields || [];
+              userDefineSchemaSettings || [];
           }
         }
 
@@ -2104,6 +2149,7 @@ const useLogs = () => {
 
   function generateHistogramData() {
     try {
+      let num_records: number = 0;
       const unparsed_x_data: any[] = [];
       const xData: number[] = [];
       const yData: number[] = [];
@@ -2112,11 +2158,12 @@ const useLogs = () => {
         searchObj.data.queryResults.hasOwnProperty("aggs") &&
         searchObj.data.queryResults.aggs
       ) {
-        searchObj.data.queryResults.aggs.histogram.map(
+        searchObj.data.queryResults.aggs.map(
           (bucket: {
             zo_sql_key: string | number | Date;
             zo_sql_num: string;
           }) => {
+            num_records = num_records + parseInt(bucket.zo_sql_num, 10);
             unparsed_x_data.push(bucket.zo_sql_key);
             // const histDate = new Date(bucket.zo_sql_key);
             xData.push(
@@ -2127,6 +2174,7 @@ const useLogs = () => {
           }
         );
       }
+      searchObj.data.queryResults.total = num_records;
 
       const chartParams = {
         title: getHistogramTitle(),