Improves pagination performance over large scale data set (#795)

* Revert accidental change

* Do not get the total size for pagination requests

server/storage-jdbc/src/main/java/org/bithon/server/storage/jdbc/metric/MetricJdbcReader.java

@@ -150,11 +150,7 @@ private String getOrderBySQL(OrderBy orderBy, String timestampColumn) {
             return "";
         }
 
-        if (orderBy.getName().equals(timestampColumn)) {
-            return StringUtils.format("ORDER BY toStartOfMinute(%s) %s, \"%s\" %s", timestampColumn, orderBy.getOrder(), orderBy.getName(), orderBy.getOrder());
-        } else {
-            return "ORDER BY \"" + orderBy.getName() + "\" " + orderBy.getOrder();
-        }
+        return "ORDER BY \"" + orderBy.getName() + "\" " + orderBy.getOrder();
     }
 
     @Override

server/web-app/src/main/resources/static/js/component/table.js

@@ -45,6 +45,7 @@ class TableComponent {
         this.mColumns = option.columns;
         this.mCreated = false;
         this.mPopoverShown = false;
+        this.mTotal = 0;
 
         this.mServerSort = option.serverSort;
         this.mHasPagination = option.pagination !== undefined && option.pagination.length > 0;
@@ -314,7 +315,21 @@ class TableComponent {
                 method: 'post',
                 contentType: "application/json",
                 showRefresh: false,
-                responseHandler: option.responseHandler,
+                responseHandler: (data, jqXHR) => {
+                    const ret = option.responseHandler(data);
+
+                    if (jqXHR.status == 200 && data.limit !== undefined && data.limit !== null) {
+                        if (data.limit.offset == 0) {
+                            this.mTotal = ret.total;
+                        } else {
+                            // For pagination request, the 'total' is not returned from the server,
+                            // But bootstrap-table requires the 'total' field tor refresh the pagination
+                            // So we use the cached 'total'
+                            ret.total = this.mTotal;
+                        }
+                    }
+                    return ret;
+                },
 
                 sidePagination: this.mPaginationSide,
                 pagination: this.mHasPagination,
@@ -337,6 +352,8 @@ class TableComponent {
                 stickyHeaderOffsetRight: stickyHeaderOffsetRight,
                 theadClasses: 'thead-light',
 
+
+
                 onLoadError: (status, jqXHR) => {
                     let message = jqXHR.responseText;
                     if (jqXHR.responseJSON !== undefined && jqXHR.responseJSON.message !== undefined) {

server/web-app/src/main/resources/static/js/component/trace-list-component.js

@@ -12,6 +12,8 @@ class TraceListComponent {
         const parent = options.parent;
         this.vTable = $(parent).append('<table></table>').find('table');
 
+        this.mTotal = 0;
+
         const excludeColumns = {};
         $.each(options.excludeColumns, (index, column) => {
             excludeColumns[column] = true;
@@ -118,6 +120,20 @@ class TraceListComponent {
                 return {};
             },
 
+            responseHandler: (data, jqXHR) => {
+                if (jqXHR.status == 200) {
+                    if (data.pageNumber == 0) {
+                        this.mTotal = data.total;
+                    } else {
+                        // For pagination request, the 'total' is not returned from the server,
+                        // But bootstrap-table requires the 'total' field tor refresh the pagination
+                        // So we use the cached 'total'
+                        data.total = this.mTotal;
+                    }
+                }
+                return data;
+            },
+
             onLoadError: (status, jqXHR) => {
                 let message;
                 if (jqXHR.responseJSON != null) {
@@ -136,6 +152,6 @@ class TraceListComponent {
     }
 
     refresh() {
-        this.vTable.bootstrapTable('refresh');
+        this.vTable.bootstrapTable('refresh', {pageNumber: 1});
     }
 }

server/web-service/src/main/java/org/bithon/server/web/service/datasource/api/GeneralQueryResponse.java

@@ -20,6 +20,7 @@
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.bithon.server.storage.datasource.query.Limit;
 
 import java.util.Collection;
 
@@ -34,10 +35,16 @@
 public class GeneralQueryResponse {
 
     /**
-     * The number of total records that satisfies the request conditions
+     * The number of total records that satisfies the request conditions.
+     * Only available when the request is performed on page 0
      */
     private Integer total;
 
+    /**
+     * The request limit parameter
+     */
+    private Limit limit;
+
     private long startTimestamp;
     private long endTimestamp;
 

server/web-service/src/main/java/org/bithon/server/web/service/datasource/api/impl/DataSourceApi.java

@@ -136,7 +136,11 @@ public GeneralQueryResponse list(GeneralQueryRequest request) throws IOException
 
         try (IDataSourceReader reader = schema.getDataStoreSpec().createReader()) {
             return GeneralQueryResponse.builder()
-                                       .total(reader.listSize(query))
+                                       // Only query the total number of records for the first page
+                                       // This also has a restriction
+                                       // that the page number is not a query parameter on web page URL
+                                       .total(query.getLimit().getOffset() == 0 ? reader.listSize(query) : 0)
+                                       .limit(query.getLimit())
                                        .data(reader.list(query))
                                        .startTimestamp(query.getInterval().getStartTime().getMilliseconds())
                                        .startTimestamp(query.getInterval().getEndTime().getMilliseconds())
@@ -285,7 +289,7 @@ private void validateQueryRequest(ISchema schema, GeneralQueryRequest request) {
                                                .orElse(null);
 
                 Preconditions.checkIfTrue(queryField != null
-                                              && schema.getColumnByName(queryField.getField()) != null,
+                                          && schema.getColumnByName(queryField.getField()) != null,
                                           "OrderBy field [%s] does not exist in the schema.",
                                           orderBy.getName());
 

server/web-service/src/main/java/org/bithon/server/web/service/tracing/api/GetTraceListResponse.java

@@ -30,5 +30,6 @@
 @AllArgsConstructor
 public class GetTraceListResponse {
     private int total;
+    private int pageNumber;
     private List<TraceSpan> rows;
 }

server/web-service/src/main/java/org/bithon/server/web/service/tracing/api/TraceApi.java

@@ -86,7 +86,8 @@ public GetTraceListResponse getTraceList(@Valid @RequestBody GetTraceListRequest
         Timestamp end = TimeSpan.fromISO8601(request.getEndTimeISO8601()).toTimestamp();
 
         return new GetTraceListResponse(
-            traceService.getTraceListSize(request.getFilters(), request.getExpression(), start, end),
+            request.getPageNumber() == 0 ? traceService.getTraceListSize(request.getFilters(), request.getExpression(), start, end) : 0,
+            request.getPageNumber(),
             traceService.getTraceList(request.getFilters(),
                                       request.getExpression(),
                                       start,