kobsio · ricoberger · Oct 31, 2021 · Oct 31, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -32,6 +32,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#185](https://github.com/kobsio/kobs/pull/185): [clickhouse] Use pagination instead of Intersection Observer API to display logs.
 - [#188](https://github.com/kobsio/kobs/pull/188): [sql] Replace the `GetQueryResults` function with the implemention used in the Clickhouse plugin, to have a proper handling for float values.
 - [#190](https://github.com/kobsio/kobs/pull/190): [core] Unify list layout across plugin.
+- [#194](https://github.com/kobsio/kobs/pull/194): [elasticsearch] Use pagination instead of infinite scrolling to display logs.
 
 ## [v0.6.0](https://github.com/kobsio/kobs/releases/tag/v0.6.0) (2021-10-11)
 

diff --git a/plugins/elasticsearch/elasticsearch.go b/plugins/elasticsearch/elasticsearch.go
@@ -43,17 +43,16 @@ func (router *Router) getInstance(name string) *instance.Instance {
 
 // getLogs returns the raw documents for a given query from Elasticsearch. The result also contains the distribution of
 // the documents in the given time range. The name of the Elasticsearch instance must be set via the name path
-// parameter, all other values like the query, scrollID, start and end time are set via query parameters. These
+// parameter, all other values like the query, start and end time are set via query parameters. These
 // parameters are then passed to the GetLogs function of the Elasticsearch instance, which returns the documents and
 // buckets.
 func (router *Router) getLogs(w http.ResponseWriter, r *http.Request) {
 	name := chi.URLParam(r, "name")
 	query := r.URL.Query().Get("query")
-	scrollID := r.URL.Query().Get("scrollID")
 	timeStart := r.URL.Query().Get("timeStart")
 	timeEnd := r.URL.Query().Get("timeEnd")
 
-	log.WithFields(logrus.Fields{"name": name, "query": query, "scrollID": scrollID, "timeStart": timeStart, "timeEnd": timeEnd}).Tracef("getLogs")
+	log.WithFields(logrus.Fields{"name": name, "query": query, "timeStart": timeStart, "timeEnd": timeEnd}).Tracef("getLogs")
 
 	i := router.getInstance(name)
 	if i == nil {
@@ -73,7 +72,7 @@ func (router *Router) getLogs(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	data, err := i.GetLogs(r.Context(), query, scrollID, parsedTimeStart, parsedTimeEnd)
+	data, err := i.GetLogs(r.Context(), query, parsedTimeStart, parsedTimeEnd)
 	if err != nil {
 		errresponse.Render(w, r, err, http.StatusInternalServerError, "Could not get logs")
 		return

diff --git a/plugins/elasticsearch/pkg/instance/instance.go b/plugins/elasticsearch/pkg/instance/instance.go
@@ -36,20 +36,14 @@ type Instance struct {
 }
 
 // GetLogs returns the raw log documents and the buckets for the distribution of the logs accross the selected time
-// range. We have to pass a query, start and end time to the function. The scrollID can be an empty string to start a
-// new query. If a scrollID is provided it will be used for pagination.
-func (i *Instance) GetLogs(ctx context.Context, query, scrollID string, timeStart, timeEnd int64) (*Data, error) {
+// range. We have to pass a query, start and end time to the function.
+func (i *Instance) GetLogs(ctx context.Context, query string, timeStart, timeEnd int64) (*Data, error) {
 	var err error
 	var body []byte
 	var url string
 
-	if scrollID == "" {
-		url = fmt.Sprintf("%s/_search?scroll=15m", i.address)
-		body = []byte(fmt.Sprintf(`{"size":100,"sort":[{"@timestamp":{"order":"desc"}}],"query":{"bool":{"must":[{"range":{"@timestamp":{"gte":"%d","lte":"%d"}}},{"query_string":{"query":"%s"}}]}},"aggs":{"logcount":{"auto_date_histogram":{"field":"@timestamp","buckets":30}}}}`, timeStart*1000, timeEnd*1000, strings.ReplaceAll(query, "\"", "\\\"")))
-	} else {
-		url = fmt.Sprintf("%s/_search/scroll", i.address)
-		body = []byte(`{"scroll" : "15m", "scroll_id" : "` + scrollID + `"}`)
-	}
+	url = fmt.Sprintf("%s/_search", i.address)
+	body = []byte(fmt.Sprintf(`{"size":1000,"sort":[{"@timestamp":{"order":"desc"}}],"query":{"bool":{"must":[{"range":{"@timestamp":{"gte":"%d","lte":"%d"}}},{"query_string":{"query":"%s"}}]}},"aggs":{"logcount":{"auto_date_histogram":{"field":"@timestamp","buckets":30}}}}`, timeStart*1000, timeEnd*1000, strings.ReplaceAll(query, "\"", "\\\"")))
 
 	log.WithFields(logrus.Fields{"query": string(body)}).Debugf("Run Elasticsearch query")
 
@@ -75,14 +69,13 @@ func (i *Instance) GetLogs(ctx context.Context, query, scrollID string, timeStar
 		}
 
 		data := &Data{
-			ScrollID:  res.ScrollID,
 			Took:      res.Took,
 			Hits:      res.Hits.Total.Value,
 			Documents: res.Hits.Hits,
 			Buckets:   res.Aggregations.LogCount.Buckets,
 		}
 
-		log.WithFields(logrus.Fields{"scrollID": data.ScrollID, "took": data.Took, "hits": data.Hits, "documents": len(data.Documents), "buckets": len(data.Buckets)}).Debugf("Elasticsearch query results")
+		log.WithFields(logrus.Fields{"took": data.Took, "hits": data.Hits, "documents": len(data.Documents), "buckets": len(data.Buckets)}).Debugf("Elasticsearch query results")
 
 		return data, nil
 	}

diff --git a/plugins/elasticsearch/pkg/instance/structs.go b/plugins/elasticsearch/pkg/instance/structs.go
@@ -50,9 +50,8 @@ type ResponseError struct {
 }
 
 // Data is the transformed Response result, which is passed to the React UI. It contains only the important fields, like
-// the scrollID, the time a request took, the number of hits, the documents and the buckets.
+// the time a request took, the number of hits, the documents and the buckets.
 type Data struct {
-	ScrollID  string                   `json:"scrollID"`
 	Took      int64                    `json:"took"`
 	Hits      int64                    `json:"hits"`
 	Documents []map[string]interface{} `json:"documents"`

diff --git a/plugins/elasticsearch/src/components/page/PageLogs.tsx b/plugins/elasticsearch/src/components/page/PageLogs.tsx
@@ -2,8 +2,6 @@ import {
   Alert,
   AlertActionLink,
   AlertVariant,
-  Button,
-  ButtonVariant,
   Card,
   CardActions,
   CardBody,
@@ -14,7 +12,7 @@ import {
   GridItem,
   Spinner,
 } from '@patternfly/react-core';
-import { InfiniteData, InfiniteQueryObserverResult, QueryObserverResult, useInfiniteQuery } from 'react-query';
+import { QueryObserverResult, useQuery } from 'react-query';
 import React from 'react';
 import { useHistory } from 'react-router-dom';
 
@@ -46,14 +44,12 @@ const PageLogs: React.FunctionComponent<IPageLogsProps> = ({
 }: IPageLogsProps) => {
   const history = useHistory();
 
-  const { isError, isFetching, isLoading, data, error, fetchNextPage, refetch } = useInfiniteQuery<ILogsData, Error>(
+  const { isError, isFetching, isLoading, data, error, refetch } = useQuery<ILogsData, Error>(
     ['elasticsearch/logs', query, times],
-    async ({ pageParam }) => {
+    async () => {
       try {
         const response = await fetch(
-          `/api/plugins/elasticsearch/logs/${name}?query=${query}&timeStart=${times.timeStart}&timeEnd=${
-            times.timeEnd
-          }&scrollID=${pageParam || ''}`,
+          `/api/plugins/elasticsearch/logs/${name}?query=${query}&timeStart=${times.timeStart}&timeEnd=${times.timeEnd}`,
           {
             method: 'get',
           },
@@ -74,7 +70,6 @@ const PageLogs: React.FunctionComponent<IPageLogsProps> = ({
       }
     },
     {
-      getNextPageParam: (lastPage, pages) => lastPage.scrollID,
       keepPreviousData: true,
     },
   );
@@ -95,7 +90,7 @@ const PageLogs: React.FunctionComponent<IPageLogsProps> = ({
         actionLinks={
           <React.Fragment>
             <AlertActionLink onClick={(): void => history.push('/')}>Home</AlertActionLink>
-            <AlertActionLink onClick={(): Promise<QueryObserverResult<InfiniteData<ILogsData>, Error>> => refetch()}>
+            <AlertActionLink onClick={(): Promise<QueryObserverResult<ILogsData, Error>> => refetch()}>
               Retry
             </AlertActionLink>
           </React.Fragment>
@@ -106,62 +101,48 @@ const PageLogs: React.FunctionComponent<IPageLogsProps> = ({
     );
   }
 
-  if (!data || data.pages.length === 0) {
+  if (!data) {
     return null;
   }
 
   return (
     <Grid hasGutter={true}>
       <GridItem sm={12} md={12} lg={3} xl={2} xl2={2}>
         <Card>
-          <PageLogsFields
-            fields={getFields(data.pages[0].documents)}
-            selectField={selectField}
-            selectedFields={fields}
-          />
+          <PageLogsFields fields={getFields(data.documents)} selectField={selectField} selectedFields={fields} />
         </Card>
       </GridItem>
       <GridItem sm={12} md={12} lg={9} xl={10} xl2={10}>
         <Card isCompact={true}>
           <CardHeader>
             <CardHeaderMain>
               <CardTitle>
-                {data.pages[0].hits} Documents in {data.pages[0].took} Milliseconds
+                {data.hits} Documents in {data.took} Milliseconds
               </CardTitle>
             </CardHeaderMain>
             <CardActions>{isFetching && <Spinner size="md" />}</CardActions>
           </CardHeader>
 
           <CardBody>
-            <LogsChart buckets={data.pages[0].buckets} changeTime={changeTime} />
+            <LogsChart buckets={data.buckets} changeTime={changeTime} />
           </CardBody>
         </Card>
+
         <p>&nbsp;</p>
-        {data.pages[0].documents.length > 0 ? (
+
+        {data.documents.length > 0 ? (
           <Card isCompact={true} style={{ maxWidth: '100%', overflowX: 'scroll' }}>
             <CardBody>
-              <LogsDocuments pages={data.pages} fields={fields} addFilter={addFilter} selectField={selectField} />
-            </CardBody>
-          </Card>
-        ) : null}
-        <p>&nbsp;</p>
-        {data.pages[0].documents.length > 0 ? (
-          <Card isCompact={true}>
-            <CardBody>
-              <Button
-                variant={ButtonVariant.primary}
-                isBlock={true}
-                isDisabled={isFetching}
-                isLoading={isFetching}
-                onClick={(): Promise<InfiniteQueryObserverResult<ILogsData, Error>> => fetchNextPage()}
-              >
-                Load more
-              </Button>
+              <LogsDocuments
+                documents={data.documents}
+                fields={fields}
+                addFilter={addFilter}
+                selectField={selectField}
+              />
             </CardBody>
           </Card>
         ) : null}
       </GridItem>
-      <p>&nbsp;</p>
     </Grid>
   );
 };

diff --git a/plugins/elasticsearch/src/components/panel/Logs.tsx b/plugins/elasticsearch/src/components/panel/Logs.tsx
@@ -2,15 +2,13 @@ import {
   Alert,
   AlertActionLink,
   AlertVariant,
-  Button,
-  ButtonVariant,
   Select,
   SelectOption,
   SelectOptionObject,
   SelectVariant,
   Spinner,
 } from '@patternfly/react-core';
-import { InfiniteData, InfiniteQueryObserverResult, QueryObserverResult, useInfiniteQuery } from 'react-query';
+import { QueryObserverResult, useQuery } from 'react-query';
 import React, { useState } from 'react';
 
 import { ILogsData, IQuery } from '../../utils/interfaces';
@@ -39,14 +37,12 @@ const Logs: React.FunctionComponent<ILogsProps> = ({
   const [showSelect, setShowSelect] = useState<boolean>(false);
   const [selectedQuery, setSelectedQuery] = useState<IQuery>(queries[0]);
 
-  const { isError, isFetching, isLoading, data, error, fetchNextPage, refetch } = useInfiniteQuery<ILogsData, Error>(
+  const { isError, isFetching, isLoading, data, error, refetch } = useQuery<ILogsData, Error>(
     ['elasticsearch/logs', selectedQuery, times],
-    async ({ pageParam }) => {
+    async () => {
       try {
         const response = await fetch(
-          `/api/plugins/elasticsearch/logs/${name}?query=${selectedQuery.query}&timeStart=${times.timeStart}&timeEnd=${
-            times.timeEnd
-          }&scrollID=${pageParam || ''}`,
+          `/api/plugins/elasticsearch/logs/${name}?query=${selectedQuery.query}&timeStart=${times.timeStart}&timeEnd=${times.timeEnd}`,
           {
             method: 'get',
           },
@@ -67,7 +63,6 @@ const Logs: React.FunctionComponent<ILogsProps> = ({
       }
     },
     {
-      getNextPageParam: (lastPage, pages) => lastPage.scrollID,
       keepPreviousData: true,
     },
   );
@@ -87,7 +82,7 @@ const Logs: React.FunctionComponent<ILogsProps> = ({
     <PluginCard
       title={title}
       description={description}
-      actions={<LogsActions name={name} queries={queries} times={times} />}
+      actions={<LogsActions name={name} queries={queries} times={times} isFetching={isFetching} />}
     >
       <div>
         {queries.length > 1 ? (
@@ -120,39 +115,24 @@ const Logs: React.FunctionComponent<ILogsProps> = ({
             title="Could not get logs"
             actionLinks={
               <React.Fragment>
-                <AlertActionLink
-                  onClick={(): Promise<QueryObserverResult<InfiniteData<ILogsData>, Error>> => refetch()}
-                >
+                <AlertActionLink onClick={(): Promise<QueryObserverResult<ILogsData, Error>> => refetch()}>
                   Retry
                 </AlertActionLink>
               </React.Fragment>
             }
           >
             <p>{error?.message}</p>
           </Alert>
-        ) : data && data.pages.length > 0 ? (
+        ) : data ? (
           <div>
             {showChart ? (
               <div>
-                <LogsChart buckets={data.pages[0].buckets} />
+                <LogsChart buckets={data.buckets} />
                 <p>&nbsp;</p>
               </div>
             ) : null}
 
-            <LogsDocuments pages={data.pages} fields={selectedQuery.fields} />
-            <p>&nbsp;</p>
-
-            {data.pages[0].documents.length > 0 ? (
-              <Button
-                variant={ButtonVariant.primary}
-                isBlock={true}
-                isDisabled={isFetching}
-                isLoading={isFetching}
-                onClick={(): Promise<InfiniteQueryObserverResult<ILogsData, Error>> => fetchNextPage()}
-              >
-                Load more
-              </Button>
-            ) : null}
+            <LogsDocuments documents={data.documents} fields={selectedQuery.fields} />
           </div>
         ) : null}
       </div>

diff --git a/plugins/elasticsearch/src/components/panel/LogsActions.tsx b/plugins/elasticsearch/src/components/panel/LogsActions.tsx
@@ -1,4 +1,4 @@
-import { CardActions, Dropdown, DropdownItem, KebabToggle } from '@patternfly/react-core';
+import { CardActions, Dropdown, DropdownItem, KebabToggle, Spinner } from '@patternfly/react-core';
 import React, { useState } from 'react';
 import { Link } from 'react-router-dom';
 
@@ -9,33 +9,43 @@ interface IActionsProps {
   name: string;
   queries: IQuery[];
   times: IPluginTimes;
+  isFetching: boolean;
 }
 
-export const Actions: React.FunctionComponent<IActionsProps> = ({ name, queries, times }: IActionsProps) => {
+export const Actions: React.FunctionComponent<IActionsProps> = ({
+  name,
+  queries,
+  times,
+  isFetching,
+}: IActionsProps) => {
   const [show, setShow] = useState<boolean>(false);
 
   return (
     <CardActions>
-      <Dropdown
-        toggle={<KebabToggle onToggle={(): void => setShow(!show)} />}
-        isOpen={show}
-        isPlain={true}
-        position="right"
-        dropdownItems={queries.map((query, index) => (
-          <DropdownItem
-            key={index}
-            component={
-              <Link
-                to={`/${name}?timeEnd=${times.timeEnd}&timeStart=${times.timeStart}&query=${query.query}${
-                  query.fields ? query.fields.map((field) => `&field=${field}`).join('') : ''
-                }`}
-              >
-                {query.name}
-              </Link>
-            }
-          />
-        ))}
-      />
+      {isFetching ? (
+        <Spinner size="md" />
+      ) : (
+        <Dropdown
+          toggle={<KebabToggle onToggle={(): void => setShow(!show)} />}
+          isOpen={show}
+          isPlain={true}
+          position="right"
+          dropdownItems={queries.map((query, index) => (
+            <DropdownItem
+              key={index}
+              component={
+                <Link
+                  to={`/${name}?timeEnd=${times.timeEnd}&timeStart=${times.timeStart}&query=${query.query}${
+                    query.fields ? query.fields.map((field) => `&field=${field}`).join('') : ''
+                  }`}
+                >
+                  {query.name}
+                </Link>
+              }
+            />
+          ))}
+        />
+      )}
     </CardActions>
   );
 };