Add Elasticsearch as datasource for Application logs

CHANGELOG.md

@@ -13,6 +13,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#4](https://github.com/kobsio/kobs/pull/4): Add Custom Resource Definition for Applications.
 - [#6](https://github.com/kobsio/kobs/pull/6): Add Prometheus as datasource for Application metrics.
 - [#8](https://github.com/kobsio/kobs/pull/8): Add new page to directly query a configured Prometheus datasource.
+- [#10](https://github.com/kobsio/kobs/pull/10): Add Elasticsearch as datasource for Application logs.
 
 ### Fixed
 

app/src/app.css

@@ -87,6 +87,11 @@
   height: 300px;
 }
 
+.kobsio-chart-container-default-small {
+  width: 100%;
+  height: 200px;
+}
+
 .kobsio-chart-container-sparkline {
   width: 100%;
   height: 150px;
@@ -104,3 +109,16 @@
   position: absolute;
   text-align: center
 }
+
+/* kobsis-table-wrapper
+ * Wrap a table component, so it looks nice within a page, but allow scrolling so the user can see all the data. */
+.kobsis-table-wrapper {
+  max-width: 100%;
+  overflow-x: scroll;
+}
+
+/* kobsio-tab-content
+ * Set a min height of 100% for the tab content. */
+.kobsio-tab-content {
+  min-height: 100%;
+}

app/src/components/applications/Application.tsx

@@ -39,6 +39,7 @@ const Applications: React.FunctionComponent = () => {
   const [tab, setTab] = useState<string>(DEFAULT_TAB);
   const refResourcesContent = useRef<HTMLElement>(null);
   const refMetricsContent = useRef<HTMLElement>(null);
+  const refLogsContent = useRef<HTMLElement>(null);
 
   const goToOverview = (): void => {
     history.push('/');
@@ -117,6 +118,7 @@ const Applications: React.FunctionComponent = () => {
           setTab={(t: string): void => setTab(t)}
           refResourcesContent={refResourcesContent}
           refMetricsContent={refMetricsContent}
+          refLogsContent={refLogsContent}
         />
       </PageSection>
 
@@ -126,6 +128,7 @@ const Applications: React.FunctionComponent = () => {
           tab={tab}
           refResourcesContent={refResourcesContent}
           refMetricsContent={refMetricsContent}
+          refLogsContent={refLogsContent}
         />
       </PageSection>
     </React.Fragment>

app/src/components/applications/details/DrawerPanel.tsx

@@ -28,6 +28,7 @@ const DrawerPanel: React.FunctionComponent<IDrawerPanelProps> = ({ application,
   const [tab, setTab] = useState<string>(DEFAULT_TAB);
   const refResourcesContent = useRef<HTMLElement>(null);
   const refMetricsContent = useRef<HTMLElement>(null);
+  const refLogsContent = useRef<HTMLElement>(null);
 
   return (
     <DrawerPanelContent minSize="50%">
@@ -61,13 +62,15 @@ const DrawerPanel: React.FunctionComponent<IDrawerPanelProps> = ({ application,
           setTab={(t: string): void => setTab(t)}
           refResourcesContent={refResourcesContent}
           refMetricsContent={refMetricsContent}
+          refLogsContent={refLogsContent}
         />
 
         <TabsContent
           application={application}
           tab={tab}
           refResourcesContent={refResourcesContent}
           refMetricsContent={refMetricsContent}
+          refLogsContent={refLogsContent}
         />
       </DrawerPanelBody>
     </DrawerPanelContent>

app/src/components/applications/details/Tabs.tsx

@@ -9,6 +9,7 @@ interface ITabsParams {
   setTab(tab: string): void;
   refResourcesContent: React.RefObject<HTMLElement>;
   refMetricsContent: React.RefObject<HTMLElement>;
+  refLogsContent: React.RefObject<HTMLElement>;
 }
 
 // Tabs renders the tabs header, which are used by the user to select a section he wants to view for an application.
@@ -19,6 +20,7 @@ const Tabs: React.FunctionComponent<ITabsParams> = ({
   setTab,
   refResourcesContent,
   refMetricsContent,
+  refLogsContent,
 }: ITabsParams) => {
   return (
     <PatternflyTabs
@@ -40,6 +42,12 @@ const Tabs: React.FunctionComponent<ITabsParams> = ({
         tabContentId="refMetrics"
         tabContentRef={refMetricsContent}
       />
+      <Tab
+        eventKey="logs"
+        title={<TabTitleText>Logs</TabTitleText>}
+        tabContentId="refLogs"
+        tabContentRef={refLogsContent}
+      />
     </PatternflyTabs>
   );
 };

app/src/components/applications/details/TabsContent.tsx

@@ -4,6 +4,7 @@ import { TabContent } from '@patternfly/react-core';
 
 import { Application } from 'generated/proto/application_pb';
 import { IDatasourceOptions } from 'utils/proto';
+import Logs from 'components/applications/details/logs/Logs';
 import Metrics from 'components/applications/details/metrics/Metrics';
 import Resources from 'components/applications/details/resources/Resources';
 
@@ -43,6 +44,7 @@ interface ITabsContent {
   tab: string;
   refResourcesContent: React.RefObject<HTMLElement>;
   refMetricsContent: React.RefObject<HTMLElement>;
+  refLogsContent: React.RefObject<HTMLElement>;
 }
 
 // TabsContent renders the content for a selected tab from the Tabs component. We also manage the datasource options,
@@ -54,6 +56,7 @@ const TabsContent: React.FunctionComponent<ITabsContent> = ({
   tab,
   refResourcesContent,
   refMetricsContent,
+  refLogsContent,
 }: ITabsContent) => {
   const history = useHistory();
   const location = useLocation();
@@ -71,6 +74,7 @@ const TabsContent: React.FunctionComponent<ITabsContent> = ({
   return (
     <React.Fragment>
       <TabContent
+        className="kobsio-tab-content"
         eventKey="resources"
         id="refResources"
         activeKey={tab}
@@ -81,7 +85,15 @@ const TabsContent: React.FunctionComponent<ITabsContent> = ({
           <Resources application={application} />
         </div>
       </TabContent>
-      <TabContent eventKey="metrics" id="refMetrics" activeKey={tab} ref={refMetricsContent} aria-label="Metrics">
+
+      <TabContent
+        className="kobsio-tab-content"
+        eventKey="metrics"
+        id="refMetrics"
+        activeKey={tab}
+        ref={refMetricsContent}
+        aria-label="Metrics"
+      >
         {/* We have to check if the refMetricsContent is not null, because otherwise the Metrics component will be shown below the resources component. */}
         <div>
           {refMetricsContent.current ? (
@@ -93,6 +105,26 @@ const TabsContent: React.FunctionComponent<ITabsContent> = ({
           ) : null}
         </div>
       </TabContent>
+
+      <TabContent
+        className="kobsio-tab-content"
+        eventKey="logs"
+        id="refLogs"
+        activeKey={tab}
+        ref={refLogsContent}
+        aria-label="Logs"
+      >
+        {/* We have to check if the refLogsContent is not null, because otherwise the Logs component will be shown below the resources component. */}
+        <div>
+          {refLogsContent.current ? (
+            <Logs
+              datasourceOptions={datasourceOptions}
+              setDatasourceOptions={changeDatasourceOptions}
+              application={application}
+            />
+          ) : null}
+        </div>
+      </TabContent>
     </React.Fragment>
   );
 };

app/src/components/applications/details/logs/Elasticsearch.tsx

@@ -0,0 +1,118 @@
+import { Alert, AlertVariant, Button } from '@patternfly/react-core';
+import React, { useCallback, useEffect, useState } from 'react';
+
+import { DatasourceLogsBucket, GetLogsRequest, GetLogsResponse } from 'generated/proto/datasources_pb';
+import { ApplicationLogsQuery } from 'generated/proto/application_pb';
+import Buckets from 'components/datasources/elasticsearch/Buckets';
+import { DatasourcesPromiseClient } from 'generated/proto/datasources_grpc_web_pb';
+import Documents from 'components/datasources/elasticsearch/Documents';
+import { IDatasourceOptions } from 'utils/proto';
+import { IDocument } from 'components/datasources/elasticsearch/helpers';
+import { apiURL } from 'utils/constants';
+import { convertDatasourceOptionsToProto } from 'utils/proto';
+
+const datasourcesService = new DatasourcesPromiseClient(apiURL, null, null);
+
+export interface IElasticsearchProps {
+  query?: string;
+  fields?: string[];
+  datasourceName: string;
+  datasourceType: string;
+  datasourceOptions: IDatasourceOptions;
+}
+
+// Elasticsearhc implements the Elasticsearch UI for kobs. It can be used to query a configured Elasticsearch instance
+// and show the logs in a table.
+const Elasticsearch: React.FunctionComponent<IElasticsearchProps> = ({
+  query,
+  fields,
+  datasourceName,
+  datasourceType,
+  datasourceOptions,
+}: IElasticsearchProps) => {
+  const [hits, setHits] = useState<number>(0);
+  const [took, setTook] = useState<number>(0);
+  const [documents, setDocuments] = useState<IDocument[]>([]);
+  const [buckets, setBuckets] = useState<DatasourceLogsBucket.AsObject[]>([]);
+  const [error, setError] = useState<string>('');
+
+  // fetchLogs fetches the logs for a given query. For the applications view, we do not care about infinite scrolling.
+  // When a user wants to see more then the fetched logs, he has to go to the datasource view.
+  const fetchLogs = useCallback(async (): Promise<void> => {
+    try {
+      if (query) {
+        const logsQuery = new ApplicationLogsQuery();
+        logsQuery.setQuery(query);
+
+        const getLogsRequest = new GetLogsRequest();
+        getLogsRequest.setName(datasourceName);
+        getLogsRequest.setScrollid('');
+        getLogsRequest.setOptions(convertDatasourceOptionsToProto(datasourceOptions));
+        getLogsRequest.setQuery(logsQuery);
+
+        const getLogsResponse: GetLogsResponse = await datasourcesService.getLogs(getLogsRequest, null);
+
+        const parsed = JSON.parse(getLogsResponse.getLogs());
+        if (parsed.length === 0) {
+          throw new Error('No documents were found');
+        } else {
+          if (getLogsResponse.toObject().bucketsList.length > 0) {
+            setBuckets(getLogsResponse.toObject().bucketsList);
+          }
+
+          setDocuments(parsed);
+          setHits(getLogsResponse.getHits());
+          setTook(getLogsResponse.getTook());
+          setError('');
+        }
+      }
+    } catch (err) {
+      setError(err.message);
+    }
+  }, [query, datasourceName, datasourceOptions]);
+
+  useEffect(() => {
+    fetchLogs();
+  }, [fetchLogs]);
+
+  return (
+    <React.Fragment>
+      {error ? (
+        <React.Fragment>
+          <p>&nbsp;</p>
+          <Alert variant={AlertVariant.danger} isInline={true} title="Could not get logs">
+            <p>{error}</p>
+          </Alert>
+        </React.Fragment>
+      ) : (
+        <React.Fragment>
+          <p>&nbsp;</p>
+
+          {buckets.length > 0 ? <Buckets hits={hits} took={took} buckets={buckets} /> : null}
+
+          <p>&nbsp;</p>
+
+          {documents.length > 0 ? (
+            <Documents selectedFields={fields ? fields : []} documents={documents} select={undefined} />
+          ) : null}
+
+          <p>&nbsp;</p>
+
+          <Button
+            component="a"
+            href={`/datasources/${datasourceType}/${datasourceName}?query=${query}${
+              fields ? `&fields=${fields.join(',')}` : ''
+            }&timeEnd=${datasourceOptions.timeEnd}&timeStart=${datasourceOptions.timeStart}`}
+            variant="primary"
+            isBlock={true}
+            target="_blank"
+          >
+            Details
+          </Button>
+        </React.Fragment>
+      )}
+    </React.Fragment>
+  );
+};
+
+export default Elasticsearch;