[WIP] Logs Sample Data (#22276)

Log sample data, check ✅

src/server/sample_data/data_sets/index.js

@@ -18,3 +18,4 @@
  */
 
 export { flightsSpecProvider } from './flights';
+export { logsSpecProvider } from './logs';

src/server/sample_data/data_sets/logs/index.js

@@ -0,0 +1,167 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import path from 'path';
+import { savedObjects } from './saved_objects';
+
+export function logsSpecProvider() {
+  return {
+    id: 'logs',
+    name: 'Sample web logs',
+    description: 'Sample data, visualizations, and dashboards for monitoring web logs.',
+    previewImagePath: '/plugins/kibana/home/sample_data_resources/logs/dashboard.png',
+    overviewDashboard: 'edf84fe0-e1a0-11e7-b6d5-4dc382ef7f5b',
+    defaultIndex: '90943e30-9a47-11e8-b64d-95841ca0b247',
+    dataPath: path.join(__dirname, './logs.json.gz'),
+    fields: {
+      request: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256
+          }
+        }
+      },
+      geo: {
+        properties: {
+          srcdest: {
+            type: 'keyword'
+          },
+          src: {
+            type: 'keyword'
+          },
+          dest: {
+            type: 'keyword'
+          },
+          coordinates: {
+            type: 'geo_point'
+          }
+        }
+      },
+      utc_time: {
+        type: 'date'
+      },
+      url: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256
+          }
+        }
+      },
+      message: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256
+          }
+        }
+      },
+      host: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256
+          }
+        }
+      },
+      clientip: {
+        type: 'ip'
+      },
+      response: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256
+          }
+        }
+      },
+      machine: {
+        properties: {
+          ram: {
+            type: 'long'
+          },
+          os: {
+            type: 'text',
+            fields: {
+              keyword: {
+                type: 'keyword',
+                ignore_above: 256
+              }
+            }
+          }
+        }
+      },
+      agent: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256
+          }
+        }
+      },
+      bytes: {
+        type: 'long'
+      },
+      tags: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256
+          }
+        }
+      },
+      referer: {
+        type: 'keyword'
+      },
+      ip: {
+        type: 'ip'
+      },
+      timestamp: {
+        type: 'date'
+      },
+      phpmemory: {
+        type: 'long'
+      },
+      memory: {
+        type: 'double'
+      },
+      extension: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256
+          }
+        }
+      }
+    },
+    timeFields: ['timestamp'],
+    currentTimeMarker: '2018-08-01T00:00:00',
+    preserveDayOfWeekTimeOfDay: true,
+    savedObjects: savedObjects,
+  };
+}

src/server/sample_data/routes/lib/load_data.test.js

@@ -19,7 +19,7 @@
 
 import { loadData } from './load_data';
 
-test('load data', done => {
+test('load flight data', done => {
   let myDocsCount = 0;
   const bulkInsertMock = (docs) => {
     myDocsCount += docs.length;
@@ -30,3 +30,15 @@ test('load data', done => {
     done();
   });
 });
+
+test('load log data', done => {
+  let myDocsCount = 0;
+  const bulkInsertMock = (docs) => {
+    myDocsCount += docs.length;
+  };
+  loadData('./src/server/sample_data/data_sets/logs/logs.json.gz', bulkInsertMock, async (err, count) => {
+    expect(myDocsCount).toBe(14005);
+    expect(count).toBe(14005);
+    done();
+  });
+});

src/server/sample_data/sample_data_mixin.js

@@ -26,6 +26,7 @@ import {
 } from './routes';
 import {
   flightsSpecProvider,
+  logsSpecProvider,
 } from './data_sets';
 
 export function sampleDataMixin(kbnServer, server) {
@@ -66,4 +67,5 @@ export function sampleDataMixin(kbnServer, server) {
   });
 
   server.registerSampleDataset(flightsSpecProvider);
+  server.registerSampleDataset(logsSpecProvider);
 }

test/functional/apps/home/_sample_data.js

@@ -32,26 +32,39 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.header.waitUntilLoadingHasFinished();
     });
 
-    it('should display registered sample data sets', async ()=> {
+    it('should display registered flights sample data sets', async ()=> {
       await retry.try(async () => {
         const exists = await PageObjects.home.doesSampleDataSetExist('flights');
         expect(exists).to.be(true);
       });
     });
 
-    it('should install sample data set', async ()=> {
+    it('should display registered logs sample data sets', async ()=> {
+      await retry.try(async () => {
+        const exists = await PageObjects.home.doesSampleDataSetExist('logs');
+        expect(exists).to.be(true);
+      });
+    });
+
+    it('should install flights sample data set', async ()=> {
       await PageObjects.home.addSampleDataSet('flights');
       const isInstalled = await PageObjects.home.isSampleDataSetInstalled('flights');
       expect(isInstalled).to.be(true);
     });
 
+    it('should install logs sample data set', async ()=> {
+      await PageObjects.home.addSampleDataSet('logs');
+      const isInstalled = await PageObjects.home.isSampleDataSetInstalled('logs');
+      expect(isInstalled).to.be(true);
+    });
+
     describe('dashboard', () => {
-      after(async () => {
+      afterEach(async () => {
         await PageObjects.common.navigateToUrl('home', 'tutorial_directory/sampleData');
         await PageObjects.header.waitUntilLoadingHasFinished();
       });
 
-      it('should launch sample data set dashboard', async ()=> {
+      it('should launch sample flights data set dashboard', async ()=> {
         await PageObjects.home.launchSampleDataSet('flights');
         await PageObjects.header.waitUntilLoadingHasFinished();
         const today = new Date();
@@ -63,6 +76,7 @@ export default function ({ getService, getPageObjects }) {
         expect(panelCount).to.be(19);
       });
 
+
       it.skip('pie charts rendered', async () => {
         await dashboardExpect.pieSliceCount(4);
       });
@@ -87,15 +101,35 @@ export default function ({ getService, getPageObjects }) {
         const tsvb = await find.existsByCssSelector('.vega-view-container');
         expect(tsvb).to.be(true);
       });
+
+      it('should launch sample logs data set dashboard', async ()=> {
+        await PageObjects.home.launchSampleDataSet('logs');
+        await PageObjects.header.waitUntilLoadingHasFinished();
+        const today = new Date();
+        const todayYearMonthDay = today.toISOString().substring(0, 10);
+        const fromTime = `${todayYearMonthDay} 00:00:00.000`;
+        const toTime = `${todayYearMonthDay} 23:59:59.999`;
+        await PageObjects.header.setAbsoluteRange(fromTime, toTime);
+        const panelCount = await PageObjects.dashboard.getPanelCount();
+        expect(panelCount).to.be(11);
+      });
+
     });
 
     // needs to be in describe block so it is run after 'dashboard describe block'
     describe('uninstall', () => {
-      it('should uninstall sample data set', async ()=> {
+      it('should uninstall flights sample data set', async ()=> {
         await PageObjects.home.removeSampleDataSet('flights');
         const isInstalled = await PageObjects.home.isSampleDataSetInstalled('flights');
         expect(isInstalled).to.be(false);
       });
+
+      it('should uninstall logs sample data set', async ()=> {
+        await PageObjects.home.removeSampleDataSet('logs');
+        const isInstalled = await PageObjects.home.isSampleDataSetInstalled('logs');
+        expect(isInstalled).to.be(false);
+      });
+
     });
   });
 }