[Telemetry] Server side fetcher (elastic#50015)

* initial push

* self code review

* ignore node-fetch type

* usageFetcher api

* user agent metric

* telemetry plugin collector

* remove extra unused method

* remove unused import

* type check

* fix collections tests

* pass kfetch as dep

* add ui metrics integration test for user agent

* dont start ui metrics when not authenticated

* user agent count always 1

* fix broken ui-metric integration tests

* try using config.get

* avoid fetching configs if sending

* type unknown -> string

* check if fetcher is causing the issue

* disable ui_metric from functional tests

* enable ui_metric back again

* ignore keyword above 256

* check requesting app first

* clean up after all the debugging :)

* fix tests

* always return 200 for ui metric reporting

* remove boom import

* logout after removing role/user

* undo some changes in tests

* inside try catch

* prevent potential race conditions in priorities with =

* use snake_case for telemetry plugin collection

* usageFetcher -> sendUsageFrom

* more replacements

* remove extra unused route

* config() -> config

* Update src/legacy/core_plugins/telemetry/index.ts

Co-Authored-By: Mike Côté <mikecote@users.noreply.github.com>

* Update src/legacy/core_plugins/ui_metric/server/routes/api/ui_metric.ts

Co-Authored-By: Mike Côté <mikecote@users.noreply.github.com>

* config() -> config

* fix SO update logic given the current changes

* fix opt in check

* triple check

* check for non boolean

* take into account older settings

* import TelemetryOptInProvider

* update test case

packages/kbn-analytics/package.json

@@ -17,6 +17,6 @@
     "@babel/cli": "7.5.5",
     "@kbn/dev-utils": "1.0.0",
     "@kbn/babel-preset": "1.0.0",
-    "typescript": "3.5.1"
+    "typescript": "3.5.3"
   }
 }

packages/kbn-analytics/src/index.ts

@@ -17,6 +17,6 @@
  * under the License.
  */
 
-export { createReporter, ReportHTTP, Reporter, ReporterConfig } from './reporter';
+export { ReportHTTP, Reporter, ReporterConfig } from './reporter';
 export { UiStatsMetricType, METRIC_TYPE } from './metrics';
 export { Report, ReportManager } from './report';

packages/kbn-analytics/src/metrics/index.ts

@@ -17,21 +17,17 @@
  * under the License.
  */
 
-import { UiStatsMetric, UiStatsMetricType } from './ui_stats';
+import { UiStatsMetric } from './ui_stats';
+import { UserAgentMetric } from './user_agent';
 
-export {
-  UiStatsMetric,
-  createUiStatsMetric,
-  UiStatsMetricReport,
-  UiStatsMetricType,
-} from './ui_stats';
+export { UiStatsMetric, createUiStatsMetric, UiStatsMetricType } from './ui_stats';
 export { Stats } from './stats';
+export { trackUsageAgent } from './user_agent';
 
-export type Metric = UiStatsMetric<UiStatsMetricType>;
-export type MetricType = keyof typeof METRIC_TYPE;
-
+export type Metric = UiStatsMetric | UserAgentMetric;
 export enum METRIC_TYPE {
   COUNT = 'count',
   LOADED = 'loaded',
   CLICK = 'click',
+  USER_AGENT = 'user_agent',
 }

packages/kbn-analytics/src/metrics/ui_stats.ts

@@ -17,37 +17,33 @@
  * under the License.
  */
 
-import { Stats } from './stats';
 import { METRIC_TYPE } from './';
 
 export type UiStatsMetricType = METRIC_TYPE.CLICK | METRIC_TYPE.LOADED | METRIC_TYPE.COUNT;
-export interface UiStatsMetricConfig<T extends UiStatsMetricType> {
-  type: T;
+export interface UiStatsMetricConfig {
+  type: UiStatsMetricType;
   appName: string;
   eventName: string;
   count?: number;
 }
 
-export interface UiStatsMetric<T extends UiStatsMetricType = UiStatsMetricType> {
-  type: T;
+export interface UiStatsMetric {
+  type: UiStatsMetricType;
   appName: string;
   eventName: string;
   count: number;
 }
 
-export function createUiStatsMetric<T extends UiStatsMetricType>({
+export function createUiStatsMetric({
   type,
   appName,
   eventName,
   count = 1,
-}: UiStatsMetricConfig<T>): UiStatsMetric<T> {
-  return { type, appName, eventName, count };
-}
-
-export interface UiStatsMetricReport {
-  key: string;
-  appName: string;
-  eventName: string;
-  type: UiStatsMetricType;
-  stats: Stats;
+}: UiStatsMetricConfig): UiStatsMetric {
+  return {
+    type,
+    appName,
+    eventName,
+    count,
+  };
 }

packages/kbn-analytics/src/metrics/user_agent.ts

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { METRIC_TYPE } from './';
+
+export interface UserAgentMetric {
+  type: METRIC_TYPE.USER_AGENT;
+  appName: string;
+  userAgent: string;
+}
+
+export function trackUsageAgent(appName: string): UserAgentMetric {
+  const userAgent = (window && window.navigator && window.navigator.userAgent) || '';
+
+  return {
+    type: METRIC_TYPE.USER_AGENT,
+    appName,
+    userAgent,
+  };
+}

packages/kbn-analytics/src/report.ts

@@ -17,28 +17,47 @@
  * under the License.
  */
 
-import { UnreachableCaseError } from './util';
-import { Metric, Stats, UiStatsMetricReport, METRIC_TYPE } from './metrics';
+import { UnreachableCaseError, wrapArray } from './util';
+import { Metric, Stats, UiStatsMetricType, METRIC_TYPE } from './metrics';
+const REPORT_VERSION = 1;
 
 export interface Report {
+  reportVersion: typeof REPORT_VERSION;
   uiStatsMetrics: {
-    [key: string]: UiStatsMetricReport;
+    [key: string]: {
+      key: string;
+      appName: string;
+      eventName: string;
+      type: UiStatsMetricType;
+      stats: Stats;
+    };
+  };
+  userAgent?: {
+    [key: string]: {
+      userAgent: string;
+      key: string;
+      type: METRIC_TYPE.USER_AGENT;
+      appName: string;
+    };
   };
 }
 
 export class ReportManager {
+  static REPORT_VERSION = REPORT_VERSION;
   public report: Report;
   constructor(report?: Report) {
     this.report = report || ReportManager.createReport();
   }
-  static createReport() {
-    return { uiStatsMetrics: {} };
+  static createReport(): Report {
+    return { reportVersion: REPORT_VERSION, uiStatsMetrics: {} };
   }
   public clearReport() {
     this.report = ReportManager.createReport();
   }
   public isReportEmpty(): boolean {
-    return Object.keys(this.report.uiStatsMetrics).length === 0;
+    const noUiStats = Object.keys(this.report.uiStatsMetrics).length === 0;
+    const noUserAgent = !this.report.userAgent || Object.keys(this.report.userAgent).length === 0;
+    return noUiStats && noUserAgent;
   }
   private incrementStats(count: number, stats?: Stats): Stats {
     const { min = 0, max = 0, sum = 0 } = stats || {};
@@ -54,28 +73,46 @@ export class ReportManager {
       sum: newSum,
     };
   }
-  assignReports(newMetrics: Metric[]) {
-    newMetrics.forEach(newMetric => this.assignReport(this.report, newMetric));
+  assignReports(newMetrics: Metric | Metric[]) {
+    wrapArray(newMetrics).forEach(newMetric => this.assignReport(this.report, newMetric));
   }
   static createMetricKey(metric: Metric): string {
     switch (metric.type) {
+      case METRIC_TYPE.USER_AGENT: {
+        const { appName, type } = metric;
+        return `${appName}-${type}`;
+      }
       case METRIC_TYPE.CLICK:
       case METRIC_TYPE.LOADED:
       case METRIC_TYPE.COUNT: {
-        const { appName, type, eventName } = metric;
+        const { appName, eventName, type } = metric;
         return `${appName}-${type}-${eventName}`;
       }
       default:
-        throw new UnreachableCaseError(metric.type);
+        throw new UnreachableCaseError(metric);
     }
   }
   private assignReport(report: Report, metric: Metric) {
+    const key = ReportManager.createMetricKey(metric);
     switch (metric.type) {
+      case METRIC_TYPE.USER_AGENT: {
+        const { appName, type, userAgent } = metric;
+        if (userAgent) {
+          this.report.userAgent = {
+            [key]: {
+              key,
+              appName,
+              type,
+              userAgent: metric.userAgent,
+            },
+          };
+        }
+        return;
+      }
       case METRIC_TYPE.CLICK:
       case METRIC_TYPE.LOADED:
       case METRIC_TYPE.COUNT: {
         const { appName, type, eventName, count } = metric;
-        const key = ReportManager.createMetricKey(metric);
         const existingStats = (report.uiStatsMetrics[key] || {}).stats;
         this.report.uiStatsMetrics[key] = {
           key,
@@ -87,7 +124,7 @@ export class ReportManager {
         return;
       }
       default:
-        throw new UnreachableCaseError(metric.type);
+        throw new UnreachableCaseError(metric);
     }
   }
 }

packages/kbn-analytics/src/reporter.ts

@@ -18,7 +18,7 @@
  */
 
 import { wrapArray } from './util';
-import { Metric, UiStatsMetric, createUiStatsMetric } from './metrics';
+import { Metric, createUiStatsMetric, trackUsageAgent, UiStatsMetricType } from './metrics';
 
 import { Storage, ReportStorageManager } from './storage';
 import { Report, ReportManager } from './report';
@@ -40,10 +40,11 @@ export class Reporter {
   private reportManager: ReportManager;
   private storageManager: ReportStorageManager;
   private debug: boolean;
+  private retryCount = 0;
+  private readonly maxRetries = 3;
 
   constructor(config: ReporterConfig) {
-    const { http, storage, debug, checkInterval = 10000, storageKey = 'analytics' } = config;
-
+    const { http, storage, debug, checkInterval = 90000, storageKey = 'analytics' } = config;
     this.http = http;
     this.checkInterval = checkInterval;
     this.interval = null;
@@ -59,18 +60,19 @@ export class Reporter {
   }
 
   private flushReport() {
+    this.retryCount = 0;
     this.reportManager.clearReport();
     this.storageManager.store(this.reportManager.report);
   }
 
-  public start() {
+  public start = () => {
     if (!this.interval) {
       this.interval = setTimeout(() => {
         this.interval = null;
         this.sendReports();
       }, this.checkInterval);
     }
-  }
+  };
 
   private log(message: any) {
     if (this.debug) {
@@ -79,36 +81,42 @@ export class Reporter {
     }
   }
 
-  public reportUiStats(
+  public reportUiStats = (
     appName: string,
-    type: UiStatsMetric['type'],
+    type: UiStatsMetricType,
     eventNames: string | string[],
     count?: number
-  ) {
+  ) => {
     const metrics = wrapArray(eventNames).map(eventName => {
-      if (this) this.log(`${type} Metric -> (${appName}:${eventName}):`);
+      this.log(`${type} Metric -> (${appName}:${eventName}):`);
       const report = createUiStatsMetric({ type, appName, eventName, count });
       this.log(report);
       return report;
     });
     this.saveToReport(metrics);
-  }
+  };
+
+  public reportUserAgent = (appName: string) => {
+    this.log(`Reporting user-agent.`);
+    const report = trackUsageAgent(appName);
+    this.saveToReport([report]);
+  };
 
-  public async sendReports() {
+  public sendReports = async () => {
     if (!this.reportManager.isReportEmpty()) {
       try {
         await this.http(this.reportManager.report);
         this.flushReport();
       } catch (err) {
         this.log(`Error Sending Metrics Report ${err}`);
+        this.retryCount = this.retryCount + 1;
+        const versionMismatch =
+          this.reportManager.report.reportVersion !== ReportManager.REPORT_VERSION;
+        if (versionMismatch || this.retryCount > this.maxRetries) {
+          this.flushReport();
+        }
       }
     }
     this.start();
-  }
-}
-
-export function createReporter(reportedConf: ReporterConfig) {
-  const reporter = new Reporter(reportedConf);
-  reporter.start();
-  return reporter;
+  };
 }

src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker

@@ -185,6 +185,7 @@ kibana_vars=(
     xpack.security.public.hostname
     xpack.security.public.port
     telemetry.enabled
+    telemetry.sendUsageFrom
 )
 
 longopts=''

src/legacy/core_plugins/telemetry/common/constants.ts

@@ -59,6 +59,12 @@ export const PRIVACY_STATEMENT_URL = `https://www.elastic.co/legal/telemetry-pri
  */
 export const KIBANA_LOCALIZATION_STATS_TYPE = 'localization';
 
+/**
+ * The type name used to publish telemetry plugin stats.
+ * @type {string}
+ */
+export const TELEMETRY_STATS_TYPE = 'telemetry';
+
 /**
  * UI metric usage type
  * @type {string}