feat(nlu-server): send data to usage API (#171)

* feat(nlu-server): send data to usage API

* rename usage of word billing for usage

* always display error message

* pr review comments

packages/nlu-cli/src/parameters/nlu-server.ts

@@ -93,5 +93,9 @@ export const parameters = asYargs({
   maxTraining: {
     description: 'The max allowed amount of simultaneous trainings on a single instance',
     type: 'number'
+  },
+  usageURL: {
+    description: 'Endpoint to send usage info to.',
+    type: 'string'
   }
 })

packages/nlu-server/src/api/index.ts

@@ -21,6 +21,7 @@ import {
   trainingDuration
 } from '../telemetry/metric'
 import { initTracing } from '../telemetry/trace'
+import { UsageClient } from '../telemetry/usage-client'
 import { InvalidRequestFormatError } from './errors'
 import { handleError, getAppId } from './http'
 
@@ -36,29 +37,39 @@ type APIOptions = {
   prometheusEnabled?: boolean
   apmEnabled?: boolean
   apmSampleRate?: number
+  usageURL?: string
 }
 
 const { modelIdService } = NLUEngine
 
+const isTrainingRunning = ({ status }: Training) => status === 'training-pending' || status === 'training'
+
 export const createAPI = async (options: APIOptions, app: Application, baseLogger: Logger): Promise<ExpressApp> => {
-  const requestLogger = baseLogger.sub('api').sub('request')
+  const apiLogger = baseLogger.sub('api')
+  const requestLogger = apiLogger.sub('request')
   const expressApp = express()
 
   expressApp.use(cors())
 
   if (options.prometheusEnabled) {
+    const prometheusLogger = apiLogger.sub('prometheus')
+    prometheusLogger.debug('prometheus metrics enabled')
+
     app.on('training_update', async (training: Training) => {
-      if (training.status !== 'canceled' && training.status !== 'done' && training.status !== 'errored') {
+      if (isTrainingRunning(training) || !training.trainingTime) {
         return
       }
 
-      if (training.trainingTime) {
-        trainingDuration.observe({ status: training.status }, training.trainingTime / 1000)
-      }
+      const trainingTime = training.trainingTime / 1000
+      prometheusLogger.debug(`adding metric "training_duration_seconds" with value: ${trainingTime}`)
+      trainingDuration.observe({ status: training.status }, trainingTime)
     })
 
     app.on('model_loaded', async (data: ModelLoadedData) => {
+      prometheusLogger.debug(`adding metric "model_storage_read_duration" with value: ${data.readTime}`)
       modelStorageReadDuration.observe(data.readTime)
+
+      prometheusLogger.debug(`adding metric "model_memory_load_duration" with value: ${data.loadTime}`)
       modelMemoryLoadDuration.observe(data.loadTime)
     })
 
@@ -68,6 +79,41 @@ export const createAPI = async (options: APIOptions, app: Application, baseLogge
     })
   }
 
+  if (options.usageURL) {
+    const usageLogger = apiLogger.sub('usage')
+    usageLogger.debug('usage endpoint enabled')
+
+    const usageClient = new UsageClient(options.usageURL)
+    app.on('training_update', async (training: Training) => {
+      if (isTrainingRunning(training) || !training.trainingTime) {
+        return
+      }
+
+      const { appId, modelId, trainingTime } = training
+      const app_id = appId
+      const model_id = modelIdService.toString(modelId)
+      const training_time = trainingTime / 1000
+      const timestamp = new Date().toISOString()
+
+      const type = 'training_time'
+      const value = {
+        app_id,
+        model_id,
+        training_time,
+        timestamp
+      }
+
+      usageLogger.debug(`sending usage ${type} with value: ${JSON.stringify(value)}`)
+
+      try {
+        await usageClient.sendUsage('nlu', type, [value])
+      } catch (thrown) {
+        const err = thrown instanceof Error ? thrown : new Error(`${thrown}`)
+        usageLogger.attachError(err).error(`an error occured when sending "${type}" usage.`)
+      }
+    })
+  }
+
   if (options.tracingEnabled) {
     await initTracing('nlu')
   }

packages/nlu-server/src/telemetry/usage-client/client.ts

@@ -0,0 +1,44 @@
+import axios from 'axios'
+import _ from 'lodash'
+import { UsagePayload, UsageData, UsageSender, UsageType } from './typings'
+
+export class UsageClient {
+  constructor(private usageURL: string) {}
+
+  public async sendUsage<S extends UsageSender, T extends UsageType>(sender: S, type: T, records: UsageData<S, T>[]) {
+    const timestamp = new Date().toISOString()
+    const usage: UsagePayload<S, T> = {
+      meta: {
+        timestamp,
+        schema_version: '1.0.0',
+        sender,
+        type
+      },
+      schema_version: '1.0.0',
+      records
+    }
+
+    try {
+      await axios.post(this.usageURL, usage)
+    } catch (err) {
+      if (axios.isAxiosError(err) && err.response?.data) {
+        const { data } = err.response
+        const message = this._serialize(data)
+        err.message += `: ${message}`
+      }
+      throw err
+    }
+  }
+
+  private _serialize = (data: any): string => {
+    if (_.isString(data)) {
+      return data
+    }
+    try {
+      const str = JSON.stringify(data)
+      return str
+    } catch (err) {
+      return `${data}`
+    }
+  }
+}

packages/nlu-server/src/telemetry/usage-client/index.ts

@@ -0,0 +1,2 @@
+export * from './typings'
+export * from './client'

packages/nlu-server/src/telemetry/usage-client/typings.ts

@@ -0,0 +1,30 @@
+type Is<X, Y> = X extends Y ? true : false
+type And<X extends boolean, Y extends boolean> = X extends false ? false : Y extends false ? false : true
+
+export type UsageSender = 'nlu' // other services might also send usage
+export type UsageType = 'training_time' // other services might also send other usage types
+
+export type UsageMetadata<S extends UsageSender, T extends UsageType> = {
+  timestamp: string
+  sender: S
+  type: T
+  schema_version: string
+}
+
+export type UsageData<S extends UsageSender, T extends UsageType> = And<
+  Is<S, 'nlu'>,
+  Is<T, 'training_time'>
+> extends true
+  ? {
+      app_id: string
+      model_id: string
+      training_time: number
+      timestamp: string
+    }
+  : never // other combination of sender + type might have other payload
+
+export type UsagePayload<S extends UsageSender, T extends UsageType> = {
+  meta: UsageMetadata<S, T>
+  schema_version: string
+  records: UsageData<S, T>[]
+}

packages/nlu-server/src/typings.d.ts

@@ -25,6 +25,7 @@ export type NLUServerOptions = {
   languageAuthToken?: string
   ducklingURL: string
   ducklingEnabled: boolean
+  usageURL?: string
 }
 
 export type CommandLineOptions = Partial<NLUServerOptions>