Merge branch 'feat/better-json-schemas-developer-experience' into dev…

…elop

.gitignore

@@ -33,3 +33,4 @@ skills/**/memory/*.json
 core/data/models/*.nlp
 package.json.backup
 .python-version
+schemas/**/*.json

core/config/voice/amazon.sample.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../schemas/voice-config-schemas/amazon.json",
   "credentials": {
     "accessKeyId": "",
     "secretAccessKey": ""

core/config/voice/google-cloud.sample.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../schemas/voice-config-schemas/google-cloud.json",
   "type": "service_account",
   "project_id": "",
   "private_key_id": "",

core/config/voice/watson-stt.sample.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../schemas/voice-config-schemas/watson-stt.json",
   "apikey": "",
   "url": "https://stream.watsonplatform.net/speech-to-text/api"
 }

core/config/voice/watson-tts.sample.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../schemas/voice-config-schemas/watson-tts.json",
   "apikey": "",
   "url": "https://stream.watsonplatform.net/text-to-speech/api"
 }

core/data/en/answers.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../schemas/global-data/global-answers.json",
   "answers": {
     "success": {},
     "errors": {

core/data/en/global-entities/color.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../../schemas/global-data/global-entity.json",
   "options": {
     "red": {
       "synonyms": ["red"],

core/data/en/global-entities/level.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../../schemas/global-data/global-entity.json",
   "options": {
     "LOW": {
       "synonyms": ["low"]

core/data/en/global-entities/partner_assistant.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../../schemas/global-data/global-entity.json",
   "options": {
     "Alexa": {
       "synonyms": ["Alexa"],

core/data/en/global-resolvers/affirmation_denial.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../../schemas/global-data/global-resolver.json",
   "name": "affirmation_denial",
   "intents": {
     "affirmation": {

core/data/fr/answers.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../schemas/global-data/global-answers.json",
   "answers": {
     "success": {},
     "errors": {

core/data/fr/global-entities/color.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../../schemas/global-data/global-entity.json",
   "options": {
     "rouge": {
       "synonyms": ["rouge"],

core/data/fr/global-entities/level.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../../schemas/global-data/global-entity.json",
   "options": {
     "bas": {
       "synonyms": ["bas", "basse"],

core/data/fr/global-entities/partner_assistant.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../../schemas/global-data/global-entity.json",
   "options": {
     "Alexa": {
       "synonyms": ["Alexa"],

core/data/fr/global-resolvers/affirmation_denial.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../../../../schemas/global-data/global-resolver.json",
   "name": "affirmation_denial",
   "intents": {
     "affirmation": {

package.json

@@ -47,6 +47,7 @@
     "prepare": "husky install",
     "generate:skills-endpoints": "ts-node scripts/generate/run-generate-skills-endpoints.js",
     "generate:http-api-key": "ts-node scripts/generate/run-generate-http-api-key.js",
+    "generate:json-schemas": "ts-node scripts/generate/run-generate-json-schemas.js",
     "build": "npm run build:app && npm run build:server",
     "build:app": "cross-env LEON_NODE_ENV=production ts-node scripts/app/run-build-app.js",
     "build:server": "npm run delete-dist:server && npm run train && npm run generate:skills-endpoints && tsc && resolve-tspaths && shx rm -rf server/dist/core server/dist/package.json && shx mv -f server/dist/server/src/* server/dist && shx rm -rf server/dist/server && shx mkdir -p server/dist/tmp",

scripts/generate/generate-json-schemas.js

@@ -0,0 +1,78 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+import { LogHelper } from '@/helpers/log-helper'
+import {
+  domainSchemaObject,
+  skillSchemaObject,
+  skillConfigSchemaObject
+} from '@/schemas/skill-schemas'
+import {
+  globalEntitySchemaObject,
+  globalResolverSchemaObject,
+  globalAnswersSchemaObject
+} from '@/schemas/global-data-schemas'
+import {
+  amazonVoiceConfiguration,
+  googleCloudVoiceConfiguration,
+  watsonVoiceConfiguration
+} from '@/schemas/voice-config-schemas'
+
+/**
+ * Generate JSON schemas
+ * @param {string} categoryName 
+ * @param {Map<string, Object>} schemas 
+ */
+export const generateSchemas = async (categoryName, schemas) => {
+  const categorySchemasPath = path.join(process.cwd(), 'schemas', categoryName)
+  await fs.promises.mkdir(categorySchemasPath, { recursive: true })
+  for (const [schemaName, schemaObject] of schemas.entries()) {
+    const schemaPath = path.join(categorySchemasPath, `${schemaName}.json`)
+    await fs.promises.writeFile(
+      schemaPath,
+      JSON.stringify(
+        {
+          $schema: 'https://json-schema.org/draft-07/schema',
+          ...schemaObject
+        },
+        null,
+        2
+      )
+    )
+  }
+}
+
+export default async () => {
+  LogHelper.info('Generating the JSON schemas...')
+  await Promise.all([
+    generateSchemas(
+      'global-data',
+      new Map([
+        ['global-entity', globalEntitySchemaObject],
+        ['global-resolver', globalResolverSchemaObject],
+        ['global-answers', globalAnswersSchemaObject]
+      ])
+    ),
+    generateSchemas(
+      'skill-schemas',
+      new Map([
+        ['domain', domainSchemaObject],
+        ['skill', skillSchemaObject],
+        ['skill-config', skillConfigSchemaObject]
+      ])
+    ),
+    generateSchemas(
+      'voice-config-schemas',
+      new Map([
+        ['amazon', amazonVoiceConfiguration],
+        [
+          'google-cloud',
+          googleCloudVoiceConfiguration
+        ],
+        ['watson-stt', watsonVoiceConfiguration],
+        ['watson-tts', watsonVoiceConfiguration]
+      ])
+    )
+  ])
+  LogHelper.success('JSON schemas generated')
+}

scripts/generate/run-generate-json-schemas.js

@@ -0,0 +1,14 @@
+import { LogHelper } from '@/helpers/log-helper'
+
+import generateJsonSchemas from './generate-json-schemas'
+
+/**
+ * Execute the generating JSON schemas script
+ */
+;(async () => {
+  try {
+    await generateJsonSchemas()
+  } catch (error) {
+    LogHelper.error(`Failed to generate the json schemas: ${error}`)
+  }
+})()

scripts/setup/setup.js

@@ -3,6 +3,7 @@ import { LogHelper } from '@/helpers/log-helper'
 
 import train from '../train/train'
 import generateHttpApiKey from '../generate/generate-http-api-key'
+import generateJsonSchemas from '../generate/generate-json-schemas'
 
 import setupDotenv from './setup-dotenv'
 import setupCore from './setup-core'
@@ -22,6 +23,7 @@ import setupPythonBinaries from './setup-python-binaries'
     LoaderHelper.stop()
     await setupPythonBinaries()
     await generateHttpApiKey()
+    await generateJsonSchemas()
     LoaderHelper.start()
     await train()
 

server/src/pre-check.ts

@@ -37,13 +37,24 @@ interface ObjectUnknown {
 }
 
 const validateSchema = (
+  schemaName: string,
   schema: ObjectUnknown,
   contentToValidate: ObjectUnknown,
   customErrorMessage: string
 ): void => {
+  const schemaFile = `${schemaName}.json`
   const validate = ajv.compile(schema)
-  const isValid = validate(contentToValidate)
+  const isValidSchemaKey =
+    typeof contentToValidate['$schema'] === 'string' &&
+    contentToValidate['$schema'].endsWith(schemaFile)
+  const isValid = validate(contentToValidate) && isValidSchemaKey
   if (!isValid) {
+    LogHelper.error(customErrorMessage)
+    if (!isValidSchemaKey) {
+      LogHelper.error(
+        `The schema key "$schema" is not valid. Expected "${schemaName}", but got "${contentToValidate['$schema']}".`
+      )
+    }
     LogHelper.error(customErrorMessage)
     const errors = new AggregateAjvError(validate.errors ?? [])
     for (const error of errors) {
@@ -88,6 +99,7 @@ const GLOBAL_DATA_SCHEMAS = {
     )
     const [configName] = file.split('.') as [keyof typeof VOICE_CONFIG_SCHEMAS]
     validateSchema(
+      `voice-config-schemas/${configName}`,
       VOICE_CONFIG_SCHEMAS[configName],
       config,
       `The voice configuration schema "${voiceConfigPath}" is not valid:`
@@ -116,6 +128,7 @@ const GLOBAL_DATA_SCHEMAS = {
         await fs.promises.readFile(globalEntityPath, 'utf8')
       )
       validateSchema(
+        'global-data/global-entity',
         globalEntitySchemaObject,
         globalEntity,
         `The global entity schema "${globalEntityPath}" is not valid:`
@@ -136,6 +149,7 @@ const GLOBAL_DATA_SCHEMAS = {
         await fs.promises.readFile(globalResolverPath, 'utf8')
       )
       validateSchema(
+        'global-data/global-resolver',
         globalResolverSchemaObject,
         globalResolver,
         `The global resolver schema "${globalResolverPath}" is not valid:`
@@ -150,6 +164,7 @@ const GLOBAL_DATA_SCHEMAS = {
       await fs.promises.readFile(globalAnswersPath, 'utf8')
     )
     validateSchema(
+      'global-data/global-answers',
       GLOBAL_DATA_SCHEMAS.answers,
       answers,
       `The global answers schema "${globalAnswersPath}" is not valid:`
@@ -173,6 +188,7 @@ const GLOBAL_DATA_SCHEMAS = {
       await fs.promises.readFile(pathToDomain, 'utf8')
     )
     validateSchema(
+      'skill-schemas/domain',
       domainSchemaObject,
       domainObject,
       `The domain schema "${pathToDomain}" is not valid:`
@@ -192,6 +208,7 @@ const GLOBAL_DATA_SCHEMAS = {
           await fs.promises.readFile(pathToSkill, 'utf8')
         )
         validateSchema(
+          'skill-schemas/skill',
           skillSchemaObject,
           skillObject,
           `The skill schema "${pathToSkill}" is not valid:`
@@ -211,6 +228,7 @@ const GLOBAL_DATA_SCHEMAS = {
             await fs.promises.readFile(skillConfigPath, 'utf8')
           )
           validateSchema(
+            'skill-schemas/skill-config',
             skillConfigSchemaObject,
             skillConfig,
             `The skill config schema "${skillConfigPath}" is not valid:`

server/src/schemas/global-data-schemas.ts

@@ -17,40 +17,37 @@ export const globalEntitySchemaObject = Type.Strict(
         )
       )
     },
-    { additionalProperties: false }
+    {
+      description:
+        'Global entities can hold data that can directly be reused in skills.'
+    }
   )
 )
 export const globalResolverSchemaObject = Type.Strict(
-  Type.Object(
-    {
-      name: Type.String(),
-      intents: Type.Record(
-        Type.String(),
-        Type.Object(
-          {
-            utterance_samples: Type.Array(Type.String()),
-            value: Type.Unknown()
-          },
-          { additionalProperties: false }
-        )
+  Type.Object({
+    name: Type.String(),
+    intents: Type.Record(
+      Type.String(),
+      Type.Object(
+        {
+          utterance_samples: Type.Array(Type.String()),
+          value: Type.Unknown()
+        },
+        { additionalProperties: false }
       )
-    },
-    { additionalProperties: false }
-  )
+    )
+  })
 )
 export const globalAnswersSchemaObject = Type.Strict(
-  Type.Object(
-    {
-      answers: Type.Record(
-        Type.String(),
-        Type.Union([
-          Type.Record(Type.String(), Type.String()),
-          Type.Array(Type.String())
-        ])
-      )
-    },
-    { additionalProperties: false }
-  )
+  Type.Object({
+    answers: Type.Record(
+      Type.String(),
+      Type.Union([
+        Type.Record(Type.String(), Type.String()),
+        Type.Array(Type.String())
+      ])
+    )
+  })
 )
 
 export type GlobalEntitySchema = Static<typeof globalEntitySchemaObject>