Register missingProperties custom function to get all variables the…

… template contains (#5518) * Register missingProperties custom function * add test * fix comments * update * add test * fix case * add functions * add * add tests * fix * init * remove some restrict * do not export LG in TemplateEngineGenerator * fix issue * refine code * remove unused using * fix test
microsoft · Jul 7, 2021 · 7472193 · 7472193
1 parent d95c163
commit 7472193
Show file tree

Hide file tree

Showing 16 changed files with 472 additions and 44 deletions.
diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/AdaptiveDialog.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/AdaptiveDialog.cs
@@ -11,11 +11,14 @@
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using AdaptiveExpressions;
 using AdaptiveExpressions.Properties;
 using Microsoft.Bot.Builder.Dialogs.Adaptive.Conditions;
+using Microsoft.Bot.Builder.Dialogs.Adaptive.Generators;
 using Microsoft.Bot.Builder.Dialogs.Adaptive.Recognizers;
 using Microsoft.Bot.Builder.Dialogs.Adaptive.Selectors;
 using Microsoft.Bot.Builder.Dialogs.Debugging;
+using Microsoft.Bot.Builder.Dialogs.Functions;
 using Microsoft.Bot.Schema;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -735,14 +738,15 @@ protected async Task<DialogTurnResult> ContinueActionsAsync(DialogContext dc, ob
         /// OnSetScopedServices provides ability to set scoped services for the current dialogContext.
         /// </summary>
         /// <remarks>
-        /// USe dialogContext.Services.Set(object) to set a scoped object that will be inherited by all children dialogContexts.
+        /// Use dialogContext.Services.Set(object) to set a scoped object that will be inherited by all children dialogContexts.
         /// </remarks>
         /// <param name="dialogContext">dialog Context.</param>
         protected virtual void OnSetScopedServices(DialogContext dialogContext)
         {
             if (Generator != null)
             {
                 dialogContext.Services.Set(this.Generator);
+                Expression.Functions.Add(MissingPropertiesFunction.Name, new MissingPropertiesFunction(dialogContext));
             }
         }
 

diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/AdaptiveDialogBot.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/AdaptiveDialogBot.cs
@@ -126,6 +126,9 @@ private async Task SetUpTurnStateAsync(ITurnContext turnContext, BotFrameworkCli
             turnContext.TurnState.Add(_languagePolicy);
             turnContext.TurnState.Add(_telemetryClient);
 
+            // put global language policy into turn scope for lg functions fallback
+            ObjectPath.SetPathValue(turnContext.TurnState, TurnPath.LanguagePolicy, _languagePolicy);
+
             // Catch "SetTestOptions" event and save into "Conversation.TestOptions".
             // Note: This is consumed by AdaptiveExpressions Extensions.RandomNext
             if (turnContext.Activity.Type == ActivityTypes.Event)

diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Functions/MissingPropertiesFunction.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Functions/MissingPropertiesFunction.cs
@@ -0,0 +1,57 @@
+// Licensed under the MIT License.
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+using System.Collections.Generic;
+using AdaptiveExpressions;
+using AdaptiveExpressions.Memory;
+using Microsoft.Bot.Builder.Dialogs.Adaptive;
+
+namespace Microsoft.Bot.Builder.Dialogs.Functions
+{
+    /// <summary>
+    /// Defines missingProperties(template) expression function.
+    /// </summary>
+    /// <remarks>
+    /// This expression will get all variables the template contains.
+    /// </remarks>
+    public class MissingPropertiesFunction : ExpressionEvaluator
+    {
+        /// <summary>
+        /// Function identifier name.
+        /// </summary>
+        public const string Name = "missingProperties";
+
+        private const string GeneratorPath = "dialogclass.generator";
+
+        private static DialogContext dialogContext;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="MissingPropertiesFunction"/> class.
+        /// </summary>
+        /// <param name="context">Dialog context.</param>
+        public MissingPropertiesFunction(DialogContext context)
+            : base(Name, Function, ReturnType.Array, FunctionUtils.ValidateUnaryString)
+        {
+            dialogContext = context;
+        }
+
+        private static (object value, string error) Function(Expression expression, IMemory state, Options options)
+        {
+            var (args, error) = FunctionUtils.EvaluateChildren(expression, state, options);
+            if (error != null)
+            {
+                return (null, error);
+            }
+
+            var templateBody = args[0]?.ToString();
+
+            if (state.TryGetValue(GeneratorPath, out var lgGenerator))
+            {
+                var generator = lgGenerator as LanguageGenerator;
+                return (generator.MissingProperties(dialogContext, templateBody, state, options), null);
+            }
+
+            return (new List<string>(), null);
+        }
+    }
+}
diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Generators/MultiLanguageGeneratorBase.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Generators/MultiLanguageGeneratorBase.cs
@@ -6,7 +6,10 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using AdaptiveExpressions;
+using AdaptiveExpressions.Memory;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace Microsoft.Bot.Builder.Dialogs.Adaptive.Generators
 {
@@ -52,64 +55,152 @@ public MultiLanguageGeneratorBase()
         /// <returns>The generator.</returns>
         public override async Task<object> GenerateAsync(DialogContext dialogContext, string template, object data, CancellationToken cancellationToken = default)
         {
-            // priority 
-            // 1. local policy
-            // 2. shared policy in turnContext
-            // 3. default policy
-            var languagePolicy = this.LanguagePolicy ?? 
-                                dialogContext.Services.Get<LanguagePolicy>() ?? 
-                                new LanguagePolicy();
-
-            // see if we have any locales that match
-            var fallbackLocales = new List<string>();
-            var targetLocale = dialogContext.GetLocale();
+            var languagePolicy = GetLanguagePolicy(dialogContext);
+            var targetLocale = GetCurrentLocale(dialogContext);
+            var fallbackLocales = GetFallbackLocales(languagePolicy, targetLocale);
+            var generators = GetGenerators(dialogContext, fallbackLocales);
 
-            if (languagePolicy.ContainsKey(targetLocale))
-            {
-                fallbackLocales.AddRange(languagePolicy[targetLocale]);
-            }
-
-            // append empty as fallback to end
-            if (targetLocale.Length != 0 && languagePolicy.ContainsKey(string.Empty))
-            {
-                fallbackLocales.AddRange(languagePolicy[string.Empty]);
-            }
-
-            if (fallbackLocales.Count == 0)
+            if (generators.Count == 0)
             {
-                throw new InvalidOperationException($"No supported language found for {targetLocale}");
+                throw new InvalidOperationException($"No generator found for language {targetLocale}");
             }
 
-            var generators = new List<LanguageGenerator>();
-            foreach (var locale in fallbackLocales)
+            var errors = new List<string>();
+            foreach (var generator in generators)
             {
-                if (this.TryGetGenerator(dialogContext, locale, out LanguageGenerator generator))
+                try
                 {
-                    generators.Add(generator);
+                    return await generator.GenerateAsync(dialogContext, template, data, cancellationToken).ConfigureAwait(false);
+                }
+#pragma warning disable CA1031 // Do not catch general exception types (catch any exception and add it to the errors list).
+                catch (Exception err)
+#pragma warning restore CA1031 // Do not catch general exception types
+                {
+                    errors.Add(err.Message);
                 }
             }
 
+            throw new InvalidOperationException(string.Join(",\n", errors.Distinct()));
+        }
+
+        /// <summary>
+        /// Method to get missing properties.
+        /// </summary>
+        /// <param name="dialogContext">dialogContext.</param>
+        /// <param name="templateBody">template or [templateId].</param>
+        /// <param name="state">Memory state.</param>
+        /// <param name="options">Options.</param>
+        /// <param name="cancellationToken">the <see cref="CancellationToken"/> for the task.</param>
+        /// <returns>Property list.</returns>
+        public override List<string> MissingProperties(DialogContext dialogContext, string templateBody, IMemory state = null, Options options = null, CancellationToken cancellationToken = default)
+        {
+            var currentLocale = GetCurrentLocale(dialogContext, state, options);
+            var languagePolicy = GetLanguagePolicy(dialogContext, state);
+            var fallbackLocales = GetFallbackLocales(languagePolicy, currentLocale);
+            var generators = GetGenerators(dialogContext, fallbackLocales);
+
             if (generators.Count == 0)
             {
-                throw new InvalidOperationException($"No generator found for language {targetLocale}");
+                generators.Add(new TemplateEngineLanguageGenerator());
             }
 
-            var errors = new List<string>();
             foreach (var generator in generators)
             {
                 try
                 {
-                    return await generator.GenerateAsync(dialogContext, template, data, cancellationToken).ConfigureAwait(false);
+                    return generator.MissingProperties(dialogContext, templateBody, state, options, cancellationToken);
                 }
 #pragma warning disable CA1031 // Do not catch general exception types (catch any exception and add it to the errors list).
-                catch (Exception err)
+                catch
 #pragma warning restore CA1031 // Do not catch general exception types
                 {
-                    errors.Add(err.Message);
+                    // ignore
                 }
             }
 
-            throw new InvalidOperationException(string.Join(",\n", errors.Distinct()));
+            return new List<string>();
+        }
+
+        private string GetCurrentLocale(DialogContext dc, IMemory memory = null, Options options = null)
+        {
+            // order
+            // 1. turn.locale
+            // 2. options.locale
+            // 3. Context.Activity.Locale
+            // 4. Thread.CurrentThread.CurrentCulture.Name
+            string currentLocale;
+            if (memory != null && memory.TryGetValue(TurnPath.Locale, out var locale))
+            {
+                currentLocale = locale.ToString();
+            }
+            else if (options != null && !string.IsNullOrEmpty(options.Locale))
+            {
+                currentLocale = options.Locale;
+            }
+            else
+            {
+                currentLocale = dc.GetLocale();
+            }
+
+            return currentLocale;
+        }
+
+        private LanguagePolicy GetLanguagePolicy(DialogContext dc, IMemory memory = null)
+        {
+            // order
+            // 1. local languagePolicy
+            // 2. turn.languagePolicy
+            // 3. dialogContext.get<LanguagePolicy>
+            // 4. Default languagepolicy
+            if (LanguagePolicy != null)
+            {
+                return LanguagePolicy;
+            }
+
+            if (memory != null && memory.TryGetValue(TurnPath.LanguagePolicy, out var languagePolicyObj))
+            {
+                return JObject.FromObject(languagePolicyObj).ToObject<LanguagePolicy>();
+            }
+
+            return dc.Services.Get<LanguagePolicy>() ??
+                                new LanguagePolicy();
+        }
+
+        private List<string> GetFallbackLocales(LanguagePolicy languagePolicy, string currentLocale)
+        {
+            var fallbackLocales = new List<string>();
+
+            if (languagePolicy.ContainsKey(currentLocale))
+            {
+                fallbackLocales.AddRange(languagePolicy[currentLocale]);
+            }
+
+            // append empty as fallback to end
+            if (currentLocale.Length != 0 && languagePolicy.ContainsKey(string.Empty))
+            {
+                fallbackLocales.AddRange(languagePolicy[string.Empty]);
+            }
+
+            if (fallbackLocales.Count == 0)
+            {
+                throw new InvalidOperationException($"No supported language found for {currentLocale}");
+            }
+
+            return fallbackLocales;
+        }
+
+        private List<LanguageGenerator> GetGenerators(DialogContext dc, List<string> fallbackLocales)
+        {
+            var generators = new List<LanguageGenerator>();
+            foreach (var locale in fallbackLocales)
+            {
+                if (TryGetGenerator(dc, locale, out var generator))
+                {
+                    generators.Add(generator);
+                }
+            }
+
+            return generators;
         }
     }
 }
diff --git a/...ries/Microsoft.Bot.Builder.Dialogs.Adaptive/Generators/TemplateEngineLanguageGenerator.cs b/...ries/Microsoft.Bot.Builder.Dialogs.Adaptive/Generators/TemplateEngineLanguageGenerator.cs
@@ -4,8 +4,11 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using AdaptiveExpressions;
+using AdaptiveExpressions.Memory;
 using Microsoft.Bot.Builder.Dialogs.Debugging;
 using Microsoft.Bot.Builder.Dialogs.Declarative.Resources;
 using Microsoft.Bot.Builder.LanguageGeneration;
@@ -139,6 +142,32 @@ public override Task<object> GenerateAsync(DialogContext dialogContext, string t
             }
         }
 
+        /// <summary>
+        /// Method to get missing properties.
+        /// </summary>
+        /// <param name="dialogContext">dialogContext.</param>
+        /// <param name="template">template or [templateId].</param>
+        /// <param name="state">Memory.</param>
+        /// <param name="options">Options.</param>
+        /// <param name="cancellationToken">the <see cref="CancellationToken"/> for the task.</param>
+        /// <returns>Property list.</returns>
+        public override List<string> MissingProperties(DialogContext dialogContext, string template, IMemory state = null, Options options = null, CancellationToken cancellationToken = default)
+        {
+            var tempTemplateName = $"{LanguageGeneration.Templates.InlineTemplateIdPrefix}{Guid.NewGuid():N}";
+
+            var multiLineMark = "```";
+
+            template = !template.Trim().StartsWith(multiLineMark, StringComparison.Ordinal) && template.Contains('\n')
+                   ? $"{multiLineMark}{template}{multiLineMark}" : template;
+
+            lg.AddTemplate(tempTemplateName, null, $"- {template}");
+            var analyzerResults = lg.AnalyzeTemplate(tempTemplateName);
+
+            // Delete it after the analyzer
+            lg.DeleteTemplate(tempTemplateName);
+            return analyzerResults.Variables;
+        }
+
         private static void RunSync(Func<Task> func)
         {
 #pragma warning disable CA2008 // Do not create tasks without passing a TaskScheduler

diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/LanguageGenerator.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/LanguageGenerator.cs
@@ -1,8 +1,11 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
+using AdaptiveExpressions;
+using AdaptiveExpressions.Memory;
 
 namespace Microsoft.Bot.Builder.Dialogs.Adaptive
 {
@@ -22,5 +25,21 @@ public abstract class LanguageGenerator
 #pragma warning disable CA1716 // Identifiers should not match keywords (we can't change the template parameter name without breaking binary compat).
         public abstract Task<object> GenerateAsync(DialogContext dialogContext, string template, object data, CancellationToken cancellationToken = default);
 #pragma warning restore CA1716 // Identifiers should not match keywords
+
+        /// <summary>
+        /// Method to get missing properties.
+        /// </summary>
+        /// <param name="dialogContext">dialogContext.</param>
+        /// <param name="template">template or [templateId].</param>
+        /// <param name="state">Memory state.</param>
+        /// <param name="options">Options.</param>
+        /// <param name="cancellationToken">the <see cref="CancellationToken"/> for the task.</param>
+        /// <returns>Property list.</returns>
+#pragma warning disable CA1716 // Identifiers should not match keywords (we can't change the template parameter name without breaking binary compat).
+        public virtual List<string> MissingProperties(DialogContext dialogContext, string template, IMemory state = null, Options options = null, CancellationToken cancellationToken = default)
+#pragma warning restore CA1716 // Identifiers should not match keywords
+        {
+            return new List<string>();
+        }
     }
 }
diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/LanguageGeneratorExtensions.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/LanguageGeneratorExtensions.cs
@@ -79,6 +79,9 @@ public static DialogManager UseLanguageGeneration(this DialogManager dialogManag
         public static DialogManager UseLanguagePolicy(this DialogManager dialogManager, LanguagePolicy policy)
         {
             dialogManager.InitialTurnState.Add<LanguagePolicy>(policy);
+
+            // put global language policy into turn scope for lg functions fallback
+            ObjectPath.SetPathValue(dialogManager.InitialTurnState, TurnPath.LanguagePolicy, policy);
             return dialogManager;
         }
     }