Convert timestamp to opt-in feature

docs/Test.md

@@ -140,6 +140,11 @@ See [Caching speech-to-text transcriptions](#caching-speech-to-text-transcriptio
 
 This is currently only used for LUIS, see the section on LUIS prebuilt entities in [Configuring prebuilt entities](LuisModelConfiguration.md#configuring-prebuilt-entities).
 
+### `--timestamp`
+(Optional) Signals whether to add a timestamp to each NLU test result.
+
+See the documentation on the [`timestamp` property](UtteranceExtensions.md#returning-timestamps-for-each-query) for more details.
+
 ### `-i, --include`
 (Optional) Path to custom NLU provider DLL. See documentation about [Specifying the include path](https://github.com/microsoft/NLU.DevOps/blob/master/docs/CliExtensions.md#specifying-the-include-path) for more details.
 

docs/UtteranceExtensions.md

@@ -22,7 +22,7 @@ When an NLU provider in NLU.DevOps returns a prediction result, the value will b
 ```
 In this case, the intent confidence score was `0.99` and the text transcription confidence score was `0.95`. This is useful context when debugging false predictions, as a low confidence score may indicate that the model could be improved with more training examples. The recognized `genre` entity also includes a confidence score of `0.80`, although it should be noted that only the LUIS provider currently returns confidence score for entity types trained from examples.
 
-## Labeled utterance timestamps
+## Returning timestamps for each query
 
 When analyzing results for a set of NLU predictions, it is often important context to understand when the test was run. For example, for Dialogflow `date` and `time` entities, the service only returns a date time string, and no indication of what token(s) triggered that entity to be recognized. For example, the result from a query like `"Call a taxi in 15 minutes"` may look like the following:
 ```json
@@ -38,7 +38,7 @@ When analyzing results for a set of NLU predictions, it is often important conte
     "timestamp": "2020-01-01T00:00:00-04:00"
 }
 ```
-Without the context provided by the `timestamp` property, we wouldn't be able to make any assertion about the correctness of the `entityValue` property for time. Currently, LUIS, Lex, and Dialogflow return a timestamp for each prediction result.
+Without the context provided by the `timestamp` property, we wouldn't be able to make any assertion about the correctness of the `entityValue` property for time. Currently, you must specify the [`--timestamp`](Test.md#--timestamp) option to ensure a timestamp is assigned to each NLU prediction result.
 
 ## Adjusting entity compare results
 

src/NLU.DevOps.CommandLine/Test/TestCommand.cs

@@ -6,7 +6,9 @@ namespace NLU.DevOps.CommandLine.Test
     using System;
     using System.Collections.Generic;
     using System.IO;
+    using System.Threading;
     using System.Threading.Tasks;
+    using Core;
     using Models;
     using Newtonsoft.Json.Linq;
     using static Serializer;
@@ -31,7 +33,8 @@ public override int Main()
 
         protected override INLUTestClient CreateNLUTestClient()
         {
-            return NLUClientFactory.CreateTestInstance(this.Options, this.Configuration, this.Options.SettingsPath);
+            var client = NLUClientFactory.CreateTestInstance(this.Options, this.Configuration, this.Options.SettingsPath);
+            return this.Options.Timestamp ? new TimestampNLUTestClient(client) : client;
         }
 
         private static void EnsureDirectory(string filePath)
@@ -129,5 +132,34 @@ private IEnumerable<(JToken Query, string SpeechFile)> LoadUtterances()
                 yield return (query, speechFile);
             }
         }
+
+        private class TimestampNLUTestClient : INLUTestClient
+        {
+            public TimestampNLUTestClient(INLUTestClient client)
+            {
+                this.Client = client;
+            }
+
+            private INLUTestClient Client { get; }
+
+            public async Task<LabeledUtterance> TestAsync(JToken query, CancellationToken cancellationToken)
+            {
+                var timestamp = DateTimeOffset.Now;
+                var result = await this.Client.TestAsync(query, cancellationToken).ConfigureAwait(false);
+                return result.WithTimestamp(timestamp);
+            }
+
+            public async Task<LabeledUtterance> TestSpeechAsync(string speechFile, JToken query, CancellationToken cancellationToken)
+            {
+                var timestamp = DateTimeOffset.Now;
+                var result = await this.Client.TestSpeechAsync(speechFile, query, cancellationToken).ConfigureAwait(false);
+                return result.WithTimestamp(timestamp);
+            }
+
+            public void Dispose()
+            {
+                this.Client.Dispose();
+            }
+        }
     }
 }

src/NLU.DevOps.CommandLine/Test/TestOptions.cs

@@ -25,5 +25,8 @@ internal class TestOptions : BaseOptions
 
         [Option('p', "parallelism", HelpText = "Numeric value to determine the numer of parallel tests.  Default value is 3.", Required = false)]
         public int Parallelism { get; set; } = 3;
+
+        [Option("timestamp", HelpText = "Assign a timestamp to each utterance result.", Required = false)]
+        public bool Timestamp { get; set; }
     }
 }

src/NLU.DevOps.Dialogflow/DialogflowNLUTestClient.cs

@@ -78,8 +78,7 @@ protected override async Task<LabeledUtterance> TestAsync(string utterance, Canc
                                 result.QueryResult.Intent.DisplayName,
                                 result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList())
                             .WithScore(result.QueryResult.IntentDetectionConfidence)
-                            .WithTextScore(result.QueryResult.SpeechRecognitionConfidence)
-                            .WithTimestamp(DateTimeOffset.Now);
+                            .WithTextScore(result.QueryResult.SpeechRecognitionConfidence);
                     },
                     cancellationToken)
                 .ConfigureAwait(false);
@@ -115,8 +114,7 @@ protected override async Task<LabeledUtterance> TestSpeechAsync(string speechFil
                                 result.QueryResult.Intent.DisplayName,
                                 result.QueryResult.Parameters?.Fields.SelectMany(GetEntities).ToList())
                             .WithScore(result.QueryResult.IntentDetectionConfidence)
-                            .WithTextScore(result.QueryResult.SpeechRecognitionConfidence)
-                            .WithTimestamp(DateTimeOffset.Now);
+                            .WithTextScore(result.QueryResult.SpeechRecognitionConfidence);
                     },
                     cancellationToken)
                 .ConfigureAwait(false);

src/NLU.DevOps.Lex.Tests/LexNLUTestClientTests.cs

@@ -86,9 +86,6 @@ public static async Task TestsWithSpeech(string slots, string entityType, string
                 // assert reads content from file (file contents are "hello world")
                 content.Should().Be("hello world");
 
-                // assert result type
-                result.Should().BeOfType<JsonLabeledUtterance>();
-
                 // assert intent and text
                 result.Intent.Should().Be(intent);
                 result.Text.Should().Be(transcript);
@@ -121,7 +118,6 @@ public static async Task CreatesLabeledUtterances()
             using (var lex = new LexNLUTestClient(string.Empty, string.Empty, mockClient.Object))
             {
                 var response = await lex.TestAsync(text).ConfigureAwait(false);
-                response.Should().BeOfType<JsonLabeledUtterance>();
                 response.Text.Should().Be(text);
                 response.Intent.Should().Be(intent);
                 response.Entities.Should().BeEmpty();

src/NLU.DevOps.Lex/LexNLUTestClient.cs

@@ -87,8 +87,7 @@ protected override async Task<LabeledUtterance> TestAsync(string utterance, Canc
                 .Select(slot => new Entity(slot.Key, slot.Value, null, 0))
                 .ToArray();
 
-            return new LabeledUtterance(utterance, postTextResponse.IntentName, entities)
-                .WithTimestamp(DateTimeOffset.Now);
+            return new LabeledUtterance(utterance, postTextResponse.IntentName, entities);
         }
 
         /// <inheritdoc />
@@ -118,8 +117,7 @@ protected override async Task<LabeledUtterance> TestSpeechAsync(string speechFil
                         .ToArray()
                     : null;
 
-                return new JsonLabeledUtterance(postContentResponse.InputTranscript, postContentResponse.IntentName, slots)
-                    .WithTimestamp(DateTimeOffset.Now);
+                return new LabeledUtterance(postContentResponse.InputTranscript, postContentResponse.IntentName, slots);
             }
         }
 

src/NLU.DevOps.Luis.Tests/LuisNLUTestClientTests.cs

@@ -66,7 +66,6 @@ public static async Task TestModel()
             using (var luis = builder.Build())
             {
                 var result = await luis.TestAsync(test).ConfigureAwait(false);
-                result.Should().BeOfType<JsonLabeledUtterance>();
                 result.Text.Should().Be(test);
                 result.Intent.Should().Be("intent");
                 result.Entities.Count.Should().Be(1);

src/NLU.DevOps.Luis/LuisNLUTestClient.cs

@@ -134,8 +134,7 @@ Entity getEntity(EntityModel entity)
                     speechLuisResult.LuisResult.Entities?.Select(getEntity).ToList())
                 .WithProperty("intents", speechLuisResult.LuisResult.Intents)
                 .WithScore(speechLuisResult.LuisResult.TopScoringIntent?.Score)
-                .WithTextScore(speechLuisResult.TextScore)
-                .WithTimestamp(DateTimeOffset.Now);
+                .WithTextScore(speechLuisResult.TextScore);
         }
     }
 }

src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs

@@ -70,7 +70,6 @@ public static async Task TestModel()
             using (var luis = builder.Build())
             {
                 var result = await luis.TestAsync(test).ConfigureAwait(false);
-                result.Should().BeOfType<JsonLabeledUtterance>();
                 result.Text.Should().Be(test);
                 result.Intent.Should().Be("intent");
                 result.Entities.Count.Should().Be(1);

src/NLU.DevOps.LuisV3/LuisNLUTestClient.cs

@@ -208,8 +208,7 @@ private LabeledUtterance LuisResultToLabeledUtterance(SpeechPredictionResponse s
             return new LabeledUtterance(query, intent, entities)
                 .WithProperty("intents", intents)
                 .WithScore(intentData?.Score)
-                .WithTextScore(speechPredictionResponse.TextScore)
-                .WithTimestamp(DateTimeOffset.Now);
+                .WithTextScore(speechPredictionResponse.TextScore);
         }
     }
 }