Forward logs of Experiment's sub MLContexts to main MLContext (#5554)

* Forward logs of Experiment's sub MLContexts to main MLContext * Adressed reviews
dotnet · Dec 16, 2020 · ae4d85c · ae4d85c
1 parent e3597cd
commit ae4d85c
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 0 deletions.
diff --git a/src/Microsoft.ML.AutoML/Experiment/Experiment.cs b/src/Microsoft.ML.AutoML/Experiment/Experiment.cs
@@ -85,6 +85,29 @@ private void MainContextCanceledEvent(object state)
             }
         }
 
+        private void RelayCurrentContextLogsToLogger(object sender, LoggingEventArgs e)
+        {
+            // Relay logs that are generated by the current MLContext to the Experiment class's
+            // _logger.
+            switch (e.Kind)
+            {
+                case ChannelMessageKind.Trace:
+                    _logger.Trace(e.Message);
+                    break;
+                case ChannelMessageKind.Info:
+                    _logger.Info(e.Message);
+                    break;
+                case ChannelMessageKind.Warning:
+                    _logger.Warning(e.Message);
+                    break;
+                case ChannelMessageKind.Error:
+                    _logger.Error(e.Message);
+                    break;
+                default:
+                    throw new NotImplementedException($"{nameof(ChannelMessageKind)}.{e.Kind} is not yet implemented.");
+            }
+        }
+
         public IList<TRunDetail> Execute()
         {
             var iterationResults = new List<TRunDetail>();
@@ -129,6 +152,7 @@ public IList<TRunDetail> Execute()
                     // context is canceled to stop further model training. The cancellation of the main MLContext
                     // a user has instantiated is not desirable, thus additional MLContexts are used.
                     _currentModelMLContext = _newContextSeedGenerator == null ? new MLContext() : new MLContext(_newContextSeedGenerator.Next());
+                    _currentModelMLContext.Log += RelayCurrentContextLogsToLogger;
                     var pipeline = PipelineSuggester.GetNextInferredPipeline(_currentModelMLContext, _history, _datasetColumnInfo, _task,
                         _optimizingMetricInfo.IsMaximizing, _experimentSettings.CacheBeforeTrainer, _logger, _trainerAllowList);
                     // break if no candidates returned, means no valid pipeline available

diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
@@ -21,10 +21,40 @@ namespace Microsoft.ML.AutoML.Test
 {
     public class AutoFitTests : BaseTestClass
     {
+        // Marker necessary for AutoFitContextLogTest to ensure that the wanted logs
+        // from Experiment's sub MLContexts were relayed to the main calling MLContext.
+        bool _markerAutoFitContextLogTest;
         public AutoFitTests(ITestOutputHelper output) : base(output)
         {
         }
 
+        private void MlContextLog(object sender, LoggingEventArgs e)
+        {
+            // Log containing ImageClassificationTrainer will only come from AutoML's sub
+            // contexts.
+            if (!_markerAutoFitContextLogTest && e.Message.Contains("[Source=ImageClassificationTrainer;"))
+                _markerAutoFitContextLogTest = true;
+        }
+
+        [TensorFlowFact]
+        public void AutoFitContextLogTest()
+        {
+            // This test confirms that logs produced from contexts made during AutoML experiment
+            // runs are correctly relayed to the main Experiment MLContext.
+            _markerAutoFitContextLogTest = false;
+            var context = new MLContext(1);
+            context.Log += MlContextLog;
+            var datasetPath = DatasetUtil.GetFlowersDataset();
+            var columnInference = context.Auto().InferColumns(datasetPath, "Label");
+            var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions);
+            var trainData = textLoader.Load(datasetPath);
+            var result = context.Auto()
+                            .CreateMulticlassClassificationExperiment(15)
+                            .Execute(trainData, columnInference.ColumnInformation);
+            Assert.True(_markerAutoFitContextLogTest, "Image classification trainer logs from Experiment's sub contexts" +
+                "were not relayed to the main MLContext.");
+        }
+
         [Fact]
         public void AutoFitBinaryTest()
         {