MDL-64787 tool_analytics: Include training data source in the log table

admin/tool/analytics/classes/output/model_logs.php

@@ -41,6 +41,11 @@ class model_logs extends \table_sql {
      */
     protected $model = null;
 
+    /**
+     * @var string|false
+     */
+    protected $evaluationmode = false;
+
     /**
      * Sets up the table_log parameters.
      *
@@ -57,21 +62,32 @@ public function __construct($uniqueid, $model) {
         $this->set_attribute('class', 'modellog generaltable generalbox');
         $this->set_attribute('aria-live', 'polite');
 
-        $this->define_columns(array('time', 'version', 'indicators', 'timesplitting', 'accuracy', 'info', 'usermodified'));
+        $this->define_columns(array('time', 'version', 'evaluationmode', 'indicators', 'timesplitting',
+            'accuracy', 'info', 'usermodified'));
         $this->define_headers(array(
             get_string('time'),
             get_string('version'),
+            get_string('evaluationmode', 'tool_analytics'),
             get_string('indicators', 'tool_analytics'),
             get_string('timesplittingmethod', 'analytics'),
             get_string('accuracy', 'tool_analytics'),
             get_string('info', 'tool_analytics'),
             get_string('fullnameuser'),
         ));
+
+        $evaluationmodehelp = new \help_icon('evaluationmode', 'tool_analytics');
+        $this->define_help_for_headers([null, null, $evaluationmodehelp, null, null, null, null, null]);
+
         $this->pageable(true);
         $this->collapsible(false);
         $this->sortable(false);
         $this->is_downloadable(false);
 
+        $this->evaluationmode = optional_param('evaluationmode', false, PARAM_ALPHANUM);
+        if ($this->evaluationmode && $this->evaluationmode != 'configuration' && $this->evaluationmode != 'trainedmodel') {
+            $this->evaluationmode = '';
+        }
+
         $this->define_baseurl($PAGE->url);
     }
 
@@ -86,6 +102,15 @@ public function col_version($log) {
         return userdate($log->version, $recenttimestr);
     }
 
+    /**
+     * Generate the evaluation mode column.
+     *
+     * @param \stdClass $log log data.
+     * @return string HTML for the evaluationmode column
+     */
+    public function col_evaluationmode($log) {
+        return get_string('evaluationmodecol' . $log->evaluationmode, 'tool_analytics');
+    }
     /**
      * Generate the time column.
      *

admin/tool/analytics/classes/output/models_list.php

@@ -201,7 +201,7 @@ public function export_for_template(\renderer_base $output) {
             }
 
             // Machine-learning-based models evaluation log.
-            if (!$model->is_static()) {
+            if (!$model->is_static() && $model->get_logs()) {
                 $urlparams['action'] = 'log';
                 $url = new \moodle_url('model.php', $urlparams);
                 $icon = new \action_menu_link_secondary($url, new \pix_icon('i/report', get_string('viewlog', 'tool_analytics')),

admin/tool/analytics/cli/evaluate_model.php

@@ -67,11 +67,17 @@
     exit(0);
 }
 
-if ($options['list'] || $options['modelid'] === false) {
+if ($options['list']) {
     \tool_analytics\clihelper::list_models();
     exit(0);
 }
 
+if ($options['modelid'] === false) {
+    // All actions but --list require a modelid.
+    echo $help;
+    exit(0);
+}
+
 // Reformat them as an array.
 if ($options['filter'] !== false) {
     $options['filter'] = explode(',', $options['filter']);
@@ -81,6 +87,10 @@
     cli_error('Error: The provided mode is not supported');
 }
 
+if ($options['mode'] == 'trainedmodel' && $options['timesplitting']) {
+    cli_error('Sorry, no time splitting method can be specified when using \'trainedmodel\' mode.');
+}
+
 // We need admin permissions.
 \core\session\manager::set_user(get_admin());
 

admin/tool/analytics/lang/en/tool_analytics.php

@@ -54,8 +54,16 @@
 $string['evaluate'] = 'Evaluate';
 $string['evaluatemodel'] = 'Evaluate model';
 $string['evaluationmode'] = 'Evaluation mode';
-$string['evaluationmodeinfo'] = 'This model has been trained using data from another site. You can evaluate the performance of the trained model on your site, or you can evaluate the performance of this model configuration using the data available on this site.';
+$string['evaluationmode_help'] = 'There are two evaluation modes:
+
+* Trained model -  Site data is used as testing data to evaluate the accuracy of the trained model.
+* Configuration - Site data is split into training and testing data, to both train and test the accuracy of the model configuration.
+
+Trained model is only available if a trained model has been imported into the site, and has not yet been re-trained using site data.';
+$string['evaluationmodeinfo'] = 'This model has been imported into the site. You can either evaluate the performance of the model, or you can evaluate the performance of the model configuration using site data.';
 $string['evaluationmodetrainedmodel'] = 'Evaluate the trained model';
+$string['evaluationmodecoltrainedmodel'] = 'Trained model';
+$string['evaluationmodecolconfiguration'] = 'Configuration';
 $string['evaluationmodeconfiguration'] = 'Evaluate the model configuration';
 $string['evaluationinbatches'] = 'The site contents are calculated and stored in batches. The evaluation process may be stopped at any time. The next time it is run, it will continue from the point when it was stopped.';
 $string['exportmodel'] = 'Export configuration';
@@ -108,7 +116,7 @@
 $string['trainingresults'] = 'Training results';
 $string['trainmodels'] = 'Train models';
 $string['versionnotsame'] = 'Imported file was from a different moodle version ({$a->importedversion}) than the current one ({$a->version})';
-$string['viewlog'] = 'Log';
+$string['viewlog'] = 'Evaluation log';
 $string['weeksenddateautomaticallyset'] = 'End date automatically set based on start date and the number of sections';
 $string['weeksenddatedefault'] = 'End date automatically calculated from the course start date.';
 $string['privacy:metadata'] = 'The Analytic models plugin does not store any personal data.';

analytics/classes/model.php

@@ -542,15 +542,22 @@ public function evaluate($options = array()) {
             $options['mode'] = 'configuration';
         }
 
-        if ($options['mode'] == 'trainedmodel') {
+        switch ($options['mode']) {
+            case 'trainedmodel':
 
-            // We are only interested on the time splitting method used by the trained model.
-            $options['timesplitting'] = $this->model->timesplitting;
+                // We are only interested on the time splitting method used by the trained model.
+                $options['timesplitting'] = $this->model->timesplitting;
 
-            // Provide the trained model directory to the ML backend if that is what we want to evaluate.
-            $trainedmodeldir = $this->get_output_dir(['execution']);
-        } else {
-            $trainedmodeldir = false;
+                // Provide the trained model directory to the ML backend if that is what we want to evaluate.
+                $trainedmodeldir = $this->get_output_dir(['execution']);
+                break;
+            case 'configuration':
+
+                $trainedmodeldir = false;
+                break;
+
+            default:
+                throw new \moodle_exception('errorunknownaction', 'analytics');
         }
 
         $this->init_analyser($options);
@@ -612,7 +619,7 @@ public function evaluate($options = array()) {
                 $dir = $predictorresult->dir;
             }
 
-            $result->logid = $this->log_result($timesplitting->get_id(), $result->score, $dir, $result->info);
+            $result->logid = $this->log_result($timesplitting->get_id(), $result->score, $dir, $result->info, $options['mode']);
 
             $results[$timesplitting->get_id()] = $result;
         }
@@ -1526,14 +1533,16 @@ protected function flag_file_as_used(\stored_file $file, $action) {
      * @param float $score
      * @param string $dir
      * @param array $info
+     * @param string $evaluationmode
      * @return int The inserted log id
      */
-    protected function log_result($timesplittingid, $score, $dir = false, $info = false) {
+    protected function log_result($timesplittingid, $score, $dir = false, $info = false, $evaluationmode = 'configuration') {
         global $DB, $USER;
 
         $log = new \stdClass();
         $log->modelid = $this->get_id();
         $log->version = $this->model->version;
+        $log->evaluationmode = $evaluationmode;
         $log->target = $this->model->target;
         $log->indicators = $this->model->indicators;
         $log->timesplitting = $timesplittingid;

lib/db/install.xml

@@ -3825,6 +3825,7 @@
         <FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
         <FIELD NAME="modelid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
         <FIELD NAME="version" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="evaluationmode" TYPE="char" LENGTH="50" NOTNULL="true" SEQUENCE="false"/>
         <FIELD NAME="target" TYPE="char" LENGTH="255" NOTNULL="true" SEQUENCE="false"/>
         <FIELD NAME="indicators" TYPE="text" NOTNULL="true" SEQUENCE="false"/>
         <FIELD NAME="timesplitting" TYPE="char" LENGTH="255" NOTNULL="false" SEQUENCE="false"/>

lib/db/upgrade.php

@@ -2753,5 +2753,32 @@ function xmldb_main_upgrade($oldversion) {
         upgrade_main_savepoint(true, 2019030100.01);
     }
 
+    if ($oldversion < 2019030700.01) {
+
+        // Define field evaluationmode to be added to analytics_models_log.
+        $table = new xmldb_table('analytics_models_log');
+        $field = new xmldb_field('evaluationmode', XMLDB_TYPE_CHAR, '50', null, null, null,
+            null, 'version');
+
+        // Conditionally launch add field evaluationmode.
+        if (!$dbman->field_exists($table, $field)) {
+            $dbman->add_field($table, $field);
+
+            $updatesql = "UPDATE {analytics_models_log}
+                             SET evaluationmode = 'configuration'";
+            $DB->execute($updatesql, []);
+
+            // Changing nullability of field evaluationmode on table block_instances to not null.
+            $field = new xmldb_field('evaluationmode', XMLDB_TYPE_CHAR, '50', null, XMLDB_NOTNULL,
+                null, null, 'version');
+
+            // Launch change of nullability for field evaluationmode.
+            $dbman->change_field_notnull($table, $field);
+        }
+
+        // Main savepoint reached.
+        upgrade_main_savepoint(true, 2019030700.01);
+    }
+
     return true;
 }

version.php

@@ -29,7 +29,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$version  = 2019030700.00;              // YYYYMMDD      = weekly release date of this DEV branch.
+$version  = 2019030700.01;              // YYYYMMDD      = weekly release date of this DEV branch.
                                         //         RR    = release increments - 00 in DEV branches.
                                         //           .XX = incremental changes.