Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
MDL-64787 tool_analytics: UI for trained models evaluation
  • Loading branch information
David Monllaó committed Mar 7, 2019
1 parent a713ed3 commit 7482393
Show file tree
Hide file tree
Showing 7 changed files with 132 additions and 4 deletions.
2 changes: 1 addition & 1 deletion admin/tool/analytics/amd/build/model.min.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

83 changes: 82 additions & 1 deletion admin/tool/analytics/amd/src/model.js
Expand Up @@ -48,6 +48,20 @@ define(['jquery', 'core/str', 'core/log', 'core/notification', 'core/modal_facto
}
};

/**
* Template to display the evaluation mode choices.
*/
var evaluationRadioHTML = '<div class="box mb-4">{{evaluationmodeinfo}}</div>' +
'<div class="form-check">' +
'<input class="form-check-input" type="radio" name="evaluationmode" id="id-mode-trainedmodel" value="trainedmodel" ' +
'checked>' +
'<label class="form-check-label" for="id-mode-trainedmodel">{{trainedmodellabel}}</label>' +
'</div>' +
'<div class="form-check">' +
'<input class="form-check-input" type="radio" name="evaluationmode" id="id-mode-configuration" value="configuration">' +
'<label class="form-check-label" for="id-mode-configuration">{{configurationlabel}}</label>' +
'</div>';

/**
* Returns the model name.
*
Expand Down Expand Up @@ -94,10 +108,77 @@ define(['jquery', 'core/str', 'core/log', 'core/notification', 'core/modal_facto
modal.getRoot().on(ModalEvents.save, function() {
window.location.href = a.attr('href');
});
modal.show();
return modal;
}).fail(Notification.exception);
});
},

/**
* Displays a select-evaluation-mode choice.
*
* @param {String} actionId
* @param {Boolean} trainedOnlyExternally
*/
selectEvaluationMode: function(actionId, trainedOnlyExternally) {
$('[data-action-id="' + actionId + '"]').on('click', function(ev) {
ev.preventDefault();

var a = $(ev.currentTarget);

if (!trainedOnlyExternally) {
// We can not evaluate trained models if the model was trained using data from this site.
// Default to evaluate the model configuration if that is the case.
window.location.href = a.attr('href');
return;
}

var stringsPromise = Str.get_strings([
{
key: 'evaluatemodel',
component: 'tool_analytics'
}, {
key: 'evaluationmode',
component: 'tool_analytics'
}, {
key: 'evaluationmodeinfo',
component: 'tool_analytics'
}, {
key: 'evaluationmodetrainedmodel',
component: 'tool_analytics'
}, {
key: 'evaluationmodeconfiguration',
component: 'tool_analytics'
}
]);
var modalPromise = ModalFactory.create({type: ModalFactory.types.SAVE_CANCEL});

$.when(stringsPromise, modalPromise).then(function(strings, modal) {


modal.getRoot().on(ModalEvents.hidden, modal.destroy.bind(modal));

modal.setTitle(strings[1]);
modal.setSaveButtonText(strings[0]);

var body = evaluationRadioHTML.replace(/{{evaluationmodeinfo}}/, strings[2])
.replace(/{{trainedmodellabel}}/, strings[3])
.replace(/{{configurationlabel}}/, strings[4]);
modal.setBody(body);

modal.getRoot().on(ModalEvents.save, function() {
var evaluationMode = $("input[name='evaluationmode']:checked").val();
if (evaluationMode == 'trainedmodel') {
a.attr('href', a.attr('href') + '&mode=trainedmodel');
}
window.location.href = a.attr('href');
return;
});

modal.show();
return modal;
}).fail(Notification.exception);
});
}
};
});
});
8 changes: 7 additions & 1 deletion admin/tool/analytics/classes/output/models_list.php
Expand Up @@ -187,10 +187,16 @@ public function export_for_template(\renderer_base $output) {

// Evaluate machine-learning-based models.
if (!$onlycli && $model->get_indicators() && !$model->is_static()) {

// Extra is_trained call as trained_locally returns false if the model has not been trained yet.
$trainedonlyexternally = !$model->trained_locally() && $model->is_trained();

$actionid = 'evaluate-' . $model->get_id();
$PAGE->requires->js_call_amd('tool_analytics/model', 'selectEvaluationMode', [$actionid, $trainedonlyexternally]);
$urlparams['action'] = 'evaluate';
$url = new \moodle_url('model.php', $urlparams);
$icon = new \action_menu_link_secondary($url, new \pix_icon('i/calc', get_string('evaluate', 'tool_analytics')),
get_string('evaluate', 'tool_analytics'));
get_string('evaluate', 'tool_analytics'), ['data-action-id' => $actionid]);
$actionsmenu->add($icon);
}

Expand Down
8 changes: 8 additions & 0 deletions admin/tool/analytics/cli/evaluate_model.php
Expand Up @@ -35,6 +35,8 @@
--non-interactive Not interactive questions
--timesplitting Restrict the evaluation to 1 single time splitting method (Optional)
--filter Analyser dependant. e.g. A courseid would evaluate the model using a single course (Optional)
--mode 'configuration' or 'trainedmodel'. You can only use mode=trainedmodel when the trained" .
" model was imported" . "
--reuse-prev-analysed Reuse recently analysed courses instead of analysing the whole site. Set it to false while" .
" coding indicators. Defaults to true (Optional)" . "
-h, --help Print out this help
Expand All @@ -50,6 +52,7 @@
'modelid' => false,
'list' => false,
'timesplitting' => false,
'mode' => 'configuration',
'reuse-prev-analysed' => true,
'non-interactive' => false,
'filter' => false
Expand All @@ -74,6 +77,10 @@
$options['filter'] = explode(',', $options['filter']);
}

if ($options['mode'] !== 'configuration' && $options['mode'] !== 'trainedmodel') {
cli_error('Error: The provided mode is not supported');
}

// We need admin permissions.
\core\session\manager::set_user(get_admin());

Expand All @@ -89,6 +96,7 @@
'filter' => $options['filter'],
'timesplitting' => $options['timesplitting'],
'reuseprevanalysed' => $options['reuse-prev-analysed'],
'mode' => $options['mode'],
);
// Evaluate its suitability to predict accurately.
$results = $model->evaluate($analyseroptions);
Expand Down
4 changes: 4 additions & 0 deletions admin/tool/analytics/lang/en/tool_analytics.php
Expand Up @@ -53,6 +53,10 @@
$string['errortrainingdataexport'] = 'The model training data could not be exported';
$string['evaluate'] = 'Evaluate';
$string['evaluatemodel'] = 'Evaluate model';
$string['evaluationmode'] = 'Evaluation mode';
$string['evaluationmodeinfo'] = 'This model has been trained using data from another site. You can evaluate the performance of the trained model on your site, or you can evaluate the performance of this model configuration using the data available on this site.';
$string['evaluationmodetrainedmodel'] = 'Evaluate the trained model';
$string['evaluationmodeconfiguration'] = 'Evaluate the model configuration';
$string['evaluationinbatches'] = 'The site contents are calculated and stored in batches. The evaluation process may be stopped at any time. The next time it is run, it will continue from the point when it was stopped.';
$string['exportmodel'] = 'Export configuration';
$string['exporttrainingdata'] = 'Export training data';
Expand Down
8 changes: 7 additions & 1 deletion admin/tool/analytics/model.php
Expand Up @@ -169,7 +169,13 @@
// Web interface is used by people who can not use CLI nor code stuff, always use
// cached stuff as they will change the model through the web interface as well
// which invalidates the previously analysed stuff.
$results = $model->evaluate(array('reuseprevanalysed' => true));
$options = ['reuseprevanalysed' => true];

$mode = optional_param('mode', false, PARAM_ALPHANUM);
if ($mode == 'trainedmodel') {
$options['mode'] = 'trainedmodel';
}
$results = $model->evaluate($options);
$renderer = $PAGE->get_renderer('tool_analytics');
echo $renderer->render_evaluate_results($results, $model->get_analyser()->get_logs());
break;
Expand Down
23 changes: 23 additions & 0 deletions analytics/classes/model.php
Expand Up @@ -1462,6 +1462,29 @@ public function get_training_data() {
return \core_analytics\dataset_manager::export_training_data($this->get_id(), $timesplittingid);
}

/**
* Has the model been trained using data from this site?
*
* This method is useful to determine if a trained model can be evaluated as
* we can not use the same data for training and for evaluation.
*
* @return bool
*/
public function trained_locally() {
global $DB;

if (!$this->is_trained() || $this->is_static()) {
// Early exit.
return false;
}

if ($DB->record_exists('analytics_train_samples', ['modelid' => $this->model->id])) {
return true;
}

return false;
}

/**
* Flag the provided file as used for training or prediction.
*
Expand Down

0 comments on commit 7482393

Please sign in to comment.