diff --git a/analytics/classes/manager.php b/analytics/classes/manager.php index fff2c50e91994..3ab5dd91f6c27 100644 --- a/analytics/classes/manager.php +++ b/analytics/classes/manager.php @@ -489,6 +489,54 @@ public static function add_builtin_models() { } } + /** + * Cleans up analytics db tables that do not directly depend on analysables that may have been deleted. + */ + public static function cleanup() { + global $DB; + + // Clean up stuff that depends on contexts that do not exist anymore. + $sql = "SELECT DISTINCT ap.contextid FROM {analytics_predictions} ap + LEFT JOIN {context} ctx ON ap.contextid = ctx.id + WHERE ctx.id IS NULL"; + $apcontexts = $DB->get_records_sql($sql); + + $sql = "SELECT DISTINCT aic.contextid FROM {analytics_indicator_calc} aic + LEFT JOIN {context} ctx ON aic.contextid = ctx.id + WHERE ctx.id IS NULL"; + $indcalccontexts = $DB->get_records_sql($sql); + + $contexts = $apcontexts + $indcalccontexts; + if ($contexts) { + list($sql, $params) = $DB->get_in_or_equal(array_keys($contexts)); + $DB->execute("DELETE FROM {analytics_prediction_actions} apa WHERE apa.predictionid IN + (SELECT ap.id FROM {analytics_predictions} ap WHERE ap.contextid $sql)", $params); + + $DB->delete_records_select('analytics_predictions', "contextid $sql", $params); + $DB->delete_records_select('analytics_indicator_calc', "contextid $sql", $params); + } + + // Clean up stuff that depends on analysable ids that do not exist anymore. + $models = self::get_all_models(); + foreach ($models as $model) { + $analyser = $model->get_analyser(array('notimesplitting' => true)); + $analysables = $analyser->get_analysables(); + if (!$analysables) { + continue; + } + + $analysableids = array_map(function($analysable) { + return $analysable->get_id(); + }, $analysables); + + list($notinsql, $params) = $DB->get_in_or_equal($analysableids, SQL_PARAMS_NAMED, 'param', false); + $params['modelid'] = $model->get_id(); + + $DB->delete_records_select('analytics_predict_samples', "modelid = :modelid AND analysableid $notinsql", $params); + $DB->delete_records_select('analytics_train_samples', "modelid = :modelid AND analysableid $notinsql", $params); + } + } + /** * Returns the provided element classes in the site. * diff --git a/analytics/tests/fixtures/test_target_course_level_shortname.php b/analytics/tests/fixtures/test_target_course_level_shortname.php new file mode 100644 index 0000000000000..1f13d463ac9fa --- /dev/null +++ b/analytics/tests/fixtures/test_target_course_level_shortname.php @@ -0,0 +1,46 @@ +. + +/** + * Test target. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once(__DIR__ . '/test_target_shortname.php'); + +/** + * Test target. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class test_target_course_level_shortname extends test_target_shortname { + + /** + * get_analyser_class + * + * @return string + */ + public function get_analyser_class() { + return '\core\analytics\analyser\courses'; + } +} diff --git a/analytics/tests/manager_test.php b/analytics/tests/manager_test.php new file mode 100644 index 0000000000000..f4c2a1c06bffe --- /dev/null +++ b/analytics/tests/manager_test.php @@ -0,0 +1,153 @@ +. + +/** + * Unit tests for the manager. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once(__DIR__ . '/fixtures/test_indicator_max.php'); +require_once(__DIR__ . '/fixtures/test_indicator_min.php'); +require_once(__DIR__ . '/fixtures/test_indicator_fullname.php'); +require_once(__DIR__ . '/fixtures/test_target_course_level_shortname.php'); + +/** + * Unit tests for the manager. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class analytics_manager_testcase extends advanced_testcase { + + /** + * test_deleted_context + */ + public function test_deleted_context() { + global $DB; + + $this->resetAfterTest(true); + $this->setAdminuser(); + set_config('enabled_stores', 'logstore_standard', 'tool_log'); + + $target = \core_analytics\manager::get_target('test_target_course_level_shortname'); + $indicators = array('test_indicator_max', 'test_indicator_min', 'test_indicator_fullname'); + foreach ($indicators as $key => $indicator) { + $indicators[$key] = \core_analytics\manager::get_indicator($indicator); + } + + $model = \core_analytics\model::create($target, $indicators); + $modelobj = $model->get_model_obj(); + + $coursepredict1 = $this->getDataGenerator()->create_course(array('visible' => 0)); + $coursepredict2 = $this->getDataGenerator()->create_course(array('visible' => 0)); + $coursetrain1 = $this->getDataGenerator()->create_course(array('visible' => 1)); + $coursetrain2 = $this->getDataGenerator()->create_course(array('visible' => 1)); + + $model->enable('\core\analytics\time_splitting\no_splitting'); + + $model->train(); + $model->predict(); + + // Generate a prediction action to confirm that it is deleted when there is an important update. + $predictions = $DB->get_records('analytics_predictions'); + $prediction = reset($predictions); + $prediction = new \core_analytics\prediction($prediction, array('whatever' => 'not used')); + $prediction->action_executed(\core_analytics\prediction::ACTION_FIXED, $model->get_target()); + + $predictioncontextid = $prediction->get_prediction_data()->contextid; + + $npredictions = $DB->count_records('analytics_predictions', array('contextid' => $predictioncontextid)); + $npredictionactions = $DB->count_records('analytics_prediction_actions', + array('predictionid' => $prediction->get_prediction_data()->id)); + $nindicatorcalc = $DB->count_records('analytics_indicator_calc', array('contextid' => $predictioncontextid)); + + \core_analytics\manager::cleanup(); + + // Nothing is incorrectly deleted. + $this->assertEquals($npredictions, $DB->count_records('analytics_predictions', + array('contextid' => $predictioncontextid))); + $this->assertEquals($npredictionactions, $DB->count_records('analytics_prediction_actions', + array('predictionid' => $prediction->get_prediction_data()->id))); + $this->assertEquals($nindicatorcalc, $DB->count_records('analytics_indicator_calc', + array('contextid' => $predictioncontextid))); + + // Now we delete a context, the course predictions and prediction actions should be deleted. + $deletedcontext = \context::instance_by_id($predictioncontextid); + delete_course($deletedcontext->instanceid, false); + + \core_analytics\manager::cleanup(); + + $this->assertEmpty($DB->count_records('analytics_predictions', array('contextid' => $predictioncontextid))); + $this->assertEmpty($DB->count_records('analytics_prediction_actions', + array('predictionid' => $prediction->get_prediction_data()->id))); + $this->assertEmpty($DB->count_records('analytics_indicator_calc', array('contextid' => $predictioncontextid))); + + set_config('enabled_stores', '', 'tool_log'); + get_log_manager(true); + } + + /** + * test_deleted_analysable + */ + public function test_deleted_analysable() { + global $DB; + + $this->resetAfterTest(true); + $this->setAdminuser(); + set_config('enabled_stores', 'logstore_standard', 'tool_log'); + + $target = \core_analytics\manager::get_target('test_target_course_level_shortname'); + $indicators = array('test_indicator_max', 'test_indicator_min', 'test_indicator_fullname'); + foreach ($indicators as $key => $indicator) { + $indicators[$key] = \core_analytics\manager::get_indicator($indicator); + } + + $model = \core_analytics\model::create($target, $indicators); + $modelobj = $model->get_model_obj(); + + $coursepredict1 = $this->getDataGenerator()->create_course(array('visible' => 0)); + $coursepredict2 = $this->getDataGenerator()->create_course(array('visible' => 0)); + $coursetrain1 = $this->getDataGenerator()->create_course(array('visible' => 1)); + $coursetrain2 = $this->getDataGenerator()->create_course(array('visible' => 1)); + + $model->enable('\core\analytics\time_splitting\no_splitting'); + + $model->train(); + $model->predict(); + + $npredictsamples = $DB->count_records('analytics_predict_samples'); + $ntrainsamples = $DB->count_records('analytics_train_samples'); + + // Now we delete an analysable, stored predict and training samples should be deleted. + $deletedcontext = \context_course::instance($coursepredict1->id); + delete_course($coursepredict1, false); + + \core_analytics\manager::cleanup(); + + $this->assertEmpty($DB->count_records('analytics_predict_samples', array('analysableid' => $coursepredict1->id))); + $this->assertEmpty($DB->count_records('analytics_train_samples', array('analysableid' => $coursepredict1->id))); + + set_config('enabled_stores', '', 'tool_log'); + get_log_manager(true); + } + +} diff --git a/lang/en/admin.php b/lang/en/admin.php index c342d025fe344..b294490d7e11b 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -1086,6 +1086,7 @@ $string['tabselectedtofront'] = 'On tables with tabs, should the row with the currently selected tab be placed at the front'; $string['tabselectedtofronttext'] = 'Bring selected tab row to front'; $string['testsiteupgradewarning'] = 'You are currently using the {$a} test site, to upgrade it properly use the command line interface tool'; +$string['taskanalyticscleanup'] = 'Analytics cleanup'; $string['taskautomatedbackup'] = 'Automated backups'; $string['taskbackupcleanup'] = 'Clean backup tables and logs'; $string['taskbadgescron'] = 'Award badges'; diff --git a/lib/classes/task/analytics_cleanup_task.php b/lib/classes/task/analytics_cleanup_task.php new file mode 100644 index 0000000000000..0824d93e29912 --- /dev/null +++ b/lib/classes/task/analytics_cleanup_task.php @@ -0,0 +1,55 @@ +. + +/** + * A scheduled task. + * + * @package core + * @copyright 2017 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\task; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Delete stale records from analytics tables. + * + * @package core + * @copyright 2017 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class analytics_cleanup_task extends \core\task\scheduled_task { + + /** + * Get a descriptive name for this task (shown to admins). + * + * @return string + */ + public function get_name() { + return get_string('taskanalyticscleanup', 'admin'); + } + + /** + * Executes the clean up task. + * + * @return void + */ + public function execute() { + $models = \core_analytics\manager::cleanup(); + } +} diff --git a/lib/db/tasks.php b/lib/db/tasks.php index bdb87087fd510..b550c54dda093 100644 --- a/lib/db/tasks.php +++ b/lib/db/tasks.php @@ -356,4 +356,13 @@ 'dayofweek' => '*', 'month' => '*' ), + array( + 'classname' => 'core\task\analytics_cleanup_task', + 'blocking' => 0, + 'minute' => 'R', + 'hour' => '*', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*' + ), );