Merge branch 'MDL-65348_master' of git://github.com/dmonllao/moodle

analytics/classes/analysis.php

@@ -67,6 +67,9 @@ public function __construct(\core_analytics\local\analyser\base $analyser, bool
         $this->analyser = $analyser;
         $this->includetarget = $includetarget;
         $this->result = $result;
+
+        // We cache the first time analysables were analysed because time-splitting methods can depend on these info.
+        self::fill_firstanalyses_cache($this->analyser->get_modelid());
     }
 
     /**
@@ -81,17 +84,15 @@ public function run() {
         // Time limit control.
         $modeltimelimit = intval(get_config('analytics', 'modeltimelimit'));
 
-        $filesbytimesplitting = array();
-
-        $alreadyprocessedanalysables = $this->get_processed_analysables();
-
         if ($this->includetarget) {
             $action = 'training';
         } else {
             $action = 'prediction';
         }
         $analysables = $this->analyser->get_analysables_iterator($action);
 
+        $processedanalysables = $this->get_processed_analysables();
+
         $inittime = microtime(true);
         foreach ($analysables as $analysable) {
             $processed = false;
@@ -121,13 +122,16 @@ public function run() {
                 }
             }
 
-            // Updated regardless of how well the analysis went.
-            if ($this->analyser->get_target()->always_update_analysis_time() || $processed) {
-                $this->update_analysable_analysed_time($alreadyprocessedanalysables, $analysable->get_id());
-            }
-
-            // Apply time limit.
             if (!$options['evaluation']) {
+
+                if (empty($processedanalysables[$analysable->get_id()]) ||
+                        $this->analyser->get_target()->always_update_analysis_time() || $processed) {
+                    // We store the list of processed analysables even if the target does not always_update_analysis_time(),
+                    // what always_update_analysis_time controls is the update of the data.
+                    $this->update_analysable_analysed_time($processedanalysables, $analysable->get_id());
+                }
+
+                // Apply time limit.
                 $timespent = microtime(true) - $inittime;
                 if ($modeltimelimit <= $timespent) {
                     break;
@@ -150,7 +154,7 @@ protected function get_processed_analysables(): array {
 
         // Weird select fields ordering for performance (analysableid key matching, analysableid is also unique by modelid).
         return $DB->get_records_select('analytics_used_analysables', $select,
-            $params, 'timeanalysed DESC', 'analysableid, modelid, action, timeanalysed, id AS primarykey');
+            $params, 'timeanalysed DESC', 'analysableid, modelid, action, firstanalysis, timeanalysed, id AS primarykey');
     }
 
     /**
@@ -590,13 +594,16 @@ protected function fill_dataset(\core_analytics\local\time_splitting\base $times
     protected function update_analysable_analysed_time(array $processedanalysables, int $analysableid) {
         global $DB;
 
+        $now = time();
+
         if (!empty($processedanalysables[$analysableid])) {
             $obj = $processedanalysables[$analysableid];
 
             $obj->id = $obj->primarykey;
             unset($obj->primarykey);
 
-            $obj->timeanalysed = time();
+            $obj->timeanalysed = $now;
+
             $DB->update_record('analytics_used_analysables', $obj);
 
         } else {
@@ -605,10 +612,54 @@ protected function update_analysable_analysed_time(array $processedanalysables,
             $obj->modelid = $this->analyser->get_modelid();
             $obj->action = ($this->includetarget) ? 'training' : 'prediction';
             $obj->analysableid = $analysableid;
-            $obj->timeanalysed = time();
+            $obj->firstanalysis = $now;
+            $obj->timeanalysed = $now;
+
+            $obj->primarykey = $DB->insert_record('analytics_used_analysables', $obj);
+
+            // Update the cache just in case it is used in the same request.
+            $key = $this->analyser->get_modelid() . '_' . $analysableid;
+            $cache = \cache::make('core', 'modelfirstanalyses');
+            $cache->set($key, $now);
+        }
+    }
+
+    /**
+     * Fills a cache containing the first time each analysable in the provided model was analysed.
+     *
+     * @param int $modelid
+     * @param int|null $analysableid
+     * @return null
+     */
+    public static function fill_firstanalyses_cache(int $modelid, ?int $analysableid = null) {
+        global $DB;
 
-            $DB->insert_record('analytics_used_analysables', $obj);
+        // Using composed keys instead of cache $identifiers because of MDL-65358.
+        $primarykey = $DB->sql_concat($modelid, "'_'", 'analysableid');
+        $sql = "SELECT $primarykey AS id, MIN(firstanalysis) AS firstanalysis
+                  FROM {analytics_used_analysables} aua
+                 WHERE modelid = :modelid";
+        $params = ['modelid' => $modelid];
+
+        if ($analysableid) {
+            $sql .= " AND analysableid = :analysableid";
+            $params['analysableid'] = $analysableid;
         }
+
+        $sql .= " GROUP BY modelid, analysableid ORDER BY analysableid";
+
+        $firstanalyses = $DB->get_records_sql($sql, $params);
+        if ($firstanalyses) {
+            $cache = \cache::make('core', 'modelfirstanalyses');
+
+            $firstanalyses = array_map(function($record) {
+                return $record->firstanalysis;
+            }, $firstanalyses);
+
+            $cache->set_many($firstanalyses);
+        }
+
+        return $firstanalyses;
     }
 
     /**

analytics/classes/local/time_splitting/periodic.php

@@ -64,12 +64,12 @@ protected function define_ranges() {
 
         $periodicity = $this->periodicity();
 
-        $now = new \DateTimeImmutable('now', \core_date::get_server_timezone_object());
-
         if ($this->analysable->get_end()) {
             $end = (new \DateTimeImmutable())->setTimestamp($this->analysable->get_end());
         }
-        $next = (new \DateTimeImmutable())->setTimestamp($this->analysable->get_start());
+        $next = (new \DateTimeImmutable())->setTimestamp($this->get_first_start());
+
+        $now = new \DateTimeImmutable('now', \core_date::get_server_timezone_object());
 
         $ranges = [];
         while ($next < $now &&
@@ -140,4 +140,13 @@ protected function get_next_range(\DateTimeImmutable $next) {
             'time' => $end
         ];
     }
+
+    /**
+     * Get the start of the first time range.
+     *
+     * @return int A timestamp.
+     */
+    protected function get_first_start() {
+        return $this->analysable->get_start();
+    }
 }

analytics/classes/local/time_splitting/upcoming_periodic.php

@@ -68,4 +68,26 @@ public function cache_indicator_calculations(): bool {
     public function valid_for_evaluation(): bool {
         return false;
     }
+
+    /**
+     * Get the start of the first time range.
+     *
+     * Overwriten to start generating predictions about upcoming stuff from time().
+     *
+     * @return int A timestamp.
+     */
+    protected function get_first_start() {
+        global $DB;
+
+        $cache = \cache::make('core', 'modelfirstanalyses');
+
+        $key = $this->modelid . '_' . $this->analysable->get_id();
+        $firstanalysis = $cache->get($key);
+        if (!empty($firstanalysis)) {
+            return $firstanalysis;
+        }
+
+        // This analysable has not yet been analysed, the start is therefore now (-1 so ready_to_predict can be executed).
+        return time() - 1;
+    }
 }

analytics/tests/analysis_test.php

@@ -0,0 +1,70 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Unit tests for the analysis class.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Unit tests for the analysis class.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class analytics_analysis_testcase extends advanced_testcase {
+
+    /**
+     * Test fill_firstanalyses_cache.
+     * @return null
+     */
+    public function test_fill_firstanalyses_cache() {
+        $this->resetAfterTest();
+
+        $this->insert_used(1, 1, 'training', 123);
+        $this->insert_used(1, 2, 'training', 124);
+        $this->insert_used(1, 1, 'prediction', 125);
+
+        $firstanalyses = \core_analytics\analysis::fill_firstanalyses_cache(1);
+        $this->assertCount(2, $firstanalyses);
+        $this->assertEquals(123, $firstanalyses['1_1']);
+        $this->assertEquals(124, $firstanalyses['1_2']);
+
+        // The cached elements gets refreshed.
+        $this->insert_used(1, 1, 'prediction', 122);
+        $firstanalyses = \core_analytics\analysis::fill_firstanalyses_cache(1, 1);
+        $this->assertCount(1, $firstanalyses);
+        $this->assertEquals(122, $firstanalyses['1_1']);
+    }
+
+    private function insert_used($modelid, $analysableid, $action, $timestamp) {
+        global $DB;
+
+        $obj = new \stdClass();
+        $obj->modelid = $modelid;
+        $obj->action = $action;
+        $obj->analysableid = $analysableid;
+        $obj->firstanalysis = $timestamp;
+        $obj->timeanalysed = $timestamp;
+        $obj->id = $DB->insert_record('analytics_used_analysables', $obj);
+    }
+}

analytics/tests/fixtures/test_timesplitting_upcoming_seconds.php

@@ -0,0 +1,52 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Test time splitting.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Test time splitting.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class test_timesplitting_upcoming_seconds extends \core_analytics\local\time_splitting\upcoming_periodic {
+
+    /**
+     * Every second.
+     * @return \DateInterval
+     */
+    public function periodicity() {
+        return new \DateInterval('PT1S');
+    }
+
+    /**
+     * Just to comply with the interface.
+     *
+     * @return \lang_string
+     */
+    public static function get_name() : \lang_string {
+        return new \lang_string('error');
+    }
+}

lang/en/cache.php

@@ -56,6 +56,7 @@
 $string['cachedef_htmlpurifier'] = 'HTML Purifier - cleaned content';
 $string['cachedef_langmenu'] = 'List of available languages';
 $string['cachedef_message_time_last_message_between_users'] = 'Time created for most recent message in a conversation';
+$string['cachedef_modelprocessedanalysables'] = 'Processed analysables in a model';
 $string['cachedef_locking'] = 'Locking';
 $string['cachedef_message_processors_enabled'] = "Message processors enabled status";
 $string['cachedef_contextwithinsights'] = 'Context with insights';

lib/db/caches.php

@@ -384,4 +384,11 @@
         'simpledata' => true,
         'ttl' => 1800
     ),
+
+    // Caches the first time we analysed models' analysables.
+    'modelfirstanalyses' => array(
+        'mode' => cache_store::MODE_REQUEST,
+        'simplekeys' => true,
+        'simpledata' => true,
+    ),
 );

lib/db/install.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<XMLDB PATH="lib/db" VERSION="20190403" COMMENT="XMLDB file for core Moodle tables"
+<XMLDB PATH="lib/db" VERSION="20190412" COMMENT="XMLDB file for core Moodle tables"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:noNamespaceSchemaLocation="../../lib/xmldb/xmldb.xsd"
 >
@@ -3978,6 +3978,7 @@
         <FIELD NAME="modelid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
         <FIELD NAME="action" TYPE="char" LENGTH="50" NOTNULL="true" SEQUENCE="false"/>
         <FIELD NAME="analysableid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="firstanalysis" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
         <FIELD NAME="timeanalysed" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
       </FIELDS>
       <KEYS>

lib/db/upgrade.php

@@ -3270,5 +3270,35 @@ function xmldb_main_upgrade($oldversion) {
         upgrade_main_savepoint(true, 2019042300.03);
     }
 
+    if ($oldversion < 2019042700.01) {
+
+        // Define field firstanalysis to be added to analytics_used_analysables.
+        $table = new xmldb_table('analytics_used_analysables');
+
+        // Declaring it as null initially (although it is NOT NULL).
+        $field = new xmldb_field('firstanalysis', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'analysableid');
+
+        // Conditionally launch add field firstanalysis.
+        if (!$dbman->field_exists($table, $field)) {
+            $dbman->add_field($table, $field);
+
+            // Set existing values to the current timeanalysed value.
+            $recordset = $DB->get_recordset('analytics_used_analysables');
+            foreach ($recordset as $record) {
+                $record->firstanalysis = $record->timeanalysed;
+                $DB->update_record('analytics_used_analysables', $record);
+            }
+            $recordset->close();
+
+            // Now make the field 'NOT NULL'.
+            $field = new xmldb_field('firstanalysis', XMLDB_TYPE_INTEGER, '10',
+                null, XMLDB_NOTNULL, null, null, 'analysableid');
+            $dbman->change_field_notnull($table, $field);
+        }
+
+        // Main savepoint reached.
+        upgrade_main_savepoint(true, 2019042700.01);
+    }
+
     return true;
 }