Add sync operation support to Kolibri plugins with migration for exam logs

kolibri/core/auth/constants/morango_sync.py

@@ -1,10 +1,14 @@
 from __future__ import unicode_literals
 
+from kolibri import __version__
 from kolibri.utils import conf
 
 
 PROFILE_FACILITY_DATA = "facilitydata"
 DATA_PORTAL_SYNCING_BASE_URL = conf.OPTIONS["Urls"]["DATA_PORTAL_SYNCING_BASE_URL"]
+CUSTOM_INSTANCE_INFO = {
+    "kolibri": __version__,
+}
 
 
 class ScopeDefinitions(object):

kolibri/core/auth/test/test_utils.py

@@ -9,13 +9,15 @@
 import factory
 import mock
 from django.core.management.base import CommandError
+from django.test import SimpleTestCase
 from django.test import TestCase
 
 from ..models import Facility
 from kolibri.core.auth.management import utils
 from kolibri.core.auth.test.test_api import FacilityFactory
 from kolibri.core.auth.test.test_api import FacilityUserFactory
 from kolibri.core.auth.utils import merge_users
+from kolibri.core.auth.utils import VersionMigrationOperation
 from kolibri.core.logger import models as log_models
 
 
@@ -481,3 +483,130 @@ def test_contentsummarylogs(self):
                     channel_id=log.channel_id,
                 ).exists()
             )
+
+
+class VersionMigrationOperationTestCase(SimpleTestCase):
+    def setUp(self):
+        super(VersionMigrationOperationTestCase, self).setUp()
+        self.operation = VersionMigrationOperation()
+        self.operation.version = "0.15.0"
+        self.upgrade = mock.Mock()
+        self.operation.upgrade = self.upgrade
+        self.downgrade = mock.Mock()
+        self.operation.downgrade = self.downgrade
+        self.context = mock.Mock()
+        self.context.is_server = False
+        self.context.sync_session.client_instance_data = {}
+        self.context.sync_session.server_instance_data = {}
+
+    def test_handle__assert_sync_session(self):
+        self.context.sync_session = None
+        with self.assertRaises(AssertionError):
+            self.operation.handle(self.context)
+
+    def test_handle__assert_version(self):
+        self.operation.version = None
+        with self.assertRaises(AssertionError):
+            self.operation.handle(self.context)
+
+    def test_handle__server__upgrade_not_needed(self):
+        self.context.is_receiver = True
+        self.context.is_server = True
+        self.context.sync_session.client_instance_data = {
+            "kolibri": self.operation.version,
+        }
+        self.assertFalse(self.operation.handle(self.context))
+        self.upgrade.assert_not_called()
+        self.downgrade.assert_not_called()
+
+    def test_handle__server__downgrade_not_needed(self):
+        self.context.is_receiver = False
+        self.context.is_server = True
+        self.context.sync_session.client_instance_data = {
+            "kolibri": self.operation.version,
+        }
+        self.assertFalse(self.operation.handle(self.context))
+        self.downgrade.assert_not_called()
+        self.upgrade.assert_not_called()
+
+    def test_handle__server__upgrade(self):
+        self.context.is_receiver = True
+        self.context.is_server = True
+        self.context.sync_session.client_instance_data = {
+            "kolibri": "0.14.7",
+        }
+        self.assertFalse(self.operation.handle(self.context))
+        self.upgrade.assert_called_once_with(self.context)
+        self.downgrade.assert_not_called()
+
+    def test_handle__server__downgrade(self):
+        self.context.is_receiver = False
+        self.context.is_server = True
+        self.context.sync_session.client_instance_data = {
+            "kolibri": "0.14.7",
+        }
+        self.assertFalse(self.operation.handle(self.context))
+        self.downgrade.assert_called_once_with(self.context)
+        self.upgrade.assert_not_called()
+
+    def test_handle__server__upgrade__no_info(self):
+        self.context.is_receiver = True
+        self.context.is_server = True
+        self.assertFalse(self.operation.handle(self.context))
+        self.upgrade.assert_called_once_with(self.context)
+        self.downgrade.assert_not_called()
+
+    def test_handle__server__downgrade__no_info(self):
+        self.context.is_receiver = False
+        self.context.is_server = True
+        self.assertFalse(self.operation.handle(self.context))
+        self.downgrade.assert_called_once_with(self.context)
+        self.upgrade.assert_not_called()
+
+    def test_handle__client__upgrade_not_needed(self):
+        self.context.is_receiver = True
+        self.context.sync_session.server_instance_data = {
+            "kolibri": self.operation.version,
+        }
+        self.assertFalse(self.operation.handle(self.context))
+        self.upgrade.assert_not_called()
+        self.downgrade.assert_not_called()
+
+    def test_handle__client__downgrade_not_needed(self):
+        self.context.is_receiver = False
+        self.context.sync_session.server_instance_data = {
+            "kolibri": self.operation.version,
+        }
+        self.assertFalse(self.operation.handle(self.context))
+        self.downgrade.assert_not_called()
+        self.upgrade.assert_not_called()
+
+    def test_handle__client__upgrade(self):
+        self.context.is_receiver = True
+        self.context.sync_session.server_instance_data = {
+            "kolibri": "0.14.7",
+        }
+        self.assertFalse(self.operation.handle(self.context))
+        self.upgrade.assert_called_once_with(self.context)
+        self.downgrade.assert_not_called()
+
+    def test_handle__client__downgrade(self):
+        self.context.is_receiver = False
+        self.context.sync_session.server_instance_data = {
+            "kolibri": "0.14.7",
+        }
+        self.assertFalse(self.operation.handle(self.context))
+        self.downgrade.assert_called_once_with(self.context)
+        self.upgrade.assert_not_called()
+
+    def test_handle__client__upgrade__no_info(self):
+        self.context.is_receiver = True
+        self.assertFalse(self.operation.handle(self.context))
+        self.upgrade.assert_called_once_with(self.context)
+        self.downgrade.assert_not_called()
+
+    def test_handle__client__downgrade__no_info(self):
+        self.context.is_receiver = False
+        self.assertFalse(self.operation.handle(self.context))
+        self.downgrade.assert_called_once_with(self.context)
+        self.upgrade.assert_not_called()

kolibri/core/auth/utils.py

@@ -16,9 +16,11 @@
 from kolibri.core.logger.models import ExamLog
 from kolibri.core.logger.models import MasteryLog
 from kolibri.core.logger.models import UserSessionLog
+from kolibri.core.logger.utils.exam_log_migration import migrate_from_exam_logs
 from kolibri.core.notifications.api import batch_process_attemptlogs
 from kolibri.core.notifications.api import batch_process_examlogs
 from kolibri.core.notifications.api import batch_process_summarylogs
+from kolibri.core.upgrade import matches_version
 
 
 def confirm_or_exit(message):
@@ -130,6 +132,63 @@ def _merge_log_data(LogModel):
     _merge_log_data(AttemptLog)
 
 
+class VersionMigrationOperation(LocalOperation):
+    """
+    Morango operation class to handle migrating data to and from other versions, assuming we're
+    handling it as the newer instance
+    """
+
+    version = None
+
+    @property
+    def version_threshold(self):
+        return "<{}".format(self.version)
+
+    def handle(self, context):
+        """
+        :type context: morango.sync.context.LocalSessionContext
+        :return: False
+        """
+        self._assert(context.sync_session is not None)
+        self._assert(self.version is not None)
+
+        # get the instance info for the other instance
+        instance_info = context.sync_session.server_instance_data
+        if context.is_server:
+            instance_info = context.sync_session.client_instance_data
+
+        # get the kolibri version, which is defined in
+        # kolibri.core.auth.constants.morango_sync:CUSTOM_INSTANCE_INFO
+        remote_version = instance_info.get("kolibri")
+
+        # pre-0.15.0 won't have the kolibri version
+        if remote_version is None or matches_version(
+            remote_version, self.version_threshold
+        ):
+            if context.is_receiver:
+                self.upgrade(context)
+            else:
+                self.downgrade(context)
+
+        return False
+
+    def upgrade(self, context):
+        """
+        Called when we're receiving data from a version older than `self.version`
+
+        :type context: morango.sync.context.LocalSessionContext
+        """
+        pass
+
+    def downgrade(self, context):
+        """
+        Called when we're producing data for a version older than `self.version`
+
+        :type context: morango.sync.context.LocalSessionContext
+        """
+        pass
+
+
 class GenerateNotifications(LocalOperation):
     """
     Generates notifications at cleanup stage (the end) of a transfer, if our instance was a
@@ -141,30 +200,32 @@ def handle(self, context):
         :type context: morango.sync.context.LocalSessionContext
         :return: False
         """
-        if not context.is_receiver:
-            raise AssertionError
-        if context.transfer_session is None:
-            raise AssertionError
+        self._assert(context.transfer_session is not None)
+        self._assert(context.is_receiver)
 
         batch_process_attemptlogs(
-            context.transfer_session.get_touched_record_ids_for_model(
-                AttemptLog.morango_model_name
-            )
+            context.transfer_session.get_touched_record_ids_for_model(AttemptLog)
         )
         batch_process_examlogs(
-            context.transfer_session.get_touched_record_ids_for_model(
-                ExamLog.morango_model_name
-            ),
-            context.transfer_session.get_touched_record_ids_for_model(
-                ExamAttemptLog.morango_model_name
-            ),
+            context.transfer_session.get_touched_record_ids_for_model(ExamLog),
+            context.transfer_session.get_touched_record_ids_for_model(ExamAttemptLog),
         )
         batch_process_summarylogs(
-            context.transfer_session.get_touched_record_ids_for_model(
-                ContentSummaryLog.morango_model_name
-            )
+            context.transfer_session.get_touched_record_ids_for_model(ContentSummaryLog)
         )
 
         # always return false, indicating that other operations in this stage's configuration
         # should also be executed
         return False
+
+
+class ExamLogsCompatibilityOperation(VersionMigrationOperation):
+    version = "0.15.0"
+
+    def upgrade(self, context):
+        """
+        Migrates exam logs to be backwards compatible with older Kolibris
+        :type context: morango.sync.context.LocalSessionContext
+        """
+        exam_logs = context.transfer_session.get_touched_record_ids_for_model(ExamLog)
+        migrate_from_exam_logs(exam_logs)

kolibri/deployment/default/settings/base.py

@@ -387,7 +387,9 @@
 
 apply_settings(sys.modules[__name__])
 
-# prepend operation to generate notifications to sync cleanup
+MORANGO_INSTANCE_INFO = "kolibri.core.auth.constants.morango_sync:CUSTOM_INSTANCE_INFO"
+# prepend custom Morango operation to handle behaviors during sync
 MORANGO_CLEANUP_OPERATIONS = (
+    "kolibri.core.auth.utils:ExamLogsCompatibilityOperation",
     "kolibri.core.auth.utils:GenerateNotifications",
 ) + morango_settings.MORANGO_CLEANUP_OPERATIONS