Merge branch 'my-awesome-changes' of github.com:abhimnc/kolibri into …

…my-awesome-changes

merged local branch to the updated branch

kolibri/core/analytics/management/commands/benchmark.py

@@ -139,7 +139,7 @@ def handle(self, *args, **options):
         self.messages.append(format_line("Device name", instance_model.hostname))
         self.messages.append(
             format_line(
-                "Free disk space", "{} Mb".format(get_free_space() / pow(2, 20))
+                "Free disk space", "{} Mb".format(get_free_space() / pow(10, 6))
             )
         )
         self.messages.append(format_line("Server time", local_now()))

kolibri/core/analytics/measurements.py

@@ -145,8 +145,8 @@ def get_machine_info():
     if not SUPPORTED_OS:
         return (None, None, None, None)
     used_cpu = str(psutil.cpu_percent())
-    used_memory = str(psutil.virtual_memory().used / pow(2, 20))  # In Megabytes
-    total_memory = str(psutil.virtual_memory().total / pow(2, 20))  # In Megabytes
+    used_memory = str(psutil.virtual_memory().used / pow(10, 6))  # In Megabytes
+    total_memory = str(psutil.virtual_memory().total / pow(10, 6))  # In Megabytes
     total_processes = str(len(psutil.pids()))
 
     return (used_cpu, used_memory, total_memory, total_processes)
@@ -200,7 +200,7 @@ def get_kolibri_use(development=False):
     if kolibri_pid:
         try:
             kolibri_proc = psutil.Process(kolibri_pid)
-            kolibri_mem = str(kolibri_proc.memory_info().rss / pow(2, 20))
+            kolibri_mem = str(kolibri_proc.memory_info().rss / pow(10, 6))
             kolibri_cpu = str(kolibri_proc.cpu_percent())
         except psutil.NoSuchProcess:
             # Kolibri server is not running

kolibri/core/analytics/utils.py

@@ -372,6 +372,10 @@ def extract_channel_statistics(channel):
         # popular_counts
         "pc": [item["count"] for item in pop],
         # storage calculated by the MB
+        # rtibbles: This is the one remaining instance of non-SI bytes units calculations that
+        # I have discovered still extant in Kolibri. As this is being used for statistics reporting
+        # I have not updated it to use SI units as with all other instances, as that would
+        # produce undesirable inconsistencies in reported statistics.
         "s": (localfiles.aggregate(Sum("file_size"))["file_size__sum"] or 0) / (2 ** 20),
         # summ_started
         "ss": summarylogs.count(),

kolibri/core/apps.py

@@ -7,14 +7,14 @@
 from django.db.backends.signals import connection_created
 from django.db.utils import DatabaseError
 
-from kolibri.core.logger.utils.data import bytes_for_humans
 from kolibri.core.sqlite.pragmas import CONNECTION_PRAGMAS
 from kolibri.core.sqlite.pragmas import START_PRAGMAS
 from kolibri.core.sqlite.utils import repair_sqlite_db
 from kolibri.core.utils.cache import process_cache
 from kolibri.core.utils.cache import RedisSettingsHelper
 from kolibri.deployment.default.sqlite_db_names import NOTIFICATIONS
 from kolibri.utils.conf import OPTIONS
+from kolibri.utils.data import bytes_for_humans
 
 logger = logging.getLogger(__name__)
 

kolibri/core/assets/src/mixins/commonCoreStrings.js

@@ -1140,7 +1140,7 @@ const nonconformingKeys = {
   BASIC_SKILLS: 'allLevelsBasicSkills',
   FOUNDATIONS: 'basicSkills',
   toolsAndSoftwareTraining: 'softwareToolsAndTraining',
-  FOUNDATIONS_LOGIC_AND_CRITICAL_THINKING: 'logicAndCriticalThinking',
+  foundationsLogicAndCriticalThinking: 'logicAndCriticalThinking',
 };
 
 /**

kolibri/core/assets/src/views/FullScreenSidePanel/index.vue

@@ -22,6 +22,8 @@
           <KIconButton
             icon="close"
             class="close-button"
+            :ariaLabel="coreString('close')"
+            :tooltip="coreString('close')"
             @click="closePanel"
           />
         </div>
@@ -65,6 +67,7 @@
 <script>
 
   import Backdrop from 'kolibri.coreVue.components.Backdrop';
+  import commonCoreStrings from 'kolibri.coreVue.mixins.commonCoreStrings';
   //import { mapState } from 'vuex';
   import responsiveWindowMixin from 'kolibri.coreVue.mixins.responsiveWindowMixin';
   //import SidePanelResourcesList from './SidePanelResourcesList';
@@ -75,7 +78,7 @@
       Backdrop,
       //SidePanelResourcesList,
     },
-    mixins: [responsiveWindowMixin],
+    mixins: [responsiveWindowMixin, commonCoreStrings],
     props: {
       closeButtonHidden: {
         type: Boolean,

kolibri/core/auth/management/utils.py

@@ -36,10 +36,10 @@
 from kolibri.core.discovery.utils.network.client import NetworkClient
 from kolibri.core.discovery.utils.network.errors import NetworkLocationNotFound
 from kolibri.core.discovery.utils.network.errors import URLParseError
-from kolibri.core.logger.utils.data import bytes_for_humans
 from kolibri.core.tasks.exceptions import UserCancelledError
 from kolibri.core.tasks.management.commands.base import AsyncCommand
 from kolibri.core.utils.lock import db_lock
+from kolibri.utils.data import bytes_for_humans
 
 
 logger = logging.getLogger(__name__)

kolibri/core/auth/middleware.py

@@ -6,6 +6,7 @@
 from django.contrib.sessions.middleware import SessionMiddleware
 from django.core.cache import cache
 from django.core.exceptions import ImproperlyConfigured
+from django.db.models.signals import post_save
 from django.utils.functional import SimpleLazyObject
 
 
@@ -41,11 +42,14 @@ def get_anonymous_user_model():
         )
 
 
+USER_SESSION_CACHE_KEY = "USER_BY_SESSION_CACHE_{}"
+
+
 def _get_user(request):
     if not hasattr(request, "_cached_user"):
         try:
             user_id = _get_user_session_key(request)
-            USER_CACHE_KEY = "USER_BY_SESSION_CACHE_{}".format(user_id)
+            USER_CACHE_KEY = USER_SESSION_CACHE_KEY.format(user_id)
             user = cache.get(USER_CACHE_KEY)
             if not user:
                 user = get_user(request)
@@ -60,6 +64,14 @@ def _get_user(request):
     return request._cached_user
 
 
+def clear_user_cache(sender, instance, created, **kwargs):
+    if not created:
+        cache.delete(USER_SESSION_CACHE_KEY.format(instance.id))
+
+
+post_save.connect(clear_user_cache, sender=settings.AUTH_USER_MODEL)
+
+
 class CustomAuthenticationMiddleware(AuthenticationMiddleware):
     """
     Adaptation of Django's ``account.middleware.AuthenticationMiddleware``

kolibri/core/content/errors.py

@@ -3,3 +3,7 @@
 
 class InvalidStorageFilenameError(KolibriError):
     pass
+
+
+class InsufficientStorageSpaceError(KolibriError):
+    pass

kolibri/core/content/management/commands/importcontent.py

@@ -9,6 +9,7 @@
 from ...utils import annotation
 from ...utils import paths
 from ...utils import transfer
+from kolibri.core.content.errors import InsufficientStorageSpaceError
 from kolibri.core.content.errors import InvalidStorageFilenameError
 from kolibri.core.content.models import ContentNode
 from kolibri.core.content.utils.file_availability import LocationError
@@ -20,6 +21,7 @@
 from kolibri.core.tasks.management.commands.base import AsyncCommand
 from kolibri.core.tasks.utils import get_current_job
 from kolibri.utils import conf
+from kolibri.utils.system import get_free_space
 
 # constants to specify the transfer method to be used
 DOWNLOAD_METHOD = "download"
@@ -245,6 +247,14 @@ def _transfer(  # noqa: max-complexity=16
                 )
             raise
 
+        if not paths.using_remote_storage():
+            free_space = get_free_space(conf.OPTIONS["Paths"]["CONTENT_DIR"])
+
+            if free_space <= total_bytes_to_transfer:
+                raise InsufficientStorageSpaceError(
+                    "Import would completely fill remaining disk space"
+                )
+
         job = get_current_job()
 
         if job:
@@ -282,6 +292,7 @@ def _transfer(  # noqa: max-complexity=16
             file_checksums_to_annotate.extend(f["id"] for f in files_to_download)
             transferred_file_size = total_bytes_to_transfer
         else:
+            remaining_bytes_to_transfer = total_bytes_to_transfer
             overall_progress_update = self.start_progress(
                 total=total_bytes_to_transfer + dummy_bytes_for_annotation
             ).update_progress
@@ -363,6 +374,14 @@ def _transfer(  # noqa: max-complexity=16
                             else:
                                 file_checksums_to_annotate.append(f["id"])
                                 transferred_file_size += f["file_size"]
+                            remaining_bytes_to_transfer -= f["file_size"]
+                            remaining_free_space = get_free_space(
+                                conf.OPTIONS["Paths"]["CONTENT_DIR"]
+                            )
+                            if remaining_free_space <= remaining_bytes_to_transfer:
+                                raise InsufficientStorageSpaceError(
+                                    "Kolibri ran out of storage space while importing content"
+                                )
                         except transfer.TransferCanceled:
                             break
                         except Exception as e:

kolibri/core/content/test/test_content_app.py

@@ -356,7 +356,9 @@ def _assert_node(self, actual, expected):
         )
 
     def _assert_nodes(self, data, nodes):
-        for actual, expected in zip(data, nodes):
+        for actual, expected in zip(
+            sorted(data, key=lambda x: x["id"]), sorted(nodes, key=lambda x: x.id)
+        ):
             self._assert_node(actual, expected)
 
     def test_contentnode_list(self):

kolibri/core/content/test/test_import_export.py

@@ -17,6 +17,7 @@
 from requests.exceptions import ReadTimeout
 from requests.exceptions import SSLError
 
+from kolibri.core.content.errors import InsufficientStorageSpaceError
 from kolibri.core.content.models import ContentNode
 from kolibri.core.content.models import File
 from kolibri.core.content.models import LocalFile
@@ -628,6 +629,105 @@ def test_remote_import_httperror_500(
             self.the_channel_id, [], node_ids=None, exclude_node_ids=None, public=False
         )
 
+    @patch("kolibri.core.content.management.commands.importcontent.get_free_space")
+    @patch(
+        "kolibri.core.content.management.commands.importcontent.transfer.FileDownload.finalize"
+    )
+    @patch(
+        "kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
+    )
+    @patch(
+        "kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
+        return_value=False,
+    )
+    def test_remote_import_no_space_at_first(
+        self,
+        is_cancelled_mock,
+        path_mock,
+        finalize_dest_mock,
+        get_free_space_mock,
+        annotation_mock,
+        get_import_export_mock,
+        channel_list_status_mock,
+    ):
+        fd1, dest_path_1 = tempfile.mkstemp()
+        fd2, dest_path_2 = tempfile.mkstemp()
+        os.close(fd1)
+        os.close(fd2)
+        path_mock.side_effect = [dest_path_1, dest_path_2]
+        LocalFile.objects.filter(pk="6bdfea4a01830fdd4a585181c0b8068c").update(
+            file_size=2201062
+        )
+        LocalFile.objects.filter(pk="211523265f53825b82f70ba19218a02e").update(
+            file_size=336974
+        )
+        get_import_export_mock.return_value = (
+            1,
+            list(
+                LocalFile.objects.filter(
+                    pk__in=[
+                        "6bdfea4a01830fdd4a585181c0b8068c",
+                        "211523265f53825b82f70ba19218a02e",
+                    ]
+                ).values("id", "file_size", "extension")
+            ),
+            10,
+        )
+        get_free_space_mock.return_value = 0
+        with self.assertRaises(InsufficientStorageSpaceError):
+            call_command("importcontent", "network", self.the_channel_id)
+
+    @patch("kolibri.core.content.management.commands.importcontent.get_free_space")
+    @patch(
+        "kolibri.core.content.management.commands.importcontent.transfer.FileDownload.finalize"
+    )
+    @patch(
+        "kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
+    )
+    @patch(
+        "kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
+        return_value=False,
+    )
+    def test_remote_import_no_space_after_first_download(
+        self,
+        is_cancelled_mock,
+        path_mock,
+        finalize_dest_mock,
+        get_free_space_mock,
+        annotation_mock,
+        get_import_export_mock,
+        channel_list_status_mock,
+    ):
+        fd1, dest_path_1 = tempfile.mkstemp()
+        fd2, dest_path_2 = tempfile.mkstemp()
+        os.close(fd1)
+        os.close(fd2)
+        path_mock.side_effect = [dest_path_1, dest_path_2]
+        LocalFile.objects.filter(pk="6bdfea4a01830fdd4a585181c0b8068c").update(
+            file_size=2201062
+        )
+        LocalFile.objects.filter(pk="211523265f53825b82f70ba19218a02e").update(
+            file_size=336974
+        )
+        get_import_export_mock.return_value = (
+            1,
+            list(
+                LocalFile.objects.filter(
+                    pk__in=[
+                        "6bdfea4a01830fdd4a585181c0b8068c",
+                        "211523265f53825b82f70ba19218a02e",
+                    ]
+                ).values("id", "file_size", "extension")
+            ),
+            2201062 + 336974,
+        )
+        get_free_space_mock.side_effect = [100000000000, 0, 0, 0, 0, 0, 0]
+        with self.assertRaises(InsufficientStorageSpaceError):
+            call_command("importcontent", "network", self.the_channel_id)
+        annotation_mock.set_content_visibility.assert_called_with(
+            self.the_channel_id, [], exclude_node_ids=None, node_ids=None, public=False
+        )
+
     @patch("kolibri.core.content.utils.transfer.sleep")
     @patch(
         "kolibri.core.content.utils.transfer.Transfer.next",

kolibri/core/device/test/test_api.py

@@ -36,6 +36,7 @@
 from kolibri.core.device.models import UserSyncStatus
 from kolibri.core.public.constants import user_sync_statuses
 from kolibri.core.public.constants.user_sync_options import DELAYED_SYNC
+from kolibri.utils.tests.helpers import override_option
 
 
 DUMMY_PASSWORD = "password"
@@ -280,6 +281,7 @@ def test_superuser_update_own_permissions(self):
         self.assertEqual(response.status_code, 403)
 
 
+@override_option("Deployment", "MINIMUM_DISK_SPACE", 0)
 class FreeSpaceTestCase(APITestCase):
     def setUp(self):
         provision_device()