diff --git a/kolibri/core/content/management/commands/importcontent.py b/kolibri/core/content/management/commands/importcontent.py index 9e2271b7301..b0d21f8b400 100644 --- a/kolibri/core/content/management/commands/importcontent.py +++ b/kolibri/core/content/management/commands/importcontent.py @@ -346,10 +346,7 @@ def _transfer( # noqa: max-complexity=16 if len(file_transfers) > 0: f, filetransfer = file_transfers.pop() future = executor.submit( - self._start_file_transfer, - f, - filetransfer, - overall_progress_update, + self._start_file_transfer, f, filetransfer ) future_file_transfers[future] = (f, filetransfer) @@ -358,7 +355,8 @@ def _transfer( # noqa: max-complexity=16 ): f, filetransfer = future_file_transfers[future] try: - status = future.result() + status, data_transferred = future.result() + overall_progress_update(data_transferred) if self.is_cancelled(): break @@ -424,24 +422,25 @@ def _transfer( # noqa: max-complexity=16 if self.is_cancelled(): self.cancel() - def _start_file_transfer(self, f, filetransfer, overall_progress_update): + def _start_file_transfer(self, f, filetransfer): """ Start to transfer the file from network/disk to the destination. Return value: * FILE_TRANSFERRED - successfully transfer the file. * FILE_SKIPPED - the file does not exist so it is skipped. """ + data_transferred = 0 + with filetransfer: for chunk in filetransfer: - length = len(chunk) - overall_progress_update(length) + data_transferred += len(chunk) # Ensure that if for some reason the total file size for the transfer # is less than what we have marked in the database that we make up # the difference so that the overall progress is never incorrect. # This could happen, for example for a local transfer if a file # has been replaced or corrupted (which we catch below) - overall_progress_update(f.file_size - filetransfer.total_size) + data_transferred += f.file_size - filetransfer.total_size # If checksum of the destination file is different from the localfile # id indicated in the database, it means that the destination file @@ -452,9 +451,9 @@ def _start_file_transfer(self, f, filetransfer, overall_progress_update): e = "File {} is corrupted.".format(filetransfer.source) logger.error("An error occurred during content import: {}".format(e)) os.remove(filetransfer.dest) - return FILE_SKIPPED + return FILE_SKIPPED, data_transferred - return FILE_TRANSFERRED + return FILE_TRANSFERRED, data_transferred def handle_async(self, *args, **options): if options["command"] == "network": diff --git a/kolibri/core/content/test/test_import_export.py b/kolibri/core/content/test/test_import_export.py index b4e45d67015..81c607b9f09 100644 --- a/kolibri/core/content/test/test_import_export.py +++ b/kolibri/core/content/test/test_import_export.py @@ -761,7 +761,6 @@ def test_local_import_source_corrupted_full_progress( local_src_path = tempfile.mkstemp()[1] with open(local_src_path, "w") as f: f.write("This is just a test") - src_file_size = os.path.getsize(local_src_path) expected_file_size = 10000 local_dest_path = tempfile.mkstemp()[1] os.remove(local_dest_path) @@ -798,7 +797,7 @@ def test_local_import_source_corrupted_full_progress( node_ids=["32a941fb77c2576e8f6b294cde4c3b0c"], ) - mock_overall_progress.assert_any_call(expected_file_size - src_file_size) + mock_overall_progress.assert_any_call(expected_file_size) @patch( "kolibri.core.content.management.commands.importcontent.transfer.FileDownload.finalize"