MDL-41838 Backup/restore: Support .tar.gz format for .mbz (2 of 2)

The new experimental setting enabletgzbackups allows backups to be
created so that the internal format for .mbz files is .tar.gz.

Restore transparently supports .mbz files with either internal
formats (.zip or .tar.gz).

The .tar.gz format has the following benefits for backup:
- Supports larger files (no limit on total size, 8GB on single file
  vs. 4GB limit on total size)
- Compresses text better, resulting in smaller .mbz files.
- Reports progress regularly during compression of single files,
  reducing the chance of timeouts during backups that include a
  very large file.

Time performance may also be improved although I haven't done a
direct comparison.

admin/settings/development.php

@@ -19,6 +19,15 @@
     $enablecssoptimiser->set_updatedcallback('theme_reset_all_caches');
     $temp->add($enablecssoptimiser);
 
+    // Backup archive .mbz format: switching to .tar.gz enables larger files, better
+    // progress reporting and possibly better performance. This is an experimental
+    // setting but if successful, should be removed and enabled by default in a future
+    // version. Note: this setting controls newly-created backups only; restore always
+    // supports both formats.
+    $temp->add(new admin_setting_configcheckbox('enabletgzbackups',
+            new lang_string('enabletgzbackups', 'admin'),
+            new lang_string('enabletgzbackups_desc', 'admin'), 0));
+
     $ADMIN->add('experimental', $temp);
 
     // "debugging" settingpage

backup/moodle2/backup_stepslib.php

@@ -1704,7 +1704,7 @@ protected function define_execution() {
         $zipfile = $basepath . '/backup.mbz';
 
         // Get the zip packer
-        $zippacker = get_file_packer('application/zip');
+        $zippacker = get_file_packer('application/vnd.moodle.backup');
 
         // Zip files
         $zippacker->archive_to_pathname($files, $zipfile, true, $this);

backup/util/helper/backup_general_helper.class.php

@@ -243,7 +243,8 @@ public static function get_backup_information_from_mbz($filepath) {
         // Extract moodle_backup.xml.
         $tmpname = 'info_from_mbz_' . time() . '_' . random_string(4);
         $tmpdir = $CFG->tempdir . '/backup/' . $tmpname;
-        $fp = get_file_packer('application/vnd.moodle.backup');
+        $packer = get_file_packer('application/vnd.moodle.backup');
+
         $extracted = $fp->extract_to_pathname($filepath, $tmpdir, array('moodle_backup.xml'));
         $moodlefile =  $tmpdir . '/' . 'moodle_backup.xml';
         if (!$extracted || !is_readable($moodlefile)) {

backup/util/ui/restore_ui_stage.class.php

@@ -212,7 +212,7 @@ protected function extract_file_to_dir() {
 
         $this->filepath = restore_controller::get_tempdir_name($this->contextid, $USER->id);
 
-        $fb = get_file_packer();
+        $fb = get_file_packer('application/vnd.moodle.backup');
         $result = $fb->extract_to_pathname("$CFG->tempdir/backup/".$this->filename,
                 "$CFG->tempdir/backup/$this->filepath/", null, $this);
 

lang/en/admin.php

@@ -487,6 +487,8 @@
 $string['enablerssfeeds'] = 'Enable RSS feeds';
 $string['enablesafebrowserintegration'] = 'Enable Safe Exam Browser integration';
 $string['enablestats'] = 'Enable statistics';
+$string['enabletgzbackups'] = 'Enable new backup format';
+$string['enabletgzbackups_desc'] = 'If enabled, future backups will be created in a new compression format for .mbz files (internally stored as a .tar.gz file). This removes the 4GB backup size restriction and may improve performance. Restore supports both formats and the difference should be transparent to users.';
 $string['enabletrusttext'] = 'Enable trusted content';
 $string['enablewebservices'] = 'Enable web services';
 $string['enablewsdocumentation'] = 'Web services documentation';

lib/filestorage/mbz_packer.php

@@ -0,0 +1,170 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Implementation of .mbz packer.
+ *
+ * This packer supports .mbz files which can be either .zip or .tar.gz format
+ * internally. A suitable format is chosen depending on system option when
+ * creating new files.
+ *
+ * Internally this packer works by wrapping the existing .zip/.tar.gz packers.
+ *
+ * Backup filenames do not contain non-ASCII characters so packers that do not
+ * support UTF-8 (like the current .tar.gz packer, and possibly external zip
+ * software in some cases if used) can be used by this packer.
+ *
+ * @package core_files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once("$CFG->libdir/filestorage/file_packer.php");
+
+/**
+ * Utility class - handles all packing/unpacking of .mbz files.
+ *
+ * @package core_files
+ * @category files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class mbz_packer extends file_packer {
+    /**
+     * Archive files and store the result in file storage.
+     *
+     * Any existing file at that location will be overwritten.
+     *
+     * @param array $files array from archive path => pathname or stored_file
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $filepath file path
+     * @param string $filename file name
+     * @param int $userid user ID
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return stored_file|bool false if error stored_file instance if ok
+     * @throws file_exception If file operations fail
+     * @throws coding_exception If any archive paths do not meet the restrictions
+     */
+    public function archive_to_storage(array $files, $contextid,
+            $component, $filearea, $itemid, $filepath, $filename,
+            $userid = null, $ignoreinvalidfiles = true, file_progress $progress = null) {
+        return $this->get_packer_for_archive_operation()->archive_to_storage($files,
+                $contextid, $component, $filearea, $itemid, $filepath, $filename,
+                $userid, $ignoreinvalidfiles, $progress);
+    }
+
+    /**
+     * Archive files and store the result in an OS file.
+     *
+     * @param array $files array from archive path => pathname or stored_file
+     * @param string $archivefile path to target zip file
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return bool true if file created, false if not
+     * @throws coding_exception If any archive paths do not meet the restrictions
+     */
+    public function archive_to_pathname(array $files, $archivefile,
+            $ignoreinvalidfiles=true, file_progress $progress = null) {
+        return $this->get_packer_for_archive_operation()->archive_to_pathname($files,
+                $archivefile, $ignoreinvalidfiles, $progress);
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param stored_file|string $archivefile full pathname of zip file or stored_file instance
+     * @param string $pathname target directory
+     * @param array $onlyfiles only extract files present in the array
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return array list of processed files (name=>true)
+     * @throws moodle_exception If error
+     */
+    public function extract_to_pathname($archivefile, $pathname,
+            array $onlyfiles = null, file_progress $progress = null) {
+        return $this->get_packer_for_read_operation($archivefile)->extract_to_pathname(
+                $archivefile, $pathname, $onlyfiles, $progress);
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param string|stored_file $archivefile full pathname of zip file or stored_file instance
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $pathbase file path
+     * @param int $userid user ID
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return array list of processed files (name=>true)
+     * @throws moodle_exception If error
+     */
+    public function extract_to_storage($archivefile, $contextid,
+            $component, $filearea, $itemid, $pathbase, $userid = null,
+            file_progress $progress = null) {
+        return $this->get_packer_for_read_operation($archivefile)->extract_to_storage(
+                $archivefile, $contextid, $component, $filearea, $itemid, $pathbase,
+                $userid, $progress);
+    }
+
+    /**
+     * Returns array of info about all files in archive.
+     *
+     * @param string|stored_file $archivefile
+     * @return array of file infos
+     */
+    public function list_files($archivefile) {
+        return $this->get_packer_for_read_operation($archivefile)->list_files($archivefile);
+    }
+
+    /**
+     * Selects appropriate packer for new archive depending on system option.
+     *
+     * @return file_packer Suitable packer
+     */
+    protected function get_packer_for_archive_operation() {
+        global $CFG;
+
+        if ($CFG->enabletgzbackups) {
+            return get_file_packer('application/x-gzip');
+        } else {
+            return get_file_packer('application/zip');
+        }
+    }
+
+    /**
+     * Selects appropriate packer for existing archive depending on file contents.
+     *
+     * @param string|stored_file $archivefile full pathname of zip file or stored_file instance
+     * @return file_packer Suitable packer
+     */
+    protected function get_packer_for_read_operation($archivefile) {
+        global $CFG;
+        require_once($CFG->dirroot . '/lib/filestorage/tgz_packer.php');
+
+        if (tgz_packer::is_tgz_file($archivefile)) {
+            return get_file_packer('application/x-gzip');
+        } else {
+            return get_file_packer('application/zip');
+        }
+    }
+}

lib/filestorage/tests/mbz_packer_test.php

@@ -0,0 +1,91 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Unit tests for /lib/filestorage/mbz_packer.php.
+ *
+ * @package core_files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->libdir . '/filestorage/file_progress.php');
+
+class core_files_mbz_packer_testcase extends advanced_testcase {
+
+    public function test_archive_with_both_options() {
+        global $CFG;
+        $this->resetAfterTest();
+
+        // Get backup packer.
+        $packer = get_file_packer('application/vnd.moodle.backup');
+
+        // Set up basic archive contents.
+        $files = array('1.txt' => array('frog'));
+
+        // Create 2 archives (each with one file in) in default mode.
+        $CFG->enabletgzbackups = false;
+        $filefalse = $CFG->tempdir . '/false.mbz';
+        $this->assertNotEmpty($packer->archive_to_pathname($files, $filefalse));
+        $context = context_system::instance();
+        $this->assertNotEmpty($storagefalse = $packer->archive_to_storage(
+                $files, $context->id, 'phpunit', 'data', 0, '/', 'false.mbz'));
+
+        // Create 2 archives in tgz mode.
+        $CFG->enabletgzbackups = true;
+        $filetrue = $CFG->tempdir . '/true.mbz';
+        $this->assertNotEmpty($packer->archive_to_pathname($files, $filetrue));
+        $context = context_system::instance();
+        $this->assertNotEmpty($storagetrue = $packer->archive_to_storage(
+                $files, $context->id, 'phpunit', 'data', 0, '/', 'false.mbz'));
+
+        // Check the sizes are different (indicating different formats).
+        $this->assertNotEquals(filesize($filefalse), filesize($filetrue));
+        $this->assertNotEquals($storagefalse->get_filesize(), $storagetrue->get_filesize());
+
+        // Extract files into storage and into filesystem from both formats.
+        // (Note: the setting does not matter, but set to false just to check.)
+        $CFG->enabletgzbackups = false;
+
+        // Extract to path (zip).
+        $packer->extract_to_pathname($filefalse, $CFG->tempdir);
+        $onefile = $CFG->tempdir . '/1.txt';
+        $this->assertEquals('frog', file_get_contents($onefile));
+        unlink($onefile);
+
+        // Extract to path (tgz).
+        $packer->extract_to_pathname($filetrue, $CFG->tempdir);
+        $onefile = $CFG->tempdir . '/1.txt';
+        $this->assertEquals('frog', file_get_contents($onefile));
+        unlink($onefile);
+
+        // Extract to storage (zip).
+        $packer->extract_to_storage($storagefalse, $context->id, 'phpunit', 'data', 1, '/');
+        $fs = get_file_storage();
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 1, '/', '1.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('frog', $out->get_content());
+
+        // Extract to storage (tgz).
+        $packer->extract_to_storage($storagetrue, $context->id, 'phpunit', 'data', 2, '/');
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/', '1.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('frog', $out->get_content());
+    }
+}

lib/moodlelib.php

@@ -6149,14 +6149,18 @@ function get_file_packer($mimetype='application/zip') {
 
     switch ($mimetype) {
         case 'application/zip':
-        case 'application/vnd.moodle.backup':
         case 'application/vnd.moodle.profiling':
             $classname = 'zip_packer';
             break;
+
         case 'application/x-gzip' :
             $classname = 'tgz_packer';
             break;
 
+        case 'application/vnd.moodle.backup':
+            $classname = 'mbz_packer';
+            break;
+
         default:
             return false;
     }