From 79c966cf213af9501a08857596ad9f28a323cc0a Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Wed, 13 Jun 2012 16:19:29 +0200 Subject: [PATCH] MDL-33710, MDL-33710 add utf-8 flag to all zip files created in moodle and unzip from storage fix --- lib/filestorage/file_packer.php | 2 +- lib/filestorage/tests/fixtures/test.txt | 1 + .../tests/fixtures/test_7zip_927.zip | Bin 0 -> 1020 bytes .../tests/fixtures/test_infozip_3.zip | Bin 0 -> 1160 bytes .../tests/fixtures/test_moodle.zip | Bin 0 -> 678 bytes .../tests/fixtures/test_moodle_22.zip | Bin 0 -> 678 bytes .../tests/fixtures/test_osx_1074.zip | Bin 0 -> 1100 bytes lib/filestorage/tests/fixtures/test_tc_8.zip | Bin 0 -> 780 bytes .../tests/fixtures/test_win8_cz.zip | Bin 0 -> 346 bytes .../tests/fixtures/test_win8_de.zip | Bin 0 -> 222 bytes .../tests/fixtures/test_winrar_421.zip | Bin 0 -> 1302 bytes .../tests/fixtures/test_winzip_165.zip | Bin 0 -> 1214 bytes .../tests/fixtures/zip_create_test_file.php | 55 ++++ lib/filestorage/tests/fixtures/zip_info.php | 302 ++++++++++++++++++ lib/filestorage/tests/zip_packer_test.php | 216 +++++++++++++ lib/filestorage/zip_archive.php | 168 +++++++++- lib/filestorage/zip_packer.php | 2 +- 17 files changed, 738 insertions(+), 8 deletions(-) create mode 100644 lib/filestorage/tests/fixtures/test.txt create mode 100644 lib/filestorage/tests/fixtures/test_7zip_927.zip create mode 100644 lib/filestorage/tests/fixtures/test_infozip_3.zip create mode 100644 lib/filestorage/tests/fixtures/test_moodle.zip create mode 100644 lib/filestorage/tests/fixtures/test_moodle_22.zip create mode 100644 lib/filestorage/tests/fixtures/test_osx_1074.zip create mode 100644 lib/filestorage/tests/fixtures/test_tc_8.zip create mode 100644 lib/filestorage/tests/fixtures/test_win8_cz.zip create mode 100644 lib/filestorage/tests/fixtures/test_win8_de.zip create mode 100644 lib/filestorage/tests/fixtures/test_winrar_421.zip create mode 100644 lib/filestorage/tests/fixtures/test_winzip_165.zip create mode 100644 lib/filestorage/tests/fixtures/zip_create_test_file.php create mode 100644 lib/filestorage/tests/fixtures/zip_info.php create mode 100644 lib/filestorage/tests/zip_packer_test.php diff --git a/lib/filestorage/file_packer.php b/lib/filestorage/file_packer.php index 7982bf209e31a..ad8e8c4aa25c0 100644 --- a/lib/filestorage/file_packer.php +++ b/lib/filestorage/file_packer.php @@ -84,7 +84,7 @@ public abstract function extract_to_storage($archivefile, $contextid, $component /** * Returns array of info about all files in archive * - * @param file_archive $archivefile + * @param string|file_archive $archivefile * @return array of file infos */ public abstract function list_files($archivefile); diff --git a/lib/filestorage/tests/fixtures/test.txt b/lib/filestorage/tests/fixtures/test.txt new file mode 100644 index 0000000000000..30d74d258442c --- /dev/null +++ b/lib/filestorage/tests/fixtures/test.txt @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/lib/filestorage/tests/fixtures/test_7zip_927.zip b/lib/filestorage/tests/fixtures/test_7zip_927.zip new file mode 100644 index 0000000000000000000000000000000000000000..08f89939f474012f1e5be499d838c53c8ed90481 GIT binary patch literal 1020 zcmWIWW@h1H0D%ol&N}ea)!$$NvO$;|hy#im(@OKw^-3yAN>YnU0&ptk1S$rp&;t=j zN;xpi<%23cy!J?MYBsXbA`A>b_5Lj99DqvT0Hprt-kj2-OY=*Q^kyI4s}FSs7PF-> z)OzRVVOj+AFc!_AV0gB*`^EaH$d1CT_QlF&FIP-PsAXibXTTMdDp2nzKxw3axly2N0u1G_v_q43~i24WnbYA$)x9mqu~_77X)I z5(lznuy}x32BYy=#)V?pN1$bh#DvZ4hq;hAhS9{EEz8KjfJl7U%#Pm+F&jn)c(bwr T9mK#6gmak~7(9U{FfafB)p_!G literal 0 HcmV?d00001 diff --git a/lib/filestorage/tests/fixtures/test_infozip_3.zip b/lib/filestorage/tests/fixtures/test_infozip_3.zip new file mode 100644 index 0000000000000000000000000000000000000000..419742edce9761981ea9ebc715729ec884a241da GIT binary patch literal 1160 zcmWIWW@h1HVBlb2*s$cR15aK34HgCl1`y_DkYNZYDm~MYR+^WtS5i?D8p6rI{O0Zr ze-JLM;AUWC`N{}XD*{wpl3H9605J;4LpO>OXcS0=9*7{)7%VOlfEtr|rZM?UPii&^ zjslvt&G(!G&~0!a1T-z`OixZ}$(f$~QlNp^m1i3D!LD0A?WR8nqj?hQK`bs)#A=pz zeja&FMGrqNpz+VPcE4CZl{7D7Gv~$1WiMAuCe9p2COKwYi9-S$kiht7c-shKq9qkp zNK(NFL8y^H4d{;JWI#638>j*)$q+COOOyzp7&i%O9BOL=Qx0d+O>jG6X;rb6_gwgDO6} z_DFAPwq8jEK^;6m9RWp$_oS8Pr4yr`3#k6t*6tVUrxL6B#mZ$bS4<{GwKUM|qkD5o zk1owGJ<^+fc(1;9ejcV*7?~ItaEBGp3?u*#xd3mdHXNab(4z}vg0vwl#-|q)eh3f` zWFm(qKAoVDM1X!E6A`Y&>)Q*|hX`SO`aq$J0Iz{eJYgN+%?e6@3~WG{#>l{+1;h*t E00KUmtN;K2 literal 0 HcmV?d00001 diff --git a/lib/filestorage/tests/fixtures/test_moodle_22.zip b/lib/filestorage/tests/fixtures/test_moodle_22.zip new file mode 100644 index 0000000000000000000000000000000000000000..dd69d7c84369afe52899285e06b440aeab6a3e34 GIT binary patch literal 678 zcmWIWW@Zs#U|`^2*s$cR15aK34K^T;1&BF;xFof>L=Qx0d+O>jG6djM%m-C`co4FPn4OprE&#rX7s!VdxBflTDk z#HSM!k_gZbWFo?qczt_;`Vb+EPai0B5#TkDi6^WByjejBkbw;d(-;{Tw1Aj_0RWIc Bnp^+? literal 0 HcmV?d00001 diff --git a/lib/filestorage/tests/fixtures/test_osx_1074.zip b/lib/filestorage/tests/fixtures/test_osx_1074.zip new file mode 100644 index 0000000000000000000000000000000000000000..4ba88a99097bd5484293ba343bdd5ddd8c826dd3 GIT binary patch literal 1100 zcmWIWW@Zs#-~hr6OU^nlAOT(m0fvC0(lZ@trFrRkB^4#15j+fU?%wbR;jfG$4BDQ$ zdW;MK-s~Jab@eycfLd9AH~^=CoInFhQj1IUKm?_R2|x|YJkywbrYAL_EuO#K2$-#0(4o2lp$= literal 0 HcmV?d00001 diff --git a/lib/filestorage/tests/fixtures/test_tc_8.zip b/lib/filestorage/tests/fixtures/test_tc_8.zip new file mode 100644 index 0000000000000000000000000000000000000000..4ba3a3da2d2076f1b44462251e3433365418a8f1 GIT binary patch literal 780 zcmWIWW@Zs#VB%n4@MAgWz<>n!fb64tb4rgc%`ZLDn|*k%egIT02g8OXXB~Lz>Tj?C z)vy4uG=^I5{Jg_!kMyQy>y=cLXnX4FF)~24Ghk}x25Ju|YD_E5OD9G-Cs27wYH^7k zh(Ix$1Ji7fH$duexCV<3E})KQTf1MZpNioa9I9WeT=sIsWMtKhOd{O4gB5525C}jh zj6jCz#qH_#vkpd)YuI8Zm#pll!uDfsbe1O+_^KwK~j z$bv@%SR2^Wm<|U;0ti5~9S5>dqXeH`P;`I*MDI@^3lTZQ>r-cBU_eAufHx~B=`yeb N;Xublt@ literal 0 HcmV?d00001 diff --git a/lib/filestorage/tests/fixtures/test_win8_cz.zip b/lib/filestorage/tests/fixtures/test_win8_cz.zip new file mode 100644 index 0000000000000000000000000000000000000000..457e990815c07867cb80722e8838d951676ff164 GIT binary patch literal 346 zcmWIWW@Zs#0D%ol&N}ea)!$$NvO$;=h)Yt7OY}el2nXO)$^%upaDHmGUP%Q3)uKSv z%W_KRLE|w*cu0APxfn(#lMj literal 0 HcmV?d00001 diff --git a/lib/filestorage/tests/fixtures/test_win8_de.zip b/lib/filestorage/tests/fixtures/test_win8_de.zip new file mode 100644 index 0000000000000000000000000000000000000000..07658a74381a00c0fda74db8140748774ff6a60d GIT binary patch literal 222 zcmWIWW@Zs#0D%ol&N}ea)!$$NvO$;|hy#im(@OKw^-3yAN>YnU0&ptk1S$rp&;t<= zrHo7>47g1Os%KziPykUtfMidAH*Sp}TR;G!Q5VR9+Zo`^$_A2R0>T0y9RcDn005WK BDhL1o literal 0 HcmV?d00001 diff --git a/lib/filestorage/tests/fixtures/test_winrar_421.zip b/lib/filestorage/tests/fixtures/test_winrar_421.zip new file mode 100644 index 0000000000000000000000000000000000000000..6c50b0777bd8dec0623ec5d11101c74d1990de55 GIT binary patch literal 1302 zcmWIWW@h1H0D%ol&N}ea)!$$NvO!pYL6|`~)GWm$NjcOaDaBN;q@tv>fS-X;^&D5y zv#s4P)=vcqm82Gz1Q0O9BGJrDIn*-I&=_t=M&9CmFIFylxneS=A$~099Dojh11<(} zhNzs`ML80$~QmLbHp%kM7MWJ-Rf%^hj^^;l28}-5|=Kf~4I$KW|}jYBt0n z3Ji?O_GuERCIM9(UVEe$q6WYBxEVwl0*V^bO7qeo1`0AT{&~RB6i{?{4@?lh37kM* zf>h{%2>eQUfWZJ#fp9)BAR_NxVgU)FyO@zlo&i_9nu9}@kwF0(!sxNg1(jiBkYHd) z+sl66^DN`T+{KCvUI-dXN(k^K)>2oX#lS>`k}9xS8ow7}DU1eNikfneEyYYX)=(b` z0O>9uMx-KS>!uhk0s9R`$8baVI8z#Ke~L1AfUSfC?nx9Yky0MA?XVDr*$$&i3kau9 zWZ$FbUv36VB&Vn_G9af^Yz~;E010gvU0Q%8HRH396PVFJ0OI*HppBrE2~OGA?DOSY z1P*o>jn8Nv21rywjGhQJ8kQ5VnQpWeVmgclMLu?dgV2r$bi3Ozj@Y*B2 z5MhKdTtL$q82nkzIe-~Zf(vL&UruRBera;{U;WYopdq|2M?M|hn^SspX@2RE-t5DB zV{kiJltBeayLW!xTex!+7#LX-Ps~I$38(^_+kj!pff=SC7eCwD{bK!8kOvVy!L9nm z%4IKCOh#7C$RxsmE5v0={l4c}#s;>lEP|K~qXWEIK?M^7I}rY0Vqn++ IG=YHu0PZt9xBvhE literal 0 HcmV?d00001 diff --git a/lib/filestorage/tests/fixtures/zip_create_test_file.php b/lib/filestorage/tests/fixtures/zip_create_test_file.php new file mode 100644 index 0000000000000..678aa3417fcb0 --- /dev/null +++ b/lib/filestorage/tests/fixtures/zip_create_test_file.php @@ -0,0 +1,55 @@ +. + +/** + * This debug script is used during zip support development. + * + * @package core_files + * @copyright 2012 Petr Skoda + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +define('CLI_SCRIPT', true); + +require(__DIR__.'/../../../../config.php'); +require_once($CFG->libdir.'/clilib.php'); + +$help = + "Create sample zip file for testing +Example: + \$php zip_create_test_file.php test.zip +"; + +if (count($_SERVER['argv']) != 2 or file_exists($_SERVER['argv'][1])) { + echo $help; + exit(0); +} + +$archive = $_SERVER['argv'][1]; + +$packer = get_file_packer('application/zip'); + +$file = __DIR__.'/test.txt'; +$files = array( + 'test.test' => $file, + 'testíček.txt' => $file, + 'Prüfung.txt' => $file, + '测试.txt' => $file, + '試験.txt' => $file, + 'Žluťoučký/Koníček.txt' => $file, +); + +$packer->archive_to_pathname($files, $archive); diff --git a/lib/filestorage/tests/fixtures/zip_info.php b/lib/filestorage/tests/fixtures/zip_info.php new file mode 100644 index 0000000000000..287466300a12f --- /dev/null +++ b/lib/filestorage/tests/fixtures/zip_info.php @@ -0,0 +1,302 @@ +. + +/** + * This debug script is used during zip support development ONLY. + * + * @package core_files + * @copyright 2012 Petr Skoda + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +define('CLI_SCRIPT', true); + +require(__DIR__.'/../../../../config.php'); +require_once($CFG->libdir.'/clilib.php'); + +if (count($_SERVER['argv']) != 2 or !file_exists($_SERVER['argv'][1])) { + cli_error("This script expects zip file name as the only parameter"); +} + +$archive = $_SERVER['argv'][1]; + +// Note: the ZIP structure is described at http://www.pkware.com/documents/casestudies/APPNOTE.TXT +if (!$filesize = filesize($archive) or !$fp = fopen($archive, 'rb+')) { + cli_error("Can not open file file $archive"); +} + +fseek($fp, 0); +$info = unpack('Vsig', fread($fp, 4)); +if ($info['sig'] !== 0x04034b50) { + fclose($fp); + cli_error("This is not a zip file: $archive"); +} + +// Find end of central directory record. +fseek($fp, $filesize - 22); +$info = unpack('Vsig', fread($fp, 4)); +if ($info['sig'] === 0x06054b50) { + // There is no comment. + fseek($fp, $filesize - 22); + $data = fread($fp, 22); +} else { + // There is some comment with 0xFF max size - that is 65557. + fseek($fp, $filesize - 65557); + $data = fread($fp, 65557); +} + +$pos = strpos($data, pack('V', 0x06054b50)); +if ($pos === false) { + // Borked ZIP structure! + fclose($fp); + cli_error("Can not find end of central directory in $archive"); +} +$centralend = unpack('Vsig/vdisk/vdisk_start/vdisk_entries/ventries/Vsize/Voffset/vcomment_length', substr($data, $pos, 22)); + +if ($centralend['disk'] !== 0 or $centralend['disk_start'] !== 0) { + cli_error("Multi-disk archives are not supported: $archive"); +} + +if ($centralend['offset'] === 0xFFFFFFFF) { + cli_error("ZIP64 archives are not supported: $archive"); +} + +if ($centralend['comment_length']) { + $centralend['comment'] = substr($data, 22, $centralend['comment_length']); +} else { + $centralend['comment'] = ''; +} + +fseek($fp, $centralend['offset']); +$data = fread($fp, $centralend['size']); +$pos = 0; +$files = array(); +for($i=0; $i<$centralend['entries']; $i++) { + $file = unpack('Vsig/vversion/vversion_req/vgeneral/vmethod/Vmodified/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length/vcomment_length/vdisk/vattr/Vattrext/Vlocal_offset', substr($data, $pos, 46)); + $file['error'] = null; + $file['central_offset'] = $centralend['offset'] + $pos; + $pos = $pos + 46; + if ($file['sig'] !== 0x02014b50) { + $files[] = array('error'=>'Invalid central file signature'); + continue; + } + $file['name'] = substr($data, $pos, $file['name_length']); + $pos = $pos + $file['name_length']; + $file['extra'] = array(); + if ($file['extra_length']) { + $extradata = substr($data, $pos, $file['extra_length']); + while (strlen($extradata) > 4) { + $extra = unpack('vid/vsize', substr($extradata, 0, 4)); + $extra['data'] = substr($extradata, 4, $extra['size']); + $extradata = substr($extradata, 4+$extra['size']); + $file['extra'][] = $extra; + } + $pos = $pos + $file['extra_length']; + } + if ($file['comment_length']) { + $file['comment'] = substr($data, $pos, $file['comment_length']); + $pos = $pos + $file['comment_length']; + } else { + $file['comment'] = ''; + } + + // Read local file header. + fseek($fp, $file['local_offset']); + $localfile = unpack('Vsig/vversion_req/vgeneral/vmethod/Vmodified/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length', fread($fp, 30)); + if ($localfile['sig'] !== 0x04034b50) { + // Borked file! + $file['error'] = 'Invalid local file signature'; + $files[] = $file; + continue; + } + if ($localfile['name_length']) { + $localfile['name'] = fread($fp, $localfile['name_length']); + } else { + $localfile['name'] = ''; + } + $localfile['extra'] = array(); + if ($localfile['extra_length']) { + $extradata = fread($fp, $localfile['extra_length']); + while (strlen($extradata) > 4) { + $extra = unpack('vid/vsize', substr($extradata, 0, 4)); + $extra['data'] = substr($extradata, 4, $extra['size']); + $extradata = substr($extradata, 4+$extra['size']); + $localfile['extra'][] = $extra; + } + } + + $file['local'] = $localfile; + $files[] = $file; +} + +echo "Archive: $archive\n"; +echo "Number of files: {$centralend['entries']}\n"; +echo "Archive comment: \"{$centralend['comment']}\" ({$centralend['comment_length']} bytes)\n"; +foreach ($files as $i=>$file) { + echo "======== File ".($i+1)." ==============================================\n"; + if (!empty($file['error'])) { + echo " ERROR: {$file['error']}\n"; + } + echo " Name: ".zip_print_name($file['name'])."\n"; + if ($file['comment'] !== '') { + echo " Comment: \"{$file['comment']}\" ({$file['comment_length']} bytes)\n"; + } + echo " Version: 0x".str_pad(dechex($file['version']), 4, '0', STR_PAD_LEFT)."\n"; + echo " Required: 0x".str_pad(dechex($file['version_req']), 4, '0', STR_PAD_LEFT)."\n"; + echo " Method: ".zip_print_method($file['method'])."\n"; + echo " General: ".zip_print_general($file['general'])."\n"; + echo " Modified: ".userdate(zip_dos2unixtime($file['modified']))."\n"; + echo " Size: ".zip_print_sizes($file['size'], $file['size_compressed'])."\n"; + echo " CRC-32: {$file['crc']}\n"; + if ($file['extra']) { + foreach($file['extra'] as $j=>$extra) { + echo " Extra ".($j+1).": ".zip_print_extra($extra)."\n"; + } + } + if (!empty($file['local']['error'])) { + echo " Local ERROR: {$file['local']['error']}\n"; + } + $localfile = $file['local']; + if ($localfile['name'] !== $file['name']) { + echo " Local name: ".zip_print_name($localfile['name'])."\n"; + } + if ($localfile['version_req'] !== $file['version_req']) { + echo " Local required: 0x".str_pad(dechex($localfile['version_req']), 4, '0', STR_PAD_LEFT)."\n"; + } + if ($localfile['method'] !== $file['method']) { + echo " Local method: ".zip_print_method($localfile['method'])."\n"; + } + if ($localfile['general'] !== $file['general']) { + echo " Local general: ".zip_print_general($localfile['general'])."\n"; + } + if ($localfile['modified'] !== $file['modified']) { + echo " Local modified: ".userdate(zip_dos2unixtime($localfile['modified']))."\n"; + } + if ($localfile['size'] !== $file['size']) { + echo " Local size: ".zip_print_sizes($localfile['size'], $localfile['size_compressed'])."\n"; + } + if ($localfile['crc'] !== $file['crc']) { + echo " Local CRC-32: {$localfile['crc']}\n"; + } + if ($localfile['extra']) { + foreach($localfile['extra'] as $j=>$extra) { + echo " Local extra ".($j+1).": ".zip_print_extra($extra)."\n"; + } + } +} + +fclose($fp); +exit(0); + +// === Some useful functions ====================================== + +function zip_print_name($name) { + $size = strlen($name); + $crc = crc32($name); + return "\"$name\" ($size bytes) - CRC $crc"; +} + +function zip_print_method($method) { + $desc = ''; + switch($method) { + case 0: $desc = 'Stored'; break; + case 1: $desc = 'Shrunk'; break; + case 2: $desc = 'Reduced factor 1'; break; + case 3: $desc = 'Reduced factor 2'; break; + case 4: $desc = 'Reduced factor 3'; break; + case 5: $desc = 'Reduced factor 4'; break; + case 6: $desc = 'Imploded'; break; + case 8: $desc = 'Deflated'; break; + case 9: $desc = 'Deflate64'; break; + case 10: $desc = 'old IBM TERSE'; break; + case 12: $desc = 'BZIP2'; break; + case 14: $desc = 'LZMA'; break; + case 18: $desc = 'IBM TERSE'; break; + case 19: $desc = 'IBM LZ77'; break; + case 97: $desc = 'WavPack'; break; + case 98: $desc = 'PPMd v1'; break; + } + if ($desc) { + $desc = " ($desc)"; + } + return "0x".str_pad(dechex($method), 4, '0', STR_PAD_LEFT).$desc; +} + +function zip_print_general($general) { + $desc = array(); + if ($general & pow(2, 0)) { + $desc[] = 'Encrypted'; + } + if ($general & pow(2, 11)) { + $desc[] = 'Unicode name'; + } + if ($desc) { + $desc = " (".implode(', ', $desc).")"; + } else { + $desc = ''; + } + return str_pad(decbin($general), 16, '0', STR_PAD_LEFT).$desc; +} + +/** + * Convert MS date+time format to unix timestamp: + * http://msdn.microsoft.com/en-us/library/windows/desktop/ms724274(v=vs.85).aspx + * + * Copied from: http://plugins.svn.wordpress.org/wp2epub/trunk/zipcreate/functions.lib.php + * author: redmonkey + * license: GPL + */ +function zip_dos2unixtime($dostime) { + $sec = 2 * ($dostime & 0x1f); + $min = ($dostime >> 5) & 0x3f; + $hrs = ($dostime >> 11) & 0x1f; + $day = ($dostime >> 16) & 0x1f; + $mon = ($dostime >> 21) & 0x0f; + $year = (($dostime >> 25) & 0x7f) + 1980; + + return mktime($hrs, $min, $sec, $mon, $day, $year); +} + +function zip_print_sizes($size, $compressed) { + return "$size ==> $compressed bytes"; +} + +function zip_print_extra($extra) { + $desc = ''; + $info = "- ".bin2hex($extra['data'])." ({$extra['size']} bytes)"; + switch($extra['id']) { + case 0x0009: $desc = 'OS/2'; break; + case 0x000a: $desc = 'NTFS'; break; + case 0x000d: $desc = 'UNIX'; break; + case 0x5455: $desc = 'Extended timestamp'; break; + case 0x5855: $desc = 'Infor-ZIP (original)'; break; + case 0x7075: + $desc = 'Info-ZIP Unicode path'; + $data = unpack('cversion/Vcrc', substr($extra['data'], 0, 5)); + $name = substr($extra['data'], 5); + $size = strlen($name); + if ($data['version'] === 1) { + $info = "- \"$name\" ($size bytes) - CRC {$data['crc']}"; + } + break; + case 0x7865: $desc = 'Info-ZIP UNIX (new)'; break; + case 0x7875: $desc = 'Info-ZIP UNIX (3rd generation)'; break; + } + if ($desc) { + $desc = " ($desc)"; + } + return "0x".str_pad(dechex($extra['id']), 4, '0', STR_PAD_LEFT)."$desc $info"; +} \ No newline at end of file diff --git a/lib/filestorage/tests/zip_packer_test.php b/lib/filestorage/tests/zip_packer_test.php new file mode 100644 index 0000000000000..996422638dca4 --- /dev/null +++ b/lib/filestorage/tests/zip_packer_test.php @@ -0,0 +1,216 @@ +. + +/** + * Unit tests for /lib/filestorage/zip_packer.php and zip_archive.php + * + * @package core_files + * @category phpunit + * @copyright 2012 Petr Skoda + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + + +class zip_packer_testcase extends advanced_testcase { + protected $testfile; + protected $files; + + protected function setUp() { + parent::setUp(); + + $this->testfile = __DIR__.'/fixtures/test.txt'; + + $fs = get_file_storage(); + $context = context_system::instance(); + if (!$file = $fs->get_file($context->id, 'phpunit', 'data', 0, '/', 'test.txt')) { + $file = $fs->create_file_from_pathname( + array('contextid'=>$context->id, 'component'=>'phpunit', 'filearea'=>'data', 'itemid'=>0, 'filepath'=>'/', 'filename'=>'test.txt'), + $this->testfile); + } + + $this->files = array( + 'test.test' => $this->testfile, + 'testíček.txt' => $this->testfile, + 'Prüfung.txt' => $this->testfile, + '测试.txt' => $this->testfile, + '試験.txt' => $this->testfile, + 'Žluťoučký/Koníček.txt' => $file, + ); + } + + public function test_get_packer() { + $this->resetAfterTest(false); + $packer = get_file_packer(); + $this->assertInstanceOf('zip_packer', $packer); + + $packer = get_file_packer('application/zip'); + $this->assertInstanceOf('zip_packer', $packer); + } + + /** + * @depends test_get_packer + */ + public function test_list_files() { + $this->resetAfterTest(false); + $moodle22 = __DIR__.'/fixtures/test_moodle_22.zip'; + $moodle = __DIR__.'/fixtures/test_moodle.zip'; + + $packer = get_file_packer('application/zip'); + + $archivefiles22 = $packer->list_files($moodle22); + $this->assertTrue(is_array($archivefiles22)); + $this->assertEquals(count($this->files), count($archivefiles22)); + foreach($archivefiles22 as $file) { + $this->assertArrayHasKey($file->pathname, $this->files); + } + + $archivefiles = $packer->list_files($moodle); + $this->assertTrue(is_array($archivefiles)); + $this->assertEquals(count($this->files), count($archivefiles)); + foreach($archivefiles as $file) { + $this->assertArrayHasKey($file->pathname, $this->files); + } + } + + /** + * @depends test_list_files + */ + public function test_archive_to_pathname() { + global $CFG; + + $this->resetAfterTest(false); + + $packer = get_file_packer('application/zip'); + $archive = "$CFG->tempdir/archive.zip"; + + $this->assertFalse(file_exists($archive)); + $result = $packer->archive_to_pathname($this->files, $archive); + $this->assertTrue($result); + $this->assertTrue(file_exists($archive)); + + $archivefiles = $packer->list_files($archive); + $this->assertTrue(is_array($archivefiles)); + $this->assertEquals(count($this->files), count($archivefiles)); + foreach($archivefiles as $file) { + $this->assertArrayHasKey($file->pathname, $this->files); + } + } + + /** + * @depends test_archive_to_pathname + */ + public function test_archive_to_storage() { + $this->resetAfterTest(false); + + $packer = get_file_packer('application/zip'); + $fs = get_file_storage(); + $context = context_system::instance(); + + $this->assertFalse($fs->file_exists($context->id, 'phpunit', 'test', 0, '/', 'archive.zip')); + $result = $packer->archive_to_storage($this->files, $context->id, 'phpunit', 'test', 0, '/', 'archive.zip'); + $this->assertInstanceOf('stored_file', $result); + $this->assertTrue($fs->file_exists($context->id, 'phpunit', 'test', 0, '/', 'archive.zip')); + + $archivefiles = $result->list_files($packer); + $this->assertTrue(is_array($archivefiles)); + $this->assertEquals(count($this->files), count($archivefiles)); + foreach($archivefiles as $file) { + $this->assertArrayHasKey($file->pathname, $this->files); + } + } + + /** + * @depends test_archive_to_storage + */ + public function test_extract_to_pathname() { + global $CFG; + + $this->resetAfterTest(false); + + $packer = get_file_packer('application/zip'); + $fs = get_file_storage(); + $context = context_system::instance(); + + $target = "$CFG->tempdir/test/"; + $testcontent = file_get_contents($this->testfile); + + @mkdir($target, $CFG->directorypermissions); + $this->assertTrue(is_dir($target)); + + $archive = "$CFG->tempdir/archive.zip"; + $this->assertTrue(file_exists($archive)); + $result = $packer->extract_to_pathname($archive, $target); + $this->assertTrue(is_array($result)); + $this->assertEquals(count($this->files), count($result)); + foreach($this->files as $file=>$unused) { + $this->assertTrue($result[$file]); + $this->assertTrue(file_exists($target.$file)); + $this->assertSame($testcontent, file_get_contents($target.$file)); + } + + $archive = $fs->get_file($context->id, 'phpunit', 'test', 0, '/', 'archive.zip'); + $this->assertNotEmpty($archive); + $result = $packer->extract_to_pathname($archive, $target); + $this->assertTrue(is_array($result)); + $this->assertEquals(count($this->files), count($result)); + foreach($this->files as $file=>$unused) { + $this->assertTrue($result[$file]); + $this->assertTrue(file_exists($target.$file)); + $this->assertSame($testcontent, file_get_contents($target.$file)); + } + } + + /** + * @depends test_archive_to_storage + */ + public function test_extract_to_storage() { + global $CFG; + + $this->resetAfterTest(true); + + $packer = get_file_packer('application/zip'); + $fs = get_file_storage(); + $context = context_system::instance(); + + $testcontent = file_get_contents($this->testfile); + + $archive = $fs->get_file($context->id, 'phpunit', 'test', 0, '/', 'archive.zip'); + $this->assertNotEmpty($archive); + $result = $packer->extract_to_storage($archive, $context->id, 'phpunit', 'target', 0, '/'); + $this->assertTrue(is_array($result)); + $this->assertEquals(count($this->files), count($result)); + foreach($this->files as $file=>$unused) { + $this->assertTrue($result[$file]); + $stored_file = $fs->get_file_by_hash(sha1("/$context->id/phpunit/target/0/$file")); + $this->assertInstanceOf('stored_file', $stored_file); + $this->assertSame($testcontent, $stored_file->get_content()); + } + + $archive = "$CFG->tempdir/archive.zip"; + $this->assertTrue(file_exists($archive)); + $result = $packer->extract_to_storage($archive, $context->id, 'phpunit', 'target', 0, '/'); + $this->assertTrue(is_array($result)); + $this->assertEquals(count($this->files), count($result)); + foreach($this->files as $file=>$unused) { + $this->assertTrue($result[$file]); + $stored_file = $fs->get_file_by_hash(sha1("/$context->id/phpunit/target/0/$file")); + $this->assertInstanceOf('stored_file', $stored_file); + $this->assertSame($testcontent, $stored_file->get_content()); + } + } +} diff --git a/lib/filestorage/zip_archive.php b/lib/filestorage/zip_archive.php index 9acd1d3484a67..4936d7b0089f0 100644 --- a/lib/filestorage/zip_archive.php +++ b/lib/filestorage/zip_archive.php @@ -46,9 +46,12 @@ class zip_archive extends file_archive { /** @var int Iteration position */ protected $pos = 0; - /** @var zip_archive TipArchive instance */ + /** @var ZipArchive instance */ protected $za; + /** @var bool was this archive modified? */ + protected $modified = false; + /** * Open or create archive (depending on $mode) * @@ -87,7 +90,7 @@ public function open($archivepathname, $mode=file_archive::CREATE, $encoding='ut } else { $this->za = null; $this->archivepathname = null; - $this->encooding = 'utf-8'; + $this->encoding = 'utf-8'; // TODO: maybe we should return some error info return false; } @@ -106,6 +109,11 @@ public function close() { $res = $this->za->close(); $this->za = null; + if ($this->modified) { + $this->fix_utf8_flags(); + $this->modified = false; + } + return $res; } @@ -241,7 +249,11 @@ public function add_file_from_pathname($localname, $pathname) { } } - return $this->za->addFile($pathname, $localname); + if (!$this->za->addFile($pathname, $localname)) { + return false; + } + $this->modified = true; + return true; } /** @@ -274,8 +286,11 @@ public function add_file_from_string($localname, $contents) { } $this->usedmem += strlen($contents); - return $this->za->addFromString($localname, $contents); - + if (!$this->za->addFromString($localname, $contents)) { + return false; + } + $this->modified = true; + return true; } /** @@ -296,7 +311,11 @@ public function add_directory($localname) { return false; } - return $this->za->addEmptyDir($localname); + if (!$this->za->addEmptyDir($localname)) { + return false; + } + $this->modified = true; + return true; } /** @@ -347,4 +366,141 @@ public function valid() { return ($this->pos < $this->count()); } + + /** + * Add unicode flag to all files in archive. + * + * NOTE: single disk archives only, no ZIP64 support. + * + * @return bool success, modifies the file contents + */ + protected function fix_utf8_flags() { + if ($this->encoding !== 'utf-8') { + return true; + } + + if (!file_exists($this->archivepathname)) { + return true; + } + + // Note: the ZIP structure is described at http://www.pkware.com/documents/casestudies/APPNOTE.TXT + if (!$fp = fopen($this->archivepathname, 'rb+')) { + return false; + } + if (!$filesize = filesize($this->archivepathname)) { + return false; + } + + // Find end of central directory record. + fseek($fp, $filesize - 22); + $info = unpack('Vsig', fread($fp, 4)); + if ($info['sig'] === 0x06054b50) { + // There is no comment. + fseek($fp, $filesize - 22); + $data = fread($fp, 22); + } else { + // There is some comment with 0xFF max size - that is 65557. + fseek($fp, $filesize - 65557); + $data = fread($fp, 65557); + } + + $pos = strpos($data, pack('V', 0x06054b50)); + if ($pos === false) { + // Borked ZIP structure! + fclose($fp); + return false; + } + $centralend = unpack('Vsig/vdisk/vdisk_start/vdisk_entries/ventries/Vsize/Voffset/vcomment_length', substr($data, $pos, 22)); + + if ($centralend['disk'] !== 0 or $centralend['disk_start'] !== 0) { + // Single disk archives only, sorry. + fclose($fp); + return false; + } + + if ($centralend['offset'] === 0xFFFFFFFF) { + // No support for ZIP64, sorry! + fclose($fp); + return false; + } + + fseek($fp, $centralend['offset']); + $data = fread($fp, $centralend['size']); + $pos = 0; + $files = array(); + for($i=0; $i<$centralend['entries']; $i++) { + $file = unpack('Vsig/vversion/vversion_req/vgeneral/vmethod/vmtime/vmdate/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length/vcomment_length/vdisk/vattr/Vattrext/Vlocal_offset', substr($data, $pos, 46)); + $file['central_offset'] = $centralend['offset'] + $pos; + $pos = $pos + 46; + if ($file['sig'] !== 0x02014b50) { + // Borked file! + fclose($fp); + return false; + } + $file['name'] = substr($data, $pos, $file['name_length']); + $pos = $pos + $file['name_length']; + if ($file['extra_length']) { + $file['extra'] = substr($data, $pos, $file['extra_length']); + $pos = $pos + $file['extra_length']; + } else { + $file['extra'] = ''; + } + if ($file['comment_length']) { + $file['comment'] = substr($data, $pos, $file['comment_length']); + $pos = $pos + $file['comment_length']; + } else { + $file['comment'] = ''; + } + + $newgeneral = $file['general'] | pow(2, 11); + if ($newgeneral === $file['general']) { + // Nothing to do with this file. + continue; + } + + if (preg_match('/^[a-zA-Z0-9_\-\.]*$/', $file['name'])) { + // ASCII file names are always ok. + continue; + } + if ($file['extra'] !== '') { + // Most probably not created by php zip ext, better to skip it. + continue; + } + if (fix_utf8($file['name']) !== $file['name']) { + // Does not look like a valid utf-8 encoded file name, skip it. + continue; + } + + // Read local file header. + fseek($fp, $file['local_offset']); + $localfile = unpack('Vsig/vversion_req/vgeneral/vmethod/vmtime/vmdate/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length', fread($fp, 30)); + if ($localfile['sig'] !== 0x04034b50) { + // Borked file! + fclose($fp); + return false; + } + + $file['local'] = $localfile; + $files[] = $file; + } + + foreach ($files as $file) { + $localfile = $file['local']; + // Add the unicode flag in central file header. + fseek($fp, $file['central_offset'] + 8); + if (ftell($fp) === $file['central_offset'] + 8) { + $newgeneral = $file['general'] | pow(2, 11); + fwrite($fp, pack('v', $newgeneral)); + } + // Modify local file header too. + fseek($fp, $file['local_offset'] + 6); + if (ftell($fp) === $file['local_offset'] + 6) { + $newgeneral = $localfile['general'] | pow(2, 11); + fwrite($fp, pack('v', $newgeneral)); + } + } + + fclose($fp); + return true; + } } diff --git a/lib/filestorage/zip_packer.php b/lib/filestorage/zip_packer.php index a7726a66dbb9e..f3b0484e6a262 100644 --- a/lib/filestorage/zip_packer.php +++ b/lib/filestorage/zip_packer.php @@ -435,7 +435,7 @@ public function extract_to_storage($archivefile, $contextid, $component, $filear /** * Returns array of info about all files in archive * - * @param file_archive $archivefile + * @param string|file_archive $archivefile * @return array of file infos */ public function list_files($archivefile) {