Allow streaming download of encrypted files (#817)

* allow streaming download of encrypted files * clean up logging * clean up logging * clean up logging * easier access to the encryption option * this is a long shot for ci * add files, try first-of-type in encrytpion test * go back to how the test was * CI is on a legacy browser * better handling of iv length for cross key version u-d-loads * the u3 d0 issue in ci * try with ss on ci * turn off ss for ci and on by default * try to use more recent chrome * more modern chrome needs test suite updates
filesender · Jul 25, 2020 · 2588e13 · 2588e13
1 parent bffdb35
commit 2588e13
Show file tree

Hide file tree

Showing 44 changed files with 7,848 additions and 254 deletions.
diff --git a/.gitignore b/.gitignore
@@ -14,3 +14,9 @@
 /push.sh
 /sync.sh
 composer.phar
+bak/
+git-switch-to-current-latest-development.sh
+import-all-poeditor-langs-and-push.sh
+scripts/client/test.php
+scripts/testing/test1.php
+
diff --git a/classes/data/Transfer.class.php b/classes/data/Transfer.class.php
@@ -986,6 +986,9 @@ public function __get($property)
         }
 
         if ($property == 'is_encrypted') {
+            if (!array_key_exists('encryption', $this->options)) {
+                return false;
+            }
             return $this->options['encryption'];
         }
         if ($property == "get_a_link") {

diff --git a/classes/storage/StorageFilesystem.class.php b/classes/storage/StorageFilesystem.class.php
@@ -354,7 +354,11 @@ public static function readChunk(File $file, $offset, $length)
         if (!file_exists($file_path)) {
             throw new StorageFilesystemFileNotFoundException($file_path, $file);
         }
-
+
+        if ($file->transfer->is_encrypted) {
+            $offset=floor($offset/Config::get('upload_chunk_size')*Config::get('upload_crypted_chunk_size'));
+        }
+
         // Open file for reading
         if ($fh = fopen($file_path, 'rb')) {
             // Sets position of file pointer
@@ -451,7 +455,7 @@ public static function completeFile(File $file)
         clearstatcache(true, $file_path);
         $size = filesize($file_path);
 
-        if ($file->transfer->options['encryption']) {
+        if ($file->transfer->is_encrypted) {
             if ($size != $file->encrypted_size) {
                 throw new FileIntegrityCheckFailedException($file, 'Expected size was '.$file->size.' but size on disk is '.$size);
             }

diff --git a/classes/storage/StorageFilesystemChunked.class.php b/classes/storage/StorageFilesystemChunked.class.php
@@ -73,7 +73,7 @@ public static function getChunkFilename($file_path, $offset)
      */
     public static function readChunk(File $file, $offset, $length)
     {
-        if ($file->transfer->options['encryption']) {
+        if ($file->transfer->is_encrypted) {
             $offset=$offset/Config::get('upload_chunk_size')*Config::get('upload_crypted_chunk_size');
         }
 

diff --git a/classes/utils/Browser.class.php b/classes/utils/Browser.class.php
@@ -44,14 +44,37 @@ class Browser
     protected $ua = '';
     protected $isChrome  = false;
     protected $isFirefox = false;
+    protected $isSafari  = false;
+    protected $allowStreamSaver  = false;
 
     public function __construct()
     {
         $ua = array_key_exists('HTTP_USER_AGENT', $_SERVER) ? $_SERVER['HTTP_USER_AGENT'] : '';
         $this->ua = $ua;
 
-        $this->isChrome  = preg_match('/[Cc]hrome/', $ua );
+        $this->isChrome  = preg_match('/[Cc]hrome/',  $ua );
         $this->isFirefox = preg_match('/[Ff]irefox/', $ua );
+        $this->isSafari  = preg_match('/[Ss]afari/',  $ua );
+        $this->isEdge    = preg_match('/[Ee]dge/',    $ua );
+
+        if( Config::get('streamsaver_enabled')) {
+
+            $this->allowStreamSaver = Config::get('streamsaver_on_unknown_browser');
+
+            if( $this->isFirefox ) {
+                $this->allowStreamSaver = Config::get('streamsaver_on_firefox');
+            }
+            if( $this->isChrome ) {
+                $this->allowStreamSaver = Config::get('streamsaver_on_chrome');
+            }
+            if( $this->isEdge ) {
+                $this->allowStreamSaver = Config::get('streamsaver_on_edge');
+            }
+            if( $this->isSafari ) {
+                $this->allowStreamSaver = Config::get('streamsaver_on_safari');
+            }
+
+        }
     }
     static function instance()
     {
@@ -63,7 +86,7 @@ static function instance()
     public function __get($property)
     {
         if (in_array($property, array(
-            'isChrome','isFirefox'
+            'isChrome','isFirefox','allowStreamSaver',
         ))) {
             return $this->$property;
         }

diff --git a/classes/utils/GUI.class.php b/classes/utils/GUI.class.php
@@ -104,24 +104,35 @@ public static function use_webasm_pbkdf2_implementation()
      */
     public static function scripts()
     {
-        $sources = array(
-            'lib/jquery/jquery.min.js',
-            'lib/jquery-ui/jquery-ui.min.js',
-            'lib/promise-polyfill/polyfill.min.js',
-            'lib/webcrypto-shim/webcrypto-shim.min.js',
-            'js/filesender.js',
-            'js/lang.js',
-            'js/client.js',
-            'js/transfer.js',
-            'js/logger.js',
-            'js/ui.js',
-            'js/FileSaver.js',
-            'js/crypter/crypto_common.js',
-            'js/crypter/crypto_blob_reader.js',
-            'js/crypter/crypto_app.js',
-            'js/pbkdf2dialog.js',
-            'lib/xregexp/xregexp-all.js'
+        $sources = array();
+
+        if( Browser::instance()->allowStreamSaver ) {
+            array_push( $sources,
+                        'lib/streamsaver/StreamSaver.js',
+                        'js/crypter/streamsaver_sink.js'
+            );
+        }
+
+        array_push( $sources,
+                    'lib/jquery/jquery.min.js',
+                    'lib/jquery-ui/jquery-ui.min.js',
+                    'lib/promise-polyfill/polyfill.min.js',
+                    'lib/web-streams-polyfill/dist/ponyfill.js',
+                    'lib/webcrypto-shim/webcrypto-shim.min.js',
+                    'js/filesender.js',
+                    'js/lang.js',
+                    'js/client.js',
+                    'js/transfer.js',
+                    'js/logger.js',
+                    'js/ui.js',
+                    'js/FileSaver.js',
+                    'js/crypter/crypto_common.js',
+                    'js/crypter/crypto_blob_reader.js',
+                    'js/crypter/crypto_app.js',
+                    'js/pbkdf2dialog.js',
+                    'lib/xregexp/xregexp-all.js'
         );
+
 
         if (Config::get('terasender_enabled')) {
             $sources[] = 'js/terasender/terasender.js';

diff --git a/classes/utils/Utilities.class.php b/classes/utils/Utilities.class.php
@@ -437,6 +437,13 @@ public static function isTrue($v)
     {
         return $v == '1' || $v == 'true';
     }
+    public static function boolToString($v)
+    {
+        if( $v == '1' || $v == 'true' ) {
+            return 'true';
+        }
+        return 'false';
+    }
 
     /**
      * This is a wrapper around the PHP http_build_query with some

diff --git a/docs/v2.0/admin/configuration/index.md b/docs/v2.0/admin/configuration/index.md
@@ -124,6 +124,13 @@ A note about colours;
 * [transfer_options_not_available_to_export_to_client](#transfer_options_not_available_to_export_to_client)
 * [chunk_upload_roundtriptoken_check_enabled](#chunk_upload_roundtriptoken_check_enabled)
 * [chunk_upload_roundtriptoken_check_accept_before](#chunk_upload_roundtriptoken_check_accept_before)
+* [streamsaver_enabled](#streamsaver_enabled)
+* [streamsaver_on_unknown_browser](#streamsaver_on_unknown_browser)
+* [streamsaver_on_firefox](#streamsaver_on_firefox)
+* [streamsaver_on_chrome](#streamsaver_on_chrome)
+* [streamsaver_on_edge](#streamsaver_edge)
+* [streamsaver_on_safari](#streamsaver_safari)
+
 
 ## Graphs
 
@@ -1244,11 +1251,67 @@ This is only for old, existing transfers which have no roundtriptoken set.
 
 
 
+### streamsaver_enabled
+* __description:__ Allow the use of StreamSaver to perform streaming download of encrypted files on supported browsers.
+* __mandatory:__ no 
+* __recommend_leaving_at_default:__ true
+* __type:__ boolean
+* __default:__ true
+* __available:__ since version 2.19
+* __comment:__ 
+
+### streamsaver_on_unknown_browser
+* __description:__ If streamsaver_enabled and the browser does not match any known browser with explicit configuration (eg streamsaver_on_firefox) then this is the default if the site should try to use streamsaver on that unknown environment.
+* __mandatory:__ no 
+* __recommend_leaving_at_default:__ true
+* __type:__ boolean
+* __default:__ false
+* __available:__ since version 2.19
+* __comment:__ 
+
+### streamsaver_on_firefox
+* __description:__ If streamsaver_enabled is true then this controls if streamsaver is used on Firefox.
+* __mandatory:__ no 
+* __recommend_leaving_at_default:__ false
+* __type:__ boolean
+* __default:__ false
+* __available:__ since version 2.19
+* __comment:__ 
+
+
+### streamsaver_on_chrome
+* __description:__ If streamsaver_enabled is true then this controls if streamsaver is used on Google Chrome.
+* __mandatory:__ no 
+* __recommend_leaving_at_default:__ true
+* __type:__ boolean
+* __default:__ true
+* __available:__ since version 2.19
+* __comment:__ 
+
+### streamsaver_on_edge
+* __description:__ If streamsaver_enabled is true then this controls if streamsaver is used on Microsoft Edge.
+* __mandatory:__ no 
+* __recommend_leaving_at_default:__ true
+* __type:__ boolean
+* __default:__ true
+* __available:__ since version 2.19
+* __comment:__ 
+
+### streamsaver_on_safari
+* __description:__ If streamsaver_enabled is true then this controls if streamsaver is used on Safari.
+* __mandatory:__ no 
+* __recommend_leaving_at_default:__ true
+* __type:__ boolean
+* __default:__ true
+* __available:__ since version 2.19
+* __comment:__ 
+
+
+
 
 
 
 
-* [transfer_options_not_available_to_export_to_client](#transfer_options_not_available_to_export_to_client)
 
 
 ---

diff --git a/includes/ConfigDefaults.php b/includes/ConfigDefaults.php
@@ -259,6 +259,14 @@
     // This allows authentication against password hashes in the local filesender db
     // with the right SAML setup.
     'using_local_saml_dbauth' => 0,
+
+    'streamsaver_enabled' => true,
+    'streamsaver_on_unknown_browser' => false,
+    'streamsaver_on_firefox' => false,
+    'streamsaver_on_chrome' => true,
+    'streamsaver_on_edge'   => true,
+    'streamsaver_on_safari' => true,
+
 
 
     'transfer_options' => array(

diff --git a/language/en_AU/lang.php b/language/en_AU/lang.php
@@ -579,3 +579,9 @@
 $lang['you_can_send_client_logs'] = 'In order to help your support team to find out what happened you can send the last log entries from your user interface by clicking this button :';
 $lang['you_generated_this_auth_secret_at'] = 'You generated this auth secret at: {datetime}';
 $lang['hit_guest_remind_rate_limit'] = 'Hit guest remind rate limit';
+$lang['use_streamsaver_for_download'] = 'Stream decrypted data directly to saved file. Streaming decrypted data directly to the file requires less memory on your computer allowing you to download larger files. Unfortunately this feature is not supported by all Web browsers. Your browser should support this feature so it has been enabled by default. If a download of encrypted files fails please uncheck this option and try again.';
+$lang['download_complete'] = 'Download complete';
+$lang['download_chunk_progress'] = 'Downloading chunk {chunkid} of {chunkcount} <br/> chunk progress: {percentofchunkcomplete} %';
+$lang['download_error'] = 'An error occurred while transferring the file.';
+$lang['download_error_abort'] = 'An ABORT error occurred while transferring the file.';
+$lang['confirm_leave_download_page'] = 'To continue to download your files you need to leave this page open.';
diff --git a/scripts/dev/www-lib-update/package.json b/scripts/dev/www-lib-update/package.json
@@ -15,6 +15,8 @@
     "jquery-ui-dist": "^1.12.1",
     "promise-polyfill": "^8.1.0",
     "reset": "^0.1.0",
+    "streamsaver": "^2.0.4",
+    "web-streams-polyfill": "^2.1.1",
     "webcrypto-shim": "^0.1.4",
     "xregexp": "^4.2.4"
   }

diff --git a/scripts/dev/www-lib-update/show-existing-versions.php b/scripts/dev/www-lib-update/show-existing-versions.php
@@ -39,10 +39,18 @@
         'path' => 'promise-polyfill/polyfill.min.js',
         'pattern' => '/v([0-9.]+)/m',
     ),
+    'streamsaver' => array(
+        'path' => 'streamsaver/StreamSaver.js',
+        'pattern' => '/v([0-9.]+)/m',
+    ),
     'webcrypto-shim' => array(
         'path' => 'webcrypto-shim/webcrypto-shim.min.js',
         'pattern' => '/ WebCrypto API shim v([0-9.]+)/m',
     ),
+    'web-streams-polyfill' => array(
+        'path' => 'web-streams-polyfill/dist/ponyfill.js',
+        'pattern' => '/v([0-9.]+)/m',
+    ),
     'xregexp' => array(
         'path' => 'xregexp/xregexp-all.js',
         'pattern' => "/XRegExp.version = '([0-9.]+)'/m",

diff --git a/scripts/dev/www-lib-update/show-new-versions.php b/scripts/dev/www-lib-update/show-new-versions.php
@@ -1,13 +1,15 @@
 <?php
 
 $packages = array(
-    'chart.js'       => array(),
-    'font-awesome'   => array(),
-    'jquery'         => array(),
-    'jquery-ui-dist' => array(),
-    'promise-polyfill' => array(),
-    'webcrypto-shim' => array(),
-    'xregexp' => array(),
+    'chart.js'             => array(),
+    'font-awesome'         => array(),
+    'jquery'               => array(),
+    'jquery-ui-dist'       => array(),
+    'promise-polyfill'     => array(),
+    'streamsaver'          => array(),
+    'webcrypto-shim'       => array(),
+    'web-streams-polyfill' => array(),
+    'xregexp'              => array(),
 );
 $json = file_get_contents(dirname(__FILE__) . '/package-lock.json');
 

diff --git a/scripts/dev/www-lib-update/update-library-versions-from-node_modules.php b/scripts/dev/www-lib-update/update-library-versions-from-node_modules.php
@@ -53,6 +53,14 @@
 
         ),
     ),
+    'streamsaver' => array(
+        'copy' => array(
+            'streamsaver/LICENSE'                        => 'streamsaver/LICENSE',
+            'streamsaver/StreamSaver.js'                 => 'streamsaver/StreamSaver.js',
+            'streamsaver/mitm.html'                      => 'streamsaver/mitm.html',
+            'streamsaver/sw.js'                          => 'streamsaver/sw.js',
+        ),
+    ),
     'webcrypto-shim' => array(
         'copy' => array(
             'webcrypto-shim/LICENSE'                    => 'webcrypto-shim/LICENSE',
@@ -61,6 +69,16 @@
 
         ),
     ),
+    'web-streams-polyfill' => array(
+        'copy' => array(
+            'web-streams-polyfill/LICENSE'              => 'web-streams-polyfill/LICENSE',
+            'web-streams-polyfill/dist/ponyfill.js'     => 'web-streams-polyfill/dist/ponyfill.js',
+            'web-streams-polyfill/dist/ponyfill.js.map'     => 'web-streams-polyfill/dist/ponyfill.js.map',
+            'web-streams-polyfill/dist/polyfill.js'     => 'web-streams-polyfill/dist/polyfill.js',
+            'web-streams-polyfill/dist/polyfill.js.map'     => 'web-streams-polyfill/dist/polyfill.js.map',
+
+        ),
+    ),
     'xregexp' => array(
         'copy' => array(
             'xregexp/LICENSE'                        => 'xregexp/LICENSE',

diff --git a/scripts/testing/README b/scripts/testing/README
@@ -3,4 +3,30 @@ of the souce tree and will copy the source code to
 /opt/filesenderetest
 and modify the config file to enable local testing.
 
+When testing upload and download of files and changes to that code
+it can be useful to have random files with specific sizes. The sizes
+allow testing around chunk boundaries. To aid in this the 
+generate-testfiles-for-current-chunk-size.php and verify-testfile-download-hash.php
+have been created.
 
+The generate-testfiles-for-current-chunk-size.php takes one parameter which is
+the location to create the test files and the script will generate a collection of files
+with random data for testing.
+
+That same location can be passed to verify-testfile-download-hash.php which also expects
+the path of a downloaded file as parameter 2. The verify script will test the md5 hash
+of your nominated file against the matching file in the generated test tree and report
+if they match or not. Note that the verify script will truncate common browser additions
+such as testfile-chunk0-1(1) back to find the base file name of testfile-chunk0-1.
+
+Common usage is as follows
+
+The same generated tree can be reused as desired
+php generate-testfiles-for-current-chunk-size.php  /tmp/xx2
+
+Files can be uploaded and downloaded in many browsers
+upload /tmp/xx2/testfile-chunk0-1
+download files to ~/Download/testfile-chunk0-1(1) testfile-chunk0-1(2) testfile-chunk0-1(3) and so on
+
+Verify downloads with
+php ./verify-testfile-download-hash.php /tmp/xx2  ~/Download/testfile-chunk0-1(3)