Support HTML5 / Javascript chunked file uploads

Summary:
Ref T7149. This adds chunking support to drag-and-drop uploads. It never activates right now unless you hack things up, since the chunk engine is still hard-coded as disabled.

The overall approach is the same as `arc upload` in D12061, with some slight changes to the API return values to avoid a few extra HTTP calls.

Test Plan:
  - Enabled chunk engine.
  - Uploaded some READMEs in a bunch of tiny 32 byte chunks.
  - Worked out of the box in Safari, Chrome, Firefox.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T7149

Differential Revision: https://secure.phabricator.com/D12066

Author: epriestley

resources/celerity/map.php

@@ -8,10 +8,10 @@
 return array(
   'names' => array(
     'core.pkg.css' => 'efdeeb14',
-    'core.pkg.js' => 'deae6907',
+    'core.pkg.js' => '31bc6546',
     'darkconsole.pkg.js' => '8ab24e01',
     'differential.pkg.css' => '1940be3f',
-    'differential.pkg.js' => '53c1ccc2',
+    'differential.pkg.js' => 'be1e5f9b',
     'diffusion.pkg.css' => '591664fa',
     'diffusion.pkg.js' => 'bfc0737b',
     'maniphest.pkg.css' => '68d4dd3d',
@@ -438,9 +438,9 @@
     'rsrc/js/application/uiexample/gesture-example.js' => '558829c2',
     'rsrc/js/application/uiexample/notification-example.js' => '8ce821c5',
     'rsrc/js/core/Busy.js' => '6453c869',
-    'rsrc/js/core/DragAndDropFileUpload.js' => '8c49f386',
+    'rsrc/js/core/DragAndDropFileUpload.js' => 'fd6ace61',
     'rsrc/js/core/DraggableList.js' => 'a16ec1c6',
-    'rsrc/js/core/FileUpload.js' => 'a4ae61bf',
+    'rsrc/js/core/FileUpload.js' => '477359c8',
     'rsrc/js/core/Hovercard.js' => '7e8468ae',
     'rsrc/js/core/KeyboardShortcut.js' => '1ae869f2',
     'rsrc/js/core/KeyboardShortcutManager.js' => 'c1700f6f',
@@ -458,13 +458,13 @@
     'rsrc/js/core/behavior-crop.js' => 'fa0f4fc2',
     'rsrc/js/core/behavior-dark-console.js' => '08883e8b',
     'rsrc/js/core/behavior-device.js' => '03d6ed07',
-    'rsrc/js/core/behavior-drag-and-drop-textarea.js' => '92eb531d',
+    'rsrc/js/core/behavior-drag-and-drop-textarea.js' => '6d49590e',
     'rsrc/js/core/behavior-error-log.js' => '6882e80a',
     'rsrc/js/core/behavior-fancy-datepicker.js' => 'c51ae228',
     'rsrc/js/core/behavior-file-tree.js' => '88236f00',
     'rsrc/js/core/behavior-form.js' => '5c54cbf3',
     'rsrc/js/core/behavior-gesture.js' => '3ab51e2c',
-    'rsrc/js/core/behavior-global-drag-and-drop.js' => '8c584f17',
+    'rsrc/js/core/behavior-global-drag-and-drop.js' => 'bbdf75ca',
     'rsrc/js/core/behavior-high-security-warning.js' => '8fc1c918',
     'rsrc/js/core/behavior-history-install.js' => '7ee2b591',
     'rsrc/js/core/behavior-hovercard.js' => 'f36e01af',
@@ -549,7 +549,7 @@
     'javelin-behavior-aphlict-status' => 'ea681761',
     'javelin-behavior-aphront-basic-tokenizer' => 'b3a4b884',
     'javelin-behavior-aphront-crop' => 'fa0f4fc2',
-    'javelin-behavior-aphront-drag-and-drop-textarea' => '92eb531d',
+    'javelin-behavior-aphront-drag-and-drop-textarea' => '6d49590e',
     'javelin-behavior-aphront-form-disable-on-submit' => '5c54cbf3',
     'javelin-behavior-aphront-more' => 'a80d0378',
     'javelin-behavior-audio-source' => '59b251eb',
@@ -588,7 +588,7 @@
     'javelin-behavior-durable-column' => 'a3ba7034',
     'javelin-behavior-error-log' => '6882e80a',
     'javelin-behavior-fancy-datepicker' => 'c51ae228',
-    'javelin-behavior-global-drag-and-drop' => '8c584f17',
+    'javelin-behavior-global-drag-and-drop' => 'bbdf75ca',
     'javelin-behavior-herald-rule-editor' => '7ebaeed3',
     'javelin-behavior-high-security-warning' => '8fc1c918',
     'javelin-behavior-history-install' => '7ee2b591',
@@ -719,11 +719,11 @@
     'phabricator-core-css' => '86bfbe8c',
     'phabricator-countdown-css' => '86b7b0a0',
     'phabricator-dashboard-css' => '17937d22',
-    'phabricator-drag-and-drop-file-upload' => '8c49f386',
+    'phabricator-drag-and-drop-file-upload' => 'fd6ace61',
     'phabricator-draggable-list' => 'a16ec1c6',
     'phabricator-fatal-config-template-css' => '8e6c6fcd',
     'phabricator-feed-css' => 'b513b5f4',
-    'phabricator-file-upload' => 'a4ae61bf',
+    'phabricator-file-upload' => '477359c8',
     'phabricator-filetree-view-css' => 'fccf9f82',
     'phabricator-flag-css' => '5337623f',
     'phabricator-hovercard' => '7e8468ae',
@@ -1122,6 +1122,11 @@
       'javelin-dom',
       'javelin-workflow',
     ),
+    '477359c8' => array(
+      'javelin-install',
+      'javelin-dom',
+      'phabricator-notification',
+    ),
     47830651 => array(
       'javelin-behavior',
       'javelin-dom',
@@ -1273,6 +1278,12 @@
       'javelin-typeahead',
       'javelin-uri',
     ),
+    '6d49590e' => array(
+      'javelin-behavior',
+      'javelin-dom',
+      'phabricator-drag-and-drop-file-upload',
+      'phabricator-textareautils',
+    ),
     '6e2de6f2' => array(
       'multirow-row-manager',
       'javelin-install',
@@ -1508,21 +1519,6 @@
       'javelin-request',
       'javelin-typeahead-source',
     ),
-    '8c49f386' => array(
-      'javelin-install',
-      'javelin-util',
-      'javelin-request',
-      'javelin-dom',
-      'javelin-uri',
-      'phabricator-file-upload',
-    ),
-    '8c584f17' => array(
-      'javelin-behavior',
-      'javelin-dom',
-      'javelin-uri',
-      'javelin-mask',
-      'phabricator-drag-and-drop-file-upload',
-    ),
     '8ce821c5' => array(
       'phabricator-notification',
       'javelin-stratcom',
@@ -1546,12 +1542,6 @@
       'javelin-uri',
       'phabricator-notification',
     ),
-    '92eb531d' => array(
-      'javelin-behavior',
-      'javelin-dom',
-      'phabricator-drag-and-drop-file-upload',
-      'phabricator-textareautils',
-    ),
     '9414ff18' => array(
       'javelin-behavior',
       'javelin-resource',
@@ -1638,11 +1628,6 @@
       'javelin-vector',
       'differential-inline-comment-editor',
     ),
-    'a4ae61bf' => array(
-      'javelin-install',
-      'javelin-dom',
-      'phabricator-notification',
-    ),
     'a80d0378' => array(
       'javelin-behavior',
       'javelin-stratcom',
@@ -1718,6 +1703,13 @@
       'javelin-stratcom',
       'javelin-dom',
     ),
+    'bbdf75ca' => array(
+      'javelin-behavior',
+      'javelin-dom',
+      'javelin-uri',
+      'javelin-mask',
+      'phabricator-drag-and-drop-file-upload',
+    ),
     'bd4c8dca' => array(
       'javelin-install',
       'javelin-util',
@@ -2007,6 +1999,14 @@
       'javelin-dom',
       'phortune-credit-card-form',
     ),
+    'fd6ace61' => array(
+      'javelin-install',
+      'javelin-util',
+      'javelin-request',
+      'javelin-dom',
+      'javelin-uri',
+      'phabricator-file-upload',
+    ),
     'fe287620' => array(
       'javelin-install',
       'javelin-dom',

src/applications/files/conduit/FileAllocateConduitAPIMethod.php

@@ -121,10 +121,20 @@ protected function execute(ConduitAPIRequest $request) {
     }
 
     // None of the storage engines can accept this file.
+    if (PhabricatorFileStorageEngine::loadWritableEngines()) {
+      $error = pht(
+        'Unable to upload file: this file is too large for any '.
+        'configured storage engine.');
+    } else {
+      $error = pht(
+        'Unable to upload file: the server is not configured with any '.
+        'writable storage engines.');
+    }
 
     return array(
       'upload' => false,
       'filePHID' => null,
+      'error' => $error,
     );
   }
 

src/applications/files/controller/PhabricatorFileDropUploadController.php

@@ -13,18 +13,82 @@ public function processRequest() {
     // NOTE: Throws if valid CSRF token is not present in the request.
     $request->validateCSRF();
 
-    $data = PhabricatorStartup::getRawInput();
     $name = $request->getStr('name');
-
+    $file_phid = $request->getStr('phid');
     // If there's no explicit view policy, make it very restrictive by default.
     // This is the correct policy for files dropped onto objects during
     // creation, comment and edit flows.
-
     $view_policy = $request->getStr('viewPolicy');
     if (!$view_policy) {
       $view_policy = $viewer->getPHID();
     }
 
+    $is_chunks = $request->getBool('querychunks');
+    if ($is_chunks) {
+      $params = array(
+        'filePHID' => $file_phid,
+      );
+
+      $result = id(new ConduitCall('file.querychunks', $params))
+        ->setUser($viewer)
+        ->execute();
+
+      return id(new AphrontAjaxResponse())->setContent($result);
+    }
+
+    $is_allocate = $request->getBool('allocate');
+    if ($is_allocate) {
+      $params = array(
+        'name' => $name,
+        'contentLength' => $request->getInt('length'),
+        'viewPolicy' => $view_policy,
+
+        // TODO: Remove.
+        // 'forceChunking' => true,
+      );
+
+      $result = id(new ConduitCall('file.allocate', $params))
+        ->setUser($viewer)
+        ->execute();
+
+      $file_phid = $result['filePHID'];
+      if ($file_phid) {
+        $file = $this->loadFile($file_phid);
+        $result += $this->getFileDictionary($file);
+      }
+
+      return id(new AphrontAjaxResponse())->setContent($result);
+    }
+
+    // Read the raw request data. We're either doing a chunk upload or a
+    // vanilla upload, so we need it.
+    $data = PhabricatorStartup::getRawInput();
+
+
+    $is_chunk_upload = $request->getBool('uploadchunk');
+    if ($is_chunk_upload) {
+      $params = array(
+        'filePHID' => $file_phid,
+        'byteStart' => $request->getInt('byteStart'),
+        'data' => $data,
+      );
+
+      $result = id(new ConduitCall('file.uploadchunk', $params))
+        ->setUser($viewer)
+        ->execute();
+
+      $file = $this->loadFile($file_phid);
+      if ($file->getIsPartial()) {
+        $result = array();
+      } else {
+        $result = array(
+          'complete' => true,
+        ) + $this->getFileDictionary($file);
+      }
+
+      return id(new AphrontAjaxResponse())->setContent($result);
+    }
+
     $file = PhabricatorFile::newFromXHRUpload(
       $data,
       array(
@@ -34,12 +98,30 @@ public function processRequest() {
         'isExplicitUpload' => true,
       ));
 
-    return id(new AphrontAjaxResponse())->setContent(
-      array(
-        'id'   => $file->getID(),
-        'phid' => $file->getPHID(),
-        'uri'  => $file->getBestURI(),
-      ));
+    $result = $this->getFileDictionary($file);
+    return id(new AphrontAjaxResponse())->setContent($result);
+  }
+
+  private function getFileDictionary(PhabricatorFile $file) {
+    return array(
+      'id'   => $file->getID(),
+      'phid' => $file->getPHID(),
+      'uri'  => $file->getBestURI(),
+    );
+  }
+
+  private function loadFile($file_phid) {
+    $viewer = $this->getViewer();
+
+    $file = id(new PhabricatorFileQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($file_phid))
+      ->executeOne();
+    if (!$file) {
+      throw new Exception(pht('Failed to load file.'));
+    }
+
+    return $file;
   }
 
 }

src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php

@@ -162,7 +162,7 @@ private function getWritableEngine() {
     return false;
   }
 
-  private function getChunkSize() {
+  public function getChunkSize() {
     // TODO: This is an artificially small size to make it easier to
     // test chunking.
     return 32;

src/applications/files/engine/PhabricatorFileStorageEngine.php

@@ -255,4 +255,40 @@ public static function loadWritableEngines() {
     return $writable;
   }
 
+
+  /**
+   * Return the largest file size which can be uploaded without chunking.
+   *
+   * Files smaller than this will always upload in one request, so clients
+   * can safely skip the allocation step.
+   *
+   * @return int|null Byte size, or `null` if there is no chunk support.
+   */
+  public static function getChunkThreshold() {
+    $engines = self::loadWritableEngines();
+
+    $min = null;
+    foreach ($engines as $engine) {
+      if (!$engine->isChunkEngine()) {
+        continue;
+      }
+
+      if (!$min) {
+        $min = $engine;
+        continue;
+      }
+
+      if ($min->getChunkSize() > $engine->getChunkSize()) {
+        $min = $engine->getChunkSize();
+      }
+    }
+
+    if (!$min) {
+      return null;
+    }
+
+    return $engine->getChunkSize();
+  }
+
+
 }

src/applications/files/view/PhabricatorGlobalUploadTargetView.php

@@ -24,11 +24,12 @@ public function render() {
     require_celerity_resource('global-drag-and-drop-css');
 
     Javelin::initBehavior('global-drag-and-drop', array(
-      'ifSupported'   => $this->showIfSupportedID,
-      'instructions'  => $instructions_id,
-      'uploadURI'     => '/file/dropupload/',
-      'browseURI'     => '/file/query/authored/',
-      'viewPolicy'    => PhabricatorPolicies::getMostOpenPolicy(),
+      'ifSupported' => $this->showIfSupportedID,
+      'instructions' => $instructions_id,
+      'uploadURI' => '/file/dropupload/',
+      'browseURI' => '/file/query/authored/',
+      'viewPolicy' => PhabricatorPolicies::getMostOpenPolicy(),
+      'chunkThreshold' => PhabricatorFileStorageEngine::getChunkThreshold(),
     ));
 
     return phutil_tag(

src/view/form/control/PhabricatorRemarkupControl.php

@@ -35,9 +35,10 @@ protected function renderInput() {
     Javelin::initBehavior(
       'aphront-drag-and-drop-textarea',
       array(
-        'target'          => $id,
-        'activatedClass'  => 'aphront-textarea-drag-and-drop',
-        'uri'             => '/file/dropupload/',
+        'target' => $id,
+        'activatedClass' => 'aphront-textarea-drag-and-drop',
+        'uri' => '/file/dropupload/',
+        'chunkThreshold' => PhabricatorFileStorageEngine::getChunkThreshold(),
       ));
 
     Javelin::initBehavior(