Feat-large-file-s3

src/Storage/Device.php

@@ -55,12 +55,13 @@ abstract public function getPath(string $filename, string $prefix = null): strin
      * @param int $chunk
      * @param int $chunks
      * @param string $tmp
+     * @param array $metadata
      * 
      * @throws \Exception
      *
      * @return int
      */
-    abstract public function upload($source, $path, $chunk = 1, $chunks = 1): int;
+    abstract public function upload($source, $path, $chunk = 1, $chunks = 1, &$metadata = []): int;
 
     /**
      * Read file by given path.

src/Storage/Device/Local.php

@@ -73,13 +73,13 @@ public function getPath($filename, $prefix = null): string
      * @param string $path
      * @param int $chunk
      * @param int $chunks
-     * @param string tmp
+     * @param array $metadata
      *
      * @throws \Exception
      *
      * @return int
      */
-    public function upload($source, $path, $chunk = 1, $chunks = 1): int
+    public function upload($source, $path, $chunk = 1, $chunks = 1, &$metadata = []): int
     {
         if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists
             if (!@\mkdir(\dirname($path), 0755, true)) {
@@ -93,48 +93,46 @@ public function upload($source, $path, $chunk = 1, $chunks = 1): int
                 throw new Exception('Can\'t upload file ' . $path);
             }
             return $chunks;
-        } else {
-            $tmp = \dirname($path) . "/tmp/chunks.log";
+        }
+        $tmp = \dirname($path) . "/tmp/chunks.log";
 
-            if (!\file_exists(\dirname($tmp))) { // Checks if directory path to file exists
-                if (!@\mkdir(\dirname($tmp), 0755, true)) {
-                    throw new Exception('Can\'t create directory: ' . \dirname($tmp));
-                }
-            }
-            if(!file_put_contents($tmp, "{$chunk}\n", FILE_APPEND)) {
-                throw new Exception('Can\'t write chunk log ' . $tmp);
+        if (!\file_exists(\dirname($tmp))) { // Checks if directory path to file exists
+            if (!@\mkdir(\dirname($tmp), 0755, true)) {
+                throw new Exception('Can\'t create directory: ' . \dirname($tmp));
             }
+        }
+        if(!file_put_contents($tmp, "{$chunk}\n", FILE_APPEND)) {
+            throw new Exception('Can\'t write chunk log ' . $tmp);
+        }
 
-            $chunkLogs = file($tmp);
-            if(!$chunkLogs) {
-                throw new Exception("Unable to read chunk log " . $tmp);
-            }
+        $chunkLogs = file($tmp);
+        if(!$chunkLogs) {
+            throw new Exception("Unable to read chunk log " . $tmp);
+        }
 
-            $chunksReceived = count(file($tmp));
+        $chunksReceived = count(file($tmp));
 
-            if(!\rename($source, dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . ".part.{$chunk}")) {
-                throw new Exception('Failed to write chunk ' . $chunk);
-            }
-
-            if ($chunks == $chunksReceived) {
-                for($i = 0; $i < $chunks; $i++) {
-                    $part = dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . ".part.{$i}";
-                    $data = file_get_contents($part);
-                    if(!$data) {
-                        throw new Exception("Failed to read chunk " . $part);
-                    }
-
-                    if(!file_put_contents($path, $data, FILE_APPEND)) {
-                        throw new Exception('Failed to append chunk ' . $part);
-                    }
-                    \unlink($part);
+        if(!\rename($source, dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . ".part.{$chunk}")) {
+            throw new Exception('Failed to write chunk ' . $chunk);
+        }
+
+        if ($chunks == $chunksReceived) {
+            for($i = 0; $i < $chunks; $i++) {
+                $part = dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . ".part.{$i}";
+                $data = file_get_contents($part);
+                if(!$data) {
+                    throw new Exception("Failed to read chunk " . $part);
+                }
+
+                if(!file_put_contents($path, $data, FILE_APPEND)) {
+                    throw new Exception('Failed to append chunk ' . $part);
                 }
-                \unlink($tmp);
-                return $chunksReceived;
+                \unlink($part);
             }
+            \unlink($tmp);
             return $chunksReceived;
         }
-        return 0;
+        return $chunksReceived;
     }
 
     /**

src/Storage/Device/S3.php

@@ -169,14 +169,124 @@ public function getPath(string $filename, $prefix = null): string
      *
      * @param string $source
      * @param string $path
+     * @param int chunk
+     * @param int chunks
+     * @param array $metadata
      *
      * @throws \Exception
      *
      * @return int
      */
-    public function upload($source, $path, $chunk = 1, $chunks = 1): int
+    public function upload($source, $path, $chunk = 1, $chunks = 1, &$metadata = []): int
     {
-        return $this->write($path, \file_get_contents($source), \mime_content_type($source));
+        if($chunk == 1 && $chunks == 1) {
+            return $this->write($path, \file_get_contents($source), \mime_content_type($source));
+        }
+        $uploadId = $metadata['uploadId'] ?? null;
+        if(empty($uploadId)) {
+            $uploadId = $this->createMultipartUpload($path, $metadata['content_type']);
+            $metadata['uploadId'] = $uploadId;
+        }
+
+        $etag = $this->uploadPart($source, $path, $chunk, $uploadId);
+        $metadata['parts'] ??= [];
+        $metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag];
+        $metadata['chunks'] ??= 0;
+        $metadata['chunks']++;
+        if($metadata['chunks'] == $chunks) {
+            $this->completeMultipartUpload($path, $uploadId, $metadata['parts']);
+        }
+        return $metadata['chunks'];
+    }
+
+    /**
+     * Start Multipart Upload
+     * 
+     * Initiate a multipart upload and return an upload ID.
+     * 
+     * @param string $path
+     * @param string $contentType
+     * 
+     * @return string
+     */
+    protected function createMultipartUpload(string $path, string $contentType): string
+    {
+        $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
+
+        $this->headers['date'] = gmdate('D, d M Y H:i:s T');
+        $this->headers['content-type'] = $contentType;
+        $this->amzHeaders['x-amz-acl'] = $this->acl;
+        $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => '']);
+        return $response->body['UploadId'];
+    }
+
+    /**
+     * Upload Part
+     * 
+     * @param string $source
+     * @param string $path
+     * @param string $uploadId
+     * @param int $chunk
+     * 
+     * @return string
+     */
+    protected function uploadPart($source, $path, $chunk, $uploadId) : string
+    {
+        $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
+
+        $data = \file_get_contents($source);
+        $this->headers['date'] = \gmdate('D, d M Y H:i:s T');
+        $this->headers['content-type'] = \mime_content_type($source);
+        $this->headers['content-md5'] = \base64_encode(md5($data, true));
+        $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data);
+        unset($this->amzHeaders['x-amz-acl']); // ACL header is not allowed in parts, only createMultipartUpload accepts this header.
+
+        $response = $this->call(self::METHOD_PUT, $uri, $data, [
+            'partNumber'=>$chunk,
+            'uploadId' => $uploadId
+        ]);
+
+        return $response->headers['etag'];
+    }
+
+    /**
+     * Complete Multipart Upload
+     * 
+     * @param string $path
+     * @param string $uploadId
+     * 
+     * @return bool
+     */
+    protected function completeMultipartUpload(string $path, string $uploadId, array $parts): bool
+    {
+        $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
+
+        $body = '<CompleteMultipartUpload>';
+        foreach ($parts as $part) {
+            $body .= "<Part><ETag>{$part['etag']}</ETag><PartNumber>{$part['partNumber']}</PartNumber></Part>";
+        }
+        $body .= '</CompleteMultipartUpload>';
+
+        $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $body);
+        $this->headers['content-md5'] = \base64_encode(md5($body, true));
+        $this->headers['date'] = \gmdate('D, d M Y H:i:s T');
+        $this->call(self::METHOD_POST, $uri, $body , ['uploadId' => $uploadId]);
+        return true;
+    }
+
+    /**
+     * Abort Multipart Upload
+     * 
+     * @param string $path
+     * @param string $uploadId
+     * 
+     * @return bool
+     */
+    protected function abortMultipartUpload(string $path, string $uploadId): bool
+    {
+        $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
+        $this->call(self::METHOD_DELETE, $uri, '', ['uploadId' => $uploadId]);
+        return true;
     }
 
     /**
@@ -191,8 +301,11 @@ public function upload($source, $path, $chunk = 1, $chunks = 1): int
     public function read(string $path, int $offset = 0, int $length = null): string
     {
         $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/';
+        if($length !== null) {
+            $end = $offset + $length - 1;
+            $this->headers['range'] = "bytes=$offset-$end";
+        }
         $response = $this->call(self::METHOD_GET, $uri);
-
         return $response->body;
     }
 
@@ -207,7 +320,7 @@ public function read(string $path, int $offset = 0, int $length = null): string
      */
     public function write(string $path, string $data, string $contentType = ''): bool
     {
-        $uri = $path !== '' ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/';
+        $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
 
         $this->headers['date'] = \gmdate('D, d M Y H:i:s T');
         $this->headers['content-type'] = $contentType;
@@ -398,7 +511,7 @@ private function getInfo(string $path): array
      * @param string $uri
      * @return string
      */
-    private function getSignatureV4(string $method, string $uri): string
+    private function getSignatureV4(string $method, string $uri, $parameters = []): string
     {
         $service = 's3';
         $region = $this->region;
@@ -420,7 +533,6 @@ private function getSignatureV4(string $method, string $uri): string
         uksort($combinedHeaders, [ & $this, 'sortMetaHeadersCmp']);
 
         // Convert null query string parameters to strings and sort
-        $parameters = [];
         uksort($parameters, [ & $this, 'sortMetaHeadersCmp']);
         $queryString = \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986);
 
@@ -470,9 +582,9 @@ private function getSignatureV4(string $method, string $uri): string
      *
      * @return  object
      */
-    private function call(string $method, string $uri, string $data = '')
+    private function call(string $method, string $uri, string $data = '', array $parameters=[])
     {
-        $url = 'https://' . $this->headers['host'] . $uri;
+        $url = 'https://' . $this->headers['host'] . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986);
         $response = new \stdClass;
         $response->body = '';
         $response->headers = [];
@@ -502,7 +614,7 @@ private function call(string $method, string $uri, string $data = '')
             }
         }
 
-        $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri);
+        $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri, $parameters);
 
         \curl_setopt($curl, CURLOPT_HTTPHEADER, $httpHeaders);
         \curl_setopt($curl, CURLOPT_HEADER, false);
@@ -552,8 +664,9 @@ private function call(string $method, string $uri, string $data = '')
         \curl_close($curl);
 
         // Parse body into XML
-        if (isset($response->headers['content-type']) && $response->headers['content-type'] == 'application/xml') {
+        if ((isset($response->headers['content-type']) && $response->headers['content-type'] == 'application/xml') || (str_starts_with($response->body, '<?xml'))) {
             $response->body = \simplexml_load_string($response->body);
+            $response->body = json_decode(json_encode($response->body), true);
         }
 
         return $response;

tests/Storage/Device/DoSpacesTest.php

@@ -137,4 +137,49 @@ public function testPartitionTotalSpace()
     {
         $this->assertEquals(-1, $this->object->getPartitionTotalSpace());
     }
+
+    public function testPartUpload() {
+        $source = __DIR__ . "/../../resources/disk-a/large_file.mp4";
+        $dest = $this->object->getPath('uploaded.mp4');
+        $totalSize = \filesize($source);
+        // AWS S3 requires each part to be at least 5MB except for last part
+        $chunkSize = 5*1024*1024;
+
+        $chunks = ceil($totalSize / $chunkSize);
+
+        $chunk = 1;
+        $start = 0;
+
+        $metadata = [
+            'parts' => [],
+            'chunks' => 0,
+            'uploadId' => null,
+            'content_type' => \mime_content_type($source),
+        ];
+        $handle = @fopen($source, "rb");
+        while ($start < $totalSize) {
+            $contents = fread($handle, $chunkSize);
+            $op = __DIR__ . '/chunk.part';
+            $cc = fopen($op, 'wb');
+            fwrite($cc, $contents);
+            fclose($cc);
+            $etag = $this->object->upload($op, $dest, $chunk, $chunks, $metadata);
+            $parts[] = ['partNumber' => $chunk, 'etag' => $etag];
+            $start += strlen($contents);
+            $chunk++;
+            fseek($handle, $start);
+        }
+        @fclose($handle);
+        unlink($op);
+
+        $this->assertEquals(\filesize($source), $this->object->getFileSize($dest));
+
+        // S3 doesnt provide a method to get a proper MD5-hash of a file created using multipart upload
+        // https://stackoverflow.com/questions/8618218/amazon-s3-checksum
+        // More info on how AWS calculates ETag for multipart upload here
+        // https://savjee.be/2015/10/Verifying-Amazon-S3-multi-part-uploads-with-ETag-hash/
+        // TODO
+        // $this->assertEquals(\md5_file($source), $this->object->getFileHash($dest));
+        $this->object->delete($dest);
+    }
 }

tests/Storage/Device/LocalTest.php

@@ -146,7 +146,18 @@ public function testPartUpload() {
         @fclose($handle);
         $this->assertEquals(\filesize($source), $this->object->getFileSize($dest));
         $this->assertEquals(\md5_file($source), $this->object->getFileHash($dest));
-        $this->object->delete($dest);
+        return $dest;
+    }
+
+    /**
+     * @depends testPartUpload
+     */
+    public function testPartRead($path) {
+        $source = __DIR__ . "/../../resources/disk-a/large_file.mp4";
+        $chunk = file_get_contents($source, false,null, 0, 500);
+        $readChunk = $this->object->read($path, 0, 500);
+        $this->assertEquals($chunk, $readChunk);
+        $this->object->delete($path);
     }
 
     public function testPartitionFreeSpace()