[TASK] Adapt FAL dumpFile to use PSR-7 response objects

A new driver method streamFile() is added (specified
in a new, internal StreamableDriverInterface).
streamFile() returns a PSR-7 response which serves
the contents of the file.

Once this interface will be marked as public, third party drivers
will be allowed to return an own response (e.g. containing a redirect
to a CDN), providing full controls to headers. It also opens
possibilties for optimizations like X-SendFile (apache) or
X-Accell-Redirect (nginx) to be used by drivers.

We also add SelfEmittableStreamInterface (marked as internal) to support
the same fast file sending using readfile() – the interface provides
a hook which is called by the AbstractApplication in sendResponse.
That means that file contents do not need to be read into memory, stored
into a stream, and then read again, but can be piped to stdout by php
directly.

For all existing drivers backward compatibility is provided by
wrapping their dumpFileContents() method into a decorator stream which
calls dumpFileContents *when* the response is sent.
That means middlewares are able to prevent/stop/enhance
the response, but the driver method dumpFileContents is still used –
it's delayed until Application::sendResponse.

The dumpFileContents method of the ResourceStorage class
is now deprecated. ResourceStorage->streamFile() should be used instead.

Change-Id: I64e707c1f8350e409ff2505b98531b92b2936e02
Releases: master
Resolves: #83793
Reviewed-on: https://review.typo3.org/55585
Reviewed-by: Benni Mack <benni@typo3.org>
Tested-by: Benni Mack <benni@typo3.org>
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: Susanne Moog <susanne.moog@typo3.org>

typo3/sysext/core/Classes/Controller/FileDumpController.php

@@ -21,7 +21,6 @@
 use TYPO3\CMS\Core\Resource\ProcessedFileRepository;
 use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Core\Utility\HttpUtility;
 
 /**
  * Class FileDumpController
@@ -73,20 +72,22 @@ public function dumpAction(ServerRequestInterface $request)
             }
 
             if ($file === null) {
-                HttpUtility::setResponseCodeAndExit(HttpUtility::HTTP_STATUS_404);
+                return (new Response)->withStatus(404);
             }
 
-            // Hook: allow some other process to do some security/access checks. Hook should issue 403 if access is rejected
+            // Hook: allow some other process to do some security/access checks. Hook should return 403 response if access is rejected, void otherwise
             foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['FileDumpEID.php']['checkFileAccess'] ?? [] as $className) {
                 $hookObject = GeneralUtility::makeInstance($className);
                 if (!$hookObject instanceof FileDumpEIDHookInterface) {
                     throw new \UnexpectedValueException($className . ' must implement interface ' . FileDumpEIDHookInterface::class, 1394442417);
                 }
-                $hookObject->checkFileAccess($file);
+                $response = $hookObject->checkFileAccess($file);
+                if ($response instanceof ResponseInterface) {
+                    return $response;
+                }
             }
-            $file->getStorage()->dumpFileContents($file);
-            // @todo Refactor FAL to not echo directly, but to implement a stream for output here and use response
-            return null;
+
+            return $file->getStorage()->streamFile($file);
         }
         return (new Response)->withStatus(403);
     }

typo3/sysext/core/Classes/Http/AbstractApplication.php

@@ -59,7 +59,7 @@ protected function createMiddlewareDispatcher(RequestHandlerInterface $requestHa
      */
     protected function sendResponse(ResponseInterface $response)
     {
-        if ($response instanceof \TYPO3\CMS\Core\Http\NullResponse) {
+        if ($response instanceof NullResponse) {
             return;
         }
 
@@ -77,7 +77,13 @@ protected function sendResponse(ResponseInterface $response)
                 header($name . ': ' . implode(', ', $values));
             }
         }
-        echo $response->getBody()->__toString();
+        $body = $response->getBody();
+        if ($body instanceof SelfEmittableStreamInterface) {
+            // Optimization for streams that use php functions like readfile() as fastpath for serving files.
+            $body->emit();
+        } else {
+            echo $body->__toString();
+        }
     }
 
     /**

typo3/sysext/core/Classes/Http/FalDumpFileContentsDecoratorStream.php

@@ -0,0 +1,108 @@
+<?php
+declare(strict_types = 1);
+
+namespace TYPO3\CMS\Core\Http;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use GuzzleHttp\Psr7\StreamDecoratorTrait;
+use Psr\Http\Message\StreamInterface;
+use TYPO3\CMS\Core\Resource\Driver\DriverInterface;
+
+/**
+ * A lazy stream, that wraps the FAL dumpFileContents() method to send file contents
+ * using emit(), as defined in SelfEmittableStreamInterface.
+ * This call will fall back to the FAL getFileContents() method if the fastpath possibility
+ * using SelfEmittableStreamInterface is not used.
+ *
+ * @internal
+ */
+class FalDumpFileContentsDecoratorStream implements StreamInterface, SelfEmittableStreamInterface
+{
+    use StreamDecoratorTrait;
+
+    /**
+     * @var string
+     */
+    protected $identifier;
+
+    /**
+     * @var DriverInterface
+     */
+    protected $driver;
+
+    /**
+     * @var int
+     */
+    protected $size;
+
+    /**
+     * @param string $identifier
+     * @param DriverInterface $driver
+     * @param int $size
+     */
+    public function __construct(string $identifier, DriverInterface $driver, int $size)
+    {
+        $this->identifier = $identifier;
+        $this->driver = $driver;
+        $this->size = $size;
+    }
+
+    /**
+     * Emit the response to stdout, as specified in SelfEmittableStreamInterface.
+     * Offload to the driver method dumpFileContents.
+     */
+    public function emit()
+    {
+        $this->driver->dumpFileContents($this->identifier);
+    }
+
+    /**
+     * Creates a stream (on demand). This method is consumed by the guzzle StreamDecoratorTrait
+     * and is used when this stream is used without the emit() fastpath.
+     *
+     * @return StreamInterface
+     */
+    protected function createStream(): StreamInterface
+    {
+        $stream = new Stream('php://temp', 'rw');
+        $stream->write($this->driver->getFileContents($this->identifier));
+        return $stream;
+    }
+
+    /**
+     * @return int
+     */
+    public function getSize(): int
+    {
+        return $this->size;
+    }
+
+    /**
+     * @return bool
+     */
+    public function isWritable(): bool
+    {
+        return false;
+    }
+
+    /**
+     * @param string $string
+     * @throws \RuntimeException on failure.
+     */
+    public function write($string)
+    {
+        throw new \RuntimeException('Cannot write to a ' . self::class, 1538331852);
+    }
+}

typo3/sysext/core/Classes/Http/SelfEmittableLazyOpenStream.php

@@ -0,0 +1,70 @@
+<?php
+declare(strict_types = 1);
+
+namespace TYPO3\CMS\Core\Http;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use GuzzleHttp\Psr7\LazyOpenStream;
+
+/**
+ * This class implements a stream that can be used like a usual PSR-7 stream
+ * but is additionally able to provide a file-serving fastpath using readfile().
+ * The file this stream refers to is opened on demand.
+ *
+ * @internal
+ */
+class SelfEmittableLazyOpenStream extends LazyOpenStream implements SelfEmittableStreamInterface
+{
+    /**
+     * @var string
+     */
+    protected $filename;
+
+    /**
+     * Constructor setting up the PHP resource
+     *
+     * @param string $filename
+     */
+    public function __construct($filename)
+    {
+        parent::__construct($filename, 'r');
+        $this->filename = $filename;
+    }
+
+    /**
+     * Output the contents of the file to the output buffer
+     */
+    public function emit()
+    {
+        readfile($this->filename, false);
+    }
+
+    /**
+     * @return bool
+     */
+    public function isWritable(): bool
+    {
+        return false;
+    }
+
+    /**
+     * @param string $string
+     * @throws \RuntimeException on failure.
+     */
+    public function write($string)
+    {
+        throw new \RuntimeException('Cannot write to a ' . self::class, 1538331833);
+    }
+}

typo3/sysext/core/Classes/Http/SelfEmittableStreamInterface.php

@@ -0,0 +1,32 @@
+<?php
+declare(strict_types = 1);
+
+namespace TYPO3\CMS\Core\Http;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use Psr\Http\Message\StreamInterface;
+
+/**
+ * A PSR-7 stream which allows to be emitted on its own.
+ *
+ * @internal
+ */
+interface SelfEmittableStreamInterface extends StreamInterface
+{
+    /**
+     * Output the contents of the stream to the output buffer
+     */
+    public function emit();
+}

typo3/sysext/core/Classes/Resource/Driver/LocalDriver.php

@@ -14,8 +14,11 @@
  * The TYPO3 project - inspiring people to share!
  */
 
+use Psr\Http\Message\ResponseInterface;
 use TYPO3\CMS\Core\Charset\CharsetConverter;
 use TYPO3\CMS\Core\Core\Environment;
+use TYPO3\CMS\Core\Http\Response;
+use TYPO3\CMS\Core\Http\SelfEmittableLazyOpenStream;
 use TYPO3\CMS\Core\Resource\Exception;
 use TYPO3\CMS\Core\Resource\FolderInterface;
 use TYPO3\CMS\Core\Resource\ResourceStorage;
@@ -26,7 +29,7 @@
 /**
  * Driver for the local file system
  */
-class LocalDriver extends AbstractHierarchicalFilesystemDriver
+class LocalDriver extends AbstractHierarchicalFilesystemDriver implements StreamableDriverInterface
 {
     /**
      * @var string
@@ -1381,6 +1384,37 @@ public function dumpFileContents($identifier)
         readfile($this->getAbsolutePath($this->canonicalizeAndCheckFileIdentifier($identifier)), 0);
     }
 
+    /**
+     * Stream file using a PSR-7 Response object.
+     *
+     * @param string $identifier
+     * @param array $properties
+     * @return ResponseInterface
+     */
+    public function streamFile(string $identifier, array $properties): ResponseInterface
+    {
+        $fileInfo = $this->getFileInfoByIdentifier($identifier, ['name', 'mimetype', 'mtime', 'size']);
+        $downloadName = $properties['filename_overwrite'] ?? $fileInfo['name'] ?? '';
+        $mimeType = $properties['mimetype_overwrite'] ?? $fileInfo['mimetype'] ?? '';
+        $contentDisposition = ($properties['as_download'] ?? false) ? 'attachment' : 'inline';
+
+        $filePath = $this->getAbsolutePath($this->canonicalizeAndCheckFileIdentifier($identifier));
+
+        return new Response(
+            new SelfEmittableLazyOpenStream($filePath),
+            200,
+            [
+                'Content-Disposition' => $contentDisposition . '; filename="' . $downloadName . '"',
+                'Content-Type' => $mimeType,
+                'Content-Length' => (string)$fileInfo['size'],
+                'Last-Modified' => gmdate('D, d M Y H:i:s', $fileInfo['mtime']) . ' GMT',
+                // Cache-Control header is needed here to solve an issue with browser IE8 and lower
+                // See for more information: http://support.microsoft.com/kb/323308
+                'Cache-Control' => '',
+            ]
+        );
+    }
+
     /**
      * Get the path of the nearest recycler folder of a given $path.
      * Return an empty string if there is no recycler folder available.

typo3/sysext/core/Classes/Resource/Driver/StreamableDriverInterface.php

@@ -0,0 +1,37 @@
+<?php
+declare(strict_types = 1);
+
+namespace TYPO3\CMS\Core\Resource\Driver;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * An interface FAL drivers have to implement to fulfil the needs
+ * of streaming files using PSR-7 Response objects.
+ *
+ * @internal
+ */
+interface StreamableDriverInterface
+{
+    /**
+     * Streams a file using a PSR-7 Response object.
+     *
+     * @param string $identifier
+     * @param array $properties
+     * @return ResponseInterface
+     */
+    public function streamFile(string $identifier, array $properties): ResponseInterface;
+}

typo3/sysext/core/Classes/Resource/Hook/FileDumpEIDHookInterface.php

@@ -14,6 +14,8 @@
  * The TYPO3 project - inspiring people to share!
  */
 
+use Psr\Http\Message\ResponseInterface;
+
 /**
  * Interface for FileDumpEID Hook to perform some custom security/access checks
  * when accessing file thought FileDumpEID
@@ -27,6 +29,7 @@ interface FileDumpEIDHookInterface
      * A 401 header must be accompanied by a www-authenticate header!
      *
      * @param \TYPO3\CMS\Core\Resource\ResourceInterface $file
+     * @return ResponseInterface|null
      */
     public function checkFileAccess(\TYPO3\CMS\Core\Resource\ResourceInterface $file);
 }