feat(storage-*): include modified headers into the response headers of files when using adapters (#12096)

This PR makes it so that `modifyResponseHeaders` is supported in our
adapters when set on the collection config. Previously it would be
ignored.

This means that users can now modify or append new headers to what's
returned by each service.

```ts
import type { CollectionConfig } from 'payload'

export const Media: CollectionConfig = {
  slug: 'media',
  upload: {
    modifyResponseHeaders: ({ headers }) => {
      const newHeaders = new Headers(headers) // Copy existing headers
      newHeaders.set('X-Frame-Options', 'DENY') // Set new header

      return newHeaders
    },
  },
}
```

Also adds support for `void` return on the `modifyResponseHeaders`
function in the case where the user just wants to use existing headers
and doesn't need more control.

eg:

```ts
import type { CollectionConfig } from 'payload'

export const Media: CollectionConfig = {
  slug: 'media',
  upload: {
    modifyResponseHeaders: ({ headers }) => {
      headers.set('X-Frame-Options', 'DENY') // You can directly set headers without returning
    },
  },
}
```

Manual testing checklist (no CI e2es setup for these envs yet):
- [x] GCS
- [x] S3
- [x] Azure
- [x] UploadThing
- [x] Vercel Blob

---------

Co-authored-by: James <james@trbl.design>

Author: paulpopus

docs/upload/overview.mdx

@@ -116,6 +116,7 @@ _An asterisk denotes that an option is required._
 | **`withMetadata`**             | If specified, appends metadata to the output image file. Accepts a boolean or a function that receives `metadata` and `req`, returning a boolean.                                                                                                  |
 | **`hideFileInputOnCreate`**    | Set to `true` to prevent the admin UI from showing file inputs during document creation, useful for programmatic file generation.                                                                                                                  |
 | **`hideRemoveFile`**           | Set to `true` to prevent the admin UI having a way to remove an existing file while editing.                                                                                                                                                       |
+| **`modifyResponseHeaders`**    | Accepts an object with existing `headers` and allows you to manipulate the response headers for media files. [More](#modifying-response-headers)                                                                                                   |
 
 ### Payload-wide Upload Options
 
@@ -453,7 +454,7 @@ To fetch files from **restricted URLs** that would otherwise be blocked by CORS,
 
 Here’s how to configure the pasteURL option to control remote URL fetching:
 
-```
+```ts
 import type { CollectionConfig } from 'payload'
 
 export const Media: CollectionConfig = {
@@ -466,7 +467,7 @@ export const Media: CollectionConfig = {
           pathname: '',
           port: '',
           protocol: 'https',
-          search: ''
+          search: '',
         },
         {
           hostname: 'example.com',
@@ -519,3 +520,44 @@ _An asterisk denotes that an option is required._
 ## Access Control
 
 All files that are uploaded to each Collection automatically support the `read` [Access Control](/docs/access-control/overview) function from the Collection itself. You can use this to control who should be allowed to see your uploads, and who should not.
+
+## Modifying response headers
+
+You can modify the response headers for files by specifying the `modifyResponseHeaders` option in your upload config. This option accepts an object with existing headers and allows you to manipulate the response headers for media files.
+
+### Modifying existing headers
+
+With this method you can directly interface with the `Headers` object and modify the existing headers to append or remove headers.
+
+```ts
+import type { CollectionConfig } from 'payload'
+
+export const Media: CollectionConfig = {
+  slug: 'media',
+  upload: {
+    modifyResponseHeaders: ({ headers }) => {
+      headers.set('X-Frame-Options', 'DENY') // You can directly set headers without returning
+    },
+  },
+}
+```
+
+### Return new headers
+
+You can also return a new `Headers` object with the modified headers. This is useful if you want to set new headers or remove existing ones.
+
+```ts
+import type { CollectionConfig } from 'payload'
+
+export const Media: CollectionConfig = {
+  slug: 'media',
+  upload: {
+    modifyResponseHeaders: ({ headers }) => {
+      const newHeaders = new Headers(headers) // Copy existing headers
+      newHeaders.set('X-Frame-Options', 'DENY') // Set new header
+
+      return newHeaders
+    },
+  },
+}
+```

packages/payload/src/uploads/endpoints/getFile.ts

@@ -38,9 +38,12 @@ export const getFileHandler: PayloadHandler = async (req) => {
 
   if (collection.config.upload.handlers?.length) {
     let customResponse: null | Response | void = null
+    const headers = new Headers()
+
     for (const handler of collection.config.upload.handlers) {
       customResponse = await handler(req, {
         doc: accessResult,
+        headers,
         params: {
           collection: collection.config.slug,
           filename,
@@ -95,7 +98,7 @@ export const getFileHandler: PayloadHandler = async (req) => {
   headers.set('Content-Type', fileTypeResult.mime)
   headers.set('Content-Length', stats.size + '')
   headers = collection.config.upload?.modifyResponseHeaders
-    ? collection.config.upload.modifyResponseHeaders({ headers })
+    ? collection.config.upload.modifyResponseHeaders({ headers }) || headers
     : headers
 
   return new Response(data, {

packages/payload/src/uploads/types.ts

@@ -211,6 +211,7 @@ export type UploadConfig = {
     req: PayloadRequest,
     args: {
       doc: TypeWithID
+      headers?: Headers
       params: { clientUploadContext?: unknown; collection: string; filename: string }
     },
   ) => Promise<Response> | Promise<void> | Response | void)[]
@@ -233,7 +234,7 @@ export type UploadConfig = {
    * Ability to modify the response headers fetching a file.
    * @default undefined
    */
-  modifyResponseHeaders?: ({ headers }: { headers: Headers }) => Headers
+  modifyResponseHeaders?: ({ headers }: { headers: Headers }) => Headers | void
   /**
    * Controls the behavior of pasting/uploading files from URLs.
    * If set to `false`, fetching from remote URLs is disabled.

packages/plugin-cloud-storage/src/types.ts

@@ -58,6 +58,7 @@ export type StaticHandler = (
   req: PayloadRequest,
   args: {
     doc?: TypeWithID
+    headers?: Headers
     params: { clientUploadContext?: unknown; collection: string; filename: string }
   },
 ) => Promise<Response> | Response

packages/storage-azure/src/staticHandler.ts

@@ -14,7 +14,7 @@ interface Args {
 }
 
 export const getHandler = ({ collection, getStorageClient }: Args): StaticHandler => {
-  return async (req, { params: { clientUploadContext, filename } }) => {
+  return async (req, { headers: incomingHeaders, params: { clientUploadContext, filename } }) => {
     try {
       const prefix = await getFilePrefix({ clientUploadContext, collection, filename, req })
       const blockBlobClient = getStorageClient().getBlockBlobClient(
@@ -30,14 +30,34 @@ export const getHandler = ({ collection, getStorageClient }: Args): StaticHandle
 
       const response = blob._response
 
+      let initHeaders: Headers = {
+        ...(response.headers.rawHeaders() as unknown as Headers),
+      }
+
+      // Typescript is difficult here with merging these types from Azure
+      if (incomingHeaders) {
+        initHeaders = {
+          ...initHeaders,
+          ...incomingHeaders,
+        }
+      }
+
+      let headers = new Headers(initHeaders)
+
       const etagFromHeaders = req.headers.get('etag') || req.headers.get('if-none-match')
       const objectEtag = response.headers.get('etag')
 
+      if (
+        collection.upload &&
+        typeof collection.upload === 'object' &&
+        typeof collection.upload.modifyResponseHeaders === 'function'
+      ) {
+        headers = collection.upload.modifyResponseHeaders({ headers }) || headers
+      }
+
       if (etagFromHeaders && etagFromHeaders === objectEtag) {
         return new Response(null, {
-          headers: new Headers({
-            ...response.headers.rawHeaders(),
-          }),
+          headers,
           status: 304,
         })
       }
@@ -63,7 +83,7 @@ export const getHandler = ({ collection, getStorageClient }: Args): StaticHandle
       })
 
       return new Response(readableStream, {
-        headers: response.headers.rawHeaders(),
+        headers,
         status: response.status,
       })
     } catch (err: unknown) {

packages/storage-gcs/src/staticHandler.ts

@@ -12,7 +12,7 @@ interface Args {
 }
 
 export const getHandler = ({ bucket, collection, getStorageClient }: Args): StaticHandler => {
-  return async (req, { params: { clientUploadContext, filename } }) => {
+  return async (req, { headers: incomingHeaders, params: { clientUploadContext, filename } }) => {
     try {
       const prefix = await getFilePrefix({ clientUploadContext, collection, filename, req })
       const file = getStorageClient().bucket(bucket).file(path.posix.join(prefix, filename))
@@ -22,13 +22,23 @@ export const getHandler = ({ bucket, collection, getStorageClient }: Args): Stat
       const etagFromHeaders = req.headers.get('etag') || req.headers.get('if-none-match')
       const objectEtag = metadata.etag
 
+      let headers = new Headers(incomingHeaders)
+
+      headers.append('Content-Length', String(metadata.size))
+      headers.append('Content-Type', String(metadata.contentType))
+      headers.append('ETag', String(metadata.etag))
+
+      if (
+        collection.upload &&
+        typeof collection.upload === 'object' &&
+        typeof collection.upload.modifyResponseHeaders === 'function'
+      ) {
+        headers = collection.upload.modifyResponseHeaders({ headers }) || headers
+      }
+
       if (etagFromHeaders && etagFromHeaders === objectEtag) {
         return new Response(null, {
-          headers: new Headers({
-            'Content-Length': String(metadata.size),
-            'Content-Type': String(metadata.contentType),
-            ETag: String(metadata.etag),
-          }),
+          headers,
           status: 304,
         })
       }
@@ -50,11 +60,7 @@ export const getHandler = ({ bucket, collection, getStorageClient }: Args): Stat
       })
 
       return new Response(readableStream, {
-        headers: new Headers({
-          'Content-Length': String(metadata.size),
-          'Content-Type': String(metadata.contentType),
-          ETag: String(metadata.etag),
-        }),
+        headers,
         status: 200,
       })
     } catch (err: unknown) {

packages/storage-s3/src/staticHandler.ts

@@ -61,7 +61,7 @@ export const getHandler = ({
   getStorageClient,
   signedDownloads,
 }: Args): StaticHandler => {
-  return async (req, { params: { clientUploadContext, filename } }) => {
+  return async (req, { headers: incomingHeaders, params: { clientUploadContext, filename } }) => {
     let object: AWS.GetObjectOutput | undefined = undefined
     try {
       const prefix = await getFilePrefix({ clientUploadContext, collection, filename, req })
@@ -94,17 +94,31 @@ export const getHandler = ({
         Key: key,
       })
 
+      if (!object.Body) {
+        return new Response(null, { status: 404, statusText: 'Not Found' })
+      }
+
+      let headers = new Headers(incomingHeaders)
+
+      headers.append('Content-Length', String(object.ContentLength))
+      headers.append('Content-Type', String(object.ContentType))
+      headers.append('Accept-Ranges', String(object.AcceptRanges))
+      headers.append('ETag', String(object.ETag))
+
       const etagFromHeaders = req.headers.get('etag') || req.headers.get('if-none-match')
       const objectEtag = object.ETag
 
+      if (
+        collection.upload &&
+        typeof collection.upload === 'object' &&
+        typeof collection.upload.modifyResponseHeaders === 'function'
+      ) {
+        headers = collection.upload.modifyResponseHeaders({ headers }) || headers
+      }
+
       if (etagFromHeaders && etagFromHeaders === objectEtag) {
         return new Response(null, {
-          headers: new Headers({
-            'Accept-Ranges': String(object.AcceptRanges),
-            'Content-Length': String(object.ContentLength),
-            'Content-Type': String(object.ContentType),
-            ETag: String(object.ETag),
-          }),
+          headers,
           status: 304,
         })
       }
@@ -125,12 +139,7 @@ export const getHandler = ({
       const bodyBuffer = await streamToBuffer(object.Body)
 
       return new Response(bodyBuffer, {
-        headers: new Headers({
-          'Accept-Ranges': String(object.AcceptRanges),
-          'Content-Length': String(object.ContentLength),
-          'Content-Type': String(object.ContentType),
-          ETag: String(object.ETag),
-        }),
+        headers,
         status: 200,
       })
     } catch (err) {

packages/storage-uploadthing/src/staticHandler.ts

@@ -9,9 +9,13 @@ type Args = {
 }
 
 export const getHandler = ({ utApi }: Args): StaticHandler => {
-  return async (req, { doc, params: { clientUploadContext, collection, filename } }) => {
+  return async (
+    req,
+    { doc, headers: incomingHeaders, params: { clientUploadContext, collection, filename } },
+  ) => {
     try {
       let key: string
+      const collectionConfig = req.payload.collections[collection]?.config
 
       if (
         clientUploadContext &&
@@ -21,7 +25,6 @@ export const getHandler = ({ utApi }: Args): StaticHandler => {
       ) {
         key = clientUploadContext.key
       } else {
-        const collectionConfig = req.payload.collections[collection]?.config
         let retrievedDoc = doc
 
         if (!retrievedDoc) {
@@ -82,23 +85,32 @@ export const getHandler = ({ utApi }: Args): StaticHandler => {
       const etagFromHeaders = req.headers.get('etag') || req.headers.get('if-none-match')
       const objectEtag = response.headers.get('etag')
 
+      let headers = new Headers(incomingHeaders)
+
+      headers.append('Content-Length', String(blob.size))
+      headers.append('Content-Type', blob.type)
+
+      if (objectEtag) {
+        headers.append('ETag', objectEtag)
+      }
+
+      if (
+        collectionConfig?.upload &&
+        typeof collectionConfig.upload === 'object' &&
+        typeof collectionConfig.upload.modifyResponseHeaders === 'function'
+      ) {
+        headers = collectionConfig.upload.modifyResponseHeaders({ headers }) || headers
+      }
+
       if (etagFromHeaders && etagFromHeaders === objectEtag) {
         return new Response(null, {
-          headers: new Headers({
-            'Content-Length': String(blob.size),
-            'Content-Type': blob.type,
-            ETag: objectEtag,
-          }),
+          headers,
           status: 304,
         })
       }
 
       return new Response(blob, {
-        headers: new Headers({
-          'Content-Length': String(blob.size),
-          'Content-Type': blob.type,
-          ETag: objectEtag!,
-        }),
+        headers,
         status: 200,
       })
     } catch (err) {