feat: azure upload file stream

dev/src/mocks/fsMock.js

@@ -0,0 +1 @@
+module.exports = { fs: { createReadStream: () => null } }

dev/src/payload.config.ts

@@ -18,6 +18,9 @@ if (process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER === 'azure') {
     allowContainerCreate: process.env.AZURE_STORAGE_ALLOW_CONTAINER_CREATE === 'true',
     baseURL: process.env.AZURE_STORAGE_ACCOUNT_BASEURL,
   })
+  // uploadOptions = {
+  //   useTempFiles: true,
+  // }
 }
 
 if (process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER === 's3') {
@@ -69,6 +72,7 @@ export default buildConfig({
             '@azure/storage-blob': path.resolve(__dirname, '../../src/adapters/azure/mock.js'),
             '@aws-sdk/client-s3': path.resolve(__dirname, '../../src/adapters/s3/mock.js'),
             '@google-cloud/storage': path.resolve(__dirname, '../../src/adapters/gcs/mock.js'),
+            fs: path.resolve(__dirname, './mocks/fsMock.js'),
           },
         },
       }

src/adapters/azure/handleUpload.ts

@@ -1,5 +1,8 @@
 import path from 'path'
+import fs from 'fs'
+import { Readable } from 'stream'
 import type { ContainerClient } from '@azure/storage-blob'
+import { AbortController } from '@azure/abort-controller'
 import type { CollectionConfig } from 'payload/types'
 import type { HandleUpload } from '../../types'
 
@@ -9,14 +12,28 @@ interface Args {
   prefix?: string
 }
 
+const multipartThreshold = 1024 * 1024 * 50 // 50MB
 export const getHandleUpload = ({ getStorageClient, prefix = '' }: Args): HandleUpload => {
   return async ({ data, file }) => {
     const blockBlobClient = getStorageClient().getBlockBlobClient(
       path.posix.join(prefix, file.filename),
     )
 
-    await blockBlobClient.upload(file.buffer, file.buffer.byteLength, {
-      blobHTTPHeaders: { blobContentType: file.mimeType },
+    // when there are no temp files, or the upload is less than the threshold size, do not stream files
+    if (!file.tempFilePath && file.buffer.length > 0 && file.buffer.length < multipartThreshold) {
+      await blockBlobClient.upload(file.buffer, file.buffer.byteLength, {
+        blobHTTPHeaders: { blobContentType: file.mimeType },
+      })
+
+      return data
+    }
+
+    const fileBufferOrStream: Readable = file.tempFilePath
+      ? fs.createReadStream(file.tempFilePath)
+      : Readable.from(file.buffer)
+
+    await blockBlobClient.uploadStream(fileBufferOrStream, 4 * 1024 * 1024, 4, {
+      abortSignal: AbortController.timeout(30 * 60 * 1000),
     })
 
     return data

src/adapters/azure/mock.js

@@ -5,3 +5,9 @@ exports.BlobServiceClient = {
     }),
   }),
 }
+
+exports.AbortController = {
+  timeout: () => null,
+}
+
+exports.Readable = { from: () => null }

src/adapters/azure/webpack.ts

@@ -8,7 +8,10 @@ export const extendWebpackConfig = (existingWebpackConfig: WebpackConfig): Webpa
       ...(existingWebpackConfig.resolve || {}),
       alias: {
         ...(existingWebpackConfig.resolve?.alias ? existingWebpackConfig.resolve.alias : {}),
+        stream: path.resolve(__dirname, './mock.js'),
         '@azure/storage-blob': path.resolve(__dirname, './mock.js'),
+        '@azure/abort-controller': path.resolve(__dirname, './mock.js'),
+        fs: path.resolve(__dirname, './fileStub.js'),
       },
     },
   }