Merge pull request #888 from chromaui/ghengeveld/ap-4034-batch-file-u…

…pload-requests-to-handle-thousands-of-files

Replace upload mechanism to use a batched mutation with a new API

node-src/index.test.ts

@@ -98,14 +98,16 @@ vi.mock('node-fetch', () => ({
       }
 
       if (query?.match('PublishBuildMutation')) {
-        if (variables.input.isolatorUrl.startsWith('http://throw-an-error')) {
-          throw new Error('fetch error');
-        }
-        publishedBuild = { id: variables.id, ...variables.input };
+        publishedBuild = {
+          id: variables.id,
+          ...variables.input,
+          storybookUrl: 'https://5d67dc0374b2e300209c41e7-pfkaemtlit.chromatic.com/',
+        };
         return {
           data: {
             publishBuild: {
               status: 'PUBLISHED',
+              storybookUrl: 'https://5d67dc0374b2e300209c41e7-pfkaemtlit.chromatic.com/',
             },
           },
         };
@@ -132,8 +134,8 @@ vi.mock('node-fetch', () => ({
                 status: 'IN_PROGRESS',
                 specCount: 1,
                 componentCount: 1,
+                storybookUrl: 'https://5d67dc0374b2e300209c41e7-pfkaemtlit.chromatic.com/',
                 webUrl: 'http://test.com',
-                cachedUrl: 'https://5d67dc0374b2e300209c41e7-pfkaemtlit.chromatic.com/iframe.html',
                 ...mockBuildFeatures,
                 app: {
                   account: {
@@ -193,7 +195,8 @@ vi.mock('node-fetch', () => ({
         };
       }
 
-      if (query?.match('GetUploadUrlsMutation')) {
+      if (query?.match('UploadBuildMutation') || query?.match('UploadMetadataMutation')) {
+        const key = query?.match('UploadBuildMutation') ? 'uploadBuild' : 'uploadMetadata';
         const contentTypes = {
           html: 'text/html',
           js: 'text/javascript',
@@ -202,13 +205,18 @@ vi.mock('node-fetch', () => ({
         };
         return {
           data: {
-            getUploadUrls: {
-              domain: 'https://chromatic.com',
-              urls: variables.paths.map((path: string) => ({
-                path,
-                url: `https://cdn.example.com/${path}`,
-                contentType: contentTypes[path.split('.').at(-1)],
-              })),
+            [key]: {
+              info: {
+                sentinelUrls: [],
+                targets: variables.files.map(({ filePath }) => ({
+                  contentType: contentTypes[filePath.split('.').at(-1)],
+                  fileKey: '',
+                  filePath,
+                  formAction: 'https://s3.amazonaws.com',
+                  formFields: {},
+                })),
+              },
+              userErrors: [],
             },
           },
         };
@@ -405,7 +413,7 @@ it('runs in simple situations', async () => {
     storybookViewLayer: 'viewLayer',
     committerEmail: 'test@test.com',
     committerName: 'tester',
-    isolatorUrl: `https://chromatic.com/iframe.html`,
+    storybookUrl: 'https://5d67dc0374b2e300209c41e7-pfkaemtlit.chromatic.com/',
   });
 });
 
@@ -465,17 +473,23 @@ it('calls out to npm build script passed and uploads files', async () => {
         contentHash: 'hash',
         contentLength: 42,
         contentType: 'text/html',
+        fileKey: '',
+        filePath: 'iframe.html',
+        formAction: 'https://s3.amazonaws.com',
+        formFields: {},
         localPath: expect.stringMatching(/\/iframe\.html$/),
         targetPath: 'iframe.html',
-        targetUrl: 'https://cdn.example.com/iframe.html',
       },
       {
         contentHash: 'hash',
         contentLength: 42,
         contentType: 'text/html',
+        fileKey: '',
+        filePath: 'index.html',
+        formAction: 'https://s3.amazonaws.com',
+        formFields: {},
         localPath: expect.stringMatching(/\/index\.html$/),
         targetPath: 'index.html',
-        targetUrl: 'https://cdn.example.com/index.html',
       },
     ],
     expect.any(Function)
@@ -494,17 +508,23 @@ it('skips building and uploads directly with storybook-build-dir', async () => {
         contentHash: 'hash',
         contentLength: 42,
         contentType: 'text/html',
+        fileKey: '',
+        filePath: 'iframe.html',
+        formAction: 'https://s3.amazonaws.com',
+        formFields: {},
         localPath: expect.stringMatching(/\/iframe\.html$/),
         targetPath: 'iframe.html',
-        targetUrl: 'https://cdn.example.com/iframe.html',
       },
       {
         contentHash: 'hash',
         contentLength: 42,
         contentType: 'text/html',
+        fileKey: '',
+        filePath: 'index.html',
+        formAction: 'https://s3.amazonaws.com',
+        formFields: {},
         localPath: expect.stringMatching(/\/index\.html$/),
         targetPath: 'index.html',
-        targetUrl: 'https://cdn.example.com/index.html',
       },
     ],
     expect.any(Function)
@@ -699,32 +719,44 @@ it('should upload metadata files if --upload-metadata is passed', async () => {
   expect(upload.mock.calls.at(-1)[1]).toEqual(
     expect.arrayContaining([
       {
-        localPath: '.storybook/main.js',
-        targetPath: '.chromatic/main.js',
         contentLength: 518,
         contentType: 'text/javascript',
-        targetUrl: 'https://cdn.example.com/.chromatic/main.js',
+        fileKey: '',
+        filePath: '.chromatic/main.js',
+        formAction: 'https://s3.amazonaws.com',
+        formFields: {},
+        localPath: '.storybook/main.js',
+        targetPath: '.chromatic/main.js',
       },
       {
-        localPath: 'storybook-out/preview-stats.trimmed.json',
-        targetPath: '.chromatic/preview-stats.trimmed.json',
         contentLength: 457,
         contentType: 'application/json',
-        targetUrl: 'https://cdn.example.com/.chromatic/preview-stats.trimmed.json',
+        fileKey: '',
+        filePath: '.chromatic/preview-stats.trimmed.json',
+        formAction: 'https://s3.amazonaws.com',
+        formFields: {},
+        localPath: 'storybook-out/preview-stats.trimmed.json',
+        targetPath: '.chromatic/preview-stats.trimmed.json',
       },
       {
-        localPath: '.storybook/preview.js',
-        targetPath: '.chromatic/preview.js',
         contentLength: 1338,
         contentType: 'text/javascript',
-        targetUrl: 'https://cdn.example.com/.chromatic/preview.js',
+        fileKey: '',
+        filePath: '.chromatic/preview.js',
+        formAction: 'https://s3.amazonaws.com',
+        formFields: {},
+        localPath: '.storybook/preview.js',
+        targetPath: '.chromatic/preview.js',
       },
       {
-        localPath: expect.any(String),
-        targetPath: '.chromatic/index.html',
         contentLength: expect.any(Number),
         contentType: 'text/html',
-        targetUrl: 'https://cdn.example.com/.chromatic/index.html',
+        fileKey: '',
+        filePath: '.chromatic/index.html',
+        formAction: 'https://s3.amazonaws.com',
+        formFields: {},
+        localPath: expect.any(String),
+        targetPath: '.chromatic/index.html',
       },
     ])
   );

node-src/index.ts

@@ -120,7 +120,7 @@ export async function run({
     code: ctx.exitCode,
     url: ctx.build?.webUrl,
     buildUrl: ctx.build?.webUrl,
-    storybookUrl: ctx.build?.cachedUrl?.replace(/iframe\.html.*$/, ''),
+    storybookUrl: ctx.build?.storybookUrl,
     specCount: ctx.build?.specCount,
     componentCount: ctx.build?.componentCount,
     testCount: ctx.build?.testCount,

node-src/lib/FileReaderBlob.ts

@@ -0,0 +1,20 @@
+import { ReadStream, createReadStream } from 'fs';
+
+export class FileReaderBlob {
+  readStream: ReadStream;
+  size: number;
+
+  constructor(filePath: string, contentLength: number, onProgress: (delta: number) => void) {
+    this.size = contentLength;
+    this.readStream = createReadStream(filePath);
+    this.readStream.on('data', (chunk: Buffer | string) => onProgress(chunk.length));
+  }
+
+  stream() {
+    return this.readStream;
+  }
+
+  get [Symbol.toStringTag]() {
+    return 'Blob';
+  }
+}

node-src/lib/compress.ts

@@ -24,7 +24,7 @@ export default async function makeZipFile(ctx: Context, files: FileDesc[]) {
     archive.pipe(sink);
 
     files.forEach(({ localPath, targetPath: name }) => {
-      ctx.log.debug({ name }, 'Adding file to zip archive');
+      ctx.log.debug(`Adding to zip archive: ${name}`);
       archive.append(createReadStream(localPath), { name });
     });