From d81a88b84b61b15ed7fcab94b58ae4b4b31e46be Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 7 Feb 2022 17:35:21 -0600 Subject: [PATCH 01/14] Update to leverage response-cache for image-optimizer Co-authored-by: Steven --- packages/next/server/base-server.ts | 10 + packages/next/server/config-shared.ts | 2 +- packages/next/server/image-optimizer.ts | 856 +++++++++--------- packages/next/server/incremental-cache.ts | 16 +- packages/next/server/next-server.ts | 96 +- packages/next/server/response-cache.ts | 27 +- .../image-optimizer/test/index.test.js | 313 +++++-- 7 files changed, 789 insertions(+), 531 deletions(-) diff --git a/packages/next/server/base-server.ts b/packages/next/server/base-server.ts index 0d28b29ae6be5..6e32793c430e5 100644 --- a/packages/next/server/base-server.ts +++ b/packages/next/server/base-server.ts @@ -58,6 +58,7 @@ import { MIDDLEWARE_ROUTE } from '../lib/constants' import { addRequestMeta, getRequestMeta } from './request-meta' import { createHeaderRoute, createRedirectRoute } from './server-route-utils' import { PrerenderManifest } from '../build' +import { ImageOptimizerCache } from './image-optimizer' export type FindComponentsResult = { components: LoadComponentsReturnType @@ -165,6 +166,7 @@ export default abstract class Server { } private incrementalCache: IncrementalCache private responseCache: ResponseCache + protected imageResponseCache: ResponseCache protected router: Router protected dynamicRoutes?: DynamicRoutes protected customRoutes: CustomRoutes @@ -360,6 +362,12 @@ export default abstract class Server { }, }) this.responseCache = new ResponseCache(this.incrementalCache) + this.imageResponseCache = new ResponseCache( + new ImageOptimizerCache({ + distDir: this.distDir, + nextConfig: this.nextConfig, + }) as any + ) } public logError(err: Error): void { @@ -1516,6 +1524,8 @@ export default abstract class Server { await handleRedirect(cachedData.props) return null } + } else if (cachedData.kind === 'IMAGE') { + throw new Error('invariant SSG should not return an image cache value') } else { return { type: isDataReq ? 'json' : 'html', diff --git a/packages/next/server/config-shared.ts b/packages/next/server/config-shared.ts index 8c6270666a26c..94878ac900adb 100644 --- a/packages/next/server/config-shared.ts +++ b/packages/next/server/config-shared.ts @@ -8,7 +8,7 @@ import { } from './image-config' export type NextConfigComplete = Required & { - images: ImageConfigComplete + images: Required typescript: Required configOrigin?: string configFile?: string diff --git a/packages/next/server/image-optimizer.ts b/packages/next/server/image-optimizer.ts index 8363070892c60..b4e531fb72ea3 100644 --- a/packages/next/server/image-optimizer.ts +++ b/packages/next/server/image-optimizer.ts @@ -1,6 +1,6 @@ import { mediaType } from 'next/dist/compiled/@hapi/accept' import { createHash } from 'crypto' -import { createReadStream, promises } from 'fs' +import { promises } from 'fs' import { getOrientation, Orientation } from 'next/dist/compiled/get-orientation' import imageSizeOf from 'next/dist/compiled/image-size' import { IncomingMessage, ServerResponse } from 'http' @@ -10,9 +10,7 @@ import contentDisposition from 'next/dist/compiled/content-disposition' import { join } from 'path' import Stream from 'stream' import nodeUrl, { UrlWithParsedQuery } from 'url' -import { NextConfig } from './config-shared' -import { fileExists } from '../lib/file-exists' -import { ImageConfig, imageConfigDefault } from './image-config' +import { NextConfigComplete } from './config-shared' import { processBuffer, decodeBuffer, Operation } from './lib/squoosh/main' import { sendEtagResponse } from './send-payload' import { getContentType, getExtension } from './serve-static' @@ -31,7 +29,6 @@ const CACHE_VERSION = 3 const ANIMATABLE_TYPES = [WEBP, PNG, GIF] const VECTOR_TYPES = [SVG] const BLUR_IMG_SIZE = 8 // should match `next-image-loader` -const inflightRequests = new Map>() let sharp: | (( @@ -48,489 +45,494 @@ try { let showSharpMissingWarning = process.env.NODE_ENV === 'production' -export async function imageOptimizer( - req: IncomingMessage, - res: ServerResponse, - parsedUrl: UrlWithParsedQuery, - nextConfig: NextConfig, - distDir: string, - render404: () => Promise, - handleRequest: ( - newReq: IncomingMessage, - newRes: ServerResponse, - newParsedUrl?: NextUrlWithParsedQuery - ) => Promise, - isDev = false -) { - const imageData: ImageConfig = nextConfig.images || imageConfigDefault - const { - deviceSizes = [], - imageSizes = [], - domains = [], - loader, - minimumCacheTTL = 60, - formats = ['image/webp'], - } = imageData - - if (loader !== 'default') { - await render404() - return { finished: true } - } +interface ParamsResult { + href: string + isAbsolute: boolean + isStatic: boolean + width: number + quality: number + mimeType: string + sizes: number[] + minimumCacheTTL: number +} - const { headers } = req - const { url, w, q } = parsedUrl.query - const mimeType = getSupportedMimeType(formats, headers.accept) - let href: string +export class ImageOptimizerCache { + private cacheDir: string + private nextConfig: NextConfigComplete + + static validateParams( + req: IncomingMessage, + query: UrlWithParsedQuery['query'], + nextConfig: NextConfigComplete, + isDev: boolean + ): ParamsResult | { errorMessage: string } { + const imageConfig = nextConfig.images + const { url, w, q } = query + let href: string + + if (!url) { + return { errorMessage: '"url" parameter is required' } + } else if (Array.isArray(url)) { + return { errorMessage: '"url" parameter cannot be an array' } + } - if (!url) { - res.statusCode = 400 - res.end('"url" parameter is required') - return { finished: true } - } else if (Array.isArray(url)) { - res.statusCode = 400 - res.end('"url" parameter cannot be an array') - return { finished: true } - } + let isAbsolute: boolean - let isAbsolute: boolean + if (url.startsWith('/')) { + href = url + isAbsolute = false + } else { + let hrefParsed: URL - if (url.startsWith('/')) { - href = url - isAbsolute = false - } else { - let hrefParsed: URL + try { + hrefParsed = new URL(url) + href = hrefParsed.toString() + isAbsolute = true + } catch (_error) { + return { errorMessage: '"url" parameter is invalid' } + } - try { - hrefParsed = new URL(url) - href = hrefParsed.toString() - isAbsolute = true - } catch (_error) { - res.statusCode = 400 - res.end('"url" parameter is invalid') - return { finished: true } + if (!['http:', 'https:'].includes(hrefParsed.protocol)) { + return { errorMessage: '"url" parameter is invalid' } + } + + if ( + !imageConfig.domains || + !imageConfig.domains.includes(hrefParsed.hostname) + ) { + return { errorMessage: '"url" parameter is not allowed' } + } } - if (!['http:', 'https:'].includes(hrefParsed.protocol)) { - res.statusCode = 400 - res.end('"url" parameter is invalid') - return { finished: true } + if (!w) { + return { errorMessage: '"w" parameter (width) is required' } + } else if (Array.isArray(w)) { + return { errorMessage: '"w" parameter (width) cannot be an array' } } - if (!domains.includes(hrefParsed.hostname)) { - res.statusCode = 400 - res.end('"url" parameter is not allowed') - return { finished: true } + if (!q) { + return { errorMessage: '"q" parameter (quality) is required' } + } else if (Array.isArray(q)) { + return { errorMessage: '"q" parameter (quality) cannot be an array' } } - } - if (!w) { - res.statusCode = 400 - res.end('"w" parameter (width) is required') - return { finished: true } - } else if (Array.isArray(w)) { - res.statusCode = 400 - res.end('"w" parameter (width) cannot be an array') - return { finished: true } - } + const width = parseInt(w, 10) - if (!q) { - res.statusCode = 400 - res.end('"q" parameter (quality) is required') - return { finished: true } - } else if (Array.isArray(q)) { - res.statusCode = 400 - res.end('"q" parameter (quality) cannot be an array') - return { finished: true } - } + if (!width || isNaN(width)) { + return { + errorMessage: '"w" parameter (width) must be a number greater than 0', + } + } - // Should match output from next-image-loader - const isStatic = url.startsWith( - `${nextConfig.basePath || ''}/_next/static/media` - ) + const sizes = [ + ...(imageConfig.deviceSizes || []), + ...(imageConfig.imageSizes || []), + ] - const width = parseInt(w, 10) + if (isDev) { + sizes.push(BLUR_IMG_SIZE) + } - if (!width || isNaN(width)) { - res.statusCode = 400 - res.end('"w" parameter (width) must be a number greater than 0') - return { finished: true } - } + if (!sizes.includes(width)) { + return { + errorMessage: `"w" parameter (width) of ${width} is not allowed`, + } + } - const sizes = [...deviceSizes, ...imageSizes] + const quality = parseInt(q) - if (isDev) { - sizes.push(BLUR_IMG_SIZE) - } + if (isNaN(quality) || quality < 1 || quality > 100) { + return { + errorMessage: + '"q" parameter (quality) must be a number between 1 and 100', + } + } - if (!sizes.includes(width)) { - res.statusCode = 400 - res.end(`"w" parameter (width) of ${width} is not allowed`) - return { finished: true } - } + const mimeType = getSupportedMimeType( + imageConfig.formats || [], + req.headers['accept'] + ) - const quality = parseInt(q) + const isStatic = url.startsWith( + `${nextConfig.basePath || ''}/_next/static/media` + ) - if (isNaN(quality) || quality < 1 || quality > 100) { - res.statusCode = 400 - res.end('"q" parameter (quality) must be a number between 1 and 100') - return { finished: true } + return { + href, + sizes, + isAbsolute, + isStatic, + width, + quality, + mimeType, + minimumCacheTTL: imageConfig.minimumCacheTTL, + } } - const hash = getHash([CACHE_VERSION, href, width, quality, mimeType]) - const imagesDir = join(distDir, 'cache', 'images') - const hashDir = join(imagesDir, hash) - const now = Date.now() - let xCache: XCacheHeader = 'MISS' + static getCacheKey({ + href, + width, + quality, + mimeType, + }: { + href: string + width: number + quality: number + mimeType: string + }): string { + return getHash([CACHE_VERSION, href, width, quality, mimeType]) + } - // If there're concurrent requests hitting the same resource and it's still - // being optimized, wait before accessing the cache. - if (inflightRequests.has(hash)) { - await inflightRequests.get(hash) + constructor({ + distDir, + nextConfig, + }: { + distDir: string + nextConfig: NextConfigComplete + }) { + this.cacheDir = join(distDir, 'cache', 'images') + this.nextConfig = nextConfig } - const dedupe = new Deferred() - inflightRequests.set(hash, dedupe.promise) - try { - if (await fileExists(hashDir, 'directory')) { - const files = await promises.readdir(hashDir) - for (let file of files) { - const [maxAgeStr, expireAtSt, etag, extension] = file.split('.') - const maxAge = Number(maxAgeStr) + async get(cacheKey: string) { + try { + const cacheDir = join(this.cacheDir, cacheKey) + const files = await promises.readdir(cacheDir) + const now = Date.now() + + for (const file of files) { + const [maxAgeSt, expireAtSt, etag, extension] = file.split('.') + const buffer = await promises.readFile(join(cacheDir, file)) const expireAt = Number(expireAtSt) - const contentType = getContentType(extension) - const fsPath = join(hashDir, file) - xCache = now < expireAt ? 'HIT' : 'STALE' - const result = setResponseHeaders( - req, - res, - url, - etag, - maxAge, - contentType, - isStatic, - isDev, - xCache - ) - if (!result.finished) { - await new Promise((resolve, reject) => { - createReadStream(fsPath) - .on('end', resolve) - .on('error', reject) - .pipe(res) - }) - } - if (xCache === 'HIT') { - return { finished: true } - } else { - await promises.unlink(fsPath) + const maxAge = Number(maxAgeSt) + const revalidate = maxAge + + return { + value: { + kind: 'IMAGE', + etag, + buffer, + extension, + revalidate, + }, + revalidate, + isStale: now > expireAt, } } + } catch (_) { + // failed to read from cache dir, treat as cache miss } + return null + } + async set( + cacheKey: string, + value: { + etag: string + buffer: Buffer + extension: string + }, + revalidate: number + ) { + const expireAt = + Math.max(revalidate, this.nextConfig.images.minimumCacheTTL) * 1000 + + Date.now() + + await writeToCacheDir( + join(this.cacheDir, cacheKey), + value.extension, + revalidate, + expireAt, + value.buffer, + value.etag + ) + } +} +export class ImageError extends Error { + statusCode?: number - let upstreamBuffer: Buffer - let upstreamType: string | null - let maxAge: number - - if (isAbsolute) { - const upstreamRes = await fetch(href) - - if (!upstreamRes.ok) { - res.statusCode = upstreamRes.status - res.end('"url" parameter is valid but upstream response is invalid') - return { finished: true } - } + constructor(message: string, statusCode?: number) { + super(message) + this.statusCode = statusCode + } +} - res.statusCode = upstreamRes.status - upstreamBuffer = Buffer.from(await upstreamRes.arrayBuffer()) - upstreamType = - detectContentType(upstreamBuffer) || - upstreamRes.headers.get('Content-Type') - maxAge = getMaxAge(upstreamRes.headers.get('Cache-Control')) - } else { - try { - const resBuffers: Buffer[] = [] - const mockRes: any = new Stream.Writable() +export async function imageOptimizer( + _req: IncomingMessage, + _res: ServerResponse, + paramsResult: ParamsResult, + nextConfig: NextConfigComplete, + handleRequest: ( + newReq: IncomingMessage, + newRes: ServerResponse, + newParsedUrl?: NextUrlWithParsedQuery + ) => Promise +): Promise<{ buffer: Buffer; contentType: string; maxAge: number }> { + let upstreamBuffer: Buffer + let upstreamType: string | null + let maxAge: number + const { isAbsolute, href, width, mimeType, quality } = paramsResult + + if (isAbsolute) { + const upstreamRes = await fetch(href) + + if (!upstreamRes.ok) { + console.error( + 'upstream image response failed for', + href, + upstreamRes.status + ) + throw new ImageError( + '"url" parameter is valid but upstream response is invalid', + upstreamRes.status + ) + } - const isStreamFinished = new Promise(function (resolve, reject) { - mockRes.on('finish', () => resolve(true)) - mockRes.on('end', () => resolve(true)) - mockRes.on('error', () => reject()) - }) + upstreamBuffer = Buffer.from(await upstreamRes.arrayBuffer()) + upstreamType = + detectContentType(upstreamBuffer) || + upstreamRes.headers.get('Content-Type') + maxAge = getMaxAge(upstreamRes.headers.get('Cache-Control')) + } else { + try { + const resBuffers: Buffer[] = [] + const mockRes: any = new Stream.Writable() - mockRes.write = (chunk: Buffer | string) => { - resBuffers.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) - } - mockRes._write = ( - chunk: Buffer | string, - _encoding: string, - callback: () => void - ) => { - mockRes.write(chunk) - // According to Node.js documentation, the callback MUST be invoked to signal that - // the write completed successfully. If this callback is not invoked, the 'finish' event - // will not be emitted. - // https://nodejs.org/docs/latest-v16.x/api/stream.html#writable_writechunk-encoding-callback - callback() - } + const isStreamFinished = new Promise(function (resolve, reject) { + mockRes.on('finish', () => resolve(true)) + mockRes.on('end', () => resolve(true)) + mockRes.on('error', () => reject()) + }) - const mockHeaders: Record = {} - - mockRes.writeHead = (_status: any, _headers: any) => - Object.assign(mockHeaders, _headers) - mockRes.getHeader = (name: string) => mockHeaders[name.toLowerCase()] - mockRes.getHeaders = () => mockHeaders - mockRes.getHeaderNames = () => Object.keys(mockHeaders) - mockRes.setHeader = (name: string, value: string | string[]) => - (mockHeaders[name.toLowerCase()] = value) - mockRes.removeHeader = (name: string) => { - delete mockHeaders[name.toLowerCase()] - } - mockRes._implicitHeader = () => {} - mockRes.connection = res.connection - mockRes.finished = false - mockRes.statusCode = 200 + mockRes.write = (chunk: Buffer | string) => { + resBuffers.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + } + mockRes._write = ( + chunk: Buffer | string, + _encoding: string, + callback: () => void + ) => { + mockRes.write(chunk) + // According to Node.js documentation, the callback MUST be invoked to signal that + // the write completed successfully. If this callback is not invoked, the 'finish' event + // will not be emitted. + // https://nodejs.org/docs/latest-v16.x/api/stream.html#writable_writechunk-encoding-callback + callback() + } - const mockReq: any = new Stream.Readable() + const mockHeaders: Record = {} + + mockRes.writeHead = (_status: any, _headers: any) => + Object.assign(mockHeaders, _headers) + mockRes.getHeader = (name: string) => mockHeaders[name.toLowerCase()] + mockRes.getHeaders = () => mockHeaders + mockRes.getHeaderNames = () => Object.keys(mockHeaders) + mockRes.setHeader = (name: string, value: string | string[]) => + (mockHeaders[name.toLowerCase()] = value) + mockRes.removeHeader = (name: string) => { + delete mockHeaders[name.toLowerCase()] + } + mockRes._implicitHeader = () => {} + mockRes.connection = _res.connection + mockRes.finished = false + mockRes.statusCode = 200 - mockReq._read = () => { - mockReq.emit('end') - mockReq.emit('close') - return Buffer.from('') - } + const mockReq: any = new Stream.Readable() - mockReq.headers = req.headers - mockReq.method = req.method - mockReq.url = href - mockReq.connection = req.connection - - await handleRequest(mockReq, mockRes, nodeUrl.parse(href, true)) - await isStreamFinished - res.statusCode = mockRes.statusCode - upstreamBuffer = Buffer.concat(resBuffers) - upstreamType = - detectContentType(upstreamBuffer) || mockRes.getHeader('Content-Type') - maxAge = getMaxAge(mockRes.getHeader('Cache-Control')) - } catch (err) { - res.statusCode = 500 - res.end('"url" parameter is valid but upstream response is invalid') - return { finished: true } + mockReq._read = () => { + mockReq.emit('end') + mockReq.emit('close') + return Buffer.from('') } - } - const expireAt = Math.max(maxAge, minimumCacheTTL) * 1000 + now - - if (upstreamType) { - const vector = VECTOR_TYPES.includes(upstreamType) - const animate = - ANIMATABLE_TYPES.includes(upstreamType) && isAnimated(upstreamBuffer) - if (vector || animate) { - await writeToCacheDir( - hashDir, - upstreamType, - maxAge, - expireAt, - upstreamBuffer - ) - sendResponse( - req, - res, - url, - maxAge, - upstreamType, - upstreamBuffer, - isStatic, - isDev, - xCache - ) - return { finished: true } - } - if (!upstreamType.startsWith('image/')) { - res.statusCode = 400 - res.end("The requested resource isn't a valid image.") - return { finished: true } - } + mockReq.headers = _req.headers + mockReq.method = _req.method + mockReq.url = href + mockReq.connection = _req.connection + + await handleRequest(mockReq, mockRes, nodeUrl.parse(href, true)) + await isStreamFinished + upstreamBuffer = Buffer.concat(resBuffers) + upstreamType = + detectContentType(upstreamBuffer) || mockRes.getHeader('Content-Type') + maxAge = getMaxAge(mockRes.getHeader('Cache-Control')) + } catch (err) { + console.error('upstream image response failed for', href, err) + throw new ImageError( + '"url" parameter is valid but upstream response is invalid', + 500 + ) } + } - let contentType: string + if (upstreamType) { + const vector = VECTOR_TYPES.includes(upstreamType) + const animate = + ANIMATABLE_TYPES.includes(upstreamType) && isAnimated(upstreamBuffer) - if (mimeType) { - contentType = mimeType - } else if ( - upstreamType?.startsWith('image/') && - getExtension(upstreamType) - ) { - contentType = upstreamType - } else { - contentType = JPEG + if (vector || animate) { + return { buffer: upstreamBuffer, contentType: upstreamType, maxAge } } - try { - let optimizedBuffer: Buffer | undefined - if (sharp) { - // Begin sharp transformation logic - const transformer = sharp(upstreamBuffer) + if (!upstreamType.startsWith('image/')) { + console.error( + "The requested resource isn't a valid image for", + href, + 'received', + upstreamType + ) + throw new ImageError("The requested resource isn't a valid image.", 400) + } + } - transformer.rotate() + let contentType: string - const { width: metaWidth } = await transformer.metadata() + if (mimeType) { + contentType = mimeType + } else if (upstreamType?.startsWith('image/') && getExtension(upstreamType)) { + contentType = upstreamType + } else { + contentType = JPEG + } + try { + let optimizedBuffer: Buffer | undefined + if (sharp) { + // Begin sharp transformation logic + const transformer = sharp(upstreamBuffer) - if (metaWidth && metaWidth > width) { - transformer.resize(width) - } + transformer.rotate() - if (contentType === AVIF) { - if (transformer.avif) { - const avifQuality = quality - 15 - transformer.avif({ - quality: Math.max(avifQuality, 0), - chromaSubsampling: '4:2:0', // same as webp - }) - } else { - console.warn( - chalk.yellow.bold('Warning: ') + - `Your installed version of the 'sharp' package does not support AVIF images. Run 'yarn add sharp@latest' to upgrade to the latest version.\n` + - 'Read more: https://nextjs.org/docs/messages/sharp-version-avif' - ) - transformer.webp({ quality }) - } - } else if (contentType === WEBP) { - transformer.webp({ quality }) - } else if (contentType === PNG) { - transformer.png({ quality }) - } else if (contentType === JPEG) { - transformer.jpeg({ quality }) - } + const { width: metaWidth } = await transformer.metadata() - optimizedBuffer = await transformer.toBuffer() - // End sharp transformation logic - } else { - if ( - showSharpMissingWarning && - nextConfig.experimental?.outputStandalone - ) { - // TODO: should we ensure squoosh also works even though we don't - // recommend it be used in production and this is a production feature - console.error( - `Error: 'sharp' is required to be installed in standalone mode for the image optimization to function correctly` - ) - req.statusCode = 500 - res.end('internal server error') - return { finished: true } - } - // Show sharp warning in production once - if (showSharpMissingWarning) { + if (metaWidth && metaWidth > width) { + transformer.resize(width) + } + + if (contentType === AVIF) { + if (transformer.avif) { + const avifQuality = quality - 15 + transformer.avif({ + quality: Math.max(avifQuality, 0), + chromaSubsampling: '4:2:0', // same as webp + }) + } else { console.warn( chalk.yellow.bold('Warning: ') + - `For production Image Optimization with Next.js, the optional 'sharp' package is strongly recommended. Run 'yarn add sharp', and Next.js will use it automatically for Image Optimization.\n` + - 'Read more: https://nextjs.org/docs/messages/sharp-missing-in-production' + `Your installed version of the 'sharp' package does not support AVIF images. Run 'yarn add sharp@latest' to upgrade to the latest version.\n` + + 'Read more: https://nextjs.org/docs/messages/sharp-version-avif' ) - showSharpMissingWarning = false + transformer.webp({ quality }) } + } else if (contentType === WEBP) { + transformer.webp({ quality }) + } else if (contentType === PNG) { + transformer.png({ quality }) + } else if (contentType === JPEG) { + transformer.jpeg({ quality }) + } - // Begin Squoosh transformation logic - const orientation = await getOrientation(upstreamBuffer) + optimizedBuffer = await transformer.toBuffer() + // End sharp transformation logic + } else { + if ( + showSharpMissingWarning && + nextConfig.experimental?.outputStandalone + ) { + // TODO: should we ensure squoosh also works even though we don't + // recommend it be used in production and this is a production feature + console.error( + `Error: 'sharp' is required to be installed in standalone mode for the image optimization to function correctly` + ) + throw new ImageError('internal server error') + } + // Show sharp warning in production once + if (showSharpMissingWarning) { + console.warn( + chalk.yellow.bold('Warning: ') + + `For production Image Optimization with Next.js, the optional 'sharp' package is strongly recommended. Run 'yarn add sharp', and Next.js will use it automatically for Image Optimization.\n` + + 'Read more: https://nextjs.org/docs/messages/sharp-missing-in-production' + ) + showSharpMissingWarning = false + } - const operations: Operation[] = [] + // Begin Squoosh transformation logic + const orientation = await getOrientation(upstreamBuffer) - if (orientation === Orientation.RIGHT_TOP) { - operations.push({ type: 'rotate', numRotations: 1 }) - } else if (orientation === Orientation.BOTTOM_RIGHT) { - operations.push({ type: 'rotate', numRotations: 2 }) - } else if (orientation === Orientation.LEFT_BOTTOM) { - operations.push({ type: 'rotate', numRotations: 3 }) - } else { - // TODO: support more orientations - // eslint-disable-next-line @typescript-eslint/no-unused-vars - // const _: never = orientation - } + const operations: Operation[] = [] - operations.push({ type: 'resize', width }) + if (orientation === Orientation.RIGHT_TOP) { + operations.push({ type: 'rotate', numRotations: 1 }) + } else if (orientation === Orientation.BOTTOM_RIGHT) { + operations.push({ type: 'rotate', numRotations: 2 }) + } else if (orientation === Orientation.LEFT_BOTTOM) { + operations.push({ type: 'rotate', numRotations: 3 }) + } else { + // TODO: support more orientations + // eslint-disable-next-line @typescript-eslint/no-unused-vars + // const _: never = orientation + } - if (contentType === AVIF) { - optimizedBuffer = await processBuffer( - upstreamBuffer, - operations, - 'avif', - quality - ) - } else if (contentType === WEBP) { - optimizedBuffer = await processBuffer( - upstreamBuffer, - operations, - 'webp', - quality - ) - } else if (contentType === PNG) { - optimizedBuffer = await processBuffer( - upstreamBuffer, - operations, - 'png', - quality - ) - } else if (contentType === JPEG) { - optimizedBuffer = await processBuffer( - upstreamBuffer, - operations, - 'jpeg', - quality - ) - } + operations.push({ type: 'resize', width }) - // End Squoosh transformation logic - } - if (optimizedBuffer) { - await writeToCacheDir( - hashDir, - contentType, - maxAge, - expireAt, - optimizedBuffer + if (contentType === AVIF) { + optimizedBuffer = await processBuffer( + upstreamBuffer, + operations, + 'avif', + quality ) - sendResponse( - req, - res, - url, - maxAge, - contentType, - optimizedBuffer, - isStatic, - isDev, - xCache + } else if (contentType === WEBP) { + optimizedBuffer = await processBuffer( + upstreamBuffer, + operations, + 'webp', + quality + ) + } else if (contentType === PNG) { + optimizedBuffer = await processBuffer( + upstreamBuffer, + operations, + 'png', + quality + ) + } else if (contentType === JPEG) { + optimizedBuffer = await processBuffer( + upstreamBuffer, + operations, + 'jpeg', + quality ) - } else { - throw new Error('Unable to optimize buffer') } - } catch (error) { - sendResponse( - req, - res, - url, - maxAge, - upstreamType, - upstreamBuffer, - isStatic, - isDev, - xCache - ) - } - return { finished: true } - } finally { - dedupe.resolve() - inflightRequests.delete(hash) + // End Squoosh transformation logic + } + if (optimizedBuffer) { + return { + buffer: optimizedBuffer, + contentType, + maxAge: Math.max(maxAge, nextConfig.images.minimumCacheTTL), + } + } else { + throw new ImageError('Unable to optimize buffer') + } + } catch (error) { + return { + buffer: upstreamBuffer, + contentType: upstreamType!, + maxAge, + } } } async function writeToCacheDir( dir: string, - contentType: string, + extension: string, maxAge: number, expireAt: number, - buffer: Buffer + buffer: Buffer, + etag: string ) { - await promises.mkdir(dir, { recursive: true }) - const extension = getExtension(contentType) - const etag = getHash([buffer]) const filename = join(dir, `${maxAge}.${expireAt}.${etag}.${extension}`) + await promises.rmdir(dir, { recursive: true }) + await promises.mkdir(dir, { recursive: true }) await promises.writeFile(filename, buffer) } @@ -549,15 +551,13 @@ function getFileNameWithExtension( return `${fileName}.${extension}` } -function setResponseHeaders( +export function setResponseHeaders( req: IncomingMessage, res: ServerResponse, url: string, etag: string, - maxAge: number, contentType: string | null, isStatic: boolean, - isDev: boolean, xCache: XCacheHeader ) { res.setHeader('Vary', 'Accept') @@ -565,7 +565,7 @@ function setResponseHeaders( 'Cache-Control', isStatic ? 'public, max-age=315360000, immutable' - : `public, max-age=${isDev ? 0 : maxAge}, must-revalidate` + : `public, max-age=0, must-revalidate` ) if (sendEtagResponse(req, res, etag)) { // already called res.end() so we're finished @@ -589,30 +589,24 @@ function setResponseHeaders( return { finished: false } } -function sendResponse( +export function sendResponse( req: IncomingMessage, res: ServerResponse, url: string, - maxAge: number, - contentType: string | null, + extension: string, buffer: Buffer, isStatic: boolean, - isDev: boolean, xCache: XCacheHeader ) { - if (xCache === 'STALE') { - return - } + const contentType = getContentType(extension) const etag = getHash([buffer]) const result = setResponseHeaders( req, res, url, etag, - maxAge, contentType, isStatic, - isDev, xCache ) if (!result.finished) { @@ -625,7 +619,7 @@ function getSupportedMimeType(options: string[], accept = ''): string { return accept.includes(mimeType) ? mimeType : '' } -function getHash(items: (string | number | Buffer)[]) { +export function getHash(items: (string | number | Buffer)[]) { const hash = createHash('sha256') for (let item of items) { if (typeof item === 'number') hash.update(String(item)) diff --git a/packages/next/server/incremental-cache.ts b/packages/next/server/incremental-cache.ts index 705af386ec4e2..78f945ca953b3 100644 --- a/packages/next/server/incremental-cache.ts +++ b/packages/next/server/incremental-cache.ts @@ -4,23 +4,24 @@ import LRUCache from 'next/dist/compiled/lru-cache' import path from 'path' import { PrerenderManifest } from '../build' import { normalizePagePath } from './normalize-page-path' +import { CachedImageValue, CachedRedirectValue } from './response-cache' function toRoute(pathname: string): string { return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/' } -interface CachedRedirectValue { - kind: 'REDIRECT' - props: Object -} - interface CachedPageValue { kind: 'PAGE' + // this needs to be a string since the cache expects to store + // the string value html: string pageData: Object } -export type IncrementalCacheValue = CachedRedirectValue | CachedPageValue +export type IncrementalCacheValue = + | CachedRedirectValue + | CachedPageValue + | CachedImageValue type IncrementalCacheEntry = { curRevalidate?: number | false @@ -82,7 +83,8 @@ export class IncrementalCache { this.cache = new LRUCache({ max, length({ value }) { - if (!value || value.kind === 'REDIRECT') return 25 + if (!value || value.kind === 'REDIRECT' || value.kind === 'IMAGE') + return 25 // rough estimate of size of cache value return value.html.length + JSON.stringify(value.pageData).length }, diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts index 38ec642e66cef..597f8bacad13e 100644 --- a/packages/next/server/next-server.ts +++ b/packages/next/server/next-server.ts @@ -47,7 +47,7 @@ import { NodeNextResponse, } from './base-http' import { PayloadOptions, sendRenderResult } from './send-payload' -import { serveStatic } from './serve-static' +import { getExtension, serveStatic } from './serve-static' import { ParsedUrlQuery } from 'querystring' import { apiResolver } from './api-utils' import { RenderOpts, renderToHTML } from './render' @@ -75,6 +75,14 @@ import { MIDDLEWARE_ROUTE } from '../lib/constants' import { loadEnvConfig } from '@next/env' import { getCustomRoute } from './server-route-utils' import { urlQueryToSearchParams } from '../shared/lib/router/utils/querystring' +import { + getHash, + ImageError, + ImageOptimizerCache, + sendResponse, + setResponseHeaders, +} from './image-optimizer' +import { CachedImageValue } from './response-cache' export * from './base-server' @@ -161,7 +169,7 @@ export default class NextNodeServer extends BaseServer { match: route('/_next/image'), type: 'route', name: '_next/image catchall', - fn: (req, res, _params, parsedUrl) => { + fn: async (req, res, _params, parsedUrl) => { if (this.minimalMode) { res.statusCode = 400 res.body('Bad Request').send() @@ -169,12 +177,77 @@ export default class NextNodeServer extends BaseServer { finished: true, } } + const imagesConfig = this.nextConfig.images - return this.imageOptimizer( - req as NodeNextRequest, - res as NodeNextResponse, - parsedUrl + if (imagesConfig.loader !== 'default') { + await this.render404(req, res) + return { finished: true } + } + const paramsResult = ImageOptimizerCache.validateParams( + req as any, + parsedUrl.query, + this.nextConfig, + !!this.renderOpts.dev ) + + if ('errorMessage' in paramsResult) { + res.statusCode = 400 + res.body(paramsResult.errorMessage).send() + return { finished: true } + } + const cacheKey = ImageOptimizerCache.getCacheKey(paramsResult) + + try { + const cacheEntry = await this.imageResponseCache.get( + cacheKey, + async () => { + const { buffer, contentType, maxAge } = + await this.imageOptimizer( + req as NodeNextRequest, + res as NodeNextResponse, + paramsResult + ) + const etag = getHash([buffer]) + + return { + value: { + kind: 'IMAGE', + buffer, + etag, + extension: getExtension(contentType) as string, + revalidate: maxAge, + }, + revalidate: maxAge, + } + } + ) + + if (cacheEntry?.value?.kind !== 'IMAGE') { + throw new Error( + 'invariant did not get entry from image response cache' + ) + } + + sendResponse( + (req as NodeNextRequest).originalRequest, + (res as NodeNextResponse).originalResponse, + paramsResult.href, + cacheEntry.value.extension, + cacheEntry.value.buffer, + paramsResult.isStatic, + cacheEntry.isMiss ? 'MISS' : cacheEntry.isStale ? 'STALE' : 'HIT' + ) + } catch (err) { + if (err instanceof ImageError) { + res.statusCode = err.statusCode + res.body(err.message).send() + return { + finished: true, + } + } + throw err + } + return { finished: true } }, }, ] @@ -482,25 +555,22 @@ export default class NextNodeServer extends BaseServer { protected async imageOptimizer( req: NodeNextRequest, res: NodeNextResponse, - parsedUrl: UrlWithParsedQuery - ): Promise<{ finished: boolean }> { + paramsResult: any + ): Promise<{ buffer: Buffer; contentType: string; maxAge: number }> { const { imageOptimizer } = require('./image-optimizer') as typeof import('./image-optimizer') return imageOptimizer( req.originalRequest, res.originalResponse, - parsedUrl, + paramsResult, this.nextConfig, - this.distDir, - () => this.render404(req, res, parsedUrl), (newReq, newRes, newParsedUrl) => this.getRequestHandler()( new NodeNextRequest(newReq), new NodeNextResponse(newRes), newParsedUrl - ), - this.renderOpts.dev + ) ) } diff --git a/packages/next/server/response-cache.ts b/packages/next/server/response-cache.ts index 2279733e721e4..c97576e0d6903 100644 --- a/packages/next/server/response-cache.ts +++ b/packages/next/server/response-cache.ts @@ -1,22 +1,39 @@ import { IncrementalCache } from './incremental-cache' import RenderResult from './render-result' -interface CachedRedirectValue { +export interface CachedRedirectValue { kind: 'REDIRECT' props: Object } interface CachedPageValue { kind: 'PAGE' + // this needs to be a RenderResult so since renderResponse + // expects that type instead of a string html: RenderResult pageData: Object } -export type ResponseCacheValue = CachedRedirectValue | CachedPageValue +export interface CachedImageValue { + kind: 'IMAGE' + etag: string + buffer: Buffer + extension: string + isMiss?: boolean + isStale?: boolean + revalidate: number +} + +export type ResponseCacheValue = + | CachedRedirectValue + | CachedPageValue + | CachedImageValue export type ResponseCacheEntry = { revalidate?: number | false value: ResponseCacheValue | null + isStale?: boolean + isMiss?: boolean } type ResponseGenerator = ( @@ -73,6 +90,7 @@ export default class ResponseCache { const cachedResponse = key ? await this.incrementalCache.get(key) : null if (cachedResponse) { resolve({ + isStale: cachedResponse.isStale, revalidate: cachedResponse.curRevalidate, value: cachedResponse.value?.kind === 'PAGE' @@ -91,7 +109,10 @@ export default class ResponseCache { } const cacheEntry = await responseGenerator(resolved) - resolve(cacheEntry) + resolve({ + ...(cacheEntry as ResponseCacheEntry), + isMiss: !cachedResponse, + }) if (key && cacheEntry && typeof cacheEntry.revalidate !== 'undefined') { await this.incrementalCache.set( diff --git a/test/integration/image-optimizer/test/index.test.js b/test/integration/image-optimizer/test/index.test.js index 5bd360511b22c..f053ef65587fc 100644 --- a/test/integration/image-optimizer/test/index.test.js +++ b/test/integration/image-optimizer/test/index.test.js @@ -3,6 +3,7 @@ import execa from 'execa' import fs from 'fs-extra' import sizeOf from 'image-size' import { + check, fetchViaHTTP, File, findPort, @@ -15,6 +16,7 @@ import { } from 'next-test-utils' import isAnimated from 'next/dist/compiled/is-animated' import { join } from 'path' +import assert from 'assert' const appDir = join(__dirname, '../app') const imagesDir = join(appDir, '.next', 'cache', 'images') @@ -82,11 +84,19 @@ async function expectAvifSmallerThanWebp(w, q) { expect(avif).toBeLessThanOrEqual(webp) } +async function fetchWithDuration(...args) { + const start = Date.now() + const res = await fetchViaHTTP(...args) + const buffer = await res.buffer() + const duration = Date.now() - start + return { duration, buffer, res } +} + function runTests({ w, isDev, domains = [], - ttl, + minimumCacheTTL, isSharp, isOutdatedSharp, avifEnabled, @@ -502,44 +512,92 @@ function runTests({ it('should use cache and stale-while-revalidate when query is the same for external image', async () => { await fs.remove(imagesDir) + const delay = 500 - const url = 'https://image-optimization-test.vercel.app/test.jpg' + const url = `https://image-optimization-test.vercel.app/api/slow?delay=${delay}` const query = { url, w, q: 39 } const opts = { headers: { accept: 'image/webp' } } - const res1 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res1.status).toBe(200) - expect(res1.headers.get('X-Nextjs-Cache')).toBe('MISS') - expect(res1.headers.get('Content-Type')).toBe('image/webp') - expect(res1.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` + const one = await fetchWithDuration(appPort, '/_next/image', query, opts) + expect(one.duration).toBeGreaterThan(delay) + expect(one.res.status).toBe(200) + expect(one.res.headers.get('X-Nextjs-Cache')).toBe('MISS') + expect(one.res.headers.get('Content-Type')).toBe('image/webp') + expect(one.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` ) - const json1 = await fsToJson(imagesDir) - expect(Object.keys(json1).length).toBe(1) - - const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res2.status).toBe(200) - expect(res2.headers.get('X-Nextjs-Cache')).toBe('HIT') - expect(res2.headers.get('Content-Type')).toBe('image/webp') - expect(res2.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` + let json1 + await check(async () => { + json1 = await fsToJson(imagesDir) + return Object.keys(json1).length === 1 ? 'success' : 'fail' + }, 'success') + + const two = await fetchWithDuration(appPort, '/_next/image', query, opts) + expect(two.res.status).toBe(200) + expect(two.res.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(two.res.headers.get('Content-Type')).toBe('image/webp') + expect(two.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` ) const json2 = await fsToJson(imagesDir) expect(json2).toStrictEqual(json1) - if (ttl) { + if (minimumCacheTTL) { // Wait until expired so we can confirm image is regenerated - await waitFor(ttl * 1000) - const res3 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res3.status).toBe(200) - expect(res3.headers.get('X-Nextjs-Cache')).toBe('STALE') - expect(res3.headers.get('Content-Type')).toBe('image/webp') - expect(res3.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` + await waitFor(minimumCacheTTL * 1000) + + const [three, four] = await Promise.all([ + fetchWithDuration(appPort, '/_next/image', query, opts), + fetchWithDuration(appPort, '/_next/image', query, opts), + ]) + + expect(three.duration).toBeLessThan(one.duration) + expect(three.res.status).toBe(200) + expect(three.res.headers.get('X-Nextjs-Cache')).toBe('STALE') + expect(three.res.headers.get('Content-Type')).toBe('image/webp') + expect(three.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` ) - const json3 = await fsToJson(imagesDir) - expect(json3).not.toStrictEqual(json1) - expect(Object.keys(json3).length).toBe(1) + + expect(four.duration).toBeLessThan(one.duration) + expect(four.res.status).toBe(200) + expect(four.res.headers.get('X-Nextjs-Cache')).toBe('STALE') + expect(four.res.headers.get('Content-Type')).toBe('image/webp') + expect(four.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + await check(async () => { + const json4 = await fsToJson(imagesDir) + try { + assert.deepStrictEqual(json4, json1) + return 'fail' + } catch (err) { + return 'success' + } + }, 'success') + + const five = await fetchWithDuration( + appPort, + '/_next/image', + query, + opts + ) + expect(five.duration).toBeLessThan(one.duration) + expect(five.res.status).toBe(200) + expect(five.res.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(five.res.headers.get('Content-Type')).toBe('image/webp') + expect(five.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + await check(async () => { + const json5 = await fsToJson(imagesDir) + try { + assert.deepStrictEqual(json5, json1) + return 'fail' + } catch (err) { + return 'success' + } + }, 'success') } }) } @@ -580,39 +638,80 @@ function runTests({ const query = { url: '/test.png', w, q: 80 } const opts = { headers: { accept: 'image/webp' } } - const res1 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res1.status).toBe(200) - expect(res1.headers.get('X-Nextjs-Cache')).toBe('MISS') - expect(res1.headers.get('Content-Type')).toBe('image/webp') - expect(res1.headers.get('Content-Disposition')).toBe( + const one = await fetchWithDuration(appPort, '/_next/image', query, opts) + expect(one.res.status).toBe(200) + expect(one.res.headers.get('X-Nextjs-Cache')).toBe('MISS') + expect(one.res.headers.get('Content-Type')).toBe('image/webp') + expect(one.res.headers.get('Content-Disposition')).toBe( `inline; filename="test.webp"` ) - const json1 = await fsToJson(imagesDir) - expect(Object.keys(json1).length).toBe(1) - - const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res2.status).toBe(200) - expect(res2.headers.get('X-Nextjs-Cache')).toBe('HIT') - expect(res2.headers.get('Content-Type')).toBe('image/webp') - expect(res2.headers.get('Content-Disposition')).toBe( + let json1 + await check(async () => { + json1 = await fsToJson(imagesDir) + return Object.keys(json1).length === 1 ? 'success' : 'fail' + }, 'success') + + const two = await fetchWithDuration(appPort, '/_next/image', query, opts) + expect(two.res.status).toBe(200) + expect(two.res.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(two.res.headers.get('Content-Type')).toBe('image/webp') + expect(two.res.headers.get('Content-Disposition')).toBe( `inline; filename="test.webp"` ) const json2 = await fsToJson(imagesDir) expect(json2).toStrictEqual(json1) - if (ttl) { + if (minimumCacheTTL) { // Wait until expired so we can confirm image is regenerated - await waitFor(ttl * 1000) - const res3 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res3.status).toBe(200) - expect(res3.headers.get('X-Nextjs-Cache')).toBe('STALE') - expect(res3.headers.get('Content-Type')).toBe('image/webp') - expect(res3.headers.get('Content-Disposition')).toBe( + await waitFor(minimumCacheTTL * 1000) + + const [three, four] = await Promise.all([ + fetchWithDuration(appPort, '/_next/image', query, opts), + fetchWithDuration(appPort, '/_next/image', query, opts), + ]) + + expect(three.duration).toBeLessThan(one.duration) + expect(three.res.status).toBe(200) + expect(three.res.headers.get('X-Nextjs-Cache')).toBe('STALE') + expect(three.res.headers.get('Content-Type')).toBe('image/webp') + expect(three.res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + + expect(four.duration).toBeLessThan(one.duration) + expect(four.res.status).toBe(200) + expect(four.res.headers.get('X-Nextjs-Cache')).toBe('STALE') + expect(four.res.headers.get('Content-Type')).toBe('image/webp') + expect(four.res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await check(async () => { + const json3 = await fsToJson(imagesDir) + try { + assert.deepStrictEqual(json3, json1) + return 'fail' + } catch (err) { + return 'success' + } + }, 'success') + + const five = await fetchWithDuration(appPort, '/_next/image', query, opts) + expect(five.duration).toBeLessThan(one.duration) + expect(five.res.status).toBe(200) + expect(five.res.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(five.res.headers.get('Content-Type')).toBe('image/webp') + expect(five.res.headers.get('Content-Disposition')).toBe( `inline; filename="test.webp"` ) - const json3 = await fsToJson(imagesDir) - expect(json3).not.toStrictEqual(json1) - expect(Object.keys(json3).length).toBe(1) + await check(async () => { + const json5 = await fsToJson(imagesDir) + try { + assert.deepStrictEqual(json5, json1) + return 'fail' + } catch (err) { + return 'success' + } + }, 'success') } }) @@ -629,8 +728,11 @@ function runTests({ expect(res1.headers.get('Content-Disposition')).toBe( `inline; filename="test.svg"` ) - const json1 = await fsToJson(imagesDir) - expect(Object.keys(json1).length).toBe(1) + let json1 + await check(async () => { + json1 = await fsToJson(imagesDir) + return Object.keys(json1).length === 1 ? 'success' : 'fail' + }, 'success') const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res2.status).toBe(200) @@ -656,8 +758,12 @@ function runTests({ expect(res1.headers.get('Content-Disposition')).toBe( `inline; filename="animated.gif"` ) - const json1 = await fsToJson(imagesDir) - expect(Object.keys(json1).length).toBe(1) + + let json1 + await check(async () => { + json1 = await fsToJson(imagesDir) + return Object.keys(json1).length === 1 ? 'success' : 'fail' + }, 'success') const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res2.status).toBe(200) @@ -716,7 +822,7 @@ function runTests({ await expectWidth(res3, w) }) - it('should proxy-pass unsupported image types and should not cache file', async () => { + it('should maintain bmp', async () => { const json1 = await fsToJson(imagesDir) expect(json1).toBeTruthy() @@ -736,8 +842,14 @@ function runTests({ `inline; filename="test.bmp"` ) - const json2 = await fsToJson(imagesDir) - expect(json2).toStrictEqual(json1) + await check(async () => { + try { + assert.deepStrictEqual(await fsToJson(imagesDir), json1) + return 'expected change, but matched' + } catch (_) { + return 'success' + } + }, 'success') }) it('should not resize if requested width is larger than original source image', async () => { @@ -840,12 +952,16 @@ function runTests({ await fs.remove(imagesDir) const query = { url: '/test.png', w, q: 80 } const opts = { headers: { accept: 'image/webp,*/*' } } - const [res1, res2] = await Promise.all([ + const [res1, res2, res3] = await Promise.all([ + fetchViaHTTP(appPort, '/_next/image', query, opts), fetchViaHTTP(appPort, '/_next/image', query, opts), fetchViaHTTP(appPort, '/_next/image', query, opts), ]) + expect(res1.status).toBe(200) expect(res2.status).toBe(200) + expect(res3.status).toBe(200) + expect(res1.headers.get('Content-Type')).toBe('image/webp') expect(res1.headers.get('Content-Disposition')).toBe( `inline; filename="test.webp"` @@ -854,21 +970,28 @@ function runTests({ expect(res2.headers.get('Content-Disposition')).toBe( `inline; filename="test.webp"` ) + expect(res3.headers.get('Content-Type')).toBe('image/webp') + expect(res3.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await expectWidth(res1, w) await expectWidth(res2, w) + await expectWidth(res3, w) - const json1 = await fsToJson(imagesDir) - expect(Object.keys(json1).length).toBe(1) + await check(async () => { + const json1 = await fsToJson(imagesDir) + return Object.keys(json1).length === 1 ? 'success' : 'fail' + }, 'success') - const xCache1 = res1.headers.get('X-Nextjs-Cache') - const xCache2 = res2.headers.get('X-Nextjs-Cache') - if (xCache1 === 'HIT') { - expect(xCache1).toBe('HIT') - expect(xCache2).toBe('MISS') - } else { - expect(xCache1).toBe('MISS') - expect(xCache2).toBe('HIT') - } + const xCache = [res1, res2, res3] + .map((r) => r.headers.get('X-Nextjs-Cache')) + .sort((a, b) => b.localeCompare(a)) + + // Since the first request is a miss it blocks + // until the cache be populated so all concurrent + // requests receive the same response + expect(xCache).toEqual(['MISS', 'MISS', 'MISS']) }) if (isDev || isSharp) { @@ -1105,13 +1228,17 @@ describe('Image Optimizer', () => { 'image-optimization-test.vercel.app', ] + // Reduce to 5 seconds so tests dont dont need to + // wait too long before testing stale responses. + const minimumCacheTTL = 5 + describe('Server support for minimumCacheTTL in next.config.js', () => { const size = 96 // defaults defined in server/config.ts - const ttl = 5 // super low ttl in seconds beforeAll(async () => { const json = JSON.stringify({ images: { - minimumCacheTTL: ttl, + domains, + minimumCacheTTL, }, }) nextOutput = '' @@ -1130,7 +1257,7 @@ describe('Image Optimizer', () => { await fs.remove(imagesDir) }) - runTests({ w: size, isDev: false, ttl }) + runTests({ w: size, isDev: false, domains, minimumCacheTTL }) }) describe('Server support for headers in next.config.js', () => { @@ -1164,17 +1291,33 @@ describe('Image Optimizer', () => { await fs.remove(imagesDir) }) - it('should set max-age header from upstream when matching next.config.js', async () => { + it('should set max-age header', async () => { const query = { url: '/test.png', w: size, q: 75 } const opts = { headers: { accept: 'image/webp' } } const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=86400, must-revalidate` + `public, max-age=0, must-revalidate` ) expect(res.headers.get('Content-Disposition')).toBe( `inline; filename="test.webp"` ) + + await check(async () => { + const files = await fsToJson(imagesDir) + + let found = false + const maxAge = '86400' + + Object.keys(files).forEach((dir) => { + if ( + Object.keys(files[dir]).some((file) => file.includes(`${maxAge}.`)) + ) { + found = true + } + }) + return found ? 'success' : 'failed' + }, 'success') }) it('should not set max-age header when not matching next.config.js', async () => { @@ -1241,18 +1384,36 @@ describe('Image Optimizer', () => { }) it('should return response when image is served from an external rewrite', async () => { + await fs.remove(imagesDir) + const query = { url: '/next-js/next-js-bg.png', w: 64, q: 75 } const opts = { headers: { accept: 'image/webp' } } const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/webp') expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=31536000, must-revalidate` + `public, max-age=0, must-revalidate` ) expect(res.headers.get('Vary')).toBe('Accept') expect(res.headers.get('Content-Disposition')).toBe( `inline; filename="next-js-bg.webp"` ) + + await check(async () => { + const files = await fsToJson(imagesDir) + + let found = false + const maxAge = '31536000' + + Object.keys(files).forEach((dir) => { + if ( + Object.keys(files[dir]).some((file) => file.includes(`${maxAge}.`)) + ) { + found = true + } + }) + return found ? 'success' : 'failed' + }, 'success') await expectWidth(res, 64) }) }) From 2bafaa1320f45fcb5f3d6e606562c526395af55d Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 7 Feb 2022 18:34:54 -0600 Subject: [PATCH 02/14] Apply suggestions --- packages/next/server/image-optimizer.ts | 44 +++++++++++------------ packages/next/server/incremental-cache.ts | 7 +++- packages/next/server/next-server.ts | 6 ++-- packages/next/server/response-cache.ts | 1 - 4 files changed, 28 insertions(+), 30 deletions(-) diff --git a/packages/next/server/image-optimizer.ts b/packages/next/server/image-optimizer.ts index b4e531fb72ea3..aaa4182e02133 100644 --- a/packages/next/server/image-optimizer.ts +++ b/packages/next/server/image-optimizer.ts @@ -45,7 +45,7 @@ try { let showSharpMissingWarning = process.env.NODE_ENV === 'production' -interface ParamsResult { +export interface ImageParamsResult { href: string isAbsolute: boolean isStatic: boolean @@ -65,8 +65,15 @@ export class ImageOptimizerCache { query: UrlWithParsedQuery['query'], nextConfig: NextConfigComplete, isDev: boolean - ): ParamsResult | { errorMessage: string } { - const imageConfig = nextConfig.images + ): ImageParamsResult | { errorMessage: string } { + const imageData = nextConfig.images + const { + deviceSizes = [], + imageSizes = [], + domains = [], + minimumCacheTTL = 60, + formats = ['image/webp'], + } = imageData const { url, w, q } = query let href: string @@ -96,10 +103,7 @@ export class ImageOptimizerCache { return { errorMessage: '"url" parameter is invalid' } } - if ( - !imageConfig.domains || - !imageConfig.domains.includes(hrefParsed.hostname) - ) { + if (!domains || !domains.includes(hrefParsed.hostname)) { return { errorMessage: '"url" parameter is not allowed' } } } @@ -124,10 +128,7 @@ export class ImageOptimizerCache { } } - const sizes = [ - ...(imageConfig.deviceSizes || []), - ...(imageConfig.imageSizes || []), - ] + const sizes = [...(deviceSizes || []), ...(imageSizes || [])] if (isDev) { sizes.push(BLUR_IMG_SIZE) @@ -148,10 +149,7 @@ export class ImageOptimizerCache { } } - const mimeType = getSupportedMimeType( - imageConfig.formats || [], - req.headers['accept'] - ) + const mimeType = getSupportedMimeType(formats || [], req.headers['accept']) const isStatic = url.startsWith( `${nextConfig.basePath || ''}/_next/static/media` @@ -165,7 +163,7 @@ export class ImageOptimizerCache { width, quality, mimeType, - minimumCacheTTL: imageConfig.minimumCacheTTL, + minimumCacheTTL: minimumCacheTTL, } } @@ -205,7 +203,6 @@ export class ImageOptimizerCache { const buffer = await promises.readFile(join(cacheDir, file)) const expireAt = Number(expireAtSt) const maxAge = Number(maxAgeSt) - const revalidate = maxAge return { value: { @@ -213,9 +210,8 @@ export class ImageOptimizerCache { etag, buffer, extension, - revalidate, }, - revalidate, + revalidate: maxAge, isStale: now > expireAt, } } @@ -248,9 +244,9 @@ export class ImageOptimizerCache { } } export class ImageError extends Error { - statusCode?: number + statusCode: number - constructor(message: string, statusCode?: number) { + constructor(message: string, statusCode: number) { super(message) this.statusCode = statusCode } @@ -259,7 +255,7 @@ export class ImageError extends Error { export async function imageOptimizer( _req: IncomingMessage, _res: ServerResponse, - paramsResult: ParamsResult, + paramsResult: ImageParamsResult, nextConfig: NextConfigComplete, handleRequest: ( newReq: IncomingMessage, @@ -441,7 +437,7 @@ export async function imageOptimizer( console.error( `Error: 'sharp' is required to be installed in standalone mode for the image optimization to function correctly` ) - throw new ImageError('internal server error') + throw new ImageError('internal server error', 500) } // Show sharp warning in production once if (showSharpMissingWarning) { @@ -511,7 +507,7 @@ export async function imageOptimizer( maxAge: Math.max(maxAge, nextConfig.images.minimumCacheTTL), } } else { - throw new ImageError('Unable to optimize buffer') + throw new ImageError('Unable to optimize buffer', 500) } } catch (error) { return { diff --git a/packages/next/server/incremental-cache.ts b/packages/next/server/incremental-cache.ts index 78f945ca953b3..5eb13f41e0d12 100644 --- a/packages/next/server/incremental-cache.ts +++ b/packages/next/server/incremental-cache.ts @@ -83,8 +83,13 @@ export class IncrementalCache { this.cache = new LRUCache({ max, length({ value }) { - if (!value || value.kind === 'REDIRECT' || value.kind === 'IMAGE') + if (!value) { return 25 + } else if (value.kind === 'REDIRECT') { + return JSON.stringify(value.props).length + } else if (value.kind === 'IMAGE') { + throw new Error('invariant image should not be incremental-cache') + } // rough estimate of size of cache value return value.html.length + JSON.stringify(value.pageData).length }, diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts index 597f8bacad13e..d69a8b0def292 100644 --- a/packages/next/server/next-server.ts +++ b/packages/next/server/next-server.ts @@ -79,10 +79,9 @@ import { getHash, ImageError, ImageOptimizerCache, + ImageParamsResult, sendResponse, - setResponseHeaders, } from './image-optimizer' -import { CachedImageValue } from './response-cache' export * from './base-server' @@ -215,7 +214,6 @@ export default class NextNodeServer extends BaseServer { buffer, etag, extension: getExtension(contentType) as string, - revalidate: maxAge, }, revalidate: maxAge, } @@ -555,7 +553,7 @@ export default class NextNodeServer extends BaseServer { protected async imageOptimizer( req: NodeNextRequest, res: NodeNextResponse, - paramsResult: any + paramsResult: ImageParamsResult ): Promise<{ buffer: Buffer; contentType: string; maxAge: number }> { const { imageOptimizer } = require('./image-optimizer') as typeof import('./image-optimizer') diff --git a/packages/next/server/response-cache.ts b/packages/next/server/response-cache.ts index c97576e0d6903..4f671025e5593 100644 --- a/packages/next/server/response-cache.ts +++ b/packages/next/server/response-cache.ts @@ -21,7 +21,6 @@ export interface CachedImageValue { extension: string isMiss?: boolean isStale?: boolean - revalidate: number } export type ResponseCacheValue = From bb1aab4044a851b9525b770ea7089baf70366839 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 7 Feb 2022 18:56:41 -0600 Subject: [PATCH 03/14] Fix skipLibCheck: false with incremental-cache --- packages/next/server/incremental-cache.ts | 35 ++++++++++++++++------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/packages/next/server/incremental-cache.ts b/packages/next/server/incremental-cache.ts index 5eb13f41e0d12..df7da3e918cf0 100644 --- a/packages/next/server/incremental-cache.ts +++ b/packages/next/server/incremental-cache.ts @@ -1,6 +1,5 @@ import type { CacheFs } from '../shared/lib/utils' -import LRUCache from 'next/dist/compiled/lru-cache' import path from 'path' import { PrerenderManifest } from '../build' import { normalizePagePath } from './normalize-page-path' @@ -10,14 +9,6 @@ function toRoute(pathname: string): string { return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/' } -interface CachedPageValue { - kind: 'PAGE' - // this needs to be a string since the cache expects to store - // the string value - html: string - pageData: Object -} - export type IncrementalCacheValue = | CachedRedirectValue | CachedPageValue @@ -31,6 +22,30 @@ type IncrementalCacheEntry = { value: IncrementalCacheValue | null } +// we need to avoid relying on lru-cache types here or +// it will break skipLibCheck: false with TypeScript +declare class LRUCacheType { + set(key: K, value: V, maxAge?: number): boolean + get(key: K): V | undefined +} +interface Constructable { + new (options?: { + max: number + length?(value: IncrementalCacheEntry, key?: string): number + }): T +} +const LRUCache = require('next/dist/compiled/lru-cache') as Constructable< + LRUCacheType +> + +interface CachedPageValue { + kind: 'PAGE' + // this needs to be a string since the cache expects to store + // the string value + html: string + pageData: Object +} + export class IncrementalCache { incrementalOptions: { flushToDisk?: boolean @@ -40,7 +55,7 @@ export class IncrementalCache { } prerenderManifest: PrerenderManifest - cache?: LRUCache + cache?: LRUCacheType locales?: string[] fs: CacheFs From 41075a5e58543bca2e841caaf870aa8200b8bfcd Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 7 Feb 2022 19:55:31 -0600 Subject: [PATCH 04/14] update types and fix fs import error --- packages/next/server/base-server.ts | 8 -------- packages/next/server/image-optimizer.ts | 5 ++++- packages/next/server/next-server.ts | 24 ++++++++++++++++-------- packages/next/server/response-cache.ts | 24 +++++++++++++++++++----- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/packages/next/server/base-server.ts b/packages/next/server/base-server.ts index 6e32793c430e5..36cb977aef674 100644 --- a/packages/next/server/base-server.ts +++ b/packages/next/server/base-server.ts @@ -58,7 +58,6 @@ import { MIDDLEWARE_ROUTE } from '../lib/constants' import { addRequestMeta, getRequestMeta } from './request-meta' import { createHeaderRoute, createRedirectRoute } from './server-route-utils' import { PrerenderManifest } from '../build' -import { ImageOptimizerCache } from './image-optimizer' export type FindComponentsResult = { components: LoadComponentsReturnType @@ -166,7 +165,6 @@ export default abstract class Server { } private incrementalCache: IncrementalCache private responseCache: ResponseCache - protected imageResponseCache: ResponseCache protected router: Router protected dynamicRoutes?: DynamicRoutes protected customRoutes: CustomRoutes @@ -362,12 +360,6 @@ export default abstract class Server { }, }) this.responseCache = new ResponseCache(this.incrementalCache) - this.imageResponseCache = new ResponseCache( - new ImageOptimizerCache({ - distDir: this.distDir, - nextConfig: this.nextConfig, - }) as any - ) } public logError(err: Error): void { diff --git a/packages/next/server/image-optimizer.ts b/packages/next/server/image-optimizer.ts index aaa4182e02133..b473555ce2848 100644 --- a/packages/next/server/image-optimizer.ts +++ b/packages/next/server/image-optimizer.ts @@ -227,8 +227,11 @@ export class ImageOptimizerCache { buffer: Buffer extension: string }, - revalidate: number + revalidate?: number | false ) { + if (typeof revalidate !== 'number') { + throw new Error('invariant revalidate must be a number for image-cache') + } const expireAt = Math.max(revalidate, this.nextConfig.images.minimumCacheTTL) * 1000 + Date.now() diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts index d69a8b0def292..f0e7e513ca38a 100644 --- a/packages/next/server/next-server.ts +++ b/packages/next/server/next-server.ts @@ -75,13 +75,7 @@ import { MIDDLEWARE_ROUTE } from '../lib/constants' import { loadEnvConfig } from '@next/env' import { getCustomRoute } from './server-route-utils' import { urlQueryToSearchParams } from '../shared/lib/router/utils/querystring' -import { - getHash, - ImageError, - ImageOptimizerCache, - ImageParamsResult, - sendResponse, -} from './image-optimizer' +import ResponseCache from '../server/response-cache' export * from './base-server' @@ -100,6 +94,8 @@ export interface NodeRequestHandler { } export default class NextNodeServer extends BaseServer { + private imageResponseCache: ResponseCache + constructor(options: Options) { // Initialize super class super(options) @@ -119,6 +115,16 @@ export default class NextNodeServer extends BaseServer { if (this.renderOpts.optimizeCss) { process.env.__NEXT_OPTIMIZE_CSS = JSON.stringify(true) } + + const { ImageOptimizerCache } = + require('./image-optimizer') as typeof import('./image-optimizer') + + this.imageResponseCache = new ResponseCache( + new ImageOptimizerCache({ + distDir: this.distDir, + nextConfig: this.nextConfig, + }) + ) } private compression = @@ -163,6 +169,8 @@ export default class NextNodeServer extends BaseServer { } protected generateImageRoutes(): Route[] { + const { getHash, ImageOptimizerCache, sendResponse, ImageError } = + require('./image-config') as typeof import('./image-optimizer') return [ { match: route('/_next/image'), @@ -553,7 +561,7 @@ export default class NextNodeServer extends BaseServer { protected async imageOptimizer( req: NodeNextRequest, res: NodeNextResponse, - paramsResult: ImageParamsResult + paramsResult: import('./image-optimizer').ImageParamsResult ): Promise<{ buffer: Buffer; contentType: string; maxAge: number }> { const { imageOptimizer } = require('./image-optimizer') as typeof import('./image-optimizer') diff --git a/packages/next/server/response-cache.ts b/packages/next/server/response-cache.ts index 4f671025e5593..ebf1df8f6659c 100644 --- a/packages/next/server/response-cache.ts +++ b/packages/next/server/response-cache.ts @@ -1,4 +1,3 @@ -import { IncrementalCache } from './incremental-cache' import RenderResult from './render-result' export interface CachedRedirectValue { @@ -39,6 +38,17 @@ type ResponseGenerator = ( hasResolved: boolean ) => Promise +interface IncrementalCache { + get: (key: string) => Promise<{ + curRevalidate?: number | false + revalidate?: number | false + value?: any | null + isStale?: boolean + isMiss?: boolean + } | null> + set: (key: string, data: any, revalidate?: number | false) => Promise +} + export default class ResponseCache { incrementalCache: IncrementalCache pendingResponses: Map> @@ -108,10 +118,14 @@ export default class ResponseCache { } const cacheEntry = await responseGenerator(resolved) - resolve({ - ...(cacheEntry as ResponseCacheEntry), - isMiss: !cachedResponse, - }) + resolve( + cacheEntry === null + ? null + : { + ...cacheEntry, + isMiss: !cachedResponse, + } + ) if (key && cacheEntry && typeof cacheEntry.revalidate !== 'undefined') { await this.incrementalCache.set( From e8c858b11f1527097da65613fb6eb52df119a1d4 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 7 Feb 2022 19:58:05 -0600 Subject: [PATCH 05/14] Revert "Fix skipLibCheck: false with incremental-cache" This reverts commit bb1aab4044a851b9525b770ea7089baf70366839. --- packages/next/server/incremental-cache.ts | 35 +++++++---------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/packages/next/server/incremental-cache.ts b/packages/next/server/incremental-cache.ts index df7da3e918cf0..5eb13f41e0d12 100644 --- a/packages/next/server/incremental-cache.ts +++ b/packages/next/server/incremental-cache.ts @@ -1,5 +1,6 @@ import type { CacheFs } from '../shared/lib/utils' +import LRUCache from 'next/dist/compiled/lru-cache' import path from 'path' import { PrerenderManifest } from '../build' import { normalizePagePath } from './normalize-page-path' @@ -9,6 +10,14 @@ function toRoute(pathname: string): string { return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/' } +interface CachedPageValue { + kind: 'PAGE' + // this needs to be a string since the cache expects to store + // the string value + html: string + pageData: Object +} + export type IncrementalCacheValue = | CachedRedirectValue | CachedPageValue @@ -22,30 +31,6 @@ type IncrementalCacheEntry = { value: IncrementalCacheValue | null } -// we need to avoid relying on lru-cache types here or -// it will break skipLibCheck: false with TypeScript -declare class LRUCacheType { - set(key: K, value: V, maxAge?: number): boolean - get(key: K): V | undefined -} -interface Constructable { - new (options?: { - max: number - length?(value: IncrementalCacheEntry, key?: string): number - }): T -} -const LRUCache = require('next/dist/compiled/lru-cache') as Constructable< - LRUCacheType -> - -interface CachedPageValue { - kind: 'PAGE' - // this needs to be a string since the cache expects to store - // the string value - html: string - pageData: Object -} - export class IncrementalCache { incrementalOptions: { flushToDisk?: boolean @@ -55,7 +40,7 @@ export class IncrementalCache { } prerenderManifest: PrerenderManifest - cache?: LRUCacheType + cache?: LRUCache locales?: string[] fs: CacheFs From 492c9408e0fda17c9b5210a4987d66d5ba355b54 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 7 Feb 2022 21:00:48 -0600 Subject: [PATCH 06/14] fix require --- packages/next/server/next-server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts index f0e7e513ca38a..1afd0bf9c210c 100644 --- a/packages/next/server/next-server.ts +++ b/packages/next/server/next-server.ts @@ -170,7 +170,7 @@ export default class NextNodeServer extends BaseServer { protected generateImageRoutes(): Route[] { const { getHash, ImageOptimizerCache, sendResponse, ImageError } = - require('./image-config') as typeof import('./image-optimizer') + require('./image-optimizer') as typeof import('./image-optimizer') return [ { match: route('/_next/image'), From 38b8439abfdabf150fd197ef50584e062f0cd506 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 7 Feb 2022 21:53:59 -0600 Subject: [PATCH 07/14] update test --- .../image-optimizer/test/index.test.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/integration/image-optimizer/test/index.test.js b/test/integration/image-optimizer/test/index.test.js index f053ef65587fc..7d9030783cfbd 100644 --- a/test/integration/image-optimizer/test/index.test.js +++ b/test/integration/image-optimizer/test/index.test.js @@ -645,10 +645,16 @@ function runTests({ expect(one.res.headers.get('Content-Disposition')).toBe( `inline; filename="test.webp"` ) + const etagOne = one.res.headers.get('etag') + let json1 await check(async () => { json1 = await fsToJson(imagesDir) - return Object.keys(json1).length === 1 ? 'success' : 'fail' + return Object.keys(json1).some((dir) => { + return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) + }) + ? 'success' + : 'fail' }, 'success') const two = await fetchWithDuration(appPort, '/_next/image', query, opts) @@ -728,10 +734,16 @@ function runTests({ expect(res1.headers.get('Content-Disposition')).toBe( `inline; filename="test.svg"` ) + const etagOne = res1.headers.get('etag') + let json1 await check(async () => { json1 = await fsToJson(imagesDir) - return Object.keys(json1).length === 1 ? 'success' : 'fail' + return Object.keys(json1).some((dir) => { + return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) + }) + ? 'success' + : 'fail' }, 'success') const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts) From ff5a125f6908013231f05a2f7714703550a320a4 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 7 Feb 2022 22:01:40 -0600 Subject: [PATCH 08/14] fix type error --- packages/next/server/next-server.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts index 8094210d07ab7..ecef2373eb0ce 100644 --- a/packages/next/server/next-server.ts +++ b/packages/next/server/next-server.ts @@ -225,7 +225,8 @@ export default class NextNodeServer extends BaseServer { }, revalidate: maxAge, } - } + }, + {} ) if (cacheEntry?.value?.kind !== 'IMAGE') { From a44187b3791fa91ac40f3a9b87485588d00d5c4e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 8 Feb 2022 10:19:19 -0600 Subject: [PATCH 09/14] update util --- packages/next/server/api-utils.ts | 24 +++++++++++++++---- packages/next/shared/lib/utils.ts | 2 +- test/e2e/prerender.test.ts | 16 ++++--------- .../prerender/pages/api/manual-revalidate.js | 12 ++++++---- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/packages/next/server/api-utils.ts b/packages/next/server/api-utils.ts index d5244556a4319..a4e5706173c45 100644 --- a/packages/next/server/api-utils.ts +++ b/packages/next/server/api-utils.ts @@ -365,15 +365,29 @@ async function unstable_revalidate( ) } + if (typeof urlPath !== 'string' || !urlPath.startsWith('/')) { + throw new Error( + `Invalid urlPath provided to revalidate(), must be a path e.g. /blog/post-1, received ${urlPath}` + ) + } + const baseUrl = context.trustHostHeader ? `https://${req.headers.host}` : `http://${context.hostname}:${context.port}` - return fetch(`${baseUrl}${urlPath}`, { - headers: { - [PRERENDER_REVALIDATE_HEADER]: context.previewModeId, - }, - }) + try { + const res = await fetch(`${baseUrl}${urlPath}`, { + headers: { + [PRERENDER_REVALIDATE_HEADER]: context.previewModeId, + }, + }) + + if (!res.ok) { + throw new Error(`Invalid response ${res.status}`) + } + } catch (err) { + throw new Error(`Failed to revalidate ${urlPath}`) + } } const COOKIE_NAME_PRERENDER_BYPASS = `__prerender_bypass` diff --git a/packages/next/shared/lib/utils.ts b/packages/next/shared/lib/utils.ts index 8bf0230b4841e..63d309f4513dd 100644 --- a/packages/next/shared/lib/utils.ts +++ b/packages/next/shared/lib/utils.ts @@ -294,7 +294,7 @@ export type NextApiResponse = ServerResponse & { ) => NextApiResponse clearPreviewData: () => NextApiResponse - unstable_revalidate: (urlPath: string) => Promise + unstable_revalidate: (urlPath: string) => Promise } /** diff --git a/test/e2e/prerender.test.ts b/test/e2e/prerender.test.ts index db468d359955e..e5cfedfc0640b 100644 --- a/test/e2e/prerender.test.ts +++ b/test/e2e/prerender.test.ts @@ -1921,10 +1921,7 @@ describe('Prerender', () => { expect(res.status).toBe(200) const revalidateData = await res.json() - const revalidatedText = revalidateData.text - const $3 = cheerio.load(revalidatedText) - expect(revalidateData.status).toBe(200) - expect($3('#time').text()).not.toBe(initialTime) + expect(revalidateData.revalidated).toBe(true) const html4 = await renderViaHTTP( next.url, @@ -1932,7 +1929,6 @@ describe('Prerender', () => { ) const $4 = cheerio.load(html4) expect($4('#time').text()).not.toBe(initialTime) - expect($3('#time').text()).toBe($4('#time').text()) }) it('should not manual revalidate for revalidate: false', async () => { @@ -1964,10 +1960,7 @@ describe('Prerender', () => { expect(res.status).toBe(200) const revalidateData = await res.json() - const revalidatedText = revalidateData.text - const $3 = cheerio.load(revalidatedText) - expect(revalidateData.status).toBe(200) - expect($3('#time').text()).toBe(initialTime) + expect(revalidateData.revalidated).toBe(true) const html4 = await renderViaHTTP( next.url, @@ -1975,7 +1968,6 @@ describe('Prerender', () => { ) const $4 = cheerio.load(html4) expect($4('#time').text()).toBe(initialTime) - expect($3('#time').text()).toBe($4('#time').text()) }) it('should handle manual revalidate for fallback: false', async () => { @@ -1998,7 +1990,7 @@ describe('Prerender', () => { expect(res2.status).toBe(200) const revalidateData = await res2.json() - expect(revalidateData.status).toBe(404) + expect(revalidateData.revalidated).toBe(false) const res3 = await fetchViaHTTP( next.url, @@ -2021,7 +2013,7 @@ describe('Prerender', () => { { redirect: 'manual' } ) expect(res5.status).toBe(200) - expect((await res5.json()).status).toBe(200) + expect((await res5.json()).revalidated).toBe(true) const res6 = await fetchViaHTTP(next.url, '/catchall-explicit/first') expect(res6.status).toBe(200) diff --git a/test/e2e/prerender/pages/api/manual-revalidate.js b/test/e2e/prerender/pages/api/manual-revalidate.js index c1d9479c3292e..dc88de2b84973 100644 --- a/test/e2e/prerender/pages/api/manual-revalidate.js +++ b/test/e2e/prerender/pages/api/manual-revalidate.js @@ -1,10 +1,14 @@ export default async function handler(req, res) { // WARNING: don't use user input in production // make sure to use trusted value for revalidating - const revalidateRes = await res.unstable_revalidate(req.query.pathname) + let revalidated = false + try { + await res.unstable_revalidate(req.query.pathname) + revalidated = true + } catch (err) { + console.error(err) + } res.json({ - revalidated: true, - status: revalidateRes.status, - text: await revalidateRes.text(), + revalidated, }) } From cfb06f1e79ff38d23480adbe198d1fce4d7733ce Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 8 Feb 2022 10:31:11 -0600 Subject: [PATCH 10/14] apply suggestions --- packages/next/server/image-optimizer.ts | 29 ++++++++++++++++--------- packages/next/server/next-server.ts | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/packages/next/server/image-optimizer.ts b/packages/next/server/image-optimizer.ts index b473555ce2848..217bfd0b43efd 100644 --- a/packages/next/server/image-optimizer.ts +++ b/packages/next/server/image-optimizer.ts @@ -163,7 +163,7 @@ export class ImageOptimizerCache { width, quality, mimeType, - minimumCacheTTL: minimumCacheTTL, + minimumCacheTTL, } } @@ -249,7 +249,7 @@ export class ImageOptimizerCache { export class ImageError extends Error { statusCode: number - constructor(message: string, statusCode: number) { + constructor(statusCode: number, message: string) { super(message) this.statusCode = statusCode } @@ -281,8 +281,8 @@ export async function imageOptimizer( upstreamRes.status ) throw new ImageError( - '"url" parameter is valid but upstream response is invalid', - upstreamRes.status + upstreamRes.status, + '"url" parameter is valid but upstream response is invalid' ) } @@ -299,7 +299,7 @@ export async function imageOptimizer( const isStreamFinished = new Promise(function (resolve, reject) { mockRes.on('finish', () => resolve(true)) mockRes.on('end', () => resolve(true)) - mockRes.on('error', () => reject()) + mockRes.on('error', (err: any) => reject(err)) }) mockRes.write = (chunk: Buffer | string) => { @@ -350,6 +350,15 @@ export async function imageOptimizer( await handleRequest(mockReq, mockRes, nodeUrl.parse(href, true)) await isStreamFinished + + if (!mockRes.statusCode) { + console.error('image response failed for', href, mockRes.statusCode) + throw new ImageError( + mockRes.statusCode, + '"url" parameter is valid but internal response is invalid' + ) + } + upstreamBuffer = Buffer.concat(resBuffers) upstreamType = detectContentType(upstreamBuffer) || mockRes.getHeader('Content-Type') @@ -357,8 +366,8 @@ export async function imageOptimizer( } catch (err) { console.error('upstream image response failed for', href, err) throw new ImageError( - '"url" parameter is valid but upstream response is invalid', - 500 + 500, + '"url" parameter is valid but upstream response is invalid' ) } } @@ -378,7 +387,7 @@ export async function imageOptimizer( 'received', upstreamType ) - throw new ImageError("The requested resource isn't a valid image.", 400) + throw new ImageError(400, "The requested resource isn't a valid image.") } } @@ -440,7 +449,7 @@ export async function imageOptimizer( console.error( `Error: 'sharp' is required to be installed in standalone mode for the image optimization to function correctly` ) - throw new ImageError('internal server error', 500) + throw new ImageError(500, 'internal server error') } // Show sharp warning in production once if (showSharpMissingWarning) { @@ -510,7 +519,7 @@ export async function imageOptimizer( maxAge: Math.max(maxAge, nextConfig.images.minimumCacheTTL), } } else { - throw new ImageError('Unable to optimize buffer', 500) + throw new ImageError(500, 'Unable to optimize buffer') } } catch (error) { return { diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts index df92da8865c83..052deb3331c07 100644 --- a/packages/next/server/next-server.ts +++ b/packages/next/server/next-server.ts @@ -192,7 +192,7 @@ export default class NextNodeServer extends BaseServer { return { finished: true } } const paramsResult = ImageOptimizerCache.validateParams( - req as any, + (req as NodeNextRequest).originalRequest, parsedUrl.query, this.nextConfig, !!this.renderOpts.dev From 494fe2ef7e5f93e1e5afe1e656c4e05d8554e0ee Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 8 Feb 2022 10:59:08 -0600 Subject: [PATCH 11/14] update types --- packages/next/server/image-optimizer.ts | 18 ++++++++------ packages/next/server/incremental-cache.ts | 23 +----------------- packages/next/server/response-cache.ts | 29 +++++++++++++++++++++-- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/packages/next/server/image-optimizer.ts b/packages/next/server/image-optimizer.ts index 217bfd0b43efd..2d5c73b6dc417 100644 --- a/packages/next/server/image-optimizer.ts +++ b/packages/next/server/image-optimizer.ts @@ -16,6 +16,7 @@ import { sendEtagResponse } from './send-payload' import { getContentType, getExtension } from './serve-static' import chalk from 'next/dist/compiled/chalk' import { NextUrlWithParsedQuery } from './request-meta' +import { IncrementalCacheEntry, IncrementalCacheValue } from './response-cache' type XCacheHeader = 'MISS' | 'HIT' | 'STALE' @@ -192,7 +193,7 @@ export class ImageOptimizerCache { this.nextConfig = nextConfig } - async get(cacheKey: string) { + async get(cacheKey: string): Promise { try { const cacheDir = join(this.cacheDir, cacheKey) const files = await promises.readdir(cacheDir) @@ -211,7 +212,10 @@ export class ImageOptimizerCache { buffer, extension, }, - revalidate: maxAge, + revalidateAfter: + Math.max(maxAge, this.nextConfig.images.minimumCacheTTL) * 1000 + + Date.now(), + curRevalidate: maxAge, isStale: now > expireAt, } } @@ -222,13 +226,13 @@ export class ImageOptimizerCache { } async set( cacheKey: string, - value: { - etag: string - buffer: Buffer - extension: string - }, + value: IncrementalCacheValue | null, revalidate?: number | false ) { + if (value?.kind !== 'IMAGE') { + throw new Error('invariant attempted to set non-image to image-cache') + } + if (typeof revalidate !== 'number') { throw new Error('invariant revalidate must be a number for image-cache') } diff --git a/packages/next/server/incremental-cache.ts b/packages/next/server/incremental-cache.ts index 5eb13f41e0d12..7b4b1a8ad6e98 100644 --- a/packages/next/server/incremental-cache.ts +++ b/packages/next/server/incremental-cache.ts @@ -4,33 +4,12 @@ import LRUCache from 'next/dist/compiled/lru-cache' import path from 'path' import { PrerenderManifest } from '../build' import { normalizePagePath } from './normalize-page-path' -import { CachedImageValue, CachedRedirectValue } from './response-cache' +import { IncrementalCacheValue, IncrementalCacheEntry } from './response-cache' function toRoute(pathname: string): string { return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/' } -interface CachedPageValue { - kind: 'PAGE' - // this needs to be a string since the cache expects to store - // the string value - html: string - pageData: Object -} - -export type IncrementalCacheValue = - | CachedRedirectValue - | CachedPageValue - | CachedImageValue - -type IncrementalCacheEntry = { - curRevalidate?: number | false - // milliseconds to revalidate after - revalidateAfter: number | false - isStale?: boolean - value: IncrementalCacheValue | null -} - export class IncrementalCache { incrementalOptions: { flushToDisk?: boolean diff --git a/packages/next/server/response-cache.ts b/packages/next/server/response-cache.ts index 6044722a285f8..66be76be07e0b 100644 --- a/packages/next/server/response-cache.ts +++ b/packages/next/server/response-cache.ts @@ -22,6 +22,27 @@ export interface CachedImageValue { isStale?: boolean } +interface IncrementalCachedPageValue { + kind: 'PAGE' + // this needs to be a string since the cache expects to store + // the string value + html: string + pageData: Object +} + +export type IncrementalCacheEntry = { + curRevalidate?: number | false + // milliseconds to revalidate after + revalidateAfter: number | false + isStale?: boolean + value: IncrementalCacheValue | null +} + +export type IncrementalCacheValue = + | CachedRedirectValue + | IncrementalCachedPageValue + | CachedImageValue + export type ResponseCacheValue = | CachedRedirectValue | CachedPageValue @@ -44,11 +65,15 @@ interface IncrementalCache { revalidateAfter?: number | false curRevalidate?: number | false revalidate?: number | false - value?: any | null + value: IncrementalCacheValue | null isStale?: boolean isMiss?: boolean } | null> - set: (key: string, data: any, revalidate?: number | false) => Promise + set: ( + key: string, + data: IncrementalCacheValue | null, + revalidate?: number | false + ) => Promise } export default class ResponseCache { From 91d963429215d63628810d2af08a9d6c9d54c5af Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 8 Feb 2022 11:28:35 -0600 Subject: [PATCH 12/14] update test --- test/integration/image-optimizer/test/index.test.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/integration/image-optimizer/test/index.test.js b/test/integration/image-optimizer/test/index.test.js index 7d9030783cfbd..863043987b19f 100644 --- a/test/integration/image-optimizer/test/index.test.js +++ b/test/integration/image-optimizer/test/index.test.js @@ -526,10 +526,16 @@ function runTests({ expect(one.res.headers.get('Content-Disposition')).toBe( `inline; filename="slow.webp"` ) + const etagOne = one.res.headers.get('etag') + let json1 await check(async () => { json1 = await fsToJson(imagesDir) - return Object.keys(json1).length === 1 ? 'success' : 'fail' + return Object.keys(json1).some((dir) => { + return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) + }) + ? 'success' + : 'fail' }, 'success') const two = await fetchWithDuration(appPort, '/_next/image', query, opts) From 96290ddc4e12780350c8f08b3f199d69a08ad87f Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 8 Feb 2022 12:45:09 -0600 Subject: [PATCH 13/14] add logs and move cache cleaning --- .../image-optimizer/test/index.test.js | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/test/integration/image-optimizer/test/index.test.js b/test/integration/image-optimizer/test/index.test.js index 863043987b19f..a6e594bc49580 100644 --- a/test/integration/image-optimizer/test/index.test.js +++ b/test/integration/image-optimizer/test/index.test.js @@ -26,6 +26,11 @@ let nextOutput let appPort let app +const cleanImagesDir = async () => { + console.warn('Cleaning', imagesDir) + await fs.remove(imagesDir) +} + const sharpMissingText = `For production Image Optimization with Next.js, the optional 'sharp' package is strongly recommended` const sharpOutdatedText = `Your installed version of the 'sharp' package does not support AVIF images. Run 'yarn add sharp@latest' to upgrade to the latest version` @@ -85,6 +90,7 @@ async function expectAvifSmallerThanWebp(w, q) { } async function fetchWithDuration(...args) { + console.warn('Fetching', args[1]) const start = Date.now() const res = await fetchViaHTTP(...args) const buffer = await res.buffer() @@ -511,7 +517,7 @@ function runTests({ }) it('should use cache and stale-while-revalidate when query is the same for external image', async () => { - await fs.remove(imagesDir) + await cleanImagesDir() const delay = 500 const url = `https://image-optimization-test.vercel.app/api/slow?delay=${delay}` @@ -639,7 +645,7 @@ function runTests({ } it('should use cache and stale-while-revalidate when query is the same for internal image', async () => { - await fs.remove(imagesDir) + await cleanImagesDir() const query = { url: '/test.png', w, q: 80 } const opts = { headers: { accept: 'image/webp' } } @@ -728,7 +734,7 @@ function runTests({ }) it('should use cached image file when parameters are the same for svg', async () => { - await fs.remove(imagesDir) + await cleanImagesDir() const query = { url: '/test.svg', w, q: 80 } const opts = { headers: { accept: 'image/webp' } } @@ -764,7 +770,7 @@ function runTests({ }) it('should use cached image file when parameters are the same for animated gif', async () => { - await fs.remove(imagesDir) + await cleanImagesDir() const query = { url: '/animated.gif', w, q: 80 } const opts = { headers: { accept: 'image/webp' } } @@ -967,7 +973,7 @@ function runTests({ }) it('should handle concurrent requests', async () => { - await fs.remove(imagesDir) + await cleanImagesDir() const query = { url: '/test.png', w, q: 80 } const opts = { headers: { accept: 'image/webp,*/*' } } const [res1, res2, res3] = await Promise.all([ @@ -1262,6 +1268,7 @@ describe('Image Optimizer', () => { nextOutput = '' nextConfig.replace('{ /* replaceme */ }', json) await nextBuild(appDir) + await cleanImagesDir() appPort = await findPort() app = await nextStart(appDir, appPort, { onStderr(msg) { @@ -1272,7 +1279,6 @@ describe('Image Optimizer', () => { afterAll(async () => { await killApp(app) nextConfig.restore() - await fs.remove(imagesDir) }) runTests({ w: size, isDev: false, domains, minimumCacheTTL }) @@ -1300,13 +1306,13 @@ describe('Image Optimizer', () => { }` ) await nextBuild(appDir) + await cleanImagesDir() appPort = await findPort() app = await nextStart(appDir, appPort) }) afterAll(async () => { await killApp(app) nextConfig.restore() - await fs.remove(imagesDir) }) it('should set max-age header', async () => { @@ -1361,13 +1367,13 @@ describe('Image Optimizer', () => { }, }) nextConfig.replace('{ /* replaceme */ }', json) + await cleanImagesDir() appPort = await findPort() app = await launchApp(appDir, appPort) }) afterAll(async () => { await killApp(app) nextConfig.restore() - await fs.remove(imagesDir) }) it('should 404 when loader is not default', async () => { const size = 384 // defaults defined in server/config.ts @@ -1392,17 +1398,17 @@ describe('Image Optimizer', () => { }` nextConfig.replace('{ /* replaceme */ }', newConfig) await nextBuild(appDir) + await cleanImagesDir() appPort = await findPort() app = await nextStart(appDir, appPort) }) afterAll(async () => { await killApp(app) nextConfig.restore() - await fs.remove(imagesDir) }) it('should return response when image is served from an external rewrite', async () => { - await fs.remove(imagesDir) + await cleanImagesDir() const query = { url: '/next-js/next-js-bg.png', w: 64, q: 75 } const opts = { headers: { accept: 'image/webp' } } @@ -1445,13 +1451,13 @@ describe('Image Optimizer', () => { }, }) nextConfig.replace('{ /* replaceme */ }', json) + await cleanImagesDir() appPort = await findPort() app = await launchApp(appDir, appPort) }) afterAll(async () => { await killApp(app) nextConfig.restore() - await fs.remove(imagesDir) }) it('should support width 8 per BLUR_IMG_SIZE with next dev', async () => { @@ -1480,10 +1486,10 @@ describe('Image Optimizer', () => { }, cwd: appDir, }) + await cleanImagesDir() }) afterAll(async () => { await killApp(app) - await fs.remove(imagesDir) }) runTests({ @@ -1509,6 +1515,7 @@ describe('Image Optimizer', () => { }) nextOutput = '' nextConfig.replace('{ /* replaceme */ }', json) + await cleanImagesDir() appPort = await findPort() app = await launchApp(appDir, appPort, { onStderr(msg) { @@ -1525,7 +1532,6 @@ describe('Image Optimizer', () => { afterAll(async () => { await killApp(app) nextConfig.restore() - await fs.remove(imagesDir) }) runTests({ @@ -1543,6 +1549,7 @@ describe('Image Optimizer', () => { beforeAll(async () => { nextOutput = '' await nextBuild(appDir) + await cleanImagesDir() appPort = await findPort() app = await nextStart(appDir, appPort, { onStderr(msg) { @@ -1558,7 +1565,6 @@ describe('Image Optimizer', () => { }) afterAll(async () => { await killApp(app) - await fs.remove(imagesDir) }) runTests({ w: size, isDev: false, domains: [], isSharp, isOutdatedSharp }) @@ -1577,6 +1583,7 @@ describe('Image Optimizer', () => { nextOutput = '' nextConfig.replace('{ /* replaceme */ }', json) await nextBuild(appDir) + await cleanImagesDir() appPort = await findPort() app = await nextStart(appDir, appPort, { onStderr(msg) { @@ -1593,7 +1600,6 @@ describe('Image Optimizer', () => { afterAll(async () => { await killApp(app) nextConfig.restore() - await fs.remove(imagesDir) }) runTests({ From 4cd09ca257d082944815210a642cdeff3f342327 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 8 Feb 2022 15:19:33 -0600 Subject: [PATCH 14/14] refactor test suite --- .../image-optimizer/test/index.test.js | 1255 +--------------- .../image-optimizer/test/old-sharp.test.js | 27 + .../image-optimizer/test/sharp.test.js | 27 + .../image-optimizer/test/squoosh.test.js | 9 + test/integration/image-optimizer/test/util.js | 1259 +++++++++++++++++ 5 files changed, 1356 insertions(+), 1221 deletions(-) create mode 100644 test/integration/image-optimizer/test/old-sharp.test.js create mode 100644 test/integration/image-optimizer/test/sharp.test.js create mode 100644 test/integration/image-optimizer/test/squoosh.test.js create mode 100644 test/integration/image-optimizer/test/util.js diff --git a/test/integration/image-optimizer/test/index.test.js b/test/integration/image-optimizer/test/index.test.js index a6e594bc49580..52797c7101667 100644 --- a/test/integration/image-optimizer/test/index.test.js +++ b/test/integration/image-optimizer/test/index.test.js @@ -1,7 +1,4 @@ /* eslint-env jest */ -import execa from 'execa' -import fs from 'fs-extra' -import sizeOf from 'image-size' import { check, fetchViaHTTP, @@ -14,1033 +11,18 @@ import { renderViaHTTP, waitFor, } from 'next-test-utils' -import isAnimated from 'next/dist/compiled/is-animated' import { join } from 'path' -import assert from 'assert' +import { cleanImagesDir, expectWidth, fsToJson, runTests } from './util' const appDir = join(__dirname, '../app') const imagesDir = join(appDir, '.next', 'cache', 'images') const nextConfig = new File(join(appDir, 'next.config.js')) const largeSize = 1080 // defaults defined in server/config.ts -let nextOutput -let appPort -let app - -const cleanImagesDir = async () => { - console.warn('Cleaning', imagesDir) - await fs.remove(imagesDir) -} - -const sharpMissingText = `For production Image Optimization with Next.js, the optional 'sharp' package is strongly recommended` -const sharpOutdatedText = `Your installed version of the 'sharp' package does not support AVIF images. Run 'yarn add sharp@latest' to upgrade to the latest version` - -async function fsToJson(dir, output = {}) { - const files = await fs.readdir(dir) - for (let file of files) { - const fsPath = join(dir, file) - const stat = await fs.stat(fsPath) - if (stat.isDirectory()) { - output[file] = {} - await fsToJson(fsPath, output[file]) - } else { - output[file] = stat.mtime.toISOString() - } - } - return output -} - -async function expectWidth(res, w) { - const buffer = await res.buffer() - const d = sizeOf(buffer) - expect(d.width).toBe(w) -} - -async function expectAvifSmallerThanWebp(w, q) { - const query = { url: '/mountains.jpg', w, q } - const res1 = await fetchViaHTTP(appPort, '/_next/image', query, { - headers: { - accept: 'image/avif', - }, - }) - expect(res1.status).toBe(200) - expect(res1.headers.get('Content-Type')).toBe('image/avif') - - const res2 = await fetchViaHTTP(appPort, '/_next/image', query, { - headers: { - accept: 'image/webp', - }, - }) - expect(res2.status).toBe(200) - expect(res2.headers.get('Content-Type')).toBe('image/webp') - - const res3 = await fetchViaHTTP(appPort, '/_next/image', query, { - headers: { - accept: 'image/jpeg', - }, - }) - expect(res3.status).toBe(200) - expect(res3.headers.get('Content-Type')).toBe('image/jpeg') - - const avif = (await res1.buffer()).byteLength - const webp = (await res2.buffer()).byteLength - const jpeg = (await res3.buffer()).byteLength - - expect(webp).toBeLessThan(jpeg) - expect(avif).toBeLessThanOrEqual(webp) -} - -async function fetchWithDuration(...args) { - console.warn('Fetching', args[1]) - const start = Date.now() - const res = await fetchViaHTTP(...args) - const buffer = await res.buffer() - const duration = Date.now() - start - return { duration, buffer, res } -} - -function runTests({ - w, - isDev, - domains = [], - minimumCacheTTL, - isSharp, - isOutdatedSharp, - avifEnabled, -}) { - it('should return home page', async () => { - const res = await fetchViaHTTP(appPort, '/', null, {}) - expect(await res.text()).toMatch(/Image Optimizer Home/m) - }) - - it('should handle non-ascii characters in image url', async () => { - const query = { w, q: 90, url: '/äöüščří.png' } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(200) - }) - - it('should maintain animated gif', async () => { - const query = { w, q: 90, url: '/animated.gif' } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(200) - expect(res.headers.get('content-type')).toContain('image/gif') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="animated.gif"` - ) - expect(isAnimated(await res.buffer())).toBe(true) - }) - - it('should maintain animated png', async () => { - const query = { w, q: 90, url: '/animated.png' } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(200) - expect(res.headers.get('content-type')).toContain('image/png') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="animated.png"` - ) - expect(isAnimated(await res.buffer())).toBe(true) - }) - - it('should maintain animated webp', async () => { - const query = { w, q: 90, url: '/animated.webp' } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(200) - expect(res.headers.get('content-type')).toContain('image/webp') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="animated.webp"` - ) - expect(isAnimated(await res.buffer())).toBe(true) - }) - - it('should maintain vector svg', async () => { - const query = { w, q: 90, url: '/test.svg' } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toContain('image/svg+xml') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - // SVG is compressible so will have accept-encoding set from - // compression - expect(res.headers.get('Vary')).toMatch(/^Accept(,|$)/) - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.svg"` - ) - const actual = await res.text() - const expected = await fs.readFile( - join(appDir, 'public', 'test.svg'), - 'utf8' - ) - expect(actual).toMatch(expected) - }) - - it('should maintain ico format', async () => { - const query = { w, q: 90, url: `/test.ico` } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toContain('image/x-icon') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toMatch(/^Accept(,|$)/) - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.ico"` - ) - const actual = await res.text() - const expected = await fs.readFile( - join(appDir, 'public', 'test.ico'), - 'utf8' - ) - expect(actual).toMatch(expected) - }) - - it('should maintain jpg format for old Safari', async () => { - const accept = - 'image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5' - const query = { w, q: 90, url: '/test.jpg' } - const opts = { headers: { accept } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toContain('image/jpeg') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.jpeg"` - ) - }) - - it('should maintain png format for old Safari', async () => { - const accept = - 'image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5' - const query = { w, q: 75, url: '/test.png' } - const opts = { headers: { accept } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toContain('image/png') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.png"` - ) - }) - - it('should fail when url is missing', async () => { - const query = { w, q: 100 } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(400) - expect(await res.text()).toBe(`"url" parameter is required`) - }) - - it('should fail when w is missing', async () => { - const query = { url: '/test.png', q: 100 } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(400) - expect(await res.text()).toBe(`"w" parameter (width) is required`) - }) - - it('should fail when q is missing', async () => { - const query = { url: '/test.png', w } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(400) - expect(await res.text()).toBe(`"q" parameter (quality) is required`) - }) - - it('should fail when q is greater than 100', async () => { - const query = { url: '/test.png', w, q: 101 } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(400) - expect(await res.text()).toBe( - `"q" parameter (quality) must be a number between 1 and 100` - ) - }) - - it('should fail when q is less than 1', async () => { - const query = { url: '/test.png', w, q: 0 } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(400) - expect(await res.text()).toBe( - `"q" parameter (quality) must be a number between 1 and 100` - ) - }) - - it('should fail when w is 0 or less', async () => { - const query = { url: '/test.png', w: 0, q: 100 } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(400) - expect(await res.text()).toBe( - `"w" parameter (width) must be a number greater than 0` - ) - }) - - it('should fail when w is not a number', async () => { - const query = { url: '/test.png', w: 'foo', q: 100 } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(400) - expect(await res.text()).toBe( - `"w" parameter (width) must be a number greater than 0` - ) - }) - - it('should fail when q is not a number', async () => { - const query = { url: '/test.png', w, q: 'foo' } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(400) - expect(await res.text()).toBe( - `"q" parameter (quality) must be a number between 1 and 100` - ) - }) - - it('should fail when domain is not defined in next.config.js', async () => { - const url = `http://vercel.com/button` - const query = { url, w, q: 100 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(400) - expect(await res.text()).toBe(`"url" parameter is not allowed`) - }) - - it('should fail when width is not in next.config.js', async () => { - const query = { url: '/test.png', w: 1000, q: 100 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(400) - expect(await res.text()).toBe( - `"w" parameter (width) of 1000 is not allowed` - ) - }) - - it('should resize relative url and webp Firefox accept header', async () => { - const query = { url: '/test.png', w, q: 80 } - const opts = { headers: { accept: 'image/webp,*/*' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/webp') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - await expectWidth(res, w) - }) - - it('should resize relative url and png accept header', async () => { - const query = { url: '/test.png', w, q: 80 } - const opts = { headers: { accept: 'image/png' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/png') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.png"` - ) - await expectWidth(res, w) - }) - - it('should resize relative url with invalid accept header as png', async () => { - const query = { url: '/test.png', w, q: 80 } - const opts = { headers: { accept: 'image/invalid' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/png') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.png"` - ) - await expectWidth(res, w) - }) - - it('should resize relative url with invalid accept header as gif', async () => { - const query = { url: '/test.gif', w, q: 80 } - const opts = { headers: { accept: 'image/invalid' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/gif') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.gif"` - ) - // FIXME: await expectWidth(res, w) - }) - - it('should resize relative url with invalid accept header as tiff', async () => { - const query = { url: '/test.tiff', w, q: 80 } - const opts = { headers: { accept: 'image/invalid' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/tiff') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.tiff"` - ) - // FIXME: await expectWidth(res, w) - }) - - it('should resize relative url and old Chrome accept header as webp', async () => { - const query = { url: '/test.png', w, q: 80 } - const opts = { - headers: { accept: 'image/webp,image/apng,image/*,*/*;q=0.8' }, - } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/webp') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - await expectWidth(res, w) - }) - - if (avifEnabled) { - it('should resize relative url and new Chrome accept header as avif', async () => { - const query = { url: '/test.png', w, q: 80 } - const opts = { - headers: { - accept: 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8', - }, - } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/avif') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.avif"` - ) - // TODO: upgrade "image-size" package to support AVIF - // See https://github.com/image-size/image-size/issues/348 - //await expectWidth(res, w) - }) - - it('should compress avif smaller than webp at q=100', async () => { - await expectAvifSmallerThanWebp(w, 100) - }) - - it('should compress avif smaller than webp at q=75', async () => { - await expectAvifSmallerThanWebp(w, 75) - }) - - it('should compress avif smaller than webp at q=50', async () => { - await expectAvifSmallerThanWebp(w, 50) - }) - } - - if (domains.includes('localhost')) { - it('should resize absolute url from localhost', async () => { - const url = `http://localhost:${appPort}/test.png` - const query = { url, w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/webp') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - await expectWidth(res, w) - }) - - it('should automatically detect image type when content-type is octet-stream', async () => { - const url = '/png-as-octet-stream' - const resOrig = await fetchViaHTTP(appPort, url) - expect(resOrig.status).toBe(200) - expect(resOrig.headers.get('Content-Type')).toBe( - 'application/octet-stream' - ) - const query = { url, w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/webp') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="png-as-octet-stream.webp"` - ) - await expectWidth(res, w) - }) - - it('should use cache and stale-while-revalidate when query is the same for external image', async () => { - await cleanImagesDir() - const delay = 500 - - const url = `https://image-optimization-test.vercel.app/api/slow?delay=${delay}` - const query = { url, w, q: 39 } - const opts = { headers: { accept: 'image/webp' } } - - const one = await fetchWithDuration(appPort, '/_next/image', query, opts) - expect(one.duration).toBeGreaterThan(delay) - expect(one.res.status).toBe(200) - expect(one.res.headers.get('X-Nextjs-Cache')).toBe('MISS') - expect(one.res.headers.get('Content-Type')).toBe('image/webp') - expect(one.res.headers.get('Content-Disposition')).toBe( - `inline; filename="slow.webp"` - ) - const etagOne = one.res.headers.get('etag') - - let json1 - await check(async () => { - json1 = await fsToJson(imagesDir) - return Object.keys(json1).some((dir) => { - return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) - }) - ? 'success' - : 'fail' - }, 'success') - - const two = await fetchWithDuration(appPort, '/_next/image', query, opts) - expect(two.res.status).toBe(200) - expect(two.res.headers.get('X-Nextjs-Cache')).toBe('HIT') - expect(two.res.headers.get('Content-Type')).toBe('image/webp') - expect(two.res.headers.get('Content-Disposition')).toBe( - `inline; filename="slow.webp"` - ) - const json2 = await fsToJson(imagesDir) - expect(json2).toStrictEqual(json1) - - if (minimumCacheTTL) { - // Wait until expired so we can confirm image is regenerated - await waitFor(minimumCacheTTL * 1000) - - const [three, four] = await Promise.all([ - fetchWithDuration(appPort, '/_next/image', query, opts), - fetchWithDuration(appPort, '/_next/image', query, opts), - ]) - - expect(three.duration).toBeLessThan(one.duration) - expect(three.res.status).toBe(200) - expect(three.res.headers.get('X-Nextjs-Cache')).toBe('STALE') - expect(three.res.headers.get('Content-Type')).toBe('image/webp') - expect(three.res.headers.get('Content-Disposition')).toBe( - `inline; filename="slow.webp"` - ) - - expect(four.duration).toBeLessThan(one.duration) - expect(four.res.status).toBe(200) - expect(four.res.headers.get('X-Nextjs-Cache')).toBe('STALE') - expect(four.res.headers.get('Content-Type')).toBe('image/webp') - expect(four.res.headers.get('Content-Disposition')).toBe( - `inline; filename="slow.webp"` - ) - await check(async () => { - const json4 = await fsToJson(imagesDir) - try { - assert.deepStrictEqual(json4, json1) - return 'fail' - } catch (err) { - return 'success' - } - }, 'success') - - const five = await fetchWithDuration( - appPort, - '/_next/image', - query, - opts - ) - expect(five.duration).toBeLessThan(one.duration) - expect(five.res.status).toBe(200) - expect(five.res.headers.get('X-Nextjs-Cache')).toBe('HIT') - expect(five.res.headers.get('Content-Type')).toBe('image/webp') - expect(five.res.headers.get('Content-Disposition')).toBe( - `inline; filename="slow.webp"` - ) - await check(async () => { - const json5 = await fsToJson(imagesDir) - try { - assert.deepStrictEqual(json5, json1) - return 'fail' - } catch (err) { - return 'success' - } - }, 'success') - } - }) - } - - it('should fail when url has file protocol', async () => { - const url = `file://localhost:${appPort}/test.png` - const query = { url, w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(400) - expect(await res.text()).toBe(`"url" parameter is invalid`) - }) - - it('should fail when url has ftp protocol', async () => { - const url = `ftp://localhost:${appPort}/test.png` - const query = { url, w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(400) - expect(await res.text()).toBe(`"url" parameter is invalid`) - }) - - if (domains.includes('localhost')) { - it('should fail when url fails to load an image', async () => { - const url = `http://localhost:${appPort}/not-an-image` - const query = { w, url, q: 100 } - const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) - expect(res.status).toBe(404) - expect(await res.text()).toBe( - `"url" parameter is valid but upstream response is invalid` - ) - }) - } - - it('should use cache and stale-while-revalidate when query is the same for internal image', async () => { - await cleanImagesDir() - - const query = { url: '/test.png', w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - - const one = await fetchWithDuration(appPort, '/_next/image', query, opts) - expect(one.res.status).toBe(200) - expect(one.res.headers.get('X-Nextjs-Cache')).toBe('MISS') - expect(one.res.headers.get('Content-Type')).toBe('image/webp') - expect(one.res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - const etagOne = one.res.headers.get('etag') - - let json1 - await check(async () => { - json1 = await fsToJson(imagesDir) - return Object.keys(json1).some((dir) => { - return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) - }) - ? 'success' - : 'fail' - }, 'success') - - const two = await fetchWithDuration(appPort, '/_next/image', query, opts) - expect(two.res.status).toBe(200) - expect(two.res.headers.get('X-Nextjs-Cache')).toBe('HIT') - expect(two.res.headers.get('Content-Type')).toBe('image/webp') - expect(two.res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - const json2 = await fsToJson(imagesDir) - expect(json2).toStrictEqual(json1) - - if (minimumCacheTTL) { - // Wait until expired so we can confirm image is regenerated - await waitFor(minimumCacheTTL * 1000) - - const [three, four] = await Promise.all([ - fetchWithDuration(appPort, '/_next/image', query, opts), - fetchWithDuration(appPort, '/_next/image', query, opts), - ]) - - expect(three.duration).toBeLessThan(one.duration) - expect(three.res.status).toBe(200) - expect(three.res.headers.get('X-Nextjs-Cache')).toBe('STALE') - expect(three.res.headers.get('Content-Type')).toBe('image/webp') - expect(three.res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - - expect(four.duration).toBeLessThan(one.duration) - expect(four.res.status).toBe(200) - expect(four.res.headers.get('X-Nextjs-Cache')).toBe('STALE') - expect(four.res.headers.get('Content-Type')).toBe('image/webp') - expect(four.res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - await check(async () => { - const json3 = await fsToJson(imagesDir) - try { - assert.deepStrictEqual(json3, json1) - return 'fail' - } catch (err) { - return 'success' - } - }, 'success') - - const five = await fetchWithDuration(appPort, '/_next/image', query, opts) - expect(five.duration).toBeLessThan(one.duration) - expect(five.res.status).toBe(200) - expect(five.res.headers.get('X-Nextjs-Cache')).toBe('HIT') - expect(five.res.headers.get('Content-Type')).toBe('image/webp') - expect(five.res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - await check(async () => { - const json5 = await fsToJson(imagesDir) - try { - assert.deepStrictEqual(json5, json1) - return 'fail' - } catch (err) { - return 'success' - } - }, 'success') - } - }) - - it('should use cached image file when parameters are the same for svg', async () => { - await cleanImagesDir() - - const query = { url: '/test.svg', w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - - const res1 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res1.status).toBe(200) - expect(res1.headers.get('X-Nextjs-Cache')).toBe('MISS') - expect(res1.headers.get('Content-Type')).toBe('image/svg+xml') - expect(res1.headers.get('Content-Disposition')).toBe( - `inline; filename="test.svg"` - ) - const etagOne = res1.headers.get('etag') - - let json1 - await check(async () => { - json1 = await fsToJson(imagesDir) - return Object.keys(json1).some((dir) => { - return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) - }) - ? 'success' - : 'fail' - }, 'success') - - const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res2.status).toBe(200) - expect(res2.headers.get('X-Nextjs-Cache')).toBe('HIT') - expect(res2.headers.get('Content-Type')).toBe('image/svg+xml') - expect(res2.headers.get('Content-Disposition')).toBe( - `inline; filename="test.svg"` - ) - const json2 = await fsToJson(imagesDir) - expect(json2).toStrictEqual(json1) - }) - - it('should use cached image file when parameters are the same for animated gif', async () => { - await cleanImagesDir() - - const query = { url: '/animated.gif', w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - - const res1 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res1.status).toBe(200) - expect(res1.headers.get('X-Nextjs-Cache')).toBe('MISS') - expect(res1.headers.get('Content-Type')).toBe('image/gif') - expect(res1.headers.get('Content-Disposition')).toBe( - `inline; filename="animated.gif"` - ) - - let json1 - await check(async () => { - json1 = await fsToJson(imagesDir) - return Object.keys(json1).length === 1 ? 'success' : 'fail' - }, 'success') - - const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res2.status).toBe(200) - expect(res2.headers.get('X-Nextjs-Cache')).toBe('HIT') - expect(res2.headers.get('Content-Type')).toBe('image/gif') - expect(res2.headers.get('Content-Disposition')).toBe( - `inline; filename="animated.gif"` - ) - const json2 = await fsToJson(imagesDir) - expect(json2).toStrictEqual(json1) - }) - - it('should set 304 status without body when etag matches if-none-match', async () => { - const query = { url: '/test.jpg', w, q: 80 } - const opts1 = { headers: { accept: 'image/webp' } } - - const res1 = await fetchViaHTTP(appPort, '/_next/image', query, opts1) - expect(res1.status).toBe(200) - expect(res1.headers.get('Content-Type')).toBe('image/webp') - expect(res1.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res1.headers.get('Vary')).toBe('Accept') - const etag = res1.headers.get('Etag') - expect(etag).toBeTruthy() - expect(res1.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - await expectWidth(res1, w) - - const opts2 = { headers: { accept: 'image/webp', 'if-none-match': etag } } - const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts2) - expect(res2.status).toBe(304) - expect(res2.headers.get('Content-Type')).toBeFalsy() - expect(res2.headers.get('Etag')).toBe(etag) - expect(res2.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res2.headers.get('Vary')).toBe('Accept') - expect(res2.headers.get('Content-Disposition')).toBeFalsy() - expect((await res2.buffer()).length).toBe(0) - - const query3 = { url: '/test.jpg', w, q: 25 } - const res3 = await fetchViaHTTP(appPort, '/_next/image', query3, opts2) - expect(res3.status).toBe(200) - expect(res3.headers.get('Content-Type')).toBe('image/webp') - expect(res3.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res3.headers.get('Vary')).toBe('Accept') - expect(res3.headers.get('Etag')).toBeTruthy() - expect(res3.headers.get('Etag')).not.toBe(etag) - expect(res3.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - await expectWidth(res3, w) - }) - - it('should maintain bmp', async () => { - const json1 = await fsToJson(imagesDir) - expect(json1).toBeTruthy() - - const query = { url: '/test.bmp', w, q: 80 } - const opts = { headers: { accept: 'image/invalid' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/bmp') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - // bmp is compressible so will have accept-encoding set from - // compression - expect(res.headers.get('Vary')).toMatch(/^Accept(,|$)/) - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.bmp"` - ) - - await check(async () => { - try { - assert.deepStrictEqual(await fsToJson(imagesDir), json1) - return 'expected change, but matched' - } catch (_) { - return 'success' - } - }, 'success') - }) - - it('should not resize if requested width is larger than original source image', async () => { - const query = { url: '/test.jpg', w: largeSize, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/webp') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('etag')).toBeTruthy() - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - await expectWidth(res, 400) - }) - - if (!isSharp) { - // this checks for specific color type output by squoosh - // which differs in sharp - it('should not change the color type of a png', async () => { - // https://github.com/vercel/next.js/issues/22929 - // A grayscaled PNG with transparent pixels. - const query = { url: '/grayscale.png', w: largeSize, q: 80 } - const opts = { headers: { accept: 'image/png' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(200) - expect(res.headers.get('Content-Type')).toBe('image/png') - expect(res.headers.get('Cache-Control')).toBe( - `public, max-age=0, must-revalidate` - ) - expect(res.headers.get('Vary')).toBe('Accept') - expect(res.headers.get('Content-Disposition')).toBe( - `inline; filename="grayscale.png"` - ) - - const png = await res.buffer() - - // Read the color type byte (offset 9 + magic number 16). - // http://www.libpng.org/pub/png/spec/1.2/PNG-Chunks.html - const colorType = png.readUIntBE(25, 1) - expect(colorType).toBe(4) - }) - } - - it('should set cache-control to immutable for static images', async () => { - if (!isDev) { - const filename = 'test' - const query = { - url: `/_next/static/media/${filename}.fab2915d.jpg`, - w, - q: 100, - } - const opts = { headers: { accept: 'image/webp' } } - - const res1 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res1.status).toBe(200) - expect(res1.headers.get('Cache-Control')).toBe( - 'public, max-age=315360000, immutable' - ) - expect(res1.headers.get('Vary')).toBe('Accept') - expect(res1.headers.get('Content-Disposition')).toBe( - `inline; filename="${filename}.webp"` - ) - await expectWidth(res1, w) - - // Ensure subsequent request also has immutable header - const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res2.status).toBe(200) - expect(res2.headers.get('Cache-Control')).toBe( - 'public, max-age=315360000, immutable' - ) - expect(res2.headers.get('Vary')).toBe('Accept') - expect(res2.headers.get('Content-Disposition')).toBe( - `inline; filename="${filename}.webp"` - ) - await expectWidth(res2, w) - } - }) - - it("should error if the resource isn't a valid image", async () => { - const query = { url: '/test.txt', w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(400) - expect(await res.text()).toBe("The requested resource isn't a valid image.") - }) - - it('should error if the image file does not exist', async () => { - const query = { url: '/does_not_exist.jpg', w, q: 80 } - const opts = { headers: { accept: 'image/webp' } } - const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) - expect(res.status).toBe(400) - expect(await res.text()).toBe("The requested resource isn't a valid image.") - }) - - it('should handle concurrent requests', async () => { - await cleanImagesDir() - const query = { url: '/test.png', w, q: 80 } - const opts = { headers: { accept: 'image/webp,*/*' } } - const [res1, res2, res3] = await Promise.all([ - fetchViaHTTP(appPort, '/_next/image', query, opts), - fetchViaHTTP(appPort, '/_next/image', query, opts), - fetchViaHTTP(appPort, '/_next/image', query, opts), - ]) - - expect(res1.status).toBe(200) - expect(res2.status).toBe(200) - expect(res3.status).toBe(200) - - expect(res1.headers.get('Content-Type')).toBe('image/webp') - expect(res1.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - expect(res2.headers.get('Content-Type')).toBe('image/webp') - expect(res2.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - expect(res3.headers.get('Content-Type')).toBe('image/webp') - expect(res3.headers.get('Content-Disposition')).toBe( - `inline; filename="test.webp"` - ) - - await expectWidth(res1, w) - await expectWidth(res2, w) - await expectWidth(res3, w) - - await check(async () => { - const json1 = await fsToJson(imagesDir) - return Object.keys(json1).length === 1 ? 'success' : 'fail' - }, 'success') - - const xCache = [res1, res2, res3] - .map((r) => r.headers.get('X-Nextjs-Cache')) - .sort((a, b) => b.localeCompare(a)) - - // Since the first request is a miss it blocks - // until the cache be populated so all concurrent - // requests receive the same response - expect(xCache).toEqual(['MISS', 'MISS', 'MISS']) - }) - - if (isDev || isSharp) { - it('should not have sharp missing warning', () => { - expect(nextOutput).not.toContain(sharpMissingText) - }) - } else { - it('should have sharp missing warning', () => { - expect(nextOutput).toContain(sharpMissingText) - }) - } - - if (isSharp && isOutdatedSharp && avifEnabled) { - it('should have sharp outdated warning', () => { - expect(nextOutput).toContain(sharpOutdatedText) - }) - } else { - it('should not have sharp outdated warning', () => { - expect(nextOutput).not.toContain(sharpOutdatedText) - }) - } -} describe('Image Optimizer', () => { describe('config checks', () => { + let app + it('should error when domains length exceeds 50', async () => { await nextConfig.replace( '{ /* replaceme */ }', @@ -1258,6 +240,14 @@ describe('Image Optimizer', () => { describe('Server support for minimumCacheTTL in next.config.js', () => { const size = 96 // defaults defined in server/config.ts + const ctx = { + w: size, + isDev: false, + domains, + minimumCacheTTL, + imagesDir, + appDir, + } beforeAll(async () => { const json = JSON.stringify({ images: { @@ -1265,27 +255,30 @@ describe('Image Optimizer', () => { minimumCacheTTL, }, }) - nextOutput = '' + ctx.nextOutput = '' nextConfig.replace('{ /* replaceme */ }', json) await nextBuild(appDir) - await cleanImagesDir() - appPort = await findPort() - app = await nextStart(appDir, appPort, { + await cleanImagesDir({ imagesDir }) + ctx.appPort = await findPort() + ctx.app = await nextStart(appDir, ctx.appPort, { onStderr(msg) { - nextOutput += msg + ctx.nextOutput += msg }, }) }) afterAll(async () => { - await killApp(app) + await killApp(ctx.app) nextConfig.restore() }) - runTests({ w: size, isDev: false, domains, minimumCacheTTL }) + runTests(ctx) }) describe('Server support for headers in next.config.js', () => { const size = 96 // defaults defined in server/config.ts + let app + let appPort + beforeAll(async () => { nextConfig.replace( '{ /* replaceme */ }', @@ -1306,7 +299,7 @@ describe('Image Optimizer', () => { }` ) await nextBuild(appDir) - await cleanImagesDir() + await cleanImagesDir({ imagesDir }) appPort = await findPort() app = await nextStart(appDir, appPort) }) @@ -1359,6 +352,9 @@ describe('Image Optimizer', () => { }) describe('dev support next.config.js cloudinary loader', () => { + let app + let appPort + beforeAll(async () => { const json = JSON.stringify({ images: { @@ -1367,7 +363,7 @@ describe('Image Optimizer', () => { }, }) nextConfig.replace('{ /* replaceme */ }', json) - await cleanImagesDir() + await cleanImagesDir({ imagesDir }) appPort = await findPort() app = await launchApp(appDir, appPort) }) @@ -1385,6 +381,9 @@ describe('Image Optimizer', () => { }) describe('External rewrite support with for serving static content in images', () => { + let app + let appPort + beforeAll(async () => { const newConfig = `{ async rewrites() { @@ -1398,7 +397,7 @@ describe('Image Optimizer', () => { }` nextConfig.replace('{ /* replaceme */ }', newConfig) await nextBuild(appDir) - await cleanImagesDir() + await cleanImagesDir({ imagesDir }) appPort = await findPort() app = await nextStart(appDir, appPort) }) @@ -1408,7 +407,7 @@ describe('Image Optimizer', () => { }) it('should return response when image is served from an external rewrite', async () => { - await cleanImagesDir() + await cleanImagesDir({ imagesDir }) const query = { url: '/next-js/next-js-bg.png', w: 64, q: 75 } const opts = { headers: { accept: 'image/webp' } } @@ -1443,6 +442,8 @@ describe('Image Optimizer', () => { }) describe('dev support for dynamic blur placeholder', () => { + let app + let appPort beforeAll(async () => { const json = JSON.stringify({ images: { @@ -1451,7 +452,7 @@ describe('Image Optimizer', () => { }, }) nextConfig.replace('{ /* replaceme */ }', json) - await cleanImagesDir() + await cleanImagesDir({ imagesDir }) appPort = await findPort() app = await launchApp(appDir, appPort) }) @@ -1468,192 +469,4 @@ describe('Image Optimizer', () => { await expectWidth(res, 8) }) }) - - const setupTests = ({ isSharp = false, isOutdatedSharp = false }) => { - describe('dev support w/o next.config.js', () => { - const size = 384 // defaults defined in server/config.ts - beforeAll(async () => { - nextOutput = '' - appPort = await findPort() - app = await launchApp(appDir, appPort, { - onStderr(msg) { - nextOutput += msg - }, - env: { - NEXT_SHARP_PATH: isSharp - ? join(appDir, 'node_modules', 'sharp') - : '', - }, - cwd: appDir, - }) - await cleanImagesDir() - }) - afterAll(async () => { - await killApp(app) - }) - - runTests({ - w: size, - isDev: true, - domains: [], - isSharp, - isOutdatedSharp, - avifEnabled: false, - }) - }) - - describe('dev support with next.config.js', () => { - const size = 400 - beforeAll(async () => { - const json = JSON.stringify({ - images: { - deviceSizes: [largeSize], - imageSizes: [size], - domains, - formats: ['image/avif', 'image/webp'], - }, - }) - nextOutput = '' - nextConfig.replace('{ /* replaceme */ }', json) - await cleanImagesDir() - appPort = await findPort() - app = await launchApp(appDir, appPort, { - onStderr(msg) { - nextOutput += msg - }, - env: { - NEXT_SHARP_PATH: isSharp - ? join(appDir, 'node_modules', 'sharp') - : '', - }, - cwd: appDir, - }) - }) - afterAll(async () => { - await killApp(app) - nextConfig.restore() - }) - - runTests({ - w: size, - isDev: true, - domains, - isSharp, - isOutdatedSharp, - avifEnabled: true, - }) - }) - - describe('Server support w/o next.config.js', () => { - const size = 384 // defaults defined in server/config.ts - beforeAll(async () => { - nextOutput = '' - await nextBuild(appDir) - await cleanImagesDir() - appPort = await findPort() - app = await nextStart(appDir, appPort, { - onStderr(msg) { - nextOutput += msg - }, - env: { - NEXT_SHARP_PATH: isSharp - ? join(appDir, 'node_modules', 'sharp') - : '', - }, - cwd: appDir, - }) - }) - afterAll(async () => { - await killApp(app) - }) - - runTests({ w: size, isDev: false, domains: [], isSharp, isOutdatedSharp }) - }) - - describe('Server support with next.config.js', () => { - const size = 399 - beforeAll(async () => { - const json = JSON.stringify({ - images: { - formats: ['image/avif', 'image/webp'], - deviceSizes: [size, largeSize], - domains, - }, - }) - nextOutput = '' - nextConfig.replace('{ /* replaceme */ }', json) - await nextBuild(appDir) - await cleanImagesDir() - appPort = await findPort() - app = await nextStart(appDir, appPort, { - onStderr(msg) { - nextOutput += msg - }, - env: { - NEXT_SHARP_PATH: isSharp - ? join(appDir, 'node_modules', 'sharp') - : '', - }, - cwd: appDir, - }) - }) - afterAll(async () => { - await killApp(app) - nextConfig.restore() - }) - - runTests({ - w: size, - isDev: false, - domains, - isSharp, - isOutdatedSharp, - avifEnabled: true, - }) - }) - } - - describe('with squoosh', () => { - setupTests({ isSharp: false, isOutdatedSharp: false }) - }) - - describe('with latest sharp', () => { - beforeAll(async () => { - await execa('yarn', ['init', '-y'], { - cwd: appDir, - stdio: 'inherit', - }) - await execa('yarn', ['add', 'sharp'], { - cwd: appDir, - stdio: 'inherit', - }) - }) - afterAll(async () => { - await fs.remove(join(appDir, 'node_modules')) - await fs.remove(join(appDir, 'yarn.lock')) - await fs.remove(join(appDir, 'package.json')) - }) - - setupTests({ isSharp: true, isOutdatedSharp: false }) - }) - - describe('with outdated sharp', () => { - beforeAll(async () => { - await execa('yarn', ['init', '-y'], { - cwd: appDir, - stdio: 'inherit', - }) - await execa('yarn', ['add', 'sharp@0.26.3'], { - cwd: appDir, - stdio: 'inherit', - }) - }) - afterAll(async () => { - await fs.remove(join(appDir, 'node_modules')) - await fs.remove(join(appDir, 'yarn.lock')) - await fs.remove(join(appDir, 'package.json')) - }) - - setupTests({ isSharp: true, isOutdatedSharp: true }) - }) }) diff --git a/test/integration/image-optimizer/test/old-sharp.test.js b/test/integration/image-optimizer/test/old-sharp.test.js new file mode 100644 index 0000000000000..c7058b89ff1f7 --- /dev/null +++ b/test/integration/image-optimizer/test/old-sharp.test.js @@ -0,0 +1,27 @@ +import execa from 'execa' +import fs from 'fs-extra' +import { join } from 'path' +import { setupTests } from './util' + +const appDir = join(__dirname, '../app') +const imagesDir = join(appDir, '.next', 'cache', 'images') + +describe('with outdated sharp', () => { + beforeAll(async () => { + await execa('yarn', ['init', '-y'], { + cwd: appDir, + stdio: 'inherit', + }) + await execa('yarn', ['add', 'sharp@0.26.3'], { + cwd: appDir, + stdio: 'inherit', + }) + }) + afterAll(async () => { + await fs.remove(join(appDir, 'node_modules')) + await fs.remove(join(appDir, 'yarn.lock')) + await fs.remove(join(appDir, 'package.json')) + }) + + setupTests({ isSharp: true, isOutdatedSharp: true, appDir, imagesDir }) +}) diff --git a/test/integration/image-optimizer/test/sharp.test.js b/test/integration/image-optimizer/test/sharp.test.js new file mode 100644 index 0000000000000..27105a870edd8 --- /dev/null +++ b/test/integration/image-optimizer/test/sharp.test.js @@ -0,0 +1,27 @@ +import execa from 'execa' +import fs from 'fs-extra' +import { join } from 'path' +import { setupTests } from './util' + +const appDir = join(__dirname, '../app') +const imagesDir = join(appDir, '.next', 'cache', 'images') + +describe('with latest sharp', () => { + beforeAll(async () => { + await execa('yarn', ['init', '-y'], { + cwd: appDir, + stdio: 'inherit', + }) + await execa('yarn', ['add', 'sharp'], { + cwd: appDir, + stdio: 'inherit', + }) + }) + afterAll(async () => { + await fs.remove(join(appDir, 'node_modules')) + await fs.remove(join(appDir, 'yarn.lock')) + await fs.remove(join(appDir, 'package.json')) + }) + + setupTests({ isSharp: true, isOutdatedSharp: false, appDir, imagesDir }) +}) diff --git a/test/integration/image-optimizer/test/squoosh.test.js b/test/integration/image-optimizer/test/squoosh.test.js new file mode 100644 index 0000000000000..ad69765403342 --- /dev/null +++ b/test/integration/image-optimizer/test/squoosh.test.js @@ -0,0 +1,9 @@ +import { join } from 'path' +import { setupTests } from './util' + +const appDir = join(__dirname, '../app') +const imagesDir = join(appDir, '.next', 'cache', 'images') + +describe('with squoosh', () => { + setupTests({ isSharp: false, isOutdatedSharp: false, appDir, imagesDir }) +}) diff --git a/test/integration/image-optimizer/test/util.js b/test/integration/image-optimizer/test/util.js new file mode 100644 index 0000000000000..47867ab3b254d --- /dev/null +++ b/test/integration/image-optimizer/test/util.js @@ -0,0 +1,1259 @@ +import http from 'http' +import fs from 'fs-extra' +import { join } from 'path' +import assert from 'assert' +import sizeOf from 'image-size' +import { + check, + fetchViaHTTP, + File, + findPort, + killApp, + launchApp, + nextBuild, + nextStart, + waitFor, +} from 'next-test-utils' +import isAnimated from 'next/dist/compiled/is-animated' + +const largeSize = 1080 // defaults defined in server/config.ts +const sharpMissingText = `For production Image Optimization with Next.js, the optional 'sharp' package is strongly recommended` +const sharpOutdatedText = `Your installed version of the 'sharp' package does not support AVIF images. Run 'yarn add sharp@latest' to upgrade to the latest version` + +export async function serveSlowImage() { + const port = await findPort() + const server = http.createServer(async (req, res) => { + const parsedUrl = new URL(req.url, 'http://localhost') + const delay = Number(parsedUrl.searchParams.get('delay')) || 500 + + console.log('delay image for', delay) + await waitFor(delay) + + res.statusCode = 200 + res.setHeader('content-type', 'image/png') + res.end(await fs.readFile(join(__dirname, '../app/public/test.png'))) + }) + + await new Promise((resolve, reject) => { + server.listen(port, (err) => { + if (err) return reject(err) + resolve() + }) + }) + console.log(`Started slow image server at ::${port}`) + return { + stop() { + server.close() + }, + port, + } +} + +export async function fsToJson(dir, output = {}) { + const files = await fs.readdir(dir) + for (let file of files) { + const fsPath = join(dir, file) + const stat = await fs.stat(fsPath) + if (stat.isDirectory()) { + output[file] = {} + await fsToJson(fsPath, output[file]) + } else { + output[file] = stat.mtime.toISOString() + } + } + return output +} + +export async function expectWidth(res, w) { + const buffer = await res.buffer() + const d = sizeOf(buffer) + expect(d.width).toBe(w) +} + +export const cleanImagesDir = async (ctx) => { + console.warn('Cleaning', ctx.imagesDir) + await fs.remove(ctx.imagesDir) +} + +async function expectAvifSmallerThanWebp(w, q, appPort) { + const query = { url: '/mountains.jpg', w, q } + const res1 = await fetchViaHTTP(appPort, '/_next/image', query, { + headers: { + accept: 'image/avif', + }, + }) + expect(res1.status).toBe(200) + expect(res1.headers.get('Content-Type')).toBe('image/avif') + + const res2 = await fetchViaHTTP(appPort, '/_next/image', query, { + headers: { + accept: 'image/webp', + }, + }) + expect(res2.status).toBe(200) + expect(res2.headers.get('Content-Type')).toBe('image/webp') + + const res3 = await fetchViaHTTP(appPort, '/_next/image', query, { + headers: { + accept: 'image/jpeg', + }, + }) + expect(res3.status).toBe(200) + expect(res3.headers.get('Content-Type')).toBe('image/jpeg') + + const avif = (await res1.buffer()).byteLength + const webp = (await res2.buffer()).byteLength + const jpeg = (await res3.buffer()).byteLength + + expect(webp).toBeLessThan(jpeg) + expect(avif).toBeLessThanOrEqual(webp) +} + +async function fetchWithDuration(...args) { + console.warn('Fetching', args[1], args[2]) + const start = Date.now() + const res = await fetchViaHTTP(...args) + const buffer = await res.buffer() + const duration = Date.now() - start + return { duration, buffer, res } +} + +export function runTests(ctx) { + let slowImageServer + beforeAll(async () => { + slowImageServer = await serveSlowImage() + }) + afterAll(async () => { + slowImageServer.stop() + }) + + it('should return home page', async () => { + const res = await fetchViaHTTP(ctx.appPort, '/', null, {}) + expect(await res.text()).toMatch(/Image Optimizer Home/m) + }) + + it('should handle non-ascii characters in image url', async () => { + const query = { w: ctx.w, q: 90, url: '/äöüščří.png' } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(200) + }) + + it('should maintain animated gif', async () => { + const query = { w: ctx.w, q: 90, url: '/animated.gif' } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(200) + expect(res.headers.get('content-type')).toContain('image/gif') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="animated.gif"` + ) + expect(isAnimated(await res.buffer())).toBe(true) + }) + + it('should maintain animated png', async () => { + const query = { w: ctx.w, q: 90, url: '/animated.png' } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(200) + expect(res.headers.get('content-type')).toContain('image/png') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="animated.png"` + ) + expect(isAnimated(await res.buffer())).toBe(true) + }) + + it('should maintain animated webp', async () => { + const query = { w: ctx.w, q: 90, url: '/animated.webp' } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(200) + expect(res.headers.get('content-type')).toContain('image/webp') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="animated.webp"` + ) + expect(isAnimated(await res.buffer())).toBe(true) + }) + + it('should maintain vector svg', async () => { + const query = { w: ctx.w, q: 90, url: '/test.svg' } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toContain('image/svg+xml') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + // SVG is compressible so will have accept-encoding set from + // compression + expect(res.headers.get('Vary')).toMatch(/^Accept(,|$)/) + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.svg"` + ) + const actual = await res.text() + const expected = await fs.readFile( + join(ctx.appDir, 'public', 'test.svg'), + 'utf8' + ) + expect(actual).toMatch(expected) + }) + + it('should maintain ico format', async () => { + const query = { w: ctx.w, q: 90, url: `/test.ico` } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toContain('image/x-icon') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toMatch(/^Accept(,|$)/) + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.ico"` + ) + const actual = await res.text() + const expected = await fs.readFile( + join(ctx.appDir, 'public', 'test.ico'), + 'utf8' + ) + expect(actual).toMatch(expected) + }) + + it('should maintain jpg format for old Safari', async () => { + const accept = + 'image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5' + const query = { w: ctx.w, q: 90, url: '/test.jpg' } + const opts = { headers: { accept } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toContain('image/jpeg') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.jpeg"` + ) + }) + + it('should maintain png format for old Safari', async () => { + const accept = + 'image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5' + const query = { w: ctx.w, q: 75, url: '/test.png' } + const opts = { headers: { accept } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toContain('image/png') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.png"` + ) + }) + + it('should fail when url is missing', async () => { + const query = { w: ctx.w, q: 100 } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(400) + expect(await res.text()).toBe(`"url" parameter is required`) + }) + + it('should fail when w is missing', async () => { + const query = { url: '/test.png', q: 100 } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(400) + expect(await res.text()).toBe(`"w" parameter (width) is required`) + }) + + it('should fail when q is missing', async () => { + const query = { url: '/test.png', w: ctx.w } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(400) + expect(await res.text()).toBe(`"q" parameter (quality) is required`) + }) + + it('should fail when q is greater than 100', async () => { + const query = { url: '/test.png', w: ctx.w, q: 101 } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(400) + expect(await res.text()).toBe( + `"q" parameter (quality) must be a number between 1 and 100` + ) + }) + + it('should fail when q is less than 1', async () => { + const query = { url: '/test.png', w: ctx.w, q: 0 } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(400) + expect(await res.text()).toBe( + `"q" parameter (quality) must be a number between 1 and 100` + ) + }) + + it('should fail when w is 0 or less', async () => { + const query = { url: '/test.png', w: 0, q: 100 } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(400) + expect(await res.text()).toBe( + `"w" parameter (width) must be a number greater than 0` + ) + }) + + it('should fail when w is not a number', async () => { + const query = { url: '/test.png', w: 'foo', q: 100 } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(400) + expect(await res.text()).toBe( + `"w" parameter (width) must be a number greater than 0` + ) + }) + + it('should fail when q is not a number', async () => { + const query = { url: '/test.png', w: ctx.w, q: 'foo' } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(400) + expect(await res.text()).toBe( + `"q" parameter (quality) must be a number between 1 and 100` + ) + }) + + it('should fail when domain is not defined in next.config.js', async () => { + const url = `http://vercel.com/button` + const query = { url, w: ctx.w, q: 100 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(400) + expect(await res.text()).toBe(`"url" parameter is not allowed`) + }) + + it('should fail when width is not in next.config.js', async () => { + const query = { url: '/test.png', w: 1000, q: 100 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(400) + expect(await res.text()).toBe( + `"w" parameter (width) of 1000 is not allowed` + ) + }) + + it('should resize relative url and webp Firefox accept header', async () => { + const query = { url: '/test.png', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp,*/*' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await expectWidth(res, ctx.w) + }) + + it('should resize relative url and png accept header', async () => { + const query = { url: '/test.png', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/png' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/png') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.png"` + ) + await expectWidth(res, ctx.w) + }) + + it('should resize relative url with invalid accept header as png', async () => { + const query = { url: '/test.png', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/invalid' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/png') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.png"` + ) + await expectWidth(res, ctx.w) + }) + + it('should resize relative url with invalid accept header as gif', async () => { + const query = { url: '/test.gif', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/invalid' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/gif') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.gif"` + ) + // FIXME: await expectWidth(res, ctx.w) + }) + + it('should resize relative url with invalid accept header as tiff', async () => { + const query = { url: '/test.tiff', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/invalid' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/tiff') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.tiff"` + ) + // FIXME: await expectWidth(res, ctx.w) + }) + + it('should resize relative url and old Chrome accept header as webp', async () => { + const query = { url: '/test.png', w: ctx.w, q: 80 } + const opts = { + headers: { accept: 'image/webp,image/apng,image/*,*/*;q=0.8' }, + } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await expectWidth(res, ctx.w) + }) + + if (ctx.avifEnabled) { + it('should resize relative url and new Chrome accept header as avif', async () => { + const query = { url: '/test.png', w: ctx.w, q: 80 } + const opts = { + headers: { + accept: 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8', + }, + } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/avif') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.avif"` + ) + // TODO: upgrade "image-size" package to support AVIF + // See https://github.com/image-size/image-size/issues/348 + //await expectWidth(res, ctx.w) + }) + + it('should compress avif smaller than webp at q=100', async () => { + await expectAvifSmallerThanWebp(ctx.w, 100, ctx.appPort) + }) + + it('should compress avif smaller than webp at q=75', async () => { + await expectAvifSmallerThanWebp(ctx.w, 75, ctx.appPort) + }) + + it('should compress avif smaller than webp at q=50', async () => { + await expectAvifSmallerThanWebp(ctx.w, 50, ctx.appPort) + }) + } + + if (ctx.domains.includes('localhost')) { + it('should resize absolute url from localhost', async () => { + const url = `http://localhost:${ctx.appPort}/test.png` + const query = { url, w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await expectWidth(res, ctx.w) + }) + + it('should automatically detect image type when content-type is octet-stream', async () => { + const url = '/png-as-octet-stream' + const resOrig = await fetchViaHTTP(ctx.appPort, url) + expect(resOrig.status).toBe(200) + expect(resOrig.headers.get('Content-Type')).toBe( + 'application/octet-stream' + ) + const query = { url, w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="png-as-octet-stream.webp"` + ) + await expectWidth(res, ctx.w) + }) + + it('should use cache and stale-while-revalidate when query is the same for external image', async () => { + await cleanImagesDir(ctx) + const delay = 500 + + const url = `http://localhost:${slowImageServer.port}/slow.png?delay=${delay}` + const query = { url, w: ctx.w, q: 39 } + const opts = { headers: { accept: 'image/webp' } } + + const one = await fetchWithDuration( + ctx.appPort, + '/_next/image', + query, + opts + ) + expect(one.duration).toBeGreaterThan(delay) + expect(one.res.status).toBe(200) + expect(one.res.headers.get('X-Nextjs-Cache')).toBe('MISS') + expect(one.res.headers.get('Content-Type')).toBe('image/webp') + expect(one.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + const etagOne = one.res.headers.get('etag') + + let json1 + await check(async () => { + json1 = await fsToJson(ctx.imagesDir) + return Object.keys(json1).some((dir) => { + return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) + }) + ? 'success' + : 'fail' + }, 'success') + + const two = await fetchWithDuration( + ctx.appPort, + '/_next/image', + query, + opts + ) + expect(two.res.status).toBe(200) + expect(two.res.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(two.res.headers.get('Content-Type')).toBe('image/webp') + expect(two.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + const json2 = await fsToJson(ctx.imagesDir) + expect(json2).toStrictEqual(json1) + + if (ctx.minimumCacheTTL) { + // Wait until expired so we can confirm image is regenerated + await waitFor(ctx.minimumCacheTTL * 1000) + + const [three, four] = await Promise.all([ + fetchWithDuration(ctx.appPort, '/_next/image', query, opts), + fetchWithDuration(ctx.appPort, '/_next/image', query, opts), + ]) + + expect(three.duration).toBeLessThan(one.duration) + expect(three.res.status).toBe(200) + expect(three.res.headers.get('X-Nextjs-Cache')).toBe('STALE') + expect(three.res.headers.get('Content-Type')).toBe('image/webp') + expect(three.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + + expect(four.duration).toBeLessThan(one.duration) + expect(four.res.status).toBe(200) + expect(four.res.headers.get('X-Nextjs-Cache')).toBe('STALE') + expect(four.res.headers.get('Content-Type')).toBe('image/webp') + expect(four.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + await check(async () => { + const json4 = await fsToJson(ctx.imagesDir) + try { + assert.deepStrictEqual(json4, json1) + return 'fail' + } catch (err) { + return 'success' + } + }, 'success') + + const five = await fetchWithDuration( + ctx.appPort, + '/_next/image', + query, + opts + ) + expect(five.duration).toBeLessThan(one.duration) + expect(five.res.status).toBe(200) + expect(five.res.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(five.res.headers.get('Content-Type')).toBe('image/webp') + expect(five.res.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + await check(async () => { + const json5 = await fsToJson(ctx.imagesDir) + try { + assert.deepStrictEqual(json5, json1) + return 'fail' + } catch (err) { + return 'success' + } + }, 'success') + } + }) + } + + it('should fail when url has file protocol', async () => { + const url = `file://localhost:${ctx.appPort}/test.png` + const query = { url, w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(400) + expect(await res.text()).toBe(`"url" parameter is invalid`) + }) + + it('should fail when url has ftp protocol', async () => { + const url = `ftp://localhost:${ctx.appPort}/test.png` + const query = { url, w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(400) + expect(await res.text()).toBe(`"url" parameter is invalid`) + }) + + if (ctx.domains.includes('localhost')) { + it('should fail when url fails to load an image', async () => { + const url = `http://localhost:${ctx.appPort}/not-an-image` + const query = { w: ctx.w, url, q: 100 } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, {}) + expect(res.status).toBe(404) + expect(await res.text()).toBe( + `"url" parameter is valid but upstream response is invalid` + ) + }) + } + + it('should use cache and stale-while-revalidate when query is the same for internal image', async () => { + await cleanImagesDir(ctx) + + const query = { url: '/test.png', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + + const one = await fetchWithDuration( + ctx.appPort, + '/_next/image', + query, + opts + ) + expect(one.res.status).toBe(200) + expect(one.res.headers.get('X-Nextjs-Cache')).toBe('MISS') + expect(one.res.headers.get('Content-Type')).toBe('image/webp') + expect(one.res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + const etagOne = one.res.headers.get('etag') + + let json1 + await check(async () => { + json1 = await fsToJson(ctx.imagesDir) + return Object.keys(json1).some((dir) => { + return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) + }) + ? 'success' + : 'fail' + }, 'success') + + const two = await fetchWithDuration( + ctx.appPort, + '/_next/image', + query, + opts + ) + expect(two.res.status).toBe(200) + expect(two.res.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(two.res.headers.get('Content-Type')).toBe('image/webp') + expect(two.res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + const json2 = await fsToJson(ctx.imagesDir) + expect(json2).toStrictEqual(json1) + + if (ctx.minimumCacheTTL) { + // Wait until expired so we can confirm image is regenerated + await waitFor(ctx.minimumCacheTTL * 1000) + + const [three, four] = await Promise.all([ + fetchWithDuration(ctx.appPort, '/_next/image', query, opts), + fetchWithDuration(ctx.appPort, '/_next/image', query, opts), + ]) + + expect(three.duration).toBeLessThan(one.duration) + expect(three.res.status).toBe(200) + expect(three.res.headers.get('X-Nextjs-Cache')).toBe('STALE') + expect(three.res.headers.get('Content-Type')).toBe('image/webp') + expect(three.res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + + expect(four.duration).toBeLessThan(one.duration) + expect(four.res.status).toBe(200) + expect(four.res.headers.get('X-Nextjs-Cache')).toBe('STALE') + expect(four.res.headers.get('Content-Type')).toBe('image/webp') + expect(four.res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await check(async () => { + const json3 = await fsToJson(ctx.imagesDir) + try { + assert.deepStrictEqual(json3, json1) + return 'fail' + } catch (err) { + return 'success' + } + }, 'success') + + const five = await fetchWithDuration( + ctx.appPort, + '/_next/image', + query, + opts + ) + expect(five.duration).toBeLessThan(one.duration) + expect(five.res.status).toBe(200) + expect(five.res.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(five.res.headers.get('Content-Type')).toBe('image/webp') + expect(five.res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await check(async () => { + const json5 = await fsToJson(ctx.imagesDir) + try { + assert.deepStrictEqual(json5, json1) + return 'fail' + } catch (err) { + return 'success' + } + }, 'success') + } + }) + + it('should use cached image file when parameters are the same for svg', async () => { + await cleanImagesDir(ctx) + + const query = { url: '/test.svg', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + + const res1 = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res1.status).toBe(200) + expect(res1.headers.get('X-Nextjs-Cache')).toBe('MISS') + expect(res1.headers.get('Content-Type')).toBe('image/svg+xml') + expect(res1.headers.get('Content-Disposition')).toBe( + `inline; filename="test.svg"` + ) + const etagOne = res1.headers.get('etag') + + let json1 + await check(async () => { + json1 = await fsToJson(ctx.imagesDir) + return Object.keys(json1).some((dir) => { + return Object.keys(json1[dir]).some((file) => file.includes(etagOne)) + }) + ? 'success' + : 'fail' + }, 'success') + + const res2 = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res2.status).toBe(200) + expect(res2.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(res2.headers.get('Content-Type')).toBe('image/svg+xml') + expect(res2.headers.get('Content-Disposition')).toBe( + `inline; filename="test.svg"` + ) + const json2 = await fsToJson(ctx.imagesDir) + expect(json2).toStrictEqual(json1) + }) + + it('should use cached image file when parameters are the same for animated gif', async () => { + await cleanImagesDir(ctx) + + const query = { url: '/animated.gif', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + + const res1 = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res1.status).toBe(200) + expect(res1.headers.get('X-Nextjs-Cache')).toBe('MISS') + expect(res1.headers.get('Content-Type')).toBe('image/gif') + expect(res1.headers.get('Content-Disposition')).toBe( + `inline; filename="animated.gif"` + ) + + let json1 + await check(async () => { + json1 = await fsToJson(ctx.imagesDir) + return Object.keys(json1).length === 1 ? 'success' : 'fail' + }, 'success') + + const res2 = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res2.status).toBe(200) + expect(res2.headers.get('X-Nextjs-Cache')).toBe('HIT') + expect(res2.headers.get('Content-Type')).toBe('image/gif') + expect(res2.headers.get('Content-Disposition')).toBe( + `inline; filename="animated.gif"` + ) + const json2 = await fsToJson(ctx.imagesDir) + expect(json2).toStrictEqual(json1) + }) + + it('should set 304 status without body when etag matches if-none-match', async () => { + const query = { url: '/test.jpg', w: ctx.w, q: 80 } + const opts1 = { headers: { accept: 'image/webp' } } + + const res1 = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts1) + expect(res1.status).toBe(200) + expect(res1.headers.get('Content-Type')).toBe('image/webp') + expect(res1.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res1.headers.get('Vary')).toBe('Accept') + const etag = res1.headers.get('Etag') + expect(etag).toBeTruthy() + expect(res1.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await expectWidth(res1, ctx.w) + + const opts2 = { headers: { accept: 'image/webp', 'if-none-match': etag } } + const res2 = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts2) + expect(res2.status).toBe(304) + expect(res2.headers.get('Content-Type')).toBeFalsy() + expect(res2.headers.get('Etag')).toBe(etag) + expect(res2.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res2.headers.get('Vary')).toBe('Accept') + expect(res2.headers.get('Content-Disposition')).toBeFalsy() + expect((await res2.buffer()).length).toBe(0) + + const query3 = { url: '/test.jpg', w: ctx.w, q: 25 } + const res3 = await fetchViaHTTP(ctx.appPort, '/_next/image', query3, opts2) + expect(res3.status).toBe(200) + expect(res3.headers.get('Content-Type')).toBe('image/webp') + expect(res3.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res3.headers.get('Vary')).toBe('Accept') + expect(res3.headers.get('Etag')).toBeTruthy() + expect(res3.headers.get('Etag')).not.toBe(etag) + expect(res3.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await expectWidth(res3, ctx.w) + }) + + it('should maintain bmp', async () => { + const json1 = await fsToJson(ctx.imagesDir) + expect(json1).toBeTruthy() + + const query = { url: '/test.bmp', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/invalid' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/bmp') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + // bmp is compressible so will have accept-encoding set from + // compression + expect(res.headers.get('Vary')).toMatch(/^Accept(,|$)/) + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.bmp"` + ) + + await check(async () => { + try { + assert.deepStrictEqual(await fsToJson(ctx.imagesDir), json1) + return 'expected change, but matched' + } catch (_) { + return 'success' + } + }, 'success') + }) + + it('should not resize if requested width is larger than original source image', async () => { + const query = { url: '/test.jpg', w: largeSize, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('etag')).toBeTruthy() + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="test.webp"` + ) + await expectWidth(res, 400) + }) + + if (!ctx.isSharp) { + // this checks for specific color type output by squoosh + // which differs in sharp + it('should not change the color type of a png', async () => { + // https://github.com/vercel/next.js/issues/22929 + // A grayscaled PNG with transparent pixels. + const query = { url: '/grayscale.png', w: largeSize, q: 80 } + const opts = { headers: { accept: 'image/png' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(200) + expect(res.headers.get('Content-Type')).toBe('image/png') + expect(res.headers.get('Cache-Control')).toBe( + `public, max-age=0, must-revalidate` + ) + expect(res.headers.get('Vary')).toBe('Accept') + expect(res.headers.get('Content-Disposition')).toBe( + `inline; filename="grayscale.png"` + ) + + const png = await res.buffer() + + // Read the color type byte (offset 9 + magic number 16). + // http://www.libpng.org/pub/png/spec/1.2/PNG-Chunks.html + const colorType = png.readUIntBE(25, 1) + expect(colorType).toBe(4) + }) + } + + it('should set cache-control to immutable for static images', async () => { + if (!ctx.isDev) { + const filename = 'test' + const query = { + url: `/_next/static/media/${filename}.fab2915d.jpg`, + w: ctx.w, + q: 100, + } + const opts = { headers: { accept: 'image/webp' } } + + const res1 = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res1.status).toBe(200) + expect(res1.headers.get('Cache-Control')).toBe( + 'public, max-age=315360000, immutable' + ) + expect(res1.headers.get('Vary')).toBe('Accept') + expect(res1.headers.get('Content-Disposition')).toBe( + `inline; filename="${filename}.webp"` + ) + await expectWidth(res1, ctx.w) + + // Ensure subsequent request also has immutable header + const res2 = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res2.status).toBe(200) + expect(res2.headers.get('Cache-Control')).toBe( + 'public, max-age=315360000, immutable' + ) + expect(res2.headers.get('Vary')).toBe('Accept') + expect(res2.headers.get('Content-Disposition')).toBe( + `inline; filename="${filename}.webp"` + ) + await expectWidth(res2, ctx.w) + } + }) + + it("should error if the resource isn't a valid image", async () => { + const query = { url: '/test.txt', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(400) + expect(await res.text()).toBe("The requested resource isn't a valid image.") + }) + + it('should error if the image file does not exist', async () => { + const query = { url: '/does_not_exist.jpg', w: ctx.w, q: 80 } + const opts = { headers: { accept: 'image/webp' } } + const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) + expect(res.status).toBe(400) + expect(await res.text()).toBe("The requested resource isn't a valid image.") + }) + + if (ctx.domains.length) { + it('should handle concurrent requests', async () => { + await cleanImagesDir(ctx) + const delay = 500 + const query = { + url: `http://localhost:${slowImageServer.port}/slow.png?delay=${delay}`, + w: ctx.w, + q: 80, + } + const opts = { headers: { accept: 'image/webp,*/*' } } + const [res1, res2, res3] = await Promise.all([ + fetchViaHTTP(ctx.appPort, '/_next/image', query, opts), + fetchViaHTTP(ctx.appPort, '/_next/image', query, opts), + fetchViaHTTP(ctx.appPort, '/_next/image', query, opts), + ]) + + if (res1.status !== 200) { + console.error(await res1.text()) + } + + expect(res1.status).toBe(200) + expect(res2.status).toBe(200) + expect(res3.status).toBe(200) + + expect(res1.headers.get('Content-Type')).toBe('image/webp') + expect(res1.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + expect(res2.headers.get('Content-Type')).toBe('image/webp') + expect(res2.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + expect(res3.headers.get('Content-Type')).toBe('image/webp') + expect(res3.headers.get('Content-Disposition')).toBe( + `inline; filename="slow.webp"` + ) + + await expectWidth(res1, ctx.w) + await expectWidth(res2, ctx.w) + await expectWidth(res3, ctx.w) + + await check(async () => { + const json1 = await fsToJson(ctx.imagesDir) + return Object.keys(json1).length === 1 ? 'success' : 'fail' + }, 'success') + + const xCache = [res1, res2, res3] + .map((r) => r.headers.get('X-Nextjs-Cache')) + .sort((a, b) => b.localeCompare(a)) + + // Since the first request is a miss it blocks + // until the cache be populated so all concurrent + // requests receive the same response + expect(xCache).toEqual(['MISS', 'MISS', 'MISS']) + }) + } + + if (ctx.isDev || ctx.isSharp) { + it('should not have sharp missing warning', () => { + expect(ctx.nextOutput).not.toContain(sharpMissingText) + }) + } else { + it('should have sharp missing warning', () => { + expect(ctx.nextOutput).toContain(sharpMissingText) + }) + } + + if (ctx.isSharp && ctx.isOutdatedSharp && ctx.avifEnabled) { + it('should have sharp outdated warning', () => { + expect(ctx.nextOutput).toContain(sharpOutdatedText) + }) + } else { + it('should not have sharp outdated warning', () => { + expect(ctx.nextOutput).not.toContain(sharpOutdatedText) + }) + } +} + +export const setupTests = (ctx) => { + const nextConfig = new File(join(ctx.appDir, 'next.config.js')) + + if (!ctx.domains) { + ctx.domains = [ + 'localhost', + 'example.com', + 'assets.vercel.com', + 'image-optimization-test.vercel.app', + ] + } + + // only run one server config with outdated sharp + if (!ctx.isOutdatedSharp) { + describe('dev support w/o next.config.js', () => { + const size = 384 // defaults defined in server/config.ts + const curCtx = { + ...ctx, + w: size, + isDev: true, + domains: [], + avifEnabled: false, + } + + beforeAll(async () => { + curCtx.nextOutput = '' + curCtx.appPort = await findPort() + curCtx.app = await launchApp(curCtx.appDir, curCtx.appPort, { + onStderr(msg) { + curCtx.nextOutput += msg + }, + env: { + NEXT_SHARP_PATH: curCtx.isSharp + ? join(curCtx.appDir, 'node_modules', 'sharp') + : '', + }, + cwd: curCtx.appDir, + }) + await cleanImagesDir(ctx) + }) + afterAll(async () => { + await killApp(curCtx.app) + }) + + runTests(curCtx) + }) + + describe('dev support with next.config.js', () => { + const size = 400 + const curCtx = { + ...ctx, + w: size, + isDev: true, + avifEnabled: true, + } + beforeAll(async () => { + const json = JSON.stringify({ + images: { + deviceSizes: [largeSize], + imageSizes: [size], + domains: curCtx.domains, + formats: ['image/avif', 'image/webp'], + }, + }) + curCtx.nextOutput = '' + nextConfig.replace('{ /* replaceme */ }', json) + await cleanImagesDir(ctx) + curCtx.appPort = await findPort() + curCtx.app = await launchApp(curCtx.appDir, curCtx.appPort, { + onStderr(msg) { + curCtx.nextOutput += msg + }, + env: { + NEXT_SHARP_PATH: curCtx.isSharp + ? join(curCtx.appDir, 'node_modules', 'sharp') + : '', + }, + cwd: curCtx.appDir, + }) + }) + afterAll(async () => { + await killApp(curCtx.app) + nextConfig.restore() + }) + + runTests(curCtx) + }) + + describe('Server support w/o next.config.js', () => { + const size = 384 // defaults defined in server/config.ts + const curCtx = { + ...ctx, + w: size, + isDev: false, + domains: [], + } + beforeAll(async () => { + curCtx.nextOutput = '' + await nextBuild(curCtx.appDir) + await cleanImagesDir(ctx) + curCtx.appPort = await findPort() + curCtx.app = await nextStart(curCtx.appDir, curCtx.appPort, { + onStderr(msg) { + curCtx.nextOutput += msg + }, + env: { + NEXT_SHARP_PATH: curCtx.isSharp + ? join(curCtx.appDir, 'node_modules', 'sharp') + : '', + }, + cwd: curCtx.appDir, + }) + }) + afterAll(async () => { + await killApp(curCtx.app) + }) + + runTests(curCtx) + }) + } + + describe('Server support with next.config.js', () => { + const size = 399 + const curCtx = { + ...ctx, + w: size, + isDev: false, + avifEnabled: true, + } + beforeAll(async () => { + const json = JSON.stringify({ + images: { + formats: ['image/avif', 'image/webp'], + deviceSizes: [size, largeSize], + domains: ctx.domains, + }, + }) + curCtx.nextOutput = '' + nextConfig.replace('{ /* replaceme */ }', json) + await nextBuild(curCtx.appDir) + await cleanImagesDir(ctx) + curCtx.appPort = await findPort() + curCtx.app = await nextStart(curCtx.appDir, curCtx.appPort, { + onStderr(msg) { + curCtx.nextOutput += msg + }, + env: { + NEXT_SHARP_PATH: curCtx.isSharp + ? join(curCtx.appDir, 'node_modules', 'sharp') + : '', + }, + cwd: curCtx.appDir, + }) + }) + afterAll(async () => { + await killApp(curCtx.app) + nextConfig.restore() + }) + + runTests(curCtx) + }) +}