From a057b7f0ea727e95e33c1763a41ae9a5121f8b61 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 4 Dec 2023 16:11:07 +0100 Subject: [PATCH 01/11] feat: provide file cdn from adapters --- .../scripts/deploy-and-run/netlify.mjs | 37 +++++++++---------- .../src/file-cdn-url-generator.ts | 11 ++++++ packages/gatsby-adapter-netlify/src/index.ts | 3 ++ .../graphql/public-url-resolver.ts | 13 ++++++- .../polyfill-remote-file/jobs/dispatchers.ts | 11 +++++- .../src/polyfill-remote-file/types.ts | 11 +++++- .../utils/url-generator.ts | 29 +++++++++------ packages/gatsby/index.d.ts | 2 + packages/gatsby/src/utils/adapter/manager.ts | 5 +++ packages/gatsby/src/utils/adapter/types.ts | 16 ++++++++ .../utils/page-ssr-module/bundle-webpack.ts | 13 +++++++ .../src/utils/page-ssr-module/lambda.ts | 6 +++ types/gatsby-monorepo/global.d.ts | 1 + 13 files changed, 124 insertions(+), 34 deletions(-) create mode 100644 packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts diff --git a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs index f3a5525f48081..f8ea6ef9cacc9 100644 --- a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs +++ b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs @@ -47,24 +47,21 @@ console.log(`Deployed to ${deployUrl}`) try { await execa(`npm`, [`run`, npmScriptToRun], { stdio: `inherit` }) } finally { - if (!process.env.GATSBY_TEST_SKIP_CLEANUP) { - console.log(`Deleting project with deploy_id ${deployInfo.deploy_id}`) - - const deleteResponse = await execa("ntl", [ - "api", - "deleteDeploy", - "--data", - `{ "deploy_id": "${deployInfo.deploy_id}" }`, - ]) - - if (deleteResponse.exitCode !== 0) { - throw new Error( - `Failed to delete project ${deleteResponse.stdout} ${deleteResponse.stderr} (${deleteResponse.exitCode})` - ) - } - - console.log( - `Successfully deleted project with deploy_id ${deployInfo.deploy_id}` - ) - } + // if (!process.env.GATSBY_TEST_SKIP_CLEANUP) { + // console.log(`Deleting project with deploy_id ${deployInfo.deploy_id}`) + // const deleteResponse = await execa("ntl", [ + // "api", + // "deleteDeploy", + // "--data", + // `{ "deploy_id": "${deployInfo.deploy_id}" }`, + // ]) + // if (deleteResponse.exitCode !== 0) { + // throw new Error( + // `Failed to delete project ${deleteResponse.stdout} ${deleteResponse.stderr} (${deleteResponse.exitCode})` + // ) + // } + // console.log( + // `Successfully deleted project with deploy_id ${deployInfo.deploy_id}` + // ) + // } } diff --git a/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts b/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts new file mode 100644 index 0000000000000..813eb457f2b9a --- /dev/null +++ b/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts @@ -0,0 +1,11 @@ +import type { FileCdnUrlGeneratorFn, FileCdnSourceImage } from "gatsby" + +export const generateImageUrl: FileCdnUrlGeneratorFn = + function generateImageUrl(source: FileCdnSourceImage): string { + const placeholderOrigin = `http://netlify.com` + const baseURL = new URL(`${placeholderOrigin}/.netlify/images`) + baseURL.searchParams.append(`url`, source.url) + return `${baseURL.pathname}${baseURL.search}` + } + +export default generateImageUrl diff --git a/packages/gatsby-adapter-netlify/src/index.ts b/packages/gatsby-adapter-netlify/src/index.ts index e741758167612..15072d323945e 100644 --- a/packages/gatsby-adapter-netlify/src/index.ts +++ b/packages/gatsby-adapter-netlify/src/index.ts @@ -142,6 +142,9 @@ const createNetlifyAdapter: AdapterInit = options => { imageCDNUrlGeneratorModulePath: useNetlifyImageCDN ? require.resolve(`./image-cdn-url-generator`) : undefined, + fileCDNUrlGeneratorModulePath: useNetlifyImageCDN + ? require.resolve(`./file-cdn-url-generator`) + : undefined, } }, } diff --git a/packages/gatsby-plugin-utils/src/polyfill-remote-file/graphql/public-url-resolver.ts b/packages/gatsby-plugin-utils/src/polyfill-remote-file/graphql/public-url-resolver.ts index 8ad35eb8d1d4b..6d831a864975b 100644 --- a/packages/gatsby-plugin-utils/src/polyfill-remote-file/graphql/public-url-resolver.ts +++ b/packages/gatsby-plugin-utils/src/polyfill-remote-file/graphql/public-url-resolver.ts @@ -15,6 +15,7 @@ export function publicUrlResolver( dispatchLocalFileServiceJob( { url: source.url, + mimeType: source.mimeType, filename: source.filename, contentDigest: source.internal.contentDigest, }, @@ -23,7 +24,17 @@ export function publicUrlResolver( ) } - return generateFileUrl({ url: source.url, filename: source.filename }, store) + return generateFileUrl( + { + url: source.url, + mimeType: source.mimeType, + filename: source.filename, + internal: { + contentDigest: source.internal.contentDigest, + }, + }, + store + ) } export function generatePublicUrlFieldConfig( diff --git a/packages/gatsby-plugin-utils/src/polyfill-remote-file/jobs/dispatchers.ts b/packages/gatsby-plugin-utils/src/polyfill-remote-file/jobs/dispatchers.ts index 858ea8f9bcbdd..e0ec8a3e5a84a 100644 --- a/packages/gatsby-plugin-utils/src/polyfill-remote-file/jobs/dispatchers.ts +++ b/packages/gatsby-plugin-utils/src/polyfill-remote-file/jobs/dispatchers.ts @@ -7,6 +7,7 @@ import { getRequestHeadersForUrl } from "../utils/get-request-headers-for-url" export function shouldDispatchLocalFileServiceJob(): boolean { return ( !( + global.__GATSBY?.fileCDNUrlGeneratorModulePath || process.env.GATSBY_CLOUD_IMAGE_CDN === `1` || process.env.GATSBY_CLOUD_IMAGE_CDN === `true` ) && process.env.NODE_ENV === `production` @@ -27,8 +28,14 @@ export function dispatchLocalFileServiceJob( { url, filename, + mimeType, contentDigest, - }: { url: string; filename: string; contentDigest: string }, + }: { + url: string + filename: string + mimeType: string + contentDigest: string + }, actions: Actions, store?: Store ): void { @@ -36,7 +43,9 @@ export function dispatchLocalFileServiceJob( const publicUrl = generateFileUrl( { url, + mimeType, filename, + internal: { contentDigest }, }, store ).split(`/`) diff --git a/packages/gatsby-plugin-utils/src/polyfill-remote-file/types.ts b/packages/gatsby-plugin-utils/src/polyfill-remote-file/types.ts index 94f8697b603c0..3479b51ebe6d1 100644 --- a/packages/gatsby-plugin-utils/src/polyfill-remote-file/types.ts +++ b/packages/gatsby-plugin-utils/src/polyfill-remote-file/types.ts @@ -101,13 +101,15 @@ export type ImageCdnTransformArgs = WidthOrHeight & { } // eslint-disable-next-line @typescript-eslint/consistent-type-definitions -export type ImageCdnSourceImage = { +type CdnSourceImage = { url: string mimeType: string filename: string internal: { contentDigest: string } } +export type ImageCdnSourceImage = CdnSourceImage + /** * The function is used to optimize image delivery by generating URLs that leverage CDN capabilities * @param {ImageCdnSourceImage} source - An object representing the source image, including properties like @@ -123,3 +125,10 @@ export type ImageCdnUrlGeneratorFn = ( imageArgs: ImageCdnTransformArgs, pathPrefix: string ) => string + +export type FileCdnSourceImage = CdnSourceImage + +export type FileCdnUrlGeneratorFn = ( + source: ImageCdnSourceImage, + pathPrefix: string +) => string diff --git a/packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/url-generator.ts b/packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/url-generator.ts index f479d18116c59..fd044a49b7ba1 100644 --- a/packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/url-generator.ts +++ b/packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/url-generator.ts @@ -7,6 +7,7 @@ import type { ImageCdnUrlGeneratorFn, ImageCdnSourceImage, ImageCdnTransformArgs, + FileCdnUrlGeneratorFn, } from "../types" import type { Store } from "gatsby" @@ -60,14 +61,13 @@ function appendUrlParamToSearchParams( const frontendHostName = process.env.IMAGE_CDN_HOSTNAME || `` +let customImageCDNUrlGenerator: ImageCdnUrlGeneratorFn | undefined = undefined +let customFileCDNUrlGenerator: FileCdnUrlGeneratorFn | undefined = undefined + +const preferDefault = (m: any): any => (m && m.default) || m + export function generateFileUrl( - { - url, - filename, - }: { - url: string - filename: string - }, + source: ImageCdnSourceImage, store?: Store ): string { const state = store?.getState() @@ -76,6 +76,17 @@ export function generateFileUrl( ? state?.config?.pathPrefix : `` + if (global.__GATSBY?.fileCDNUrlGeneratorModulePath) { + if (!customFileCDNUrlGenerator) { + customFileCDNUrlGenerator = preferDefault( + require(global.__GATSBY.fileCDNUrlGeneratorModulePath) + ) as FileCdnUrlGeneratorFn + } + return customFileCDNUrlGenerator(source, pathPrefix) + } + + const { url, filename } = source + const fileExt = extname(filename) const filenameWithoutExt = basename(filename, fileExt) @@ -93,10 +104,6 @@ export function generateFileUrl( return `${frontendHostName}${parsedURL.pathname}${parsedURL.search}` } -let customImageCDNUrlGenerator: ImageCdnUrlGeneratorFn | undefined = undefined - -const preferDefault = (m: any): any => (m && m.default) || m - export function generateImageUrl( source: ImageCdnSourceImage, imageArgs: ImageCdnTransformArgs, diff --git a/packages/gatsby/index.d.ts b/packages/gatsby/index.d.ts index a450eed930408..c95953df30f8f 100644 --- a/packages/gatsby/index.d.ts +++ b/packages/gatsby/index.d.ts @@ -48,6 +48,8 @@ export { ImageCdnUrlGeneratorFn, ImageCdnSourceImage, ImageCdnTransformArgs, + FileCdnUrlGeneratorFn, + FileCdnSourceImage, } from "./dist/utils/adapter/types" export const useScrollRestoration: (key: string) => { diff --git a/packages/gatsby/src/utils/adapter/manager.ts b/packages/gatsby/src/utils/adapter/manager.ts index 6d6023dee7e4f..8e97c6c737ae7 100644 --- a/packages/gatsby/src/utils/adapter/manager.ts +++ b/packages/gatsby/src/utils/adapter/manager.ts @@ -256,6 +256,11 @@ export async function initAdapterManager(): Promise { global.__GATSBY.imageCDNUrlGeneratorModulePath = configFromAdapter.imageCDNUrlGeneratorModulePath } + + if (configFromAdapter?.fileCDNUrlGeneratorModulePath) { + global.__GATSBY.fileCDNUrlGeneratorModulePath = + configFromAdapter.fileCDNUrlGeneratorModulePath + } } return { diff --git a/packages/gatsby/src/utils/adapter/types.ts b/packages/gatsby/src/utils/adapter/types.ts index 38a3607f529e1..4cb15f935f23d 100644 --- a/packages/gatsby/src/utils/adapter/types.ts +++ b/packages/gatsby/src/utils/adapter/types.ts @@ -6,12 +6,16 @@ import type { ImageCdnUrlGeneratorFn, ImageCdnSourceImage, ImageCdnTransformArgs, + FileCdnUrlGeneratorFn, + FileCdnSourceImage, } from "gatsby-plugin-utils/dist/polyfill-remote-file/types" export type { ImageCdnUrlGeneratorFn, ImageCdnSourceImage, ImageCdnTransformArgs, + FileCdnUrlGeneratorFn, + FileCdnSourceImage, } interface IBaseRoute { @@ -171,6 +175,18 @@ export interface IAdapterConfig { * example for the Netlify adapter. */ imageCDNUrlGeneratorModulePath?: string + /** + * @todo: specify that image cdn does image transformation (like resizing) while file cdn does not + * (but might do content negotiation) + * Path to a CommonJS module that implements an file CDN URL generation function. The function + * is used to optimize image delivery by generating URLs that leverage CDN capabilities. This module + * should have a default export function that conforms to the {@link FileCdnUrlGeneratorFn} type: + * + * Adapters should provide an absolute path to this module. + * See 'packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts' as an implementation + * example for the Netlify adapter. + */ + fileCDNUrlGeneratorModulePath?: string } type WithRequired = T & { [P in K]-?: T[P] } diff --git a/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts b/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts index 1590c27cbcf5d..c0641a5336d84 100644 --- a/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts +++ b/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts @@ -231,6 +231,15 @@ export async function createPageSSRBundle({ IMAGE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH = `./image-cdn-url-generator.js` } + let FILE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH = `` + if (global.__GATSBY?.fileCDNUrlGeneratorModulePath) { + await fs.copyFile( + global.__GATSBY.fileCDNUrlGeneratorModulePath, + path.join(outputDir, `file-cdn-url-generator.js`) + ) + FILE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH = `./file-cdn-url-generator.js` + } + let functionCode = await fs.readFile( path.join(__dirname, `lambda.js`), `utf-8` @@ -248,6 +257,10 @@ export async function createPageSSRBundle({ `%IMAGE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH%`, IMAGE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH ) + .replaceAll( + `%FILE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH%`, + FILE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH + ) await fs.outputFile(path.join(outputDir, `lambda.js`), functionCode) diff --git a/packages/gatsby/src/utils/page-ssr-module/lambda.ts b/packages/gatsby/src/utils/page-ssr-module/lambda.ts index 3b03052f64cc8..c60cb5f37e77f 100644 --- a/packages/gatsby/src/utils/page-ssr-module/lambda.ts +++ b/packages/gatsby/src/utils/page-ssr-module/lambda.ts @@ -98,6 +98,12 @@ if (`%IMAGE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH%`) { `%IMAGE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH%` ) } +// eslint-disable-next-line no-constant-condition +if (`%FILE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH%`) { + global.__GATSBY.fileCDNUrlGeneratorModulePath = require.resolve( + `%FILE_CDN_URL_GENERATOR_MODULE_RELATIVE_PATH%` + ) +} const dbPath = setupFsWrapper() diff --git a/types/gatsby-monorepo/global.d.ts b/types/gatsby-monorepo/global.d.ts index 1fe3271d47df8..cbca12bca9189 100644 --- a/types/gatsby-monorepo/global.d.ts +++ b/types/gatsby-monorepo/global.d.ts @@ -8,6 +8,7 @@ declare module NodeJS { buildId: string root: string imageCDNUrlGeneratorModulePath?: string + fileCDNUrlGeneratorModulePath?: string } _polyfillRemoteFileCache?: import("gatsby").GatsbyCache From d1ae4afca67471b0d3582d2e62170cb61685396d Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 4 Dec 2023 16:12:19 +0100 Subject: [PATCH 02/11] update test --- e2e-tests/adapters/cypress/e2e/remote-file.cy.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts b/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts index 2a2f347365e50..d269f813c7df3 100644 --- a/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts +++ b/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts @@ -31,7 +31,7 @@ const configs = [ { title: `remote-file (SSR, Page Query)`, pagePath: `/routes/ssr/remote-file/`, - fileCDN: false, + fileCDN: true, placeholders: false, }, ] @@ -110,8 +110,11 @@ for (const config of configs) { ) for (const url of urls) { - // using OSS implementation for publicURL for now - expect(url).to.match(new RegExp(`^${PATH_PREFIX}/_gatsby/file`)) + const { href, origin } = new URL(url) + const urlWithoutOrigin = href.replace(origin, ``) + + // using Netlify Image CDN + expect(urlWithoutOrigin).to.match(/^\/.netlify\/images/) const res = await fetch(url, { method: "HEAD", }) From ece0984d0b95d4c5fb472ded67f70bd148ae0e5f Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 4 Dec 2023 17:06:07 +0100 Subject: [PATCH 03/11] fix tests --- e2e-tests/adapters/cypress/e2e/remote-file.cy.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts b/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts index d269f813c7df3..203ad6876630a 100644 --- a/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts +++ b/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts @@ -109,13 +109,11 @@ for (const config of configs) { $urls.map((_, $url) => $url.getAttribute("href")) ) - for (const url of urls) { - const { href, origin } = new URL(url) - const urlWithoutOrigin = href.replace(origin, ``) - + // urls is array of href attribute, not absolute urls, so it already is stripped of origin + for (const urlWithoutOrigin of urls) { // using Netlify Image CDN expect(urlWithoutOrigin).to.match(/^\/.netlify\/images/) - const res = await fetch(url, { + const res = await fetch(urlWithoutOrigin, { method: "HEAD", }) expect(res.ok).to.be.true From 343c0aff7821a7cd54929a10c664fcc44dc29e09 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 5 Dec 2023 15:50:17 +0100 Subject: [PATCH 04/11] use edge function for non-image File CDN --- .../adapters/cypress/e2e/remote-file.cy.ts | 259 ++++++++++-------- e2e-tests/adapters/gatsby-node.ts | 35 ++- e2e-tests/adapters/src/pages/index.jsx | 6 +- .../adapters/src/pages/routes/remote-file.jsx | 25 +- .../src/pages/routes/ssr/remote-file.jsx | 25 +- .../templates/remote-file-from-context.jsx | 15 +- packages/gatsby-adapter-netlify/package.json | 1 + .../src/__tests__/file-cdn-url-generator.ts | 198 +++++++++++++ .../src/__tests__/image-cdn-url-generator.ts | 91 +++--- .../src/file-cdn-handler.ts | 87 ++++++ .../src/file-cdn-url-generator.ts | 36 ++- packages/gatsby-adapter-netlify/src/index.ts | 26 +- .../__tests__/public-resolver.ts | 8 + .../utils/__tests__/url-generator.ts | 24 ++ yarn.lock | 5 + 15 files changed, 659 insertions(+), 182 deletions(-) create mode 100644 packages/gatsby-adapter-netlify/src/__tests__/file-cdn-url-generator.ts create mode 100644 packages/gatsby-adapter-netlify/src/file-cdn-handler.ts diff --git a/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts b/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts index 203ad6876630a..81b3acb000e2c 100644 --- a/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts +++ b/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts @@ -19,19 +19,16 @@ const configs = [ { title: `remote-file (SSG, Page Query)`, pagePath: `/routes/remote-file/`, - fileCDN: true, placeholders: true, }, { title: `remote-file (SSG, Page Context)`, pagePath: `/routes/remote-file-data-from-context/`, - fileCDN: true, placeholders: true, }, { title: `remote-file (SSR, Page Query)`, pagePath: `/routes/ssr/remote-file/`, - fileCDN: true, placeholders: false, }, ] @@ -57,54 +54,54 @@ for (const config of configs) { cy.wait(600) }) - async function testImages(images, expectations) { - for (let i = 0; i < images.length; i++) { - const expectation = expectations[i] + describe(`Image CDN`, () => { + async function testImages(images, expectations) { + for (let i = 0; i < images.length; i++) { + const expectation = expectations[i] - const url = images[i].currentSrc + const url = images[i].currentSrc - const { href, origin } = new URL(url) - const urlWithoutOrigin = href.replace(origin, ``) + const { href, origin } = new URL(url) + const urlWithoutOrigin = href.replace(origin, ``) - // using Netlify Image CDN - expect(urlWithoutOrigin).to.match(/^\/.netlify\/images/) + // using Netlify Image CDN + expect(urlWithoutOrigin).to.match(/^\/.netlify\/images/) - const res = await fetch(url, { - method: "HEAD", - }) - expect(res.ok).to.be.true + const res = await fetch(url, { + method: "HEAD", + }) + expect(res.ok).to.be.true - const expectedNaturalWidth = - expectation.naturalWidth ?? expectation.width - const expectedNaturalHeight = - expectation.naturalHeight ?? expectation.height + const expectedNaturalWidth = + expectation.naturalWidth ?? expectation.width + const expectedNaturalHeight = + expectation.naturalHeight ?? expectation.height - if (expectation.width) { - expect( - Math.ceil(images[i].getBoundingClientRect().width) - ).to.be.equal(expectation.width) - } - if (expectation.height) { - expect( - Math.ceil(images[i].getBoundingClientRect().height) - ).to.be.equal(expectation.height) - } - if (expectedNaturalWidth) { - expect(Math.ceil(images[i].naturalWidth)).to.be.equal( - expectedNaturalWidth - ) - } - if (expectedNaturalHeight) { - expect(Math.ceil(images[i].naturalHeight)).to.be.equal( - expectedNaturalHeight - ) + if (expectation.width) { + expect( + Math.ceil(images[i].getBoundingClientRect().width) + ).to.be.equal(expectation.width) + } + if (expectation.height) { + expect( + Math.ceil(images[i].getBoundingClientRect().height) + ).to.be.equal(expectation.height) + } + if (expectedNaturalWidth) { + expect(Math.ceil(images[i].naturalWidth)).to.be.equal( + expectedNaturalWidth + ) + } + if (expectedNaturalHeight) { + expect(Math.ceil(images[i].naturalHeight)).to.be.equal( + expectedNaturalHeight + ) + } } } - } - it(`should render correct dimensions`, () => { - if (config.fileCDN) { - cy.get('[data-testid="public"]').then(async $urls => { + it(`should render correct dimensions`, () => { + cy.get('[data-testid="image-public"]').then(async $urls => { const urls = Array.from( $urls.map((_, $url) => $url.getAttribute("href")) ) @@ -119,28 +116,8 @@ for (const config of configs) { expect(res.ok).to.be.true } }) - } - - cy.get(".resize").then({ timeout: 60000 }, async $imgs => { - await testImages(Array.from($imgs), [ - { - width: 100, - height: 133, - }, - { - width: 100, - height: 160, - }, - { - width: 100, - height: 67, - }, - ]) - }) - cy.get(".fixed img:not([aria-hidden=true])").then( - { timeout: 60000 }, - async $imgs => { + cy.get(".resize").then({ timeout: 60000 }, async $imgs => { await testImages(Array.from($imgs), [ { width: 100, @@ -155,72 +132,112 @@ for (const config of configs) { height: 67, }, ]) - } - ) + }) - cy.get(".constrained img:not([aria-hidden=true])").then( - { timeout: 60000 }, - async $imgs => { - await testImages(Array.from($imgs), [ - { - width: 300, - height: 400, - }, - { - width: 300, - height: 481, - }, - { - width: 300, - height: 200, - }, - ]) - } - ) + cy.get(".fixed img:not([aria-hidden=true])").then( + { timeout: 60000 }, + async $imgs => { + await testImages(Array.from($imgs), [ + { + width: 100, + height: 133, + }, + { + width: 100, + height: 160, + }, + { + width: 100, + height: 67, + }, + ]) + } + ) + + cy.get(".constrained img:not([aria-hidden=true])").then( + { timeout: 60000 }, + async $imgs => { + await testImages(Array.from($imgs), [ + { + width: 300, + height: 400, + }, + { + width: 300, + height: 481, + }, + { + width: 300, + height: 200, + }, + ]) + } + ) + + cy.get(".full img:not([aria-hidden=true])").then( + { timeout: 60000 }, + async $imgs => { + await testImages(Array.from($imgs), [ + { + naturalHeight: 1333, + }, + { + naturalHeight: 1603, + }, + { + naturalHeight: 666, + }, + ]) + } + ) + }) - cy.get(".full img:not([aria-hidden=true])").then( - { timeout: 60000 }, - async $imgs => { - await testImages(Array.from($imgs), [ - { - naturalHeight: 1333, - }, - { - naturalHeight: 1603, - }, - { - naturalHeight: 666, - }, - ]) + it(`should render a placeholder`, () => { + if (config.placeholders) { + cy.get(".fixed [data-placeholder-image]") + .first() + .should("have.css", "background-color", "rgb(232, 184, 8)") + cy.get(".constrained [data-placeholder-image]") + .first() + .should($el => { + expect($el.prop("tagName")).to.be.equal("IMG") + expect($el.prop("src")).to.contain("data:image/jpg;base64") + }) + cy.get(".constrained_traced [data-placeholder-image]") + .first() + .should($el => { + // traced falls back to DOMINANT_COLOR + expect($el.prop("tagName")).to.be.equal("DIV") + expect($el).to.be.empty + }) } - ) - }) - - it(`should render a placeholder`, () => { - if (config.placeholders) { - cy.get(".fixed [data-placeholder-image]") - .first() - .should("have.css", "background-color", "rgb(232, 184, 8)") - cy.get(".constrained [data-placeholder-image]") - .first() - .should($el => { - expect($el.prop("tagName")).to.be.equal("IMG") - expect($el.prop("src")).to.contain("data:image/jpg;base64") - }) - cy.get(".constrained_traced [data-placeholder-image]") + cy.get(".full [data-placeholder-image]") .first() .should($el => { - // traced falls back to DOMINANT_COLOR expect($el.prop("tagName")).to.be.equal("DIV") expect($el).to.be.empty }) - } - cy.get(".full [data-placeholder-image]") - .first() - .should($el => { - expect($el.prop("tagName")).to.be.equal("DIV") - expect($el).to.be.empty - }) + }) + }) + + it(`File CDN`, () => { + cy.get('[data-testid="file-public"]').then(async $urls => { + const urls = Array.from( + $urls.map((_, $url) => $url.getAttribute("href")) + ) + + // urls is array of href attribute, not absolute urls, so it already is stripped of origin + for (const urlWithoutOrigin of urls) { + // using Netlify Image CDN + expect(urlWithoutOrigin).to.match( + new RegExp(`^${PATH_PREFIX}/_gatsby/file`) + ) + const res = await fetch(urlWithoutOrigin, { + method: "HEAD", + }) + expect(res.ok).to.be.true + } + }) }) } ) diff --git a/e2e-tests/adapters/gatsby-node.ts b/e2e-tests/adapters/gatsby-node.ts index 3c876c4f8efe7..e054bf7e56693 100644 --- a/e2e-tests/adapters/gatsby-node.ts +++ b/e2e-tests/adapters/gatsby-node.ts @@ -12,7 +12,7 @@ export const createPages: GatsbyNode["createPages"] = async ({ }) => { const { data: ImageCDNRemoteFileFromPageContextData } = await graphql(` query ImageCDNGatsbyNode { - allMyRemoteFile { + allMyRemoteImage { nodes { id url @@ -41,6 +41,14 @@ export const createPages: GatsbyNode["createPages"] = async ({ full: gatsbyImage(layout: FULL_WIDTH, width: 500, placeholder: NONE) } } + allMyRemoteFile { + nodes { + id + url + filename + publicUrl + } + } } `) @@ -111,6 +119,21 @@ export const createPages: GatsbyNode["createPages"] = async ({ // Image CDN export const createSchemaCustomization: GatsbyNode["createSchemaCustomization"] = function createSchemaCustomization({ actions, schema, store }) { + actions.createTypes( + addRemoteFilePolyfillInterface( + schema.buildObjectType({ + name: "MyRemoteImage", + fields: {}, + interfaces: ["Node", "RemoteFile"], + }), + { + schema, + actions, + store, + } + ) + ) + actions.createTypes( addRemoteFilePolyfillInterface( schema.buildObjectType({ @@ -161,6 +184,14 @@ export const sourceNodes: GatsbyNode["sourceNodes"] = function sourceNodes({ width: 2000, height: 1333, }, + { + // svg is not considered for image cdn - file cdn will be used + name: "fileA.svg", + url: "https://www.gatsbyjs.com/Gatsby-Logo.svg", + mimeType: "image/svg+xml", + filename: "Gatsby-Logo.svg", + type: `MyRemoteFile`, + }, ] items.forEach((item, index) => { @@ -168,7 +199,7 @@ export const sourceNodes: GatsbyNode["sourceNodes"] = function sourceNodes({ id: createNodeId(`remote-file-${index}`), ...item, internal: { - type: "MyRemoteFile", + type: item.type ?? "MyRemoteImage", contentDigest: createContentDigest(item.url), }, }) diff --git a/e2e-tests/adapters/src/pages/index.jsx b/e2e-tests/adapters/src/pages/index.jsx index d0a2cad54df27..0032374b21b5f 100644 --- a/e2e-tests/adapters/src/pages/index.jsx +++ b/e2e-tests/adapters/src/pages/index.jsx @@ -40,15 +40,15 @@ const routes = [ url: "/routes/client-only/named-wildcard/corinno/fenring", }, { - text: "RemoteFile (ImageCDN) (SSG, Page Query)", + text: "RemoteFile (ImageCDN and FileCDN) (SSG, Page Query)", url: "/routes/remote-file", }, { - text: "RemoteFile (ImageCDN) (SSG, Page Context)", + text: "RemoteFile (ImageCDN and FileCDN) (SSG, Page Context)", url: "/routes/remote-file-data-from-context", }, { - text: "RemoteFile (ImageCDN) (SSR, Page Query)", + text: "RemoteFile (ImageCDN and FileCDN) (SSR, Page Query)", url: "/routes/ssr/remote-file", }, ] diff --git a/e2e-tests/adapters/src/pages/routes/remote-file.jsx b/e2e-tests/adapters/src/pages/routes/remote-file.jsx index d82c8c5030651..c93c3f7eff70b 100644 --- a/e2e-tests/adapters/src/pages/routes/remote-file.jsx +++ b/e2e-tests/adapters/src/pages/routes/remote-file.jsx @@ -7,11 +7,11 @@ import Layout from "../../components/layout" const RemoteFile = ({ data }) => { return ( - {data.allMyRemoteFile.nodes.map(node => { + {data.allMyRemoteImage.nodes.map(node => { return (

- + {node.filename}

@@ -39,13 +39,24 @@ const RemoteFile = ({ data }) => {
) })} + {data.allMyRemoteFile.nodes.map(node => { + return ( + + ) + })} ) } export const pageQuery = graphql` query SSGImageCDNPageQuery { - allMyRemoteFile { + allMyRemoteImage { nodes { id url @@ -74,6 +85,14 @@ export const pageQuery = graphql` full: gatsbyImage(layout: FULL_WIDTH, width: 500, placeholder: NONE) } } + allMyRemoteFile { + nodes { + id + url + filename + publicUrl + } + } } ` diff --git a/e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx b/e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx index a838f7948b5ee..923831a179b6d 100644 --- a/e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx +++ b/e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx @@ -7,11 +7,11 @@ import Layout from "../../../components/layout" const RemoteFile = ({ data }) => { return ( - {data.allMyRemoteFile.nodes.map(node => { + {data.allMyRemoteImage.nodes.map(node => { return (

- + {node.filename}

@@ -39,13 +39,24 @@ const RemoteFile = ({ data }) => {
) })} + {data.allMyRemoteFile.nodes.map(node => { + return ( + + ) + })} ) } export const pageQuery = graphql` query SSRImageCDNPageQuery { - allMyRemoteFile { + allMyRemoteImage { nodes { id url @@ -81,6 +92,14 @@ export const pageQuery = graphql` full: gatsbyImage(layout: FULL_WIDTH, width: 500, placeholder: NONE) } } + allMyRemoteFile { + nodes { + id + url + filename + publicUrl + } + } } ` diff --git a/e2e-tests/adapters/src/templates/remote-file-from-context.jsx b/e2e-tests/adapters/src/templates/remote-file-from-context.jsx index 2e2d8af24496f..d6532903dce06 100644 --- a/e2e-tests/adapters/src/templates/remote-file-from-context.jsx +++ b/e2e-tests/adapters/src/templates/remote-file-from-context.jsx @@ -6,11 +6,11 @@ import Layout from "../components/layout" const RemoteFile = ({ pageContext: data }) => { return ( - {data.allMyRemoteFile.nodes.map(node => { + {data.allMyRemoteImage.nodes.map(node => { return (

- + {node.filename}

@@ -38,6 +38,17 @@ const RemoteFile = ({ pageContext: data }) => {
) })} + {data.allMyRemoteFile.nodes.map(node => { + return ( + + ) + })} ) } diff --git a/packages/gatsby-adapter-netlify/package.json b/packages/gatsby-adapter-netlify/package.json index bb0c6bfebd5b6..0dd2e1552ae59 100644 --- a/packages/gatsby-adapter-netlify/package.json +++ b/packages/gatsby-adapter-netlify/package.json @@ -41,6 +41,7 @@ "devDependencies": { "@babel/cli": "^7.20.7", "@babel/core": "^7.20.12", + "@netlify/edge-functions": "^2.2.0", "babel-preset-gatsby-package": "^3.13.0-next.0", "cross-env": "^7.0.3", "memfs": "^4.6.0", diff --git a/packages/gatsby-adapter-netlify/src/__tests__/file-cdn-url-generator.ts b/packages/gatsby-adapter-netlify/src/__tests__/file-cdn-url-generator.ts new file mode 100644 index 0000000000000..97c8338d181b5 --- /dev/null +++ b/packages/gatsby-adapter-netlify/src/__tests__/file-cdn-url-generator.ts @@ -0,0 +1,198 @@ +import { generateFileUrl } from "../file-cdn-url-generator" + +describe(`generateFileUrl`, () => { + describe(`image`, () => { + // pathPrefix is not used for images + const pathPrefix = `/prefix` + + it(`should return a file based url`, () => { + const source = { + url: `https://example.com/file.jpg`, + filename: `file.jpg`, + mimeType: `image/jpeg`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/.netlify/images?url=https%3A%2F%2Fexample.com%2Ffile.jpg&cd=1234"` + ) + }) + + it(`should handle special characters`, () => { + const source = { + url: `https://example.com/file-éà.jpg`, + filename: `file-éà.jpg`, + mimeType: `image/jpeg`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/.netlify/images?url=https%3A%2F%2Fexample.com%2Ffile-%C3%A9%C3%A0.jpg&cd=1234"` + ) + }) + + it(`should handle spaces`, () => { + const source = { + url: `https://example.com/file test.jpg`, + filename: `file test.jpg`, + mimeType: `image/jpeg`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/.netlify/images?url=https%3A%2F%2Fexample.com%2Ffile+test.jpg&cd=1234"` + ) + }) + + it(`should handle html encoded urls`, () => { + const source = { + url: `https://example.com/file%20test.jpg`, + filename: `file test.jpg`, + mimeType: `image/jpeg`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/.netlify/images?url=https%3A%2F%2Fexample.com%2Ffile%2520test.jpg&cd=1234"` + ) + }) + }) + + describe(`no image`, () => { + describe(`no pathPrefix`, () => { + const pathPrefix = `` + + it(`should return a file based url`, () => { + const source = { + url: `https://example.com/file.pdf`, + filename: `file.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/_gatsby/file/9f2eba7a1dbc78363c52aeb0daec9031/file.pdf?url=https%3A%2F%2Fexample.com%2Ffile.pdf&cd=1234"` + ) + }) + + it(`should handle special characters`, () => { + const source = { + url: `https://example.com/file-éà.pdf`, + filename: `file-éà.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/_gatsby/file/8802451220032a66565f179e89e00a83/file-%C3%A9%C3%A0.pdf?url=https%3A%2F%2Fexample.com%2Ffile-%C3%A9%C3%A0.pdf&cd=1234"` + ) + }) + + it(`should handle spaces`, () => { + const source = { + url: `https://example.com/file test.pdf`, + filename: `file test.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/_gatsby/file/6e41758c045f4509e19938d738d2a23c/file%20test.pdf?url=https%3A%2F%2Fexample.com%2Ffile+test.pdf&cd=1234"` + ) + }) + + it(`should handle html encoded urls`, () => { + const source = { + url: `https://example.com/file%20test.pdf`, + filename: `file test.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/_gatsby/file/799c0b15477311f5b8d9f635594671f2/file%20test.pdf?url=https%3A%2F%2Fexample.com%2Ffile%2520test.pdf&cd=1234"` + ) + }) + }) + + describe(`with pathPrefix`, () => { + const pathPrefix = `/prefix` + + it(`should return a file based url`, () => { + const source = { + url: `https://example.com/file.pdf`, + filename: `file.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/prefix/_gatsby/file/9f2eba7a1dbc78363c52aeb0daec9031/file.pdf?url=https%3A%2F%2Fexample.com%2Ffile.pdf&cd=1234"` + ) + }) + + it(`should handle special characters`, () => { + const source = { + url: `https://example.com/file-éà.pdf`, + filename: `file-éà.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/prefix/_gatsby/file/8802451220032a66565f179e89e00a83/file-%C3%A9%C3%A0.pdf?url=https%3A%2F%2Fexample.com%2Ffile-%C3%A9%C3%A0.pdf&cd=1234"` + ) + }) + + it(`should handle spaces`, () => { + const source = { + url: `https://example.com/file test.pdf`, + filename: `file test.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/prefix/_gatsby/file/6e41758c045f4509e19938d738d2a23c/file%20test.pdf?url=https%3A%2F%2Fexample.com%2Ffile+test.pdf&cd=1234"` + ) + }) + + it(`should handle html encoded urls`, () => { + const source = { + url: `https://example.com/file%20test.pdf`, + filename: `file test.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, + } + + expect(generateFileUrl(source, pathPrefix)).toMatchInlineSnapshot( + `"/prefix/_gatsby/file/799c0b15477311f5b8d9f635594671f2/file%20test.pdf?url=https%3A%2F%2Fexample.com%2Ffile%2520test.pdf&cd=1234"` + ) + }) + }) + }) +}) diff --git a/packages/gatsby-adapter-netlify/src/__tests__/image-cdn-url-generator.ts b/packages/gatsby-adapter-netlify/src/__tests__/image-cdn-url-generator.ts index 9b940d7c7f036..58567e42e3f06 100644 --- a/packages/gatsby-adapter-netlify/src/__tests__/image-cdn-url-generator.ts +++ b/packages/gatsby-adapter-netlify/src/__tests__/image-cdn-url-generator.ts @@ -1,6 +1,11 @@ +import type { ImageCdnTransformArgs } from "gatsby" + import { generateImageUrl, generateImageArgs } from "../image-cdn-url-generator" describe(`generateImageUrl`, () => { + // pathPrefix is not used for images + const pathPrefix = `/prefix` + const source = { url: `https://example.com/image.jpg`, filename: `image.jpg`, @@ -12,13 +17,17 @@ describe(`generateImageUrl`, () => { it(`should return an image based url`, () => { expect( - generateImageUrl(source, { - width: 100, - height: 100, - cropFocus: `top`, - format: `webp`, - quality: 80, - }) + generateImageUrl( + source, + { + width: 100, + height: 100, + cropFocus: `top`, + format: `webp`, + quality: 80, + }, + pathPrefix + ) ).toMatchInlineSnapshot( `"/.netlify/images?w=100&h=100&fit=cover&position=top&fm=webp&q=80&url=https%3A%2F%2Fexample.com%2Fimage.jpg&cd=1234"` ) @@ -35,13 +44,17 @@ describe(`generateImageUrl`, () => { } expect( - generateImageUrl(source, { - width: 100, - height: 100, - cropFocus: `top`, - format: `webp`, - quality: 80, - }) + generateImageUrl( + source, + { + width: 100, + height: 100, + cropFocus: `top`, + format: `webp`, + quality: 80, + }, + pathPrefix + ) ).toMatchInlineSnapshot( `"/.netlify/images?w=100&h=100&fit=cover&position=top&fm=webp&q=80&url=https%3A%2F%2Fexample.com%2Fimage-%C3%A9%C3%A0.jpg&cd=1234"` ) @@ -58,13 +71,17 @@ describe(`generateImageUrl`, () => { } expect( - generateImageUrl(source, { - width: 100, - height: 100, - cropFocus: `top`, - format: `webp`, - quality: 80, - }) + generateImageUrl( + source, + { + width: 100, + height: 100, + cropFocus: `top`, + format: `webp`, + quality: 80, + }, + pathPrefix + ) ).toMatchInlineSnapshot( `"/.netlify/images?w=100&h=100&fit=cover&position=top&fm=webp&q=80&url=https%3A%2F%2Fexample.com%2Fimage+test.jpg&cd=1234"` ) @@ -81,13 +98,17 @@ describe(`generateImageUrl`, () => { } expect( - generateImageUrl(source, { - width: 100, - height: 100, - cropFocus: `top`, - format: `webp`, - quality: 80, - }) + generateImageUrl( + source, + { + width: 100, + height: 100, + cropFocus: `top`, + format: `webp`, + quality: 80, + }, + pathPrefix + ) ).toMatchInlineSnapshot( `"/.netlify/images?w=100&h=100&fit=cover&position=top&fm=webp&q=80&url=https%3A%2F%2Fexample.com%2Fimage%2520test.jpg&cd=1234"` ) @@ -99,15 +120,19 @@ describe(`generateImageUrl`, () => { [`cropFocus`, `position`, `center,right`], [`format`, `fm`, `webp`], [`quality`, `q`, 60], - ] as Array<[keyof ImageArgs, string, ImageArgs[keyof ImageArgs]]>)( + ] as Array<[keyof ImageCdnTransformArgs, string, ImageCdnTransformArgs[keyof ImageCdnTransformArgs]]>)( `should set %s in image args`, (key, queryKey, value) => { const url = new URL( // @ts-ignore remove typings - `https://netlify.com${generateImageUrl(source, { - format: `webp`, - [key]: value, - })}` + `https://netlify.com${generateImageUrl( + source, + { + format: `webp`, + [key]: value, + }, + pathPrefix + )}` ) expect(url.searchParams.get(queryKey)).toEqual(value.toString()) diff --git a/packages/gatsby-adapter-netlify/src/file-cdn-handler.ts b/packages/gatsby-adapter-netlify/src/file-cdn-handler.ts new file mode 100644 index 0000000000000..cc8482c790c04 --- /dev/null +++ b/packages/gatsby-adapter-netlify/src/file-cdn-handler.ts @@ -0,0 +1,87 @@ +import fs from "fs-extra" +import * as path from "path" + +import packageJson from "gatsby-adapter-netlify/package.json" + +export interface IFunctionManifest { + version: 1 + functions: Array< + | { + function: string + name?: string + path: string + cache?: "manual" + generator: string + } + | { + function: string + name?: string + pattern: string + cache?: "manual" + generator: string + } + > + layers?: Array<{ name: `https://${string}/mod.ts`; flag: string }> + import_map?: string +} + +export async function prepareFileCdnHandler({ + pathPrefix, +}: { + pathPrefix: string +}): Promise { + const functionId = `file-cdn` + + const edgeFunctionsManifestPath = path.join( + process.cwd(), + `.netlify`, + `edge-functions`, + `manifest.json` + ) + + const fileCdnEdgeFunction = path.join( + process.cwd(), + `.netlify`, + `edge-functions`, + `${functionId}`, + `${functionId}.mts` + ) + + const handlerSource = /* typescript */ ` + import type { Context } from "@netlify/edge-functions" + + export default async (req: Request, context: Context): Promise => { + const url = new URL(req.url) + const remoteUrl = url.searchParams.get("url") + + // @todo: use allowed remote urls to decide wether request should be allowed + // blocked by https://github.com/gatsbyjs/gatsby/pull/38719 + const isAllowed = true + if (isAllowed) { + console.log(\`URL allowed\`, { remoteUrl }) + return fetch(remoteUrl); + } else { + console.error(\`URL not allowed: \${remoteUrl}\`) + return new Response("Not allowed", { status: 403 }) + } + } + ` + + await fs.outputFileSync(fileCdnEdgeFunction, handlerSource) + + const manifest: IFunctionManifest = { + functions: [ + { + path: `${pathPrefix}/_gatsby/file/*`, + function: functionId, + generator: `gatsby-adapter-netlify@${ + packageJson?.version ?? `unknown` + }`, + }, + ], + layers: [], + version: 1, + } + + await fs.outputJSON(edgeFunctionsManifestPath, manifest) +} diff --git a/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts b/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts index 813eb457f2b9a..1bc3e37abfe64 100644 --- a/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts +++ b/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts @@ -1,11 +1,37 @@ +import crypto from "crypto" +import { basename } from "path" + import type { FileCdnUrlGeneratorFn, FileCdnSourceImage } from "gatsby" -export const generateImageUrl: FileCdnUrlGeneratorFn = - function generateImageUrl(source: FileCdnSourceImage): string { - const placeholderOrigin = `http://netlify.com` - const baseURL = new URL(`${placeholderOrigin}/.netlify/images`) +function isImage(node: FileCdnSourceImage): boolean { + return node.mimeType.startsWith(`image/`) && node.mimeType !== `image/svg+xml` +} + +const placeholderOrigin = `http://netlify.com` + +export const generateFileUrl: FileCdnUrlGeneratorFn = function generateFileUrl( + source: FileCdnSourceImage, + pathPrefix: string +): string { + // use image cdn for images and file lambda for other files + let baseURL: URL + if (isImage(source)) { + baseURL = new URL(`${placeholderOrigin}/.netlify/images`) + baseURL.searchParams.append(`url`, source.url) + baseURL.searchParams.append(`cd`, source.internal.contentDigest) + } else { + baseURL = new URL( + `${placeholderOrigin}${pathPrefix}/_gatsby/file/${crypto + .createHash(`md5`) + .update(source.url) + .digest(`hex`)}/${basename(source.filename)}` + ) + baseURL.searchParams.append(`url`, source.url) + baseURL.searchParams.append(`cd`, source.internal.contentDigest) return `${baseURL.pathname}${baseURL.search}` } + return `${baseURL.pathname}${baseURL.search}` +} -export default generateImageUrl +export default generateFileUrl diff --git a/packages/gatsby-adapter-netlify/src/index.ts b/packages/gatsby-adapter-netlify/src/index.ts index 15072d323945e..09c802a27fcf4 100644 --- a/packages/gatsby-adapter-netlify/src/index.ts +++ b/packages/gatsby-adapter-netlify/src/index.ts @@ -1,6 +1,7 @@ import { join } from "path" import type { AdapterInit, IAdapterConfig } from "gatsby" import { prepareFunctionVariants } from "./lambda-handler" +import { prepareFileCdnHandler } from "./file-cdn-handler" import { handleRoutesManifest } from "./route-handler" import packageJson from "gatsby-adapter-netlify/package.json" @@ -35,6 +36,16 @@ async function getCacheUtils(): Promise { } const createNetlifyAdapter: AdapterInit = options => { + let useNetlifyImageCDN = options?.imageCDN + if ( + typeof useNetlifyImageCDN === `undefined` && + typeof process.env.NETLIFY_IMAGE_CDN !== `undefined` + ) { + useNetlifyImageCDN = + process.env.NETLIFY_IMAGE_CDN === `true` || + process.env.NETLIFY_IMAGE_CDN === `1` + } + return { name: `gatsby-adapter-netlify`, cache: { @@ -72,7 +83,12 @@ const createNetlifyAdapter: AdapterInit = options => { routesManifest, functionsManifest, headerRoutes, + pathPrefix, }): Promise { + if (useNetlifyImageCDN) { + await prepareFileCdnHandler({ pathPrefix }) + } + const { lambdasThatUseCaching } = await handleRoutesManifest( routesManifest, headerRoutes @@ -118,16 +134,6 @@ const createNetlifyAdapter: AdapterInit = options => { excludeDatastoreFromEngineFunction = false } - let useNetlifyImageCDN = options?.imageCDN - if ( - typeof useNetlifyImageCDN === `undefined` && - typeof process.env.NETLIFY_IMAGE_CDN !== `undefined` - ) { - useNetlifyImageCDN = - process.env.NETLIFY_IMAGE_CDN === `true` || - process.env.NETLIFY_IMAGE_CDN === `1` - } - return { excludeDatastoreFromEngineFunction, deployURL, diff --git a/packages/gatsby-plugin-utils/src/polyfill-remote-file/__tests__/public-resolver.ts b/packages/gatsby-plugin-utils/src/polyfill-remote-file/__tests__/public-resolver.ts index b0098b9faadd8..615c7140c6e68 100644 --- a/packages/gatsby-plugin-utils/src/polyfill-remote-file/__tests__/public-resolver.ts +++ b/packages/gatsby-plugin-utils/src/polyfill-remote-file/__tests__/public-resolver.ts @@ -38,6 +38,10 @@ describe(`publicResolver`, () => { generateFileUrl({ filename: source.filename, url: source.url, + mimeType: source.mimeType, + internal: { + contentDigest: source.internal.contentDigest, + }, }) ) }) @@ -61,6 +65,10 @@ describe(`publicResolver`, () => { generateFileUrl({ filename: source.filename, url: source.url, + mimeType: source.mimeType, + internal: { + contentDigest: source.internal.contentDigest, + }, }) ) }) diff --git a/packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/__tests__/url-generator.ts b/packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/__tests__/url-generator.ts index 4223bc230ca15..606a300eb7373 100644 --- a/packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/__tests__/url-generator.ts +++ b/packages/gatsby-plugin-utils/src/polyfill-remote-file/utils/__tests__/url-generator.ts @@ -56,6 +56,10 @@ describe(`url-generator`, () => { const fileSource = { url: `https://example.com/file.pdf`, filename: `file.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, } expect( @@ -108,6 +112,10 @@ describe(`url-generator`, () => { file: generateFileUrl({ url: fileUrlToEncrypt, filename: `file.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, }), image: generateImageUrl(imageNode, resizeArgs), }[type] @@ -186,6 +194,10 @@ describe(`url-generator`, () => { const source = { url: `https://example.com/file.pdf`, filename: `file.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, } expect(generateFileUrl(source)).toMatchInlineSnapshot( @@ -197,6 +209,10 @@ describe(`url-generator`, () => { const source = { url: `https://example.com/file-éà.pdf`, filename: `file-éà.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, } expect(generateFileUrl(source)).toMatchInlineSnapshot( @@ -208,6 +224,10 @@ describe(`url-generator`, () => { const source = { url: `https://example.com/file test.pdf`, filename: `file test.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, } expect(generateFileUrl(source)).toMatchInlineSnapshot( @@ -219,6 +239,10 @@ describe(`url-generator`, () => { const source = { url: `https://example.com/file%20test.pdf`, filename: `file test.pdf`, + mimeType: `application/pdf`, + internal: { + contentDigest: `1234`, + }, } expect(generateFileUrl(source)).toMatchInlineSnapshot( diff --git a/yarn.lock b/yarn.lock index c8e4b42adcf29..edca74a7eaf04 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3262,6 +3262,11 @@ path-exists "^5.0.0" readdirp "^3.4.0" +"@netlify/edge-functions@^2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@netlify/edge-functions/-/edge-functions-2.2.0.tgz#5f7f5c7602a7f98888a4b4421576ca609dad2083" + integrity sha512-8UeKA2nUDB0oWE+Z0gLpA7wpLq8nM+NrZEQMfSdzfMJNvmYVabil/mS07rb0EBrUxM9PCKidKenaiCRnPTBSKw== + "@netlify/functions@^1.6.0": version "1.6.0" resolved "https://registry.yarnpkg.com/@netlify/functions/-/functions-1.6.0.tgz#c373423e6fef0e6f7422ac0345e8bbf2cb692366" From 20861ba97a368b09140c00ae7d33b2cac3d9144d Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 5 Dec 2023 16:59:45 +0100 Subject: [PATCH 05/11] why edge function was not deployed? --- e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs index f8ea6ef9cacc9..afea0d50c2d70 100644 --- a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs +++ b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs @@ -1,5 +1,6 @@ // @ts-check import { execa } from "execa" +import { inspect } from "util" // only set NETLIFY_SITE_ID from E2E_ADAPTERS_NETLIFY_SITE_ID if it's set if (process.env.E2E_ADAPTERS_NETLIFY_SITE_ID) { @@ -20,7 +21,7 @@ await execa(`npm`, [`run`, `clean`], { stdio: `inherit` }) const deployResults = await execa( "ntl", - ["deploy", "--build", "--json", "--message", deployTitle], + ["deploy", "--build", "--json", "--cwd=.", "--message", deployTitle], { reject: false, } @@ -42,6 +43,7 @@ const deployInfo = JSON.parse(deployResults.stdout) const deployUrl = deployInfo.deploy_url + (process.env.PATH_PREFIX ?? ``) process.env.DEPLOY_URL = deployUrl +console.log(inspect({ deployInfo }, { depth: Infinity })) console.log(`Deployed to ${deployUrl}`) try { From ba1d8882bf03c8e57ff761a3d0b8a4ea1a6d6393 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 5 Dec 2023 17:41:13 +0100 Subject: [PATCH 06/11] bump netlify-cli --- e2e-tests/adapters/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/adapters/package.json b/e2e-tests/adapters/package.json index 38a970a0e77b2..b9aa2149e4cd8 100644 --- a/e2e-tests/adapters/package.json +++ b/e2e-tests/adapters/package.json @@ -36,7 +36,7 @@ "cypress": "^12.14.0", "dotenv": "^8.6.0", "gatsby-cypress": "^3.11.0", - "netlify-cli": "^15.8.0", + "netlify-cli": "^17.9.0", "npm-run-all": "^4.1.5", "start-server-and-test": "^2.0.0", "typescript": "^5.1.6" From 747b8744626feb65942e4f49fc0acf6c963ebb6f Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 5 Dec 2023 18:19:38 +0100 Subject: [PATCH 07/11] ? --- e2e-tests/adapters/package.json | 1 + e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/e2e-tests/adapters/package.json b/e2e-tests/adapters/package.json index b9aa2149e4cd8..1d4641907bf5f 100644 --- a/e2e-tests/adapters/package.json +++ b/e2e-tests/adapters/package.json @@ -35,6 +35,7 @@ "cross-env": "^7.0.3", "cypress": "^12.14.0", "dotenv": "^8.6.0", + "tree-cli": "0.6.7", "gatsby-cypress": "^3.11.0", "netlify-cli": "^17.9.0", "npm-run-all": "^4.1.5", diff --git a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs index afea0d50c2d70..886e0663f2e62 100644 --- a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs +++ b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs @@ -1,6 +1,7 @@ // @ts-check import { execa } from "execa" import { inspect } from "util" +import tree from "tree-cli" // only set NETLIFY_SITE_ID from E2E_ADAPTERS_NETLIFY_SITE_ID if it's set if (process.env.E2E_ADAPTERS_NETLIFY_SITE_ID) { @@ -43,6 +44,14 @@ const deployInfo = JSON.parse(deployResults.stdout) const deployUrl = deployInfo.deploy_url + (process.env.PATH_PREFIX ?? ``) process.env.DEPLOY_URL = deployUrl +const { report } = await tree({ + base: ".netlify", + noreport: true, // this just avoid outputting by default, still is generated + l: Infinity, +}) + +console.log(report) + console.log(inspect({ deployInfo }, { depth: Infinity })) console.log(`Deployed to ${deployUrl}`) From 697ea61049d5b808cbeb80f4c6dfd98b758ce53b Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 6 Dec 2023 07:05:52 +0100 Subject: [PATCH 08/11] bump node for adapters tests --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e17a153324ebc..b9d39a3f86996 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -455,6 +455,8 @@ jobs: e2e_tests_adapters: <<: *e2e-executor + docker: + - image: cypress/browsers:node-18.16.1-chrome-114.0.5735.133-1-ff-114.0.2-edge-114.0.1823.51-1 steps: - run: echo 'export CYPRESS_RECORD_KEY="${CY_CLOUD_ADAPTERS}"' >> "$BASH_ENV" - e2e-test: From 4c20ceb64f52b8445b3ab3da4935c5caf9a9cdc5 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 6 Dec 2023 07:34:48 +0100 Subject: [PATCH 09/11] add execa to dev deps --- e2e-tests/adapters/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-tests/adapters/package.json b/e2e-tests/adapters/package.json index 1d4641907bf5f..fea84269501ed 100644 --- a/e2e-tests/adapters/package.json +++ b/e2e-tests/adapters/package.json @@ -36,6 +36,7 @@ "cypress": "^12.14.0", "dotenv": "^8.6.0", "tree-cli": "0.6.7", + "execa": "^6.1.0", "gatsby-cypress": "^3.11.0", "netlify-cli": "^17.9.0", "npm-run-all": "^4.1.5", From d0cbd0acc38157adee667a62cdba3b953e777fc6 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 6 Dec 2023 08:29:23 +0100 Subject: [PATCH 10/11] cleanup --- e2e-tests/adapters/package.json | 1 - .../scripts/deploy-and-run/netlify.mjs | 45 +++++++------------ 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/e2e-tests/adapters/package.json b/e2e-tests/adapters/package.json index fea84269501ed..1a752be74f4ef 100644 --- a/e2e-tests/adapters/package.json +++ b/e2e-tests/adapters/package.json @@ -35,7 +35,6 @@ "cross-env": "^7.0.3", "cypress": "^12.14.0", "dotenv": "^8.6.0", - "tree-cli": "0.6.7", "execa": "^6.1.0", "gatsby-cypress": "^3.11.0", "netlify-cli": "^17.9.0", diff --git a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs index 886e0663f2e62..1326826e6b7ba 100644 --- a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs +++ b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs @@ -1,7 +1,5 @@ // @ts-check import { execa } from "execa" -import { inspect } from "util" -import tree from "tree-cli" // only set NETLIFY_SITE_ID from E2E_ADAPTERS_NETLIFY_SITE_ID if it's set if (process.env.E2E_ADAPTERS_NETLIFY_SITE_ID) { @@ -44,35 +42,26 @@ const deployInfo = JSON.parse(deployResults.stdout) const deployUrl = deployInfo.deploy_url + (process.env.PATH_PREFIX ?? ``) process.env.DEPLOY_URL = deployUrl -const { report } = await tree({ - base: ".netlify", - noreport: true, // this just avoid outputting by default, still is generated - l: Infinity, -}) - -console.log(report) - -console.log(inspect({ deployInfo }, { depth: Infinity })) console.log(`Deployed to ${deployUrl}`) try { await execa(`npm`, [`run`, npmScriptToRun], { stdio: `inherit` }) } finally { - // if (!process.env.GATSBY_TEST_SKIP_CLEANUP) { - // console.log(`Deleting project with deploy_id ${deployInfo.deploy_id}`) - // const deleteResponse = await execa("ntl", [ - // "api", - // "deleteDeploy", - // "--data", - // `{ "deploy_id": "${deployInfo.deploy_id}" }`, - // ]) - // if (deleteResponse.exitCode !== 0) { - // throw new Error( - // `Failed to delete project ${deleteResponse.stdout} ${deleteResponse.stderr} (${deleteResponse.exitCode})` - // ) - // } - // console.log( - // `Successfully deleted project with deploy_id ${deployInfo.deploy_id}` - // ) - // } + if (!process.env.GATSBY_TEST_SKIP_CLEANUP) { + console.log(`Deleting project with deploy_id ${deployInfo.deploy_id}`) + const deleteResponse = await execa("ntl", [ + "api", + "deleteDeploy", + "--data", + `{ "deploy_id": "${deployInfo.deploy_id}" }`, + ]) + if (deleteResponse.exitCode !== 0) { + throw new Error( + `Failed to delete project ${deleteResponse.stdout} ${deleteResponse.stderr} (${deleteResponse.exitCode})` + ) + } + console.log( + `Successfully deleted project with deploy_id ${deployInfo.deploy_id}` + ) + } } From c89d499c5a8018c7488f2a17eb21b929f16e66e7 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 6 Dec 2023 09:07:34 +0100 Subject: [PATCH 11/11] some jsdocs updates --- .../src/polyfill-remote-file/types.ts | 10 +++++++++- packages/gatsby/src/utils/adapter/types.ts | 5 +---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/gatsby-plugin-utils/src/polyfill-remote-file/types.ts b/packages/gatsby-plugin-utils/src/polyfill-remote-file/types.ts index 3479b51ebe6d1..f6cea08de015a 100644 --- a/packages/gatsby-plugin-utils/src/polyfill-remote-file/types.ts +++ b/packages/gatsby-plugin-utils/src/polyfill-remote-file/types.ts @@ -128,7 +128,15 @@ export type ImageCdnUrlGeneratorFn = ( export type FileCdnSourceImage = CdnSourceImage +/** + * The function is used to optimize image delivery by generating URLs that leverage CDN capabilities + * @param {FileCdnSourceImage} source - An object representing the source file, including properties like + * URL, filename, and MIME type. + * @param {string} pathPrefix - A string representing the path prefix to be prepended to the + * generated URL. + * @returns {string} A string representing the generated URL for the file on the CDN. + */ export type FileCdnUrlGeneratorFn = ( - source: ImageCdnSourceImage, + source: FileCdnSourceImage, pathPrefix: string ) => string diff --git a/packages/gatsby/src/utils/adapter/types.ts b/packages/gatsby/src/utils/adapter/types.ts index 4cb15f935f23d..d5f7b8d1a617b 100644 --- a/packages/gatsby/src/utils/adapter/types.ts +++ b/packages/gatsby/src/utils/adapter/types.ts @@ -176,10 +176,7 @@ export interface IAdapterConfig { */ imageCDNUrlGeneratorModulePath?: string /** - * @todo: specify that image cdn does image transformation (like resizing) while file cdn does not - * (but might do content negotiation) - * Path to a CommonJS module that implements an file CDN URL generation function. The function - * is used to optimize image delivery by generating URLs that leverage CDN capabilities. This module + * Path to a CommonJS module that implements an file CDN URL generation function. This module * should have a default export function that conforms to the {@link FileCdnUrlGeneratorFn} type: * * Adapters should provide an absolute path to this module.