From bbdddd74063161c838c652e49448301fef5987c9 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 25 Jan 2024 15:48:07 +0100 Subject: [PATCH] fix: add missing fs method rewrites to handle fetchRemoteFile in dsg/ssr engine (#38822) * test(adapters-e2e): enable placeholder tests in ssr remote-file * fix(gatsby-adapter-netlify): bundling file-cdn * fix(gatsby): set pathPrefix in engines * fix(gatsby): add missing fs method rewrites to handle fetchRemoteFile in dsg/ssr engine --- .../adapters/cypress/e2e/remote-file.cy.ts | 2 +- .../src/pages/routes/ssr/remote-file.jsx | 26 ++-- .../src/file-cdn-url-generator.ts | 5 +- .../schema/graphql-engine/bundle-webpack.ts | 8 +- .../gatsby/src/schema/graphql-engine/entry.ts | 22 ++++ .../src/utils/page-ssr-module/lambda.ts | 121 +++++++++++++++++- 6 files changed, 160 insertions(+), 24 deletions(-) diff --git a/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts b/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts index 5f168eab76c80..56adae8cec9b1 100644 --- a/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts +++ b/e2e-tests/adapters/cypress/e2e/remote-file.cy.ts @@ -29,7 +29,7 @@ const configs = [ { title: `remote-file (SSR, Page Query)`, pagePath: `/routes/ssr/remote-file/`, - placeholders: false, + placeholders: true, }, ] diff --git a/e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx b/e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx index 87477e2ea3b9c..2afa15d950340 100644 --- a/e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx +++ b/e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx @@ -4,7 +4,7 @@ import React from "react" import { GatsbyImage } from "gatsby-plugin-image" import Layout from "../../../components/layout" -const RemoteFile = ({ data }) => { +const RemoteFile = ({ data, serverData }) => { return ( {data.allMyRemoteImage.nodes.map(node => { @@ -54,6 +54,7 @@ const RemoteFile = ({ data }) => { ) })} + 
{JSON.stringify(serverData, null, 2)}
 ) } @@ -65,8 +66,7 @@ export const pageQuery = graphql` id url filename - # FILE_CDN is not supported in SSR/DSG yet - # publicUrl + publicUrl resize(width: 100) { height width @@ -75,23 +75,17 @@ export const pageQuery = graphql` fixed: gatsbyImage( layout: FIXED width: 100 - # only NONE placeholder is supported in SSR/DSG - # placeholder: DOMINANT_COLOR - placeholder: NONE + placeholder: DOMINANT_COLOR ) constrained: gatsbyImage( layout: CONSTRAINED width: 300 - # only NONE placeholder is supported in SSR/DSG - # placeholder: DOMINANT_COLOR - placeholder: NONE + placeholder: BLURRED ) constrained_traced: gatsbyImage( layout: CONSTRAINED width: 300 - # only NONE placeholder is supported in SSR/DSG - # placeholder: DOMINANT_COLOR - placeholder: NONE + placeholder: TRACED_SVG ) full: gatsbyImage(layout: FULL_WIDTH, width: 500, placeholder: NONE) } @@ -109,3 +103,11 @@ export const pageQuery = graphql` ` export default RemoteFile + +export function getServerData() { + return { + props: { + ssr: true, + }, + } +} diff --git a/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts b/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts index f24efd9c8281c..4cb22b0fa07f6 100644 --- a/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts +++ b/packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts @@ -1,4 +1,4 @@ -import crypto from "crypto" +import { createHash } from "crypto" import { basename } from "path" import type { FileCdnUrlGeneratorFn, FileCdnSourceImage } from "gatsby" @@ -21,8 +21,7 @@ export const generateFileUrl: FileCdnUrlGeneratorFn = function generateFileUrl( baseURL.searchParams.append(`cd`, source.internal.contentDigest) } else { baseURL = new URL( - `${placeholderOrigin}${pathPrefix}/_gatsby/file/${crypto - .createHash(`md5`) + `${placeholderOrigin}${pathPrefix}/_gatsby/file/${createHash(`md5`) .update(source.url) .digest(`hex`)}/${basename(source.filename)}` ) diff --git a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts index 4b87d2e5115a9..bf73d6021cdd3 100644 --- a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts +++ b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts @@ -129,6 +129,11 @@ export async function createGraphqlEngineBundle( reporter: Reporter, isVerbose?: boolean ): Promise { + const state = store.getState() + const pathPrefix = state.program.prefixPaths + ? state.config.pathPrefix ?? `` + : `` + const schemaSnapshotString = await fs.readFile( path.join(rootDir, `.cache`, `schema.gql`), `utf-8` @@ -151,7 +156,7 @@ export async function createGraphqlEngineBundle( // We force a specific lmdb binary module if we detected a broken lmdb installation or if we detect the presence of an adapter let forcedLmdbBinaryModule: string | undefined = undefined - if (store.getState().adapter.instance) { + if (state.adapter.instance) { forcedLmdbBinaryModule = `${lmdbPackage}/node.abi83.glibc.node` } // We always force the binary if we've installed an alternative path @@ -317,6 +322,7 @@ export async function createGraphqlEngineBundle( "process.env.GATSBY_SKIP_WRITING_SCHEMA_TO_FILE": `true`, "process.env.NODE_ENV": JSON.stringify(`production`), SCHEMA_SNAPSHOT: JSON.stringify(schemaSnapshotString), + PATH_PREFIX: JSON.stringify(pathPrefix), "process.env.GATSBY_LOGGER": JSON.stringify(`yurnalist`), "process.env.GATSBY_SLICES": JSON.stringify( !!process.env.GATSBY_SLICES diff --git a/packages/gatsby/src/schema/graphql-engine/entry.ts b/packages/gatsby/src/schema/graphql-engine/entry.ts index f64a93df57561..c44ce3dcaded6 100644 --- a/packages/gatsby/src/schema/graphql-engine/entry.ts +++ b/packages/gatsby/src/schema/graphql-engine/entry.ts @@ -43,12 +43,34 @@ export class GraphQLEngine { this.getRunner() } + private setupPathPrefix(pathPrefix: string): void { + if (pathPrefix) { + store.dispatch({ + type: `SET_PROGRAM`, + payload: { + prefixPaths: true, + }, + }) + + store.dispatch({ + type: `SET_SITE_CONFIG`, + payload: { + ...store.getState().config, + pathPrefix, + }, + }) + } + } + private async _doGetRunner(): Promise { await tracerReadyPromise const wrapActivity = reporter.phantomActivity(`Initializing GraphQL Engine`) wrapActivity.start() try { + // @ts-ignore PATH_PREFIX is being "inlined" by bundler + this.setupPathPrefix(PATH_PREFIX) + // @ts-ignore SCHEMA_SNAPSHOT is being "inlined" by bundler store.dispatch(actions.createTypes(SCHEMA_SNAPSHOT)) diff --git a/packages/gatsby/src/utils/page-ssr-module/lambda.ts b/packages/gatsby/src/utils/page-ssr-module/lambda.ts index c60cb5f37e77f..81ae8f6938eff 100644 --- a/packages/gatsby/src/utils/page-ssr-module/lambda.ts +++ b/packages/gatsby/src/utils/page-ssr-module/lambda.ts @@ -10,7 +10,7 @@ import { promisify } from "util" import type { IGatsbyPage } from "../../internal" import type { ISSRData } from "./entry" -import { link } from "linkfs" +import { link, rewritableMethods as linkRewritableMethods } from "linkfs" const cdnDatastore = `%CDN_DATASTORE_PATH%` const PATH_PREFIX = `%PATH_PREFIX%` @@ -30,7 +30,7 @@ function setupFsWrapper(): string { global.__GATSBY.root = TEMP_DIR // TODO: don't hardcode this - const cacheDir = `/var/task/.cache` + const cacheDir = `${process.cwd()}/.cache` // we need to rewrite fs const rewrites = [ @@ -47,8 +47,63 @@ function setupFsWrapper(): string { to: TEMP_CACHE_DIR, rewrites, }) + + // copied from https://github.com/streamich/linkfs/blob/master/src/index.ts#L126-L142 + function mapPathUsingRewrites(fsPath: fs.PathLike): string { + let filename = path.resolve(String(fsPath)) + for (const [from, to] of rewrites) { + if (filename.indexOf(from) === 0) { + const rootRegex = /(?:^[a-zA-Z]:\\$)|(?:^\/$)/ // C:\ vs / + const isRoot = from.match(rootRegex) + const baseRegex = `^(` + from.replace(/\\/g, `\\\\`) + `)` + + if (isRoot) { + const regex = new RegExp(baseRegex) + filename = filename.replace(regex, () => to + path.sep) + } else { + const regex = new RegExp(baseRegex + `(\\\\|/|$)`) + filename = filename.replace(regex, (_match, _p1, p2) => to + p2) + } + } + } + return filename + } + + function applyRename< + T = typeof import("fs") | typeof import("fs").promises + >(fsToRewrite: T, lfs: T, method: "rename" | "renameSync"): void { + const original = fsToRewrite[method] + if (original) { + // @ts-ignore - complains about __promisify__ which doesn't actually exist in runtime + lfs[method] = ( + ...args: Parameters + ): ReturnType => { + args[0] = mapPathUsingRewrites(args[0]) + args[1] = mapPathUsingRewrites(args[1]) + // @ts-ignore - can't decide which signature this is, but we just preserve the original + // signature here and mostly care about first 2 arguments being PathLike + return original.apply(fsToRewrite, args) + } + } + } + + // linkfs doesn't handle following methods, so we need to add them manually + linkRewritableMethods.push(`createWriteStream`, `createReadStream`, `rm`) + + function createLinkedFS< + T = typeof import("fs") | typeof import("fs").promises + >(fsToRewrite: T): T { + const linkedFS = link(fsToRewrite, rewrites) as T + + // linkfs doesn't handle `to` argument in `rename` and `renameSync` methods + applyRename(fsToRewrite, linkedFS, `rename`) + applyRename(fsToRewrite, linkedFS, `renameSync`) + + return linkedFS + } + // Alias the cache dir paths to the temp dir - const lfs = link(fs, rewrites) as typeof import("fs") + const lfs = createLinkedFS(fs) // linkfs doesn't pass across the `native` prop, which graceful-fs needs for (const key in lfs) { @@ -56,16 +111,68 @@ function setupFsWrapper(): string { lfs[key].native = fs[key].native } } - - const dbPath = path.join(TEMP_CACHE_DIR, `data`, `datastore`) - // 'promises' is not initially linked within the 'linkfs' // package, and is needed by underlying Gatsby code (the // @graphql-tools/code-file-loader) - lfs.promises = link(fs.promises, rewrites) + lfs.promises = createLinkedFS(fs.promises) + + const originalWritesStream = fs.WriteStream + function LinkedWriteStream( + this: fs.WriteStream, + ...args: Parameters<(typeof fs)["createWriteStream"]> + ): fs.WriteStream { + args[0] = mapPathUsingRewrites(args[0]) + args[1] = + typeof args[1] === `string` + ? { + flags: args[1], + // @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options) + fs: lfs, + } + : { + ...(args[1] || {}), + // @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options) + fs: lfs, + } + + // @ts-ignore TS doesn't like extending prototype "classes" + return originalWritesStream.apply(this, args) + } + LinkedWriteStream.prototype = Object.create(originalWritesStream.prototype) + // @ts-ignore TS doesn't like extending prototype "classes" + lfs.WriteStream = LinkedWriteStream + + const originalReadStream = fs.ReadStream + function LinkedReadStream( + this: fs.ReadStream, + ...args: Parameters<(typeof fs)["createReadStream"]> + ): fs.ReadStream { + args[0] = mapPathUsingRewrites(args[0]) + args[1] = + typeof args[1] === `string` + ? { + flags: args[1], + // @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options) + fs: lfs, + } + : { + ...(args[1] || {}), + // @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options) + fs: lfs, + } + + // @ts-ignore TS doesn't like extending prototype "classes" + return originalReadStream.apply(this, args) + } + LinkedReadStream.prototype = Object.create(originalReadStream.prototype) + // @ts-ignore TS doesn't like extending prototype "classes" + lfs.ReadStream = LinkedReadStream + + const dbPath = path.join(TEMP_CACHE_DIR, `data`, `datastore`) // Gatsby uses this instead of fs if present // eslint-disable-next-line no-underscore-dangle + // @ts-ignore __promisify__ stuff global._fsWrapper = lfs if (!cdnDatastore) {