From 5427cfcfcf18781cc136b8fa17ccdf213152a510 Mon Sep 17 00:00:00 2001 From: Vacilando Date: Thu, 13 Oct 2022 22:40:03 +0200 Subject: [PATCH 01/15] [gatsby-source-drupal] provide proxyUrl in addition to baseUrl to allow using CDN, API gateway, etc. #36811 --- packages/gatsby-source-drupal/README.md | 21 +++++++++++++++++++ .../gatsby-source-drupal/src/gatsby-node.js | 10 ++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md index 4327018e346a1..ddba438df7e3a 100644 --- a/packages/gatsby-source-drupal/README.md +++ b/packages/gatsby-source-drupal/README.md @@ -250,6 +250,27 @@ module.exports = { } ``` +## CDN + +You can add optional CDN or API gateway URL `proxyUrl` param. The URL can be a proxy of the Drupal `baseUrl` +or another URL where the Drupal json paths can be retrieved. + +```javascript +// In your gatsby-config.js +module.exports = { + plugins: [ + { + resolve: `gatsby-source-drupal`, + options: { + baseUrl: `https://live-contentacms.pantheonsite.io/`, + baseUrl: `https://xyz.cloudfront.net/`, + apiBase: `api`, // optional, defaults to `jsonapi` + }, + }, + ], +} +``` + ## GET Search Params You can append optional GET request params to the request url using `params` option. diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index cfe34cb110ba5..05fa4d4303070 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -155,6 +155,7 @@ exports.sourceNodes = async ( globalReporter = reporter const { baseUrl, + proxyUrl = baseUrl, apiBase = `jsonapi`, basicAuth = {}, filters, @@ -534,7 +535,12 @@ ${JSON.stringify(webhookBody, null, 4)}` } } - let d + // If proxyUrl is defined, use it instead of baseUrl to get the content. + if (proxyUrl !== baseUrl) { + url = url.replace(baseUrl, proxyUrl) + } + + let d try { d = await requestQueue.push([ url, @@ -833,6 +839,8 @@ exports.pluginOptionsSchema = ({ Joi }) => baseUrl: Joi.string() .required() .description(`The URL to root of your Drupal instance`), + proxyUrl: Joi.string() + .description(`The CDN URL equivalent to your baseUrl`), apiBase: Joi.string().description( `The path to the root of the JSONAPI — defaults to "jsonapi"` ), From 2cd261c29bed686cd1c037fd5a1b96791258f681 Mon Sep 17 00:00:00 2001 From: Vacilando Date: Thu, 13 Oct 2022 22:50:29 +0200 Subject: [PATCH 02/15] [gatsby-source-drupal] provide proxyUrl in addition to baseUrl to allow using CDN, API gateway, etc. #36811 --- packages/gatsby-source-drupal/src/gatsby-node.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index 05fa4d4303070..f4ae49544704b 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -535,12 +535,14 @@ ${JSON.stringify(webhookBody, null, 4)}` } } - // If proxyUrl is defined, use it instead of baseUrl to get the content. - if (proxyUrl !== baseUrl) { - url = url.replace(baseUrl, proxyUrl) - } + // If proxyUrl is defined, use it instead of baseUrl to get the content. + if (proxyUrl !== baseUrl) { + console.log('url A: ' + url) + url = url.replace(baseUrl, proxyUrl) + console.log('url B: ' + url) + } - let d + let d try { d = await requestQueue.push([ url, @@ -841,7 +843,7 @@ exports.pluginOptionsSchema = ({ Joi }) => .description(`The URL to root of your Drupal instance`), proxyUrl: Joi.string() .description(`The CDN URL equivalent to your baseUrl`), - apiBase: Joi.string().description( + apiBase: Joi.string().description( `The path to the root of the JSONAPI — defaults to "jsonapi"` ), basicAuth: Joi.object({ From d6d63afc82af52d67eac4cd30a626ff8897b6966 Mon Sep 17 00:00:00 2001 From: Vacilando Date: Thu, 13 Oct 2022 22:52:14 +0200 Subject: [PATCH 03/15] [gatsby-source-drupal] provide proxyUrl in addition to baseUrl to allow using CDN, API gateway, etc. #36811 --- packages/gatsby-source-drupal/src/gatsby-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index f4ae49544704b..05694099ffa16 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -843,7 +843,7 @@ exports.pluginOptionsSchema = ({ Joi }) => .description(`The URL to root of your Drupal instance`), proxyUrl: Joi.string() .description(`The CDN URL equivalent to your baseUrl`), - apiBase: Joi.string().description( + apiBase: Joi.string().description( `The path to the root of the JSONAPI — defaults to "jsonapi"` ), basicAuth: Joi.object({ From 93148365f8481c86f70231292c3e0a62d7c4eee8 Mon Sep 17 00:00:00 2001 From: Vacilando Date: Thu, 13 Oct 2022 22:55:18 +0200 Subject: [PATCH 04/15] [gatsby-source-drupal] improve README.me about the proxyUrl param --- packages/gatsby-source-drupal/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md index ddba438df7e3a..ab1cdf68f398c 100644 --- a/packages/gatsby-source-drupal/README.md +++ b/packages/gatsby-source-drupal/README.md @@ -252,8 +252,8 @@ module.exports = { ## CDN -You can add optional CDN or API gateway URL `proxyUrl` param. The URL can be a proxy of the Drupal `baseUrl` -or another URL where the Drupal json paths can be retrieved. +You can add optional CDN or API gateway URL `proxyUrl` param. The URL can be a simple proxy of the Drupal +`baseUrl`, or another URL (even containing a path) where the Drupal JSON API resources can be retrieved. ```javascript // In your gatsby-config.js @@ -263,7 +263,7 @@ module.exports = { resolve: `gatsby-source-drupal`, options: { baseUrl: `https://live-contentacms.pantheonsite.io/`, - baseUrl: `https://xyz.cloudfront.net/`, + proxyUrl: `https://xyz.cloudfront.net/`, apiBase: `api`, // optional, defaults to `jsonapi` }, }, From 0cb00baf08ba2ddbef3ccfb8d4fdb07530ca4ea5 Mon Sep 17 00:00:00 2001 From: Vacilando Date: Thu, 13 Oct 2022 23:05:19 +0200 Subject: [PATCH 05/15] [gatsby-source-drupal] remove debug commands --- packages/gatsby-source-drupal/src/gatsby-node.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index 05694099ffa16..8036ab92259cc 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -537,9 +537,7 @@ ${JSON.stringify(webhookBody, null, 4)}` // If proxyUrl is defined, use it instead of baseUrl to get the content. if (proxyUrl !== baseUrl) { - console.log('url A: ' + url) url = url.replace(baseUrl, proxyUrl) - console.log('url B: ' + url) } let d From 7601aff108c099a0ec4b9397c642725281babe7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20F=C3=BCl=C3=B6pp?= Date: Thu, 13 Oct 2022 23:16:00 +0200 Subject: [PATCH 06/15] Update packages/gatsby-source-drupal/README.md Co-authored-by: Kyle Mathews --- packages/gatsby-source-drupal/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md index ab1cdf68f398c..384f59c3786a6 100644 --- a/packages/gatsby-source-drupal/README.md +++ b/packages/gatsby-source-drupal/README.md @@ -252,7 +252,7 @@ module.exports = { ## CDN -You can add optional CDN or API gateway URL `proxyUrl` param. The URL can be a simple proxy of the Drupal +You can add an optional CDN or API gateway URL `proxyUrl` param. The URL can be a simple proxy of the Drupal `baseUrl`, or another URL (even containing a path) where the Drupal JSON API resources can be retrieved. ```javascript From b125620aea5db2e7d3d93c42e7ac37acf9e464e5 Mon Sep 17 00:00:00 2001 From: Vacilando Date: Thu, 13 Oct 2022 23:21:47 +0200 Subject: [PATCH 07/15] [gatsby-source-drupal] fixing lint based on circleci feedback --- packages/gatsby-source-drupal/src/gatsby-node.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index 8036ab92259cc..ab9d72d1fc4d3 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -839,8 +839,9 @@ exports.pluginOptionsSchema = ({ Joi }) => baseUrl: Joi.string() .required() .description(`The URL to root of your Drupal instance`), - proxyUrl: Joi.string() - .description(`The CDN URL equivalent to your baseUrl`), + proxyUrl: Joi.string().description( + `The CDN URL equivalent to your baseUrl` + ), apiBase: Joi.string().description( `The path to the root of the JSONAPI — defaults to "jsonapi"` ), From 6ed98124c7c23a81c2e890c99a63fe4e751b7fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20F=C3=BCl=C3=B6pp?= Date: Fri, 14 Oct 2022 00:23:26 +0200 Subject: [PATCH 08/15] Update packages/gatsby-source-drupal/README.md Co-authored-by: Kyle Mathews --- packages/gatsby-source-drupal/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md index 384f59c3786a6..eb923fe9e38e2 100644 --- a/packages/gatsby-source-drupal/README.md +++ b/packages/gatsby-source-drupal/README.md @@ -255,6 +255,8 @@ module.exports = { You can add an optional CDN or API gateway URL `proxyUrl` param. The URL can be a simple proxy of the Drupal `baseUrl`, or another URL (even containing a path) where the Drupal JSON API resources can be retrieved. +This option is required as Drupal doesn't know about the CDN so it returns URLs pointing to the `baseUrl`. With `proxyUrl` set, the plugin will rewrite URLs returned from Drupal to keep pointing at the `proxyUrl` + ```javascript // In your gatsby-config.js module.exports = { From ab070eb64c6b6bcf0bfbc6830a383d74b2bf9f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20F=C3=BCl=C3=B6pp?= Date: Fri, 14 Oct 2022 18:25:09 +0200 Subject: [PATCH 09/15] Update README.md --- packages/gatsby-source-drupal/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md index eb923fe9e38e2..046a4a4e91e27 100644 --- a/packages/gatsby-source-drupal/README.md +++ b/packages/gatsby-source-drupal/README.md @@ -265,7 +265,7 @@ module.exports = { resolve: `gatsby-source-drupal`, options: { baseUrl: `https://live-contentacms.pantheonsite.io/`, - proxyUrl: `https://xyz.cloudfront.net/`, + proxyUrl: `https://xyz.cloudfront.net/`, // optional, defaults to the value of baseUrl apiBase: `api`, // optional, defaults to `jsonapi` }, }, From bf5b0b137773d0b45b343166c58d451e23ea603c Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 22 Nov 2022 10:17:42 -0800 Subject: [PATCH 10/15] add proxy url test --- .../src/__tests__/index.js | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index 59ff60f390c68..3bcf1d1db933d 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -1,5 +1,12 @@ +const baseUrl = `http://fixture` +const proxyUrl = `http://fixture-proxy` + jest.mock(`got`, () => jest.fn(path => { + if (path.includes(proxyUrl)) { + path = path.replace(proxyUrl, baseUrl) + } + let last = `` if (path.includes(`i18n-test`)) { last = `i18n-test-` @@ -59,7 +66,6 @@ const { handleWebhookUpdate } = require(`../utils`) describe(`gatsby-source-drupal`, () => { let nodes = {} const createNodeId = id => `generated-id-${id}` - const baseUrl = `http://fixture` const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) const { objectContaining } = expect const actions = { @@ -450,6 +456,19 @@ describe(`gatsby-source-drupal`, () => { expect(nodes[createNodeId(`und.article-3`)]).toBeDefined() }) + it(`Can use the proxyUrl plugin option to use a different API url for sourcing`, async () => { + nodes = {} + await sourceNodes(args, { baseUrl, proxyUrl }) + expect(Object.keys(nodes).length).not.toEqual(0) + expect(nodes[createNodeId(`und.file-1`)]).toBeDefined() + expect(nodes[createNodeId(`und.file-2`)]).toBeDefined() + expect(nodes[createNodeId(`und.tag-1`)]).toBeDefined() + expect(nodes[createNodeId(`und.tag-2`)]).toBeDefined() + expect(nodes[createNodeId(`und.article-1`)]).toBeDefined() + expect(nodes[createNodeId(`und.article-2`)]).toBeDefined() + expect(nodes[createNodeId(`und.article-3`)]).toBeDefined() + }) + it(`Verify JSON:API includes relationships`, async () => { // Reset nodes and test includes relationships. Object.keys(nodes).forEach(key => delete nodes[key]) From 837f9a32d1dd703d015f7277f190c18eb9c80f87 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 22 Nov 2022 10:25:24 -0800 Subject: [PATCH 11/15] expect got to be called with string containing proxy url --- packages/gatsby-source-drupal/src/__tests__/index.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index 3bcf1d1db933d..f042a0cf3ed64 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -1,3 +1,5 @@ +import got from "got" + const baseUrl = `http://fixture` const proxyUrl = `http://fixture-proxy` @@ -457,8 +459,13 @@ describe(`gatsby-source-drupal`, () => { }) it(`Can use the proxyUrl plugin option to use a different API url for sourcing`, async () => { + got.mockClear() nodes = {} await sourceNodes(args, { baseUrl, proxyUrl }) + expect(got).toHaveBeenCalledWith( + expect.stringContaining(proxyUrl), + expect.anything() + ) expect(Object.keys(nodes).length).not.toEqual(0) expect(nodes[createNodeId(`und.file-1`)]).toBeDefined() expect(nodes[createNodeId(`und.file-2`)]).toBeDefined() From 8bb277423374b65d1906aaca540734461ecdf88b Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 22 Nov 2022 10:30:58 -0800 Subject: [PATCH 12/15] expect each got call to have proxyUrl except for the first one --- .../gatsby-source-drupal/src/__tests__/index.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index f042a0cf3ed64..f97f2bd3b80a4 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -462,10 +462,19 @@ describe(`gatsby-source-drupal`, () => { got.mockClear() nodes = {} await sourceNodes(args, { baseUrl, proxyUrl }) - expect(got).toHaveBeenCalledWith( - expect.stringContaining(proxyUrl), - expect.anything() - ) + + for (const [index] of got.mock.calls.entries()) { + if (index === 0) { + // we don't proxy 'http://fixture/jsonapi' which is the first call + continue + } + expect(got).toHaveBeenNthCalledWith( + index + 1, + expect.stringContaining(proxyUrl), + expect.anything() + ) + } + expect(Object.keys(nodes).length).not.toEqual(0) expect(nodes[createNodeId(`und.file-1`)]).toBeDefined() expect(nodes[createNodeId(`und.file-2`)]).toBeDefined() From 36dec910622594288d1456f7a55f8f01d6f41449 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 22 Nov 2022 10:31:46 -0800 Subject: [PATCH 13/15] Update index.js --- packages/gatsby-source-drupal/src/__tests__/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index f97f2bd3b80a4..d7a409ed93eb3 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -463,11 +463,11 @@ describe(`gatsby-source-drupal`, () => { nodes = {} await sourceNodes(args, { baseUrl, proxyUrl }) - for (const [index] of got.mock.calls.entries()) { - if (index === 0) { - // we don't proxy 'http://fixture/jsonapi' which is the first call + for (const [index, call] of got.mock.calls.entries()) { + if (call[0] === `http://fixture/jsonapi`) { continue } + expect(got).toHaveBeenNthCalledWith( index + 1, expect.stringContaining(proxyUrl), From 7421b2d6e5e5961ef733876d18d5815fbf44fa6f Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 22 Nov 2022 10:33:16 -0800 Subject: [PATCH 14/15] make sure we only skip once --- packages/gatsby-source-drupal/src/__tests__/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index d7a409ed93eb3..3470fce36b7a5 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -463,8 +463,10 @@ describe(`gatsby-source-drupal`, () => { nodes = {} await sourceNodes(args, { baseUrl, proxyUrl }) + let callSkipCount = 0 for (const [index, call] of got.mock.calls.entries()) { if (call[0] === `http://fixture/jsonapi`) { + callSkipCount++ continue } @@ -475,6 +477,8 @@ describe(`gatsby-source-drupal`, () => { ) } + expect(callSkipCount).toBe(1) + expect(Object.keys(nodes).length).not.toEqual(0) expect(nodes[createNodeId(`und.file-1`)]).toBeDefined() expect(nodes[createNodeId(`und.file-2`)]).toBeDefined() From c200fec567f2cd33cd44a7982c671e981bd81a19 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 22 Nov 2022 10:34:02 -0800 Subject: [PATCH 15/15] Update index.js --- packages/gatsby-source-drupal/src/__tests__/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index 3470fce36b7a5..ef8ded945c761 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -478,6 +478,7 @@ describe(`gatsby-source-drupal`, () => { } expect(callSkipCount).toBe(1) + expect(got).toBeCalledTimes(8) expect(Object.keys(nodes).length).not.toEqual(0) expect(nodes[createNodeId(`und.file-1`)]).toBeDefined()