From d6b4943764989c0e89df2d6875cd19691566dfb3 Mon Sep 17 00:00:00 2001 From: Julien Barbay Date: Thu, 17 Aug 2023 18:10:50 +0700 Subject: [PATCH] feat(assets): support remote images (#7778) Co-authored-by: Sarah Rainsberger Co-authored-by: Princesseuh Co-authored-by: Erika <3019731+Princesseuh@users.noreply.github.com> --- .changeset/itchy-pants-grin.md | 5 + .changeset/sour-frogs-shout.md | 27 +++ packages/astro/package.json | 2 + packages/astro/src/@types/astro.ts | 68 ++++++- packages/astro/src/assets/build/generate.ts | 174 ++++++++++++++++++ packages/astro/src/assets/build/remote.ts | 48 +++++ packages/astro/src/assets/generate.ts | 127 ------------- packages/astro/src/assets/image-endpoint.ts | 17 +- packages/astro/src/assets/internal.ts | 39 +++- packages/astro/src/assets/services/service.ts | 55 ++++-- .../astro/src/assets/utils/remotePattern.ts | 63 +++++++ .../astro/src/assets/utils/transformToPath.ts | 11 +- .../astro/src/assets/vite-plugin-assets.ts | 12 +- packages/astro/src/core/build/generate.ts | 2 +- packages/astro/src/core/config/schema.ts | 26 +++ packages/astro/test/core-image.test.js | 35 +++- .../core-image-ssg/src/pages/remote.astro | 7 + packages/astro/test/test-image-service.js | 4 +- .../test/units/assets/remote-pattern.test.js | 111 +++++++++++ .../vercel/src/image/build-service.ts | 2 +- .../vercel/src/image/dev-service.ts | 2 +- .../integrations/vercel/src/image/shared.ts | 4 +- pnpm-lock.yaml | 6 + 23 files changed, 657 insertions(+), 190 deletions(-) create mode 100644 .changeset/itchy-pants-grin.md create mode 100644 .changeset/sour-frogs-shout.md create mode 100644 packages/astro/src/assets/build/generate.ts create mode 100644 packages/astro/src/assets/build/remote.ts delete mode 100644 packages/astro/src/assets/generate.ts create mode 100644 packages/astro/src/assets/utils/remotePattern.ts create mode 100644 packages/astro/test/fixtures/core-image-ssg/src/pages/remote.astro create mode 100644 packages/astro/test/units/assets/remote-pattern.test.js diff --git a/.changeset/itchy-pants-grin.md b/.changeset/itchy-pants-grin.md new file mode 100644 index 000000000000..2ab292f27d30 --- /dev/null +++ b/.changeset/itchy-pants-grin.md @@ -0,0 +1,5 @@ +--- +'@astrojs/vercel': patch +--- + +Update image support to work with latest version of Astro diff --git a/.changeset/sour-frogs-shout.md b/.changeset/sour-frogs-shout.md new file mode 100644 index 000000000000..9006914f1061 --- /dev/null +++ b/.changeset/sour-frogs-shout.md @@ -0,0 +1,27 @@ +--- +'astro': patch +--- + +Added support for optimizing remote images from authorized sources when using `astro:assets`. This comes with two new parameters to specify which domains (`image.domains`) and host patterns (`image.remotePatterns`) are authorized for remote images. + +For example, the following configuration will only allow remote images from `astro.build` to be optimized: + +```ts +// astro.config.mjs +export default defineConfig({ + image: { + domains: ["astro.build"], + } +}); +``` + +The following configuration will only allow remote images from HTTPS hosts: + +```ts +// astro.config.mjs +export default defineConfig({ + image: { + remotePatterns: [{ protocol: "https" }], + } +}); +``` diff --git a/packages/astro/package.json b/packages/astro/package.json index 19c9437ea1a3..c79eccc6052f 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -147,6 +147,7 @@ "github-slugger": "^2.0.0", "gray-matter": "^4.0.3", "html-escaper": "^3.0.3", + "http-cache-semantics": "^4.1.1", "js-yaml": "^4.1.0", "kleur": "^4.1.4", "magic-string": "^0.30.2", @@ -186,6 +187,7 @@ "@types/estree": "^0.0.51", "@types/hast": "^2.3.4", "@types/html-escaper": "^3.0.0", + "@types/http-cache-semantics": "^4.0.1", "@types/js-yaml": "^4.0.5", "@types/mime": "^2.0.3", "@types/mocha": "^9.1.1", diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index e35ad48636a0..cc5ddea7f3a1 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -13,6 +13,7 @@ import type { AddressInfo } from 'node:net'; import type * as rollup from 'rollup'; import type { TsConfigJson } from 'tsconfig-resolver'; import type * as vite from 'vite'; +import type { RemotePattern } from '../assets/utils/remotePattern'; import type { SerializedSSRManifest } from '../core/app/types'; import type { PageBuildData } from '../core/build/types'; import type { AstroConfigType } from '../core/config'; @@ -43,6 +44,7 @@ export type { ImageQualityPreset, ImageTransform, } from '../assets/types'; +export type { RemotePattern } from '../assets/utils/remotePattern'; export type { SSRManifest } from '../core/app/types'; export type { AstroCookies } from '../core/cookies'; @@ -366,10 +368,10 @@ export interface ViteUserConfig extends vite.UserConfig { ssr?: vite.SSROptions; } -export interface ImageServiceConfig { +export interface ImageServiceConfig = Record> { // eslint-disable-next-line @typescript-eslint/ban-types entrypoint: 'astro/assets/services/sharp' | 'astro/assets/services/squoosh' | (string & {}); - config?: Record; + config?: T; } /** @@ -1010,6 +1012,68 @@ export interface AstroUserConfig { * ``` */ service: ImageServiceConfig; + + /** + * @docs + * @name image.domains (Experimental) + * @type {string[]} + * @default `{domains: []}` + * @version 2.10.10 + * @description + * Defines a list of permitted image source domains for local image optimization. No other remote images will be optimized by Astro. + * + * This option requires an array of individual domain names as strings. Wildcards are not permitted. Instead, use [`image.remotePatterns`](#imageremotepatterns-experimental) to define a list of allowed source URL patterns. + * + * ```js + * // astro.config.mjs + * { + * image: { + * // Example: Allow remote image optimization from a single domain + * domains: ['astro.build'], + * }, + * } + * ``` + */ + domains?: string[]; + + /** + * @docs + * @name image.remotePatterns (Experimental) + * @type {RemotePattern[]} + * @default `{remotePatterns: []}` + * @version 2.10.10 + * @description + * Defines a list of permitted image source URL patterns for local image optimization. + * + * `remotePatterns` can be configured with four properties: + * 1. protocol + * 2. hostname + * 3. port + * 4. pathname + * + * ```js + * { + * image: { + * // Example: allow processing all images from your aws s3 bucket + * remotePatterns: [{ + * protocol: 'https', + * hostname: '**.amazonaws.com', + * }], + * }, + * } + * ``` + * + * You can use wildcards to define the permitted `hostname` and `pathname` values as described below. Otherwise, only the exact values provided will be configured: + * `hostname`: + * - Start with '**.' to allow all subdomains ('endsWith'). + * - Start with '*.' to allow only one level of subdomain. + * + * `pathname`: + * - End with '/**' to allow all sub-routes ('startsWith'). + * - End with '/*' to allow only one level of sub-route. + + */ + remotePatterns?: Partial[]; }; /** diff --git a/packages/astro/src/assets/build/generate.ts b/packages/astro/src/assets/build/generate.ts new file mode 100644 index 000000000000..b78800a4382a --- /dev/null +++ b/packages/astro/src/assets/build/generate.ts @@ -0,0 +1,174 @@ +import fs, { readFileSync } from 'node:fs'; +import { basename, join } from 'node:path/posix'; +import type { StaticBuildOptions } from '../../core/build/types.js'; +import { warn } from '../../core/logger/core.js'; +import { prependForwardSlash } from '../../core/path.js'; +import { isServerLikeOutput } from '../../prerender/utils.js'; +import { getConfiguredImageService, isESMImportedImage } from '../internal.js'; +import type { LocalImageService } from '../services/service.js'; +import type { ImageMetadata, ImageTransform } from '../types.js'; +import { loadRemoteImage, type RemoteCacheEntry } from './remote.js'; + +interface GenerationDataUncached { + cached: false; + weight: { + before: number; + after: number; + }; +} + +interface GenerationDataCached { + cached: true; +} + +type GenerationData = GenerationDataUncached | GenerationDataCached; + +export async function generateImage( + buildOpts: StaticBuildOptions, + options: ImageTransform, + filepath: string +): Promise { + let useCache = true; + const assetsCacheDir = new URL('assets/', buildOpts.settings.config.cacheDir); + + // Ensure that the cache directory exists + try { + await fs.promises.mkdir(assetsCacheDir, { recursive: true }); + } catch (err) { + warn( + buildOpts.logging, + 'astro:assets', + `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${err}` + ); + useCache = false; + } + + let serverRoot: URL, clientRoot: URL; + if (isServerLikeOutput(buildOpts.settings.config)) { + serverRoot = buildOpts.settings.config.build.server; + clientRoot = buildOpts.settings.config.build.client; + } else { + serverRoot = buildOpts.settings.config.outDir; + clientRoot = buildOpts.settings.config.outDir; + } + + const isLocalImage = isESMImportedImage(options.src); + + const finalFileURL = new URL('.' + filepath, clientRoot); + const finalFolderURL = new URL('./', finalFileURL); + + // For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server + const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json'); + const cachedFileURL = new URL(cacheFile, assetsCacheDir); + + await fs.promises.mkdir(finalFolderURL, { recursive: true }); + + // Check if we have a cached entry first + try { + if (isLocalImage) { + await fs.promises.copyFile(cachedFileURL, finalFileURL); + + return { + cached: true, + }; + } else { + const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry; + + // If the cache entry is not expired, use it + if (JSONData.expires < Date.now()) { + await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64')); + + return { + cached: true, + }; + } + } + } catch (e: any) { + if (e.code !== 'ENOENT') { + throw new Error(`An error was encountered while reading the cache file. Error: ${e}`); + } + // If the cache file doesn't exist, just move on, and we'll generate it + } + + // The original filepath or URL from the image transform + const originalImagePath = isLocalImage + ? (options.src as ImageMetadata).src + : (options.src as string); + + let imageData; + let resultData: { data: Buffer | undefined; expires: number | undefined } = { + data: undefined, + expires: undefined, + }; + + // If the image is local, we can just read it directly, otherwise we need to download it + if (isLocalImage) { + imageData = await fs.promises.readFile( + new URL( + '.' + + prependForwardSlash( + join(buildOpts.settings.config.build.assets, basename(originalImagePath)) + ), + serverRoot + ) + ); + } else { + const remoteImage = await loadRemoteImage(originalImagePath); + resultData.expires = remoteImage.expires; + imageData = remoteImage.data; + } + + const imageService = (await getConfiguredImageService()) as LocalImageService; + resultData.data = ( + await imageService.transform( + imageData, + { ...options, src: originalImagePath }, + buildOpts.settings.config.image + ) + ).data; + + try { + // Write the cache entry + if (useCache) { + if (isLocalImage) { + await fs.promises.writeFile(cachedFileURL, resultData.data); + } else { + await fs.promises.writeFile( + cachedFileURL, + JSON.stringify({ + data: Buffer.from(resultData.data).toString('base64'), + expires: resultData.expires, + }) + ); + } + } + } catch (e) { + warn( + buildOpts.logging, + 'astro:assets', + `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}` + ); + } finally { + // Write the final file + await fs.promises.writeFile(finalFileURL, resultData.data); + } + + return { + cached: false, + weight: { + // Divide by 1024 to get size in kilobytes + before: Math.trunc(imageData.byteLength / 1024), + after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024), + }, + }; +} + +export function getStaticImageList(): Iterable< + [string, { path: string; options: ImageTransform }] +> { + if (!globalThis?.astroAsset?.staticImages) { + return []; + } + + return globalThis.astroAsset.staticImages?.entries(); +} diff --git a/packages/astro/src/assets/build/remote.ts b/packages/astro/src/assets/build/remote.ts new file mode 100644 index 000000000000..c3d4bb9bae32 --- /dev/null +++ b/packages/astro/src/assets/build/remote.ts @@ -0,0 +1,48 @@ +import CachePolicy from 'http-cache-semantics'; + +export type RemoteCacheEntry = { data: string; expires: number }; + +export async function loadRemoteImage(src: string) { + const req = new Request(src); + const res = await fetch(req); + + if (!res.ok) { + throw new Error( + `Failed to load remote image ${src}. The request did not return a 200 OK response. (received ${res.status}))` + ); + } + + // calculate an expiration date based on the response's TTL + const policy = new CachePolicy(webToCachePolicyRequest(req), webToCachePolicyResponse(res)); + const expires = policy.storable() ? policy.timeToLive() : 0; + + return { + data: Buffer.from(await res.arrayBuffer()), + expires: Date.now() + expires, + }; +} + +function webToCachePolicyRequest({ url, method, headers: _headers }: Request): CachePolicy.Request { + let headers: CachePolicy.Headers = {}; + // Be defensive here due to a cookie header bug in node@18.14.1 + undici + try { + headers = Object.fromEntries(_headers.entries()); + } catch {} + return { + method, + url, + headers, + }; +} + +function webToCachePolicyResponse({ status, headers: _headers }: Response): CachePolicy.Response { + let headers: CachePolicy.Headers = {}; + // Be defensive here due to a cookie header bug in node@18.14.1 + undici + try { + headers = Object.fromEntries(_headers.entries()); + } catch {} + return { + status, + headers, + }; +} diff --git a/packages/astro/src/assets/generate.ts b/packages/astro/src/assets/generate.ts deleted file mode 100644 index d6cb02e560f3..000000000000 --- a/packages/astro/src/assets/generate.ts +++ /dev/null @@ -1,127 +0,0 @@ -import fs from 'node:fs'; -import { basename, join } from 'node:path/posix'; -import type { StaticBuildOptions } from '../core/build/types.js'; -import { warn } from '../core/logger/core.js'; -import { prependForwardSlash } from '../core/path.js'; -import { isServerLikeOutput } from '../prerender/utils.js'; -import { getConfiguredImageService, isESMImportedImage } from './internal.js'; -import type { LocalImageService } from './services/service.js'; -import type { ImageTransform } from './types.js'; - -interface GenerationDataUncached { - cached: false; - weight: { - before: number; - after: number; - }; -} - -interface GenerationDataCached { - cached: true; -} - -type GenerationData = GenerationDataUncached | GenerationDataCached; - -export async function generateImage( - buildOpts: StaticBuildOptions, - options: ImageTransform, - filepath: string -): Promise { - if (!isESMImportedImage(options.src)) { - return undefined; - } - - let useCache = true; - const assetsCacheDir = new URL('assets/', buildOpts.settings.config.cacheDir); - - // Ensure that the cache directory exists - try { - await fs.promises.mkdir(assetsCacheDir, { recursive: true }); - } catch (err) { - warn( - buildOpts.logging, - 'astro:assets', - `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${err}` - ); - useCache = false; - } - - let serverRoot: URL, clientRoot: URL; - if (isServerLikeOutput(buildOpts.settings.config)) { - serverRoot = buildOpts.settings.config.build.server; - clientRoot = buildOpts.settings.config.build.client; - } else { - serverRoot = buildOpts.settings.config.outDir; - clientRoot = buildOpts.settings.config.outDir; - } - - const finalFileURL = new URL('.' + filepath, clientRoot); - const finalFolderURL = new URL('./', finalFileURL); - const cachedFileURL = new URL(basename(filepath), assetsCacheDir); - - try { - await fs.promises.copyFile(cachedFileURL, finalFileURL); - - return { - cached: true, - }; - } catch (e) { - // no-op - } - - // The original file's path (the `src` attribute of the ESM imported image passed by the user) - const originalImagePath = options.src.src; - - const fileData = await fs.promises.readFile( - new URL( - '.' + - prependForwardSlash( - join(buildOpts.settings.config.build.assets, basename(originalImagePath)) - ), - serverRoot - ) - ); - - const imageService = (await getConfiguredImageService()) as LocalImageService; - const resultData = await imageService.transform( - fileData, - { ...options, src: originalImagePath }, - buildOpts.settings.config.image.service.config - ); - - await fs.promises.mkdir(finalFolderURL, { recursive: true }); - - if (useCache) { - try { - await fs.promises.writeFile(cachedFileURL, resultData.data); - await fs.promises.copyFile(cachedFileURL, finalFileURL); - } catch (e) { - warn( - buildOpts.logging, - 'astro:assets', - `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}` - ); - await fs.promises.writeFile(finalFileURL, resultData.data); - } - } else { - await fs.promises.writeFile(finalFileURL, resultData.data); - } - - return { - cached: false, - weight: { - before: Math.trunc(fileData.byteLength / 1024), - after: Math.trunc(resultData.data.byteLength / 1024), - }, - }; -} - -export function getStaticImageList(): Iterable< - [string, { path: string; options: ImageTransform }] -> { - if (!globalThis?.astroAsset?.staticImages) { - return []; - } - - return globalThis.astroAsset.staticImages?.entries(); -} diff --git a/packages/astro/src/assets/image-endpoint.ts b/packages/astro/src/assets/image-endpoint.ts index 0553272c219c..fa62cbdd128a 100644 --- a/packages/astro/src/assets/image-endpoint.ts +++ b/packages/astro/src/assets/image-endpoint.ts @@ -1,11 +1,11 @@ import mime from 'mime/lite.js'; import type { APIRoute } from '../@types/astro.js'; import { isRemotePath } from '../core/path.js'; -import { getConfiguredImageService } from './internal.js'; +import { getConfiguredImageService, isRemoteAllowed } from './internal.js'; import { isLocalService } from './services/service.js'; import { etag } from './utils/etag.js'; // @ts-expect-error -import { imageServiceConfig } from 'astro:assets'; +import { imageConfig } from 'astro:assets'; async function loadRemoteImage(src: URL) { try { @@ -33,7 +33,7 @@ export const get: APIRoute = async ({ request }) => { } const url = new URL(request.url); - const transform = await imageService.parseURL(url, imageServiceConfig); + const transform = await imageService.parseURL(url, imageConfig); if (!transform?.src) { throw new Error('Incorrect transform returned by `parseURL`'); @@ -45,17 +45,18 @@ export const get: APIRoute = async ({ request }) => { const sourceUrl = isRemotePath(transform.src) ? new URL(transform.src) : new URL(transform.src, url.origin); + + if (isRemotePath(transform.src) && isRemoteAllowed(transform.src, imageConfig) === false) { + return new Response('Forbidden', { status: 403 }); + } + inputBuffer = await loadRemoteImage(sourceUrl); if (!inputBuffer) { return new Response('Not Found', { status: 404 }); } - const { data, format } = await imageService.transform( - inputBuffer, - transform, - imageServiceConfig - ); + const { data, format } = await imageService.transform(inputBuffer, transform, imageConfig); return new Response(data, { status: 200, diff --git a/packages/astro/src/assets/internal.ts b/packages/astro/src/assets/internal.ts index 06e4f8cc0c89..ffc27333fa20 100644 --- a/packages/astro/src/assets/internal.ts +++ b/packages/astro/src/assets/internal.ts @@ -1,4 +1,5 @@ -import type { AstroSettings } from '../@types/astro.js'; +import { isRemotePath } from '@astrojs/internal-helpers/path'; +import type { AstroConfig, AstroSettings } from '../@types/astro.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { isLocalService, type ImageService } from './services/service.js'; import type { @@ -7,6 +8,7 @@ import type { ImageTransform, UnresolvedImageTransform, } from './types.js'; +import { matchHostname, matchPattern } from './utils/remotePattern.js'; export function injectImageEndpoint(settings: AstroSettings) { settings.injectedRoutes.push({ @@ -22,6 +24,26 @@ export function isESMImportedImage(src: ImageMetadata | string): src is ImageMet return typeof src === 'object'; } +export function isRemoteImage(src: ImageMetadata | string): src is string { + return typeof src === 'string'; +} + +export function isRemoteAllowed( + src: string, + { + domains = [], + remotePatterns = [], + }: Partial> +): boolean { + if (!isRemotePath(src)) return false; + + const url = new URL(src); + return ( + domains.some((domain) => matchHostname(url, domain)) || + remotePatterns.some((remotePattern) => matchPattern(url, remotePattern)) + ); +} + export async function getConfiguredImageService(): Promise { if (!globalThis?.astroAsset?.imageService) { const { default: service }: { default: ImageService } = await import( @@ -43,7 +65,7 @@ export async function getConfiguredImageService(): Promise { export async function getImage( options: ImageTransform | UnresolvedImageTransform, - serviceConfig: Record + imageConfig: AstroConfig['image'] ): Promise { if (!options || typeof options !== 'object') { throw new AstroError({ @@ -64,13 +86,18 @@ export async function getImage( }; const validatedOptions = service.validateOptions - ? await service.validateOptions(resolvedOptions, serviceConfig) + ? await service.validateOptions(resolvedOptions, imageConfig) : resolvedOptions; - let imageURL = await service.getURL(validatedOptions, serviceConfig); + let imageURL = await service.getURL(validatedOptions, imageConfig); // In build and for local services, we need to collect the requested parameters so we can generate the final images - if (isLocalService(service) && globalThis.astroAsset.addStaticImage) { + if ( + isLocalService(service) && + globalThis.astroAsset.addStaticImage && + // If `getURL` returned the same URL as the user provided, it means the service doesn't need to do anything + !(isRemoteImage(validatedOptions.src) && imageURL === validatedOptions.src) + ) { imageURL = globalThis.astroAsset.addStaticImage(validatedOptions); } @@ -80,7 +107,7 @@ export async function getImage( src: imageURL, attributes: service.getHTMLAttributes !== undefined - ? service.getHTMLAttributes(validatedOptions, serviceConfig) + ? service.getHTMLAttributes(validatedOptions, imageConfig) : {}, }; } diff --git a/packages/astro/src/assets/services/service.ts b/packages/astro/src/assets/services/service.ts index d3479c8803fa..5af4a898be8b 100644 --- a/packages/astro/src/assets/services/service.ts +++ b/packages/astro/src/assets/services/service.ts @@ -1,7 +1,8 @@ +import type { AstroConfig } from '../../@types/astro.js'; import { AstroError, AstroErrorData } from '../../core/errors/index.js'; import { joinPaths } from '../../core/path.js'; import { VALID_SUPPORTED_FORMATS } from '../consts.js'; -import { isESMImportedImage } from '../internal.js'; +import { isESMImportedImage, isRemoteAllowed } from '../internal.js'; import type { ImageOutputFormat, ImageTransform } from '../types.js'; export type ImageService = LocalImageService | ExternalImageService; @@ -23,7 +24,11 @@ export function parseQuality(quality: string): string | number { return result; } -interface SharedServiceProps { +type ImageConfig = Omit & { + service: { entrypoint: string; config: T }; +}; + +interface SharedServiceProps = Record> { /** * Return the URL to the endpoint or URL your images are generated from. * @@ -32,7 +37,7 @@ interface SharedServiceProps { * For external services, this should point to the URL your images are coming from, for instance, `/_vercel/image` * */ - getURL: (options: ImageTransform, serviceConfig: Record) => string | Promise; + getURL: (options: ImageTransform, imageConfig: ImageConfig) => string | Promise; /** * Return any additional HTML attributes separate from `src` that your service requires to show the image properly. * @@ -41,7 +46,7 @@ interface SharedServiceProps { */ getHTMLAttributes?: ( options: ImageTransform, - serviceConfig: Record + imageConfig: ImageConfig ) => Record | Promise>; /** * Validate and return the options passed by the user. @@ -53,18 +58,20 @@ interface SharedServiceProps { */ validateOptions?: ( options: ImageTransform, - serviceConfig: Record + imageConfig: ImageConfig ) => ImageTransform | Promise; } -export type ExternalImageService = SharedServiceProps; +export type ExternalImageService = Record> = + SharedServiceProps; export type LocalImageTransform = { src: string; [key: string]: any; }; -export interface LocalImageService extends SharedServiceProps { +export interface LocalImageService = Record> + extends SharedServiceProps { /** * Parse the requested parameters passed in the URL from `getURL` back into an object to be used later by `transform`. * @@ -72,7 +79,7 @@ export interface LocalImageService extends SharedServiceProps { */ parseURL: ( url: URL, - serviceConfig: Record + imageConfig: ImageConfig ) => LocalImageTransform | undefined | Promise | Promise; /** * Performs the image transformations on the input image and returns both the binary data and @@ -81,7 +88,7 @@ export interface LocalImageService extends SharedServiceProps { transform: ( inputBuffer: Buffer, transform: LocalImageTransform, - serviceConfig: Record + imageConfig: ImageConfig ) => Promise<{ data: Buffer; format: ImageOutputFormat }>; } @@ -202,21 +209,31 @@ export const baseService: Omit = { decoding: attributes.decoding ?? 'async', }; }, - getURL(options: ImageTransform) { - // Both our currently available local services don't handle remote images, so we return the path as is. - if (!isESMImportedImage(options.src)) { + getURL(options, imageConfig) { + const searchParams = new URLSearchParams(); + + if (isESMImportedImage(options.src)) { + searchParams.append('href', options.src.src); + } else if (isRemoteAllowed(options.src, imageConfig)) { + searchParams.append('href', options.src); + } else { + // If it's not an imported image, nor is it allowed using the current domains or remote patterns, we'll just return the original URL return options.src; } - const searchParams = new URLSearchParams(); - searchParams.append('href', options.src.src); + const params: Record = { + w: 'width', + h: 'height', + q: 'quality', + f: 'format', + }; - options.width && searchParams.append('w', options.width.toString()); - options.height && searchParams.append('h', options.height.toString()); - options.quality && searchParams.append('q', options.quality.toString()); - options.format && searchParams.append('f', options.format); + Object.entries(params).forEach(([param, key]) => { + options[key] && searchParams.append(param, options[key].toString()); + }); - return joinPaths(import.meta.env.BASE_URL, '/_image?') + searchParams; + const imageEndpoint = joinPaths(import.meta.env.BASE_URL, '/_image'); + return `${imageEndpoint}?${searchParams}`; }, parseURL(url) { const params = url.searchParams; diff --git a/packages/astro/src/assets/utils/remotePattern.ts b/packages/astro/src/assets/utils/remotePattern.ts new file mode 100644 index 000000000000..7708b42e7380 --- /dev/null +++ b/packages/astro/src/assets/utils/remotePattern.ts @@ -0,0 +1,63 @@ +export type RemotePattern = { + hostname?: string; + pathname?: string; + protocol?: string; + port?: string; +}; + +export function matchPattern(url: URL, remotePattern: RemotePattern) { + return ( + matchProtocol(url, remotePattern.protocol) && + matchHostname(url, remotePattern.hostname, true) && + matchPort(url, remotePattern.port) && + matchPathname(url, remotePattern.pathname, true) + ); +} + +export function matchPort(url: URL, port?: string) { + return !port || port === url.port; +} + +export function matchProtocol(url: URL, protocol?: string) { + return !protocol || protocol === url.protocol.slice(0, -1); +} + +export function matchHostname(url: URL, hostname?: string, allowWildcard?: boolean) { + if (!hostname) { + return true; + } else if (!allowWildcard || !hostname.startsWith('*')) { + return hostname === url.hostname; + } else if (hostname.startsWith('**.')) { + const slicedHostname = hostname.slice(2); // ** length + return slicedHostname !== url.hostname && url.hostname.endsWith(slicedHostname); + } else if (hostname.startsWith('*.')) { + const slicedHostname = hostname.slice(1); // * length + const additionalSubdomains = url.hostname + .replace(slicedHostname, '') + .split('.') + .filter(Boolean); + return additionalSubdomains.length === 1; + } + + return false; +} + +export function matchPathname(url: URL, pathname?: string, allowWildcard?: boolean) { + if (!pathname) { + return true; + } else if (!allowWildcard || !pathname.endsWith('*')) { + return pathname === url.pathname; + } else if (pathname.endsWith('/**')) { + const slicedPathname = pathname.slice(0, -2); // ** length + return slicedPathname !== url.pathname && url.pathname.startsWith(slicedPathname); + } else if (pathname.endsWith('/*')) { + const slicedPathname = pathname.slice(0, -1); // * length + const additionalPathChunks = url.pathname + .replace(slicedPathname, '') + .split('/') + .filter(Boolean); + return additionalPathChunks.length === 1; + } + + return false; +} diff --git a/packages/astro/src/assets/utils/transformToPath.ts b/packages/astro/src/assets/utils/transformToPath.ts index 04ddee0a193f..d5535137be04 100644 --- a/packages/astro/src/assets/utils/transformToPath.ts +++ b/packages/astro/src/assets/utils/transformToPath.ts @@ -5,14 +5,13 @@ import { isESMImportedImage } from '../internal.js'; import type { ImageTransform } from '../types.js'; export function propsToFilename(transform: ImageTransform, hash: string) { - if (!isESMImportedImage(transform.src)) { - return transform.src; - } - - let filename = removeQueryString(transform.src.src); + let filename = removeQueryString( + isESMImportedImage(transform.src) ? transform.src.src : transform.src + ); const ext = extname(filename); filename = basename(filename, ext); - const outputExt = transform.format ? `.${transform.format}` : ext; + + let outputExt = transform.format ? `.${transform.format}` : ext; return `/${filename}_${hash}${outputExt}`; } diff --git a/packages/astro/src/assets/vite-plugin-assets.ts b/packages/astro/src/assets/vite-plugin-assets.ts index 56525300167c..0f00e0ecb569 100644 --- a/packages/astro/src/assets/vite-plugin-assets.ts +++ b/packages/astro/src/assets/vite-plugin-assets.ts @@ -12,7 +12,6 @@ import { removeQueryString, } from '../core/path.js'; import { VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js'; -import { isESMImportedImage } from './internal.js'; import { emitESMImage } from './utils/emitAsset.js'; import { hashTransform, propsToFilename } from './utils/transformToPath.js'; @@ -85,8 +84,8 @@ export default function assets({ import { getImage as getImageInternal } from "astro/assets"; export { default as Image } from "astro/components/Image.astro"; - export const imageServiceConfig = ${JSON.stringify(settings.config.image.service.config)}; - export const getImage = async (options) => await getImageInternal(options, imageServiceConfig); + export const imageConfig = ${JSON.stringify(settings.config.image)}; + export const getImage = async (options) => await getImageInternal(options, imageConfig); `; } }, @@ -109,15 +108,10 @@ export default function assets({ if (globalThis.astroAsset.staticImages.has(hash)) { filePath = globalThis.astroAsset.staticImages.get(hash)!.path; } else { - // If the image is not imported, we can return the path as-is, since static references - // should only point ot valid paths for builds or remote images - if (!isESMImportedImage(options.src)) { - return options.src; - } - filePath = prependForwardSlash( joinPaths(settings.config.build.assets, propsToFilename(options, hash)) ); + globalThis.astroAsset.staticImages.set(hash, { path: filePath, options: options }); } diff --git a/packages/astro/src/core/build/generate.ts b/packages/astro/src/core/build/generate.ts index 708295296399..a78a46883ab8 100644 --- a/packages/astro/src/core/build/generate.ts +++ b/packages/astro/src/core/build/generate.ts @@ -19,7 +19,7 @@ import type { import { generateImage as generateImageInternal, getStaticImageList, -} from '../../assets/generate.js'; +} from '../../assets/build/generate.js'; import { eachPageDataFromEntryPoint, eachRedirectPageData, diff --git a/packages/astro/src/core/config/schema.ts b/packages/astro/src/core/config/schema.ts index 51b9a6d48654..87ff7ba9f5b2 100644 --- a/packages/astro/src/core/config/schema.ts +++ b/packages/astro/src/core/config/schema.ts @@ -182,9 +182,35 @@ export const AstroConfigSchema = z.object({ ]), config: z.record(z.any()).default({}), }), + domains: z.array(z.string()).default([]), + remotePatterns: z + .array( + z.object({ + protocol: z.string().optional(), + hostname: z + .string() + .refine( + (val) => !val.includes('*') || val.startsWith('*.') || val.startsWith('**.'), + { + message: 'wildcards can only be placed at the beginning of the hostname', + } + ) + .optional(), + port: z.string().optional(), + pathname: z + .string() + .refine((val) => !val.includes('*') || val.endsWith('/*') || val.endsWith('/**'), { + message: 'wildcards can only be placed at the end of a pathname', + }) + .optional(), + }) + ) + .default([]), }) .default({ service: { entrypoint: 'astro/assets/services/squoosh', config: {} }, + domains: [], + remotePatterns: [], }), markdown: z .object({ diff --git a/packages/astro/test/core-image.test.js b/packages/astro/test/core-image.test.js index 8c09de245211..5d656a6f6a49 100644 --- a/packages/astro/test/core-image.test.js +++ b/packages/astro/test/core-image.test.js @@ -25,6 +25,7 @@ describe('astro:image', () => { }, image: { service: testImageService({ foo: 'bar' }), + domains: ['avatars.githubusercontent.com'], }, }); @@ -198,6 +199,15 @@ describe('astro:image', () => { $ = cheerio.load(html); }); + it('has proper link and works', async () => { + let $img = $('#remote img'); + + let src = $img.attr('src'); + expect(src.startsWith('/_image?')).to.be.true; + const imageRequest = await fixture.fetch(src); + expect(imageRequest.status).to.equal(200); + }); + it('includes the provided alt', async () => { let $img = $('#remote img'); expect($img.attr('alt')).to.equal('fred'); @@ -587,6 +597,7 @@ describe('astro:image', () => { }, image: { service: testImageService(), + domains: ['astro.build'], }, }); // Remove cache directory @@ -604,6 +615,15 @@ describe('astro:image', () => { expect(data).to.be.an.instanceOf(Buffer); }); + it('writes out allowed remote images', async () => { + const html = await fixture.readFile('/remote/index.html'); + const $ = cheerio.load(html); + const src = $('#remote img').attr('src'); + expect(src.length).to.be.greaterThan(0); + const data = await fixture.readFile(src, null); + expect(data).to.be.an.instanceOf(Buffer); + }); + it('writes out images to dist folder with proper extension if no format was passed', async () => { const html = await fixture.readFile('/index.html'); const $ = cheerio.load(html); @@ -708,12 +728,15 @@ describe('astro:image', () => { }); it('has cache entries', async () => { - const generatedImages = (await fixture.glob('_astro/**/*.webp')).map((path) => - basename(path) - ); - const cachedImages = (await fixture.glob('../node_modules/.astro/assets/**/*.webp')).map( - (path) => basename(path) - ); + const generatedImages = (await fixture.glob('_astro/**/*.webp')) + .map((path) => basename(path)) + .sort(); + const cachedImages = [ + ...(await fixture.glob('../node_modules/.astro/assets/**/*.webp')), + ...(await fixture.glob('../node_modules/.astro/assets/**/*.json')), + ] + .map((path) => basename(path).replace('.webp.json', '.webp')) + .sort(); expect(generatedImages).to.deep.equal(cachedImages); }); diff --git a/packages/astro/test/fixtures/core-image-ssg/src/pages/remote.astro b/packages/astro/test/fixtures/core-image-ssg/src/pages/remote.astro new file mode 100644 index 000000000000..727a15ff0868 --- /dev/null +++ b/packages/astro/test/fixtures/core-image-ssg/src/pages/remote.astro @@ -0,0 +1,7 @@ +--- +import { Image } from "astro:assets"; +--- + +
+fred +
diff --git a/packages/astro/test/test-image-service.js b/packages/astro/test/test-image-service.js index ebdbb0765d40..bcf623caa809 100644 --- a/packages/astro/test/test-image-service.js +++ b/packages/astro/test/test-image-service.js @@ -17,8 +17,8 @@ export default { ...baseService, getHTMLAttributes(options, serviceConfig) { options['data-service'] = 'my-custom-service'; - if (serviceConfig.foo) { - options['data-service-config'] = serviceConfig.foo; + if (serviceConfig.service.config.foo) { + options['data-service-config'] = serviceConfig.service.config.foo; } return baseService.getHTMLAttributes(options); }, diff --git a/packages/astro/test/units/assets/remote-pattern.test.js b/packages/astro/test/units/assets/remote-pattern.test.js new file mode 100644 index 000000000000..62a411e3a659 --- /dev/null +++ b/packages/astro/test/units/assets/remote-pattern.test.js @@ -0,0 +1,111 @@ +import { expect } from 'chai'; +import { + matchProtocol, + matchPort, + matchHostname, + matchPathname, + matchPattern, +} from '../../../dist/assets/utils/remotePattern.js'; + +describe('astro/src/assets/utils/remotePattern', () => { + const url1 = new URL('https://docs.astro.build/en/getting-started'); + const url2 = new URL('http://preview.docs.astro.build:8080/'); + const url3 = new URL('https://astro.build/'); + const url4 = new URL('https://example.co/'); + + describe('remote pattern matchers', () => { + it('matches protocol', async () => { + // undefined + expect(matchProtocol(url1)).to.be.true; + + // defined, true/false + expect(matchProtocol(url1, 'http')).to.be.false; + expect(matchProtocol(url1, 'https')).to.be.true; + }); + + it('matches port', async () => { + // undefined + expect(matchPort(url1)).to.be.true; + + // defined, but port is empty (default port used in URL) + expect(matchPort(url1, '')).to.be.true; + + // defined and port is custom + expect(matchPort(url2, '8080')).to.be.true; + }); + + it('matches hostname (no wildcards)', async () => { + // undefined + expect(matchHostname(url1)).to.be.true; + + // defined, true/false + expect(matchHostname(url1, 'astro.build')).to.be.false; + expect(matchHostname(url1, 'docs.astro.build')).to.be.true; + }); + + it('matches hostname (with wildcards)', async () => { + // defined, true/false + expect(matchHostname(url1, 'docs.astro.build', true)).to.be.true; + expect(matchHostname(url1, '**.astro.build', true)).to.be.true; + expect(matchHostname(url1, '*.astro.build', true)).to.be.true; + + expect(matchHostname(url2, '*.astro.build', true)).to.be.false; + expect(matchHostname(url2, '**.astro.build', true)).to.be.true; + + expect(matchHostname(url3, 'astro.build', true)).to.be.true; + expect(matchHostname(url3, '*.astro.build', true)).to.be.false; + expect(matchHostname(url3, '**.astro.build', true)).to.be.false; + }); + + it('matches pathname (no wildcards)', async () => { + // undefined + expect(matchPathname(url1)).to.be.true; + + // defined, true/false + expect(matchPathname(url1, '/')).to.be.false; + expect(matchPathname(url1, '/en/getting-started')).to.be.true; + }); + + it('matches pathname (with wildcards)', async () => { + // defined, true/false + expect(matchPathname(url1, '/en/**', true)).to.be.true; + expect(matchPathname(url1, '/en/*', true)).to.be.true; + expect(matchPathname(url1, '/**', true)).to.be.true; + + expect(matchPathname(url2, '/**', true)).to.be.false; + expect(matchPathname(url2, '/*', true)).to.be.false; + }); + + it('matches patterns', async () => { + expect(matchPattern(url1, {})).to.be.true; + + expect( + matchPattern(url1, { + protocol: 'https', + }) + ).to.be.true; + + expect( + matchPattern(url1, { + protocol: 'https', + hostname: '**.astro.build', + }) + ).to.be.true; + + expect( + matchPattern(url1, { + protocol: 'https', + hostname: '**.astro.build', + pathname: '/en/**', + }) + ).to.be.true; + + expect( + matchPattern(url4, { + protocol: 'https', + hostname: 'example.com', + }) + ).to.be.false; + }); + }); +}); diff --git a/packages/integrations/vercel/src/image/build-service.ts b/packages/integrations/vercel/src/image/build-service.ts index 973ceb22a7aa..63a37a5fee89 100644 --- a/packages/integrations/vercel/src/image/build-service.ts +++ b/packages/integrations/vercel/src/image/build-service.ts @@ -3,7 +3,7 @@ import { isESMImportedImage, sharedValidateOptions } from './shared'; const service: ExternalImageService = { validateOptions: (options, serviceOptions) => - sharedValidateOptions(options, serviceOptions, 'production'), + sharedValidateOptions(options, serviceOptions.service.config, 'production'), getHTMLAttributes(options) { const { inputtedWidth, ...props } = options; diff --git a/packages/integrations/vercel/src/image/dev-service.ts b/packages/integrations/vercel/src/image/dev-service.ts index 04df9932a504..be6360fe3f8b 100644 --- a/packages/integrations/vercel/src/image/dev-service.ts +++ b/packages/integrations/vercel/src/image/dev-service.ts @@ -5,7 +5,7 @@ import { sharedValidateOptions } from './shared'; const service: LocalImageService = { validateOptions: (options, serviceOptions) => - sharedValidateOptions(options, serviceOptions, 'development'), + sharedValidateOptions(options, serviceOptions.service.config, 'development'), getHTMLAttributes(options, serviceOptions) { const { inputtedWidth, ...props } = options; diff --git a/packages/integrations/vercel/src/image/shared.ts b/packages/integrations/vercel/src/image/shared.ts index 0b6db2037805..473750fae404 100644 --- a/packages/integrations/vercel/src/image/shared.ts +++ b/packages/integrations/vercel/src/image/shared.ts @@ -89,10 +89,10 @@ export function getImageConfig( export function sharedValidateOptions( options: ImageTransform, - serviceOptions: Record, + serviceConfig: Record, mode: 'development' | 'production' ) { - const vercelImageOptions = serviceOptions as VercelImageConfig; + const vercelImageOptions = serviceConfig as VercelImageConfig; if ( mode === 'development' && diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5f0529643a73..15bc70591a90 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -578,6 +578,9 @@ importers: html-escaper: specifier: ^3.0.3 version: 3.0.3 + http-cache-semantics: + specifier: ^4.1.1 + version: 4.1.1 js-yaml: specifier: ^4.1.0 version: 4.1.0 @@ -690,6 +693,9 @@ importers: '@types/html-escaper': specifier: ^3.0.0 version: 3.0.0 + '@types/http-cache-semantics': + specifier: ^4.0.1 + version: 4.0.1 '@types/js-yaml': specifier: ^4.0.5 version: 4.0.5