-
-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(assets): support remote images (#7778)
Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca> Co-authored-by: Princesseuh <princssdev@gmail.com> Co-authored-by: Erika <3019731+Princesseuh@users.noreply.github.com>
- Loading branch information
4 people
committed
Aug 17, 2023
1 parent
2145960
commit d6b4943
Showing
23 changed files
with
657 additions
and
190 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| '@astrojs/vercel': patch | ||
| --- | ||
|
|
||
| Update image support to work with latest version of Astro |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| --- | ||
| 'astro': patch | ||
| --- | ||
|
|
||
| Added support for optimizing remote images from authorized sources when using `astro:assets`. This comes with two new parameters to specify which domains (`image.domains`) and host patterns (`image.remotePatterns`) are authorized for remote images. | ||
|
|
||
| For example, the following configuration will only allow remote images from `astro.build` to be optimized: | ||
|
|
||
| ```ts | ||
| // astro.config.mjs | ||
| export default defineConfig({ | ||
| image: { | ||
| domains: ["astro.build"], | ||
| } | ||
| }); | ||
| ``` | ||
|
|
||
| The following configuration will only allow remote images from HTTPS hosts: | ||
|
|
||
| ```ts | ||
| // astro.config.mjs | ||
| export default defineConfig({ | ||
| image: { | ||
| remotePatterns: [{ protocol: "https" }], | ||
| } | ||
| }); | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,174 @@ | ||
| import fs, { readFileSync } from 'node:fs'; | ||
| import { basename, join } from 'node:path/posix'; | ||
| import type { StaticBuildOptions } from '../../core/build/types.js'; | ||
| import { warn } from '../../core/logger/core.js'; | ||
| import { prependForwardSlash } from '../../core/path.js'; | ||
| import { isServerLikeOutput } from '../../prerender/utils.js'; | ||
| import { getConfiguredImageService, isESMImportedImage } from '../internal.js'; | ||
| import type { LocalImageService } from '../services/service.js'; | ||
| import type { ImageMetadata, ImageTransform } from '../types.js'; | ||
| import { loadRemoteImage, type RemoteCacheEntry } from './remote.js'; | ||
|
|
||
| interface GenerationDataUncached { | ||
| cached: false; | ||
| weight: { | ||
| before: number; | ||
| after: number; | ||
| }; | ||
| } | ||
|
|
||
| interface GenerationDataCached { | ||
| cached: true; | ||
| } | ||
|
|
||
| type GenerationData = GenerationDataUncached | GenerationDataCached; | ||
|
|
||
| export async function generateImage( | ||
| buildOpts: StaticBuildOptions, | ||
| options: ImageTransform, | ||
| filepath: string | ||
| ): Promise<GenerationData | undefined> { | ||
| let useCache = true; | ||
| const assetsCacheDir = new URL('assets/', buildOpts.settings.config.cacheDir); | ||
|
|
||
| // Ensure that the cache directory exists | ||
| try { | ||
| await fs.promises.mkdir(assetsCacheDir, { recursive: true }); | ||
| } catch (err) { | ||
| warn( | ||
| buildOpts.logging, | ||
| 'astro:assets', | ||
| `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${err}` | ||
| ); | ||
| useCache = false; | ||
| } | ||
|
|
||
| let serverRoot: URL, clientRoot: URL; | ||
| if (isServerLikeOutput(buildOpts.settings.config)) { | ||
| serverRoot = buildOpts.settings.config.build.server; | ||
| clientRoot = buildOpts.settings.config.build.client; | ||
| } else { | ||
| serverRoot = buildOpts.settings.config.outDir; | ||
| clientRoot = buildOpts.settings.config.outDir; | ||
| } | ||
|
|
||
| const isLocalImage = isESMImportedImage(options.src); | ||
|
|
||
| const finalFileURL = new URL('.' + filepath, clientRoot); | ||
| const finalFolderURL = new URL('./', finalFileURL); | ||
|
|
||
| // For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server | ||
| const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json'); | ||
| const cachedFileURL = new URL(cacheFile, assetsCacheDir); | ||
|
|
||
| await fs.promises.mkdir(finalFolderURL, { recursive: true }); | ||
|
|
||
| // Check if we have a cached entry first | ||
| try { | ||
| if (isLocalImage) { | ||
| await fs.promises.copyFile(cachedFileURL, finalFileURL); | ||
|
|
||
| return { | ||
| cached: true, | ||
| }; | ||
| } else { | ||
| const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry; | ||
|
|
||
| // If the cache entry is not expired, use it | ||
| if (JSONData.expires < Date.now()) { | ||
| await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64')); | ||
|
|
||
| return { | ||
| cached: true, | ||
| }; | ||
| } | ||
| } | ||
| } catch (e: any) { | ||
| if (e.code !== 'ENOENT') { | ||
| throw new Error(`An error was encountered while reading the cache file. Error: ${e}`); | ||
| } | ||
| // If the cache file doesn't exist, just move on, and we'll generate it | ||
| } | ||
|
|
||
| // The original filepath or URL from the image transform | ||
| const originalImagePath = isLocalImage | ||
| ? (options.src as ImageMetadata).src | ||
| : (options.src as string); | ||
|
|
||
| let imageData; | ||
| let resultData: { data: Buffer | undefined; expires: number | undefined } = { | ||
| data: undefined, | ||
| expires: undefined, | ||
| }; | ||
|
|
||
| // If the image is local, we can just read it directly, otherwise we need to download it | ||
| if (isLocalImage) { | ||
| imageData = await fs.promises.readFile( | ||
| new URL( | ||
| '.' + | ||
| prependForwardSlash( | ||
| join(buildOpts.settings.config.build.assets, basename(originalImagePath)) | ||
| ), | ||
| serverRoot | ||
| ) | ||
| ); | ||
| } else { | ||
| const remoteImage = await loadRemoteImage(originalImagePath); | ||
| resultData.expires = remoteImage.expires; | ||
| imageData = remoteImage.data; | ||
| } | ||
|
|
||
| const imageService = (await getConfiguredImageService()) as LocalImageService; | ||
| resultData.data = ( | ||
| await imageService.transform( | ||
| imageData, | ||
| { ...options, src: originalImagePath }, | ||
| buildOpts.settings.config.image | ||
| ) | ||
| ).data; | ||
|
|
||
| try { | ||
| // Write the cache entry | ||
| if (useCache) { | ||
| if (isLocalImage) { | ||
| await fs.promises.writeFile(cachedFileURL, resultData.data); | ||
| } else { | ||
| await fs.promises.writeFile( | ||
| cachedFileURL, | ||
| JSON.stringify({ | ||
| data: Buffer.from(resultData.data).toString('base64'), | ||
| expires: resultData.expires, | ||
| }) | ||
| ); | ||
| } | ||
| } | ||
| } catch (e) { | ||
| warn( | ||
| buildOpts.logging, | ||
| 'astro:assets', | ||
| `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}` | ||
| ); | ||
| } finally { | ||
| // Write the final file | ||
| await fs.promises.writeFile(finalFileURL, resultData.data); | ||
| } | ||
|
|
||
| return { | ||
| cached: false, | ||
| weight: { | ||
| // Divide by 1024 to get size in kilobytes | ||
| before: Math.trunc(imageData.byteLength / 1024), | ||
| after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024), | ||
| }, | ||
| }; | ||
| } | ||
|
|
||
| export function getStaticImageList(): Iterable< | ||
| [string, { path: string; options: ImageTransform }] | ||
| > { | ||
| if (!globalThis?.astroAsset?.staticImages) { | ||
| return []; | ||
| } | ||
|
|
||
| return globalThis.astroAsset.staticImages?.entries(); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| import CachePolicy from 'http-cache-semantics'; | ||
|
|
||
| export type RemoteCacheEntry = { data: string; expires: number }; | ||
|
|
||
| export async function loadRemoteImage(src: string) { | ||
| const req = new Request(src); | ||
| const res = await fetch(req); | ||
|
|
||
| if (!res.ok) { | ||
| throw new Error( | ||
| `Failed to load remote image ${src}. The request did not return a 200 OK response. (received ${res.status}))` | ||
| ); | ||
| } | ||
|
|
||
| // calculate an expiration date based on the response's TTL | ||
| const policy = new CachePolicy(webToCachePolicyRequest(req), webToCachePolicyResponse(res)); | ||
| const expires = policy.storable() ? policy.timeToLive() : 0; | ||
|
|
||
| return { | ||
| data: Buffer.from(await res.arrayBuffer()), | ||
| expires: Date.now() + expires, | ||
| }; | ||
| } | ||
|
|
||
| function webToCachePolicyRequest({ url, method, headers: _headers }: Request): CachePolicy.Request { | ||
| let headers: CachePolicy.Headers = {}; | ||
| // Be defensive here due to a cookie header bug in node@18.14.1 + undici | ||
| try { | ||
| headers = Object.fromEntries(_headers.entries()); | ||
| } catch {} | ||
| return { | ||
| method, | ||
| url, | ||
| headers, | ||
| }; | ||
| } | ||
|
|
||
| function webToCachePolicyResponse({ status, headers: _headers }: Response): CachePolicy.Response { | ||
| let headers: CachePolicy.Headers = {}; | ||
| // Be defensive here due to a cookie header bug in node@18.14.1 + undici | ||
| try { | ||
| headers = Object.fromEntries(_headers.entries()); | ||
| } catch {} | ||
| return { | ||
| status, | ||
| headers, | ||
| }; | ||
| } |
Oops, something went wrong.