From d9def90ce153be86ab538dd282005d3a4db19271 Mon Sep 17 00:00:00 2001 From: Felix Haus Date: Fri, 12 Feb 2021 20:29:01 +0100 Subject: [PATCH 1/3] Adds option to fetch images from S3 bucket --- lib/image-optimizer.ts | 45 ++++++++++++++++++++++++++++++------ test/e2e.test.ts | 2 +- test/image-optimizer.test.ts | 28 ++++++++++++++++++---- test/utils/s3-public-dir.ts | 2 +- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/lib/image-optimizer.ts b/lib/image-optimizer.ts index bb75608..d586bba 100644 --- a/lib/image-optimizer.ts +++ b/lib/image-optimizer.ts @@ -4,6 +4,7 @@ import { ImageConfig } from 'next/dist/next-server/server/image-config'; import nodeFetch, { RequestInfo, RequestInit } from 'node-fetch'; import { UrlWithParsedQuery } from 'url'; import Server from 'next/dist/next-server/server/next-server'; +import S3 from 'aws-sdk/clients/s3'; let originCacheControl: string | null; @@ -21,11 +22,17 @@ async function fetchPolyfill(url: RequestInfo, init?: RequestInit) { // Polyfill fetch used by nextImageOptimizer global.fetch = fetchPolyfill; +interface S3Config { + s3: S3; + bucket: string; +} + async function imageOptimizer( imageConfig: ImageConfig, req: IncomingMessage, res: ServerResponse, - parsedUrl: UrlWithParsedQuery + parsedUrl: UrlWithParsedQuery, + s3Config?: S3Config ) { // Create Next Server mock const server = ({ @@ -38,11 +45,35 @@ async function imageOptimizer( res: ServerResponse, url: UrlWithParsedQuery ) => { - // TODO: When deployed together with Terraform Next.js we can use - // AWS SDK here to fetch the image directly from S3 instead of - // using node - fetch + if (s3Config) { + // S3 expects keys without leading `/` + const trimmedKey = url.href.startsWith('/') + ? url.href.substring(1) + : url.href; + + const object = await s3Config.s3 + .getObject({ + Key: trimmedKey, + Bucket: s3Config.bucket, + }) + .promise(); + + if (!object.Body) { + throw new Error(`Could not fetch image ${trimmedKey} from bucket.`); + } + + res.statusCode = 200; + + if (object.ContentType) { + res.setHeader('Content-Type', object.ContentType); + } + + if (object.CacheControl) { + originCacheControl = object.CacheControl; + } - if (headers.referer) { + res.end(object.Body); + } else if (headers.referer) { let upstreamBuffer: Buffer; let upstreamType: string | null; @@ -54,7 +85,7 @@ async function imageOptimizer( const upstreamRes = await nodeFetch(origin); if (!upstreamRes.ok) { - throw new Error(`Could not fetch image from ${origin}`); + throw new Error(`Could not fetch image from ${origin}.`); } res.statusCode = upstreamRes.status; @@ -78,4 +109,4 @@ async function imageOptimizer( }; } -export { imageOptimizer }; +export { S3Config, imageOptimizer }; diff --git a/test/e2e.test.ts b/test/e2e.test.ts index 69b70d3..e618f66 100644 --- a/test/e2e.test.ts +++ b/test/e2e.test.ts @@ -1,6 +1,6 @@ import * as path from 'path'; import { LambdaSAM, generateSAM } from '@dealmore/sammy'; -import { S3 } from 'aws-sdk'; +import S3 from 'aws-sdk/clients/s3'; import { URLSearchParams } from 'url'; import { s3PublicDir } from './utils/s3-public-dir'; diff --git a/test/image-optimizer.test.ts b/test/image-optimizer.test.ts index 607ff95..8897a8d 100644 --- a/test/image-optimizer.test.ts +++ b/test/image-optimizer.test.ts @@ -7,10 +7,10 @@ import { ImageConfig, imageConfigDefault, } from 'next/dist/next-server/server/image-config'; -import { S3 } from 'aws-sdk'; +import S3 from 'aws-sdk/clients/s3'; import * as path from 'path'; -import { imageOptimizer } from '../lib/image-optimizer'; +import { imageOptimizer, S3Config } from '../lib/image-optimizer'; import { createDeferred } from './utils'; import { s3PublicDir } from './utils/s3-public-dir'; import { acceptAllFixtures, acceptWebpFixtures } from './constants'; @@ -39,7 +39,8 @@ function generateParams(url: string, options: Options = {}) { async function runOptimizer( params: ReturnType, imageConfig: ImageConfig, - requestHeaders: Record + requestHeaders: Record, + s3Config?: S3Config ) { // Mock request & response const request = createRequest({ @@ -68,7 +69,8 @@ async function runOptimizer( imageConfig, request, response, - params.parsedUrl + params.parsedUrl, + s3Config ); return { @@ -109,6 +111,24 @@ describe('[unit]', () => { bucketName = upload.bucketName; }); + test('Fetch internal image from S3', async () => { + const fixture = acceptAllFixtures[0]; + const fixturePath = `/${fixture[0]}`; + const params = generateParams(fixturePath, optimizerParams); + + const { result, headers } = await runOptimizer( + params, + imageConfig, + { + accept: '*/*', + }, + { s3, bucket: bucketName } + ); + + expect(result.finished).toBe(true); + expect(headers['content-type']).toBe(fixture[1]['content-type']); + }); + test('Custom image size', async () => { const fixture = acceptAllFixtures[0]; const fixturePath = `http://${s3Endpoint}/${bucketName}/${fixture[0]}`; diff --git a/test/utils/s3-public-dir.ts b/test/utils/s3-public-dir.ts index 1162e67..b4aa37c 100644 --- a/test/utils/s3-public-dir.ts +++ b/test/utils/s3-public-dir.ts @@ -1,4 +1,4 @@ -import { S3 } from 'aws-sdk'; +import S3 from 'aws-sdk/clients/s3'; import { randomBytes } from 'crypto'; import { promises as fs, createReadStream } from 'fs'; import * as path from 'path'; From 23364569cf467b17fe49274357d745a010df393f Mon Sep 17 00:00:00 2001 From: Felix Haus Date: Sat, 13 Feb 2021 16:47:45 +0100 Subject: [PATCH 2/3] Adds image resizing from S3 source to handler --- lib/declarations.d.ts | 2 + lib/handler.ts | 34 ++++- main.tf | 11 +- test/e2e.test.ts | 264 +++++++++++++++++++++++------------- test/utils/s3-public-dir.ts | 28 ++-- variables.tf | 12 ++ 6 files changed, 238 insertions(+), 113 deletions(-) diff --git a/lib/declarations.d.ts b/lib/declarations.d.ts index 25ff48b..836ffee 100644 --- a/lib/declarations.d.ts +++ b/lib/declarations.d.ts @@ -7,5 +7,7 @@ declare module NodeJS { TF_NEXTIMAGE_DOMAINS?: string; TF_NEXTIMAGE_DEVICE_SIZES?: string; TF_NEXTIMAGE_IMAGE_SIZES?: string; + TF_NEXTIMAGE_SOURCE_BUCKET?: string; + __DEBUG__USE_LOCAL_BUCKET?: string; } } diff --git a/lib/handler.ts b/lib/handler.ts index c69ce61..6d4b058 100644 --- a/lib/handler.ts +++ b/lib/handler.ts @@ -8,10 +8,31 @@ import { APIGatewayProxyStructuredResultV2, } from 'aws-lambda'; import { Writable } from 'stream'; +import S3 from 'aws-sdk/clients/s3'; -import { imageOptimizer } from './image-optimizer'; +import { imageOptimizer, S3Config } from './image-optimizer'; import { normalizeHeaders } from './normalized-headers'; +function generateS3Config(bucketName?: string): S3Config | undefined { + let s3: S3; + + if (!bucketName) { + return undefined; + } + + // Only for testing purposes when connecting against a local S3 backend + if (process.env.__DEBUG__USE_LOCAL_BUCKET) { + s3 = new S3(JSON.parse(process.env.__DEBUG__USE_LOCAL_BUCKET)); + } else { + s3 = new S3(); + } + + return { + s3, + bucket: bucketName, + }; +} + function parseFromEnv(key: string, defaultValue: T) { try { if (key in process.env) { @@ -38,6 +59,7 @@ const imageSizes = parseFromEnv( 'TF_NEXTIMAGE_IMAGE_SIZES', imageConfigDefault.deviceSizes ); +const sourceBucket = process.env.TF_NEXTIMAGE_SOURCE_BUCKET ?? undefined; const imageConfig: ImageConfig = { ...imageConfigDefault, @@ -49,6 +71,8 @@ const imageConfig: ImageConfig = { export async function handler( event: APIGatewayProxyEventV2 ): Promise { + const s3Config = generateS3Config(sourceBucket); + const reqMock: any = { headers: normalizeHeaders(event.headers), method: event.requestContext.http.method, @@ -75,7 +99,13 @@ export async function handler( resMock._implicitHeader = () => {}; const parsedUrl = parseUrl(reqMock.url, true); - const result = await imageOptimizer(imageConfig, reqMock, resMock, parsedUrl); + const result = await imageOptimizer( + imageConfig, + reqMock, + resMock, + parsedUrl, + s3Config + ); const normalizedHeaders: Record = {}; for (const [headerKey, headerValue] of Object.entries(mockHeaders)) { diff --git a/main.tf b/main.tf index 7be684e..e3eb41f 100644 --- a/main.tf +++ b/main.tf @@ -29,10 +29,11 @@ module "image_optimizer" { publish = true environment_variables = { - NODE_ENV = "production", - TF_NEXTIMAGE_DOMAINS = jsonencode(var.next_image_domains) - TF_NEXTIMAGE_DEVICE_SIZES = var.next_image_device_sizes != null ? jsonencode(var.next_image_device_sizes) : null - TF_NEXTIMAGE_IMAGE_SIZES = var.next_image_image_sizes != null ? jsonencode(var.next_image_image_sizes) : null + NODE_ENV = "production", + TF_NEXTIMAGE_DOMAINS = jsonencode(var.next_image_domains) + TF_NEXTIMAGE_DEVICE_SIZES = var.next_image_device_sizes != null ? jsonencode(var.next_image_device_sizes) : null + TF_NEXTIMAGE_IMAGE_SIZES = var.next_image_image_sizes != null ? jsonencode(var.next_image_image_sizes) : null + TF_NEXTIMAGE_SOURCE_BUCKET = var.source_bucket_id } create_package = false @@ -47,6 +48,8 @@ module "image_optimizer" { cloudwatch_logs_retention_in_days = 30 + attach_policy_json = var.lambda_policy_json != null + policy_json = var.lambda_policy_json role_permissions_boundary = var.lambda_role_permissions_boundary tags = var.tags diff --git a/test/e2e.test.ts b/test/e2e.test.ts index e618f66..cc0d986 100644 --- a/test/e2e.test.ts +++ b/test/e2e.test.ts @@ -15,36 +15,9 @@ describe('[e2e]', () => { const fixturesDir = path.resolve(__dirname, './fixtures'); const cacheControlHeader = 'public, max-age=123456'; let fixtureBucketName: string; - let lambdaSAM: LambdaSAM; let s3: S3; beforeAll(async () => { - // Generate SAM for the worker lambda - lambdaSAM = await generateSAM({ - lambdas: { - imageOptimizer: { - filename: 'dist.zip', - handler: 'handler.handler', - runtime: 'nodejs12.x', - memorySize: 1024, - route, - method: 'get', - environment: { - TF_NEXTIMAGE_DOMAINS: JSON.stringify([hostIpAddress]), - }, - }, - }, - cwd: pathToWorker, - onData(data) { - console.log(data.toString()); - }, - onError(data) { - console.log(data.toString()); - }, - }); - - await lambdaSAM.start(); - // Upload the fixtures to public S3 server const S3options: S3.Types.ClientConfiguration = { accessKeyId: 'test', @@ -60,92 +33,195 @@ describe('[e2e]', () => { fixtureBucketName = upload.bucketName; }); - afterAll(async () => { - await lambdaSAM.stop(); + describe('Without S3', () => { + let lambdaSAM: LambdaSAM; + + beforeAll(async () => { + // Generate SAM for the worker lambda + lambdaSAM = await generateSAM({ + lambdas: { + imageOptimizer: { + filename: 'dist.zip', + handler: 'handler.handler', + runtime: 'nodejs12.x', + memorySize: 1024, + route, + method: 'get', + environment: { + TF_NEXTIMAGE_DOMAINS: JSON.stringify([hostIpAddress]), + }, + }, + }, + cwd: pathToWorker, + onData(data) { + console.log(data.toString()); + }, + onError(data) { + console.log(data.toString()); + }, + }); + + await lambdaSAM.start(); + }); + + afterAll(async () => { + await lambdaSAM.stop(); + }); + + test.each(acceptAllFixtures)( + 'External: Accept */*: %s', + async (filePath, fixtureResponse) => { + const publicPath = `http://${s3Endpoint}/${fixtureBucketName}/${filePath}`; + const optimizerParams = new URLSearchParams({ + url: publicPath, + w: '2048', + q: '75', + }); + const optimizerPrefix = `external_accept_all_w-${optimizerParams.get( + 'w' + )}_q-${optimizerParams.get('q')}_`; + const snapshotFileName = path.join( + __dirname, + '__snapshots__/e2e/', + `${optimizerPrefix}${filePath.replace('/', '_')}.${ + fixtureResponse.ext + }` + ); + + const response = await lambdaSAM.sendApiGwRequest( + `${route}?${optimizerParams.toString()}` + ); + const body = await response + .text() + .then((text) => Buffer.from(text, 'base64')); + + expect(response.status).toBe(200); + expect(body).toMatchFile(snapshotFileName); + expect(response.headers.get('Content-Type')).toBe( + fixtureResponse['content-type'] + ); + expect(response.headers.get('Cache-Control')).toBe(cacheControlHeader); + + // Header settings needed for CloudFront compression + expect(response.headers.has('Content-Length')).toBeTruthy(); + expect(response.headers.has('Content-Encoding')).toBeFalsy(); + } + ); + + test.each(acceptAllFixtures)( + 'Internal: Accept */*: %s', + async (filePath, fixtureResponse) => { + const publicPath = `/${fixtureBucketName}/${filePath}`; + const optimizerParams = new URLSearchParams({ + url: publicPath, + w: '2048', + q: '75', + }); + const optimizerPrefix = `internal_accept_all_w-${optimizerParams.get( + 'w' + )}_q-${optimizerParams.get('q')}_`; + const snapshotFileName = path.join( + __dirname, + '__snapshots__/e2e/', + `${optimizerPrefix}${filePath.replace('/', '_')}.${ + fixtureResponse.ext + }` + ); + + const response = await lambdaSAM.sendApiGwRequest( + `${route}?${optimizerParams.toString()}`, + { + headers: { + Accept: '*/*', + Referer: `http://${s3Endpoint}/`, + }, + } + ); + + const body = await response + .text() + .then((text) => Buffer.from(text, 'base64')); + + expect(response.status).toBe(200); + expect(body).toMatchFile(snapshotFileName); + expect(response.headers.get('Content-Type')).toBe( + fixtureResponse['content-type'] + ); + expect(response.headers.get('Cache-Control')).toBe(cacheControlHeader); + + // Header settings needed for CloudFront compression + expect(response.headers.has('Content-Length')).toBeTruthy(); + expect(response.headers.has('Content-Encoding')).toBeFalsy(); + } + ); }); - test.each(acceptAllFixtures)( - 'External: Accept */*: %s', - async (filePath, fixtureResponse) => { - const publicPath = `http://${s3Endpoint}/${fixtureBucketName}/${filePath}`; - const optimizerParams = new URLSearchParams({ - url: publicPath, - w: '2048', - q: '75', + describe('With S3', () => { + let lambdaSAM: LambdaSAM; + + beforeAll(async () => { + // Generate SAM for the worker lambda + lambdaSAM = await generateSAM({ + lambdas: { + imageOptimizer: { + filename: 'dist.zip', + handler: 'handler.handler', + runtime: 'nodejs12.x', + memorySize: 1024, + route, + method: 'get', + environment: { + TF_NEXTIMAGE_SOURCE_BUCKET: fixtureBucketName, + __DEBUG__USE_LOCAL_BUCKET: JSON.stringify({ + accessKeyId: 'test', + secretAccessKey: 'testtest', + endpoint: s3Endpoint, + s3ForcePathStyle: true, + signatureVersion: 'v4', + sslEnabled: false, + }), + }, + }, + }, + cwd: pathToWorker, + onData(data) { + console.log(data.toString()); + }, + onError(data) { + console.log(data.toString()); + }, }); - const optimizerPrefix = `external_accept_all_w-${optimizerParams.get( - 'w' - )}_q-${optimizerParams.get('q')}_`; - const snapshotFileName = path.join( - __dirname, - '__snapshots__/e2e/', - `${optimizerPrefix}${filePath.replace('/', '_')}.${fixtureResponse.ext}` - ); - const response = await lambdaSAM.sendApiGwRequest( - `${route}?${optimizerParams.toString()}` - ); - const body = await response - .text() - .then((text) => Buffer.from(text, 'base64')); + await lambdaSAM.start(); + }); - expect(response.status).toBe(200); - expect(body).toMatchFile(snapshotFileName); - expect(response.headers.get('Content-Type')).toBe( - fixtureResponse['content-type'] - ); - expect(response.headers.get('Cache-Control')).toBe(cacheControlHeader); - - // Header settings needed for CloudFront compression - expect(response.headers.has('Content-Length')).toBeTruthy(); - expect(response.headers.has('Content-Encoding')).toBeFalsy(); - } - ); - - test.each(acceptAllFixtures)( - 'Internal: Accept */*: %s', - async (filePath, fixtureResponse) => { - const publicPath = `/${fixtureBucketName}/${filePath}`; + afterAll(async () => { + await lambdaSAM.stop(); + }); + + test('Internal: Fetch image from S3', async () => { + const fixture = acceptAllFixtures[0]; + const publicPath = `/${fixture[0]}`; const optimizerParams = new URLSearchParams({ url: publicPath, w: '2048', q: '75', }); - const optimizerPrefix = `internal_accept_all_w-${optimizerParams.get( - 'w' - )}_q-${optimizerParams.get('q')}_`; - const snapshotFileName = path.join( - __dirname, - '__snapshots__/e2e/', - `${optimizerPrefix}${filePath.replace('/', '_')}.${fixtureResponse.ext}` - ); const response = await lambdaSAM.sendApiGwRequest( - `${route}?${optimizerParams.toString()}`, - { - headers: { - Accept: '*/*', - Referer: `http://${s3Endpoint}/`, - }, - } + `${route}?${optimizerParams.toString()}` ); const body = await response .text() .then((text) => Buffer.from(text, 'base64')); - expect(response.status).toBe(200); - expect(body).toMatchFile(snapshotFileName); - expect(response.headers.get('Content-Type')).toBe( - fixtureResponse['content-type'] + expect(response.ok).toBeTruthy(); + expect(response.headers.get('content-type')).toBe( + fixture[1]['content-type'] ); - expect(response.headers.get('Cache-Control')).toBe(cacheControlHeader); - - // Header settings needed for CloudFront compression - expect(response.headers.has('Content-Length')).toBeTruthy(); - expect(response.headers.has('Content-Encoding')).toBeFalsy(); - } - ); + }); + }); test.todo('Run test against domain that is not on the list'); }); diff --git a/test/utils/s3-public-dir.ts b/test/utils/s3-public-dir.ts index b4aa37c..787ed2e 100644 --- a/test/utils/s3-public-dir.ts +++ b/test/utils/s3-public-dir.ts @@ -25,19 +25,21 @@ async function uploadDir( const files = (await getFiles(s3Path)) as string[]; await Promise.all( - files.map((filePath) => { - // Restore the relative structure - const objectKey = path.relative(s3Path, filePath); - return s3 - .putObject({ - Key: objectKey, - Bucket: bucketName, - Body: createReadStream(filePath), - CacheControl: cacheControl, - ContentType: getType(filePath) ?? undefined, - }) - .promise(); - }) + files + .filter((filePath) => !filePath.includes('.DS_Store')) + .map((filePath) => { + // Restore the relative structure + const objectKey = path.relative(s3Path, filePath); + return s3 + .putObject({ + Key: objectKey, + Bucket: bucketName, + Body: createReadStream(filePath), + CacheControl: cacheControl, + ContentType: getType(filePath) ?? undefined, + }) + .promise(); + }) ); return files; diff --git a/variables.tf b/variables.tf index 240cdcc..3d136c1 100644 --- a/variables.tf +++ b/variables.tf @@ -47,12 +47,24 @@ variable "lambda_timeout" { } } +variable "lambda_policy_json" { + description = "Additional policy document as JSON to attach to the Lambda Function role." + type = string + default = null +} + variable "lambda_role_permissions_boundary" { description = "ARN of IAM policy that scopes aws_iam_role access for the lambda." type = string default = null } +variable "source_bucket_id" { + description = "When your static files are deployed to a Bucket (e.g. with Terraform Next.js) the optimizer can pull the source from the bucket rather than over the internet." + type = string + default = null +} + ##################### # CloudFront settings ##################### From 62d64b43495a5611f1f5798233db4254d308b17e Mon Sep 17 00:00:00 2001 From: Felix Haus Date: Sat, 13 Feb 2021 17:30:58 +0100 Subject: [PATCH 3/3] Fixes lambda role policy attachment --- main.tf | 2 +- variables.tf | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index e3eb41f..40bd021 100644 --- a/main.tf +++ b/main.tf @@ -48,7 +48,7 @@ module "image_optimizer" { cloudwatch_logs_retention_in_days = 30 - attach_policy_json = var.lambda_policy_json != null + attach_policy_json = var.lambda_attach_policy_json policy_json = var.lambda_policy_json role_permissions_boundary = var.lambda_role_permissions_boundary diff --git a/variables.tf b/variables.tf index 3d136c1..aaf146f 100644 --- a/variables.tf +++ b/variables.tf @@ -47,10 +47,16 @@ variable "lambda_timeout" { } } +variable "lambda_attach_policy_json" { + description = "Controls whether lambda_policy_json should be added to IAM role for Lambda function." + type = bool + default = false +} + variable "lambda_policy_json" { description = "Additional policy document as JSON to attach to the Lambda Function role." type = string - default = null + default = "" } variable "lambda_role_permissions_boundary" {