feat: added SQS worker server to revalidate expired pages

package.json

@@ -62,11 +62,12 @@
     "@aws-sdk/client-cloudfront": "3.590.0",
     "@aws-sdk/client-elastic-beanstalk": "3.590.0",
     "@aws-sdk/client-s3": "3.591.0",
+    "@aws-sdk/client-sqs": "3.682.0",
     "@aws-sdk/client-sts": "3.590.0",
     "@aws-sdk/credential-providers": "3.590.0",
     "@aws-sdk/util-endpoints": "3.587.0",
     "@dbbs/next-cache-handler-core": "1.2.0",
-    "@dbbs/next-cache-handler-s3": "1.1.4",
+    "@dbbs/next-cache-handler-s3": "1.2.0",
     "aws-cdk-lib": "2.144.0",
     "aws-sdk": "2.1635.0",
     "cdk-assets": "2.144.0",

src/build/cache/handlers/appRouterRevalidate.ts

@@ -0,0 +1,15 @@
+export default `import { NextResponse, NextRequest } from 'next/server'
+import { revalidatePath } from 'next/cache'
+
+export const POST = async (req: NextRequest) => {
+  try {
+    const { path } = await req.json()
+
+    revalidatePath(path)
+
+    return NextResponse.json({ message: 'Revalidated' }, { status: 200 })
+  } catch (err) {
+    return NextResponse.json({ error: (err as Error).message }, { status: 500 })
+  }
+}
+`

src/build/next.ts

@@ -1,13 +1,16 @@
 import childProcess from 'node:child_process'
-import fs from 'node:fs'
+import fs from 'fs/promises'
 import path from 'node:path'
 import { type ProjectPackager, type ProjectSettings, loadFile } from '../common/project'
 import loadConfig from '../commands/helpers/loadConfig'
+import appRouterRevalidate from './cache/handlers/appRouterRevalidate'
 
 interface BuildOptions {
   packager: ProjectPackager
   nextConfigPath: string
   s3BucketName: string
+  isAppDir: boolean
+  projectPath: string
 }
 
 interface BuildAppOptions {
@@ -18,7 +21,7 @@ interface BuildAppOptions {
 
 export const OUTPUT_FOLDER = 'serverless-next'
 
-const setNextOptions = async (nextConfigPath: string, s3BucketName: string) => {
+const setNextOptions = async (nextConfigPath: string, s3BucketName: string): Promise<() => Promise<void>> => {
   // set s3 bucket name for cache handler during build time
   process.env.STATIC_BUCKET_NAME = s3BucketName
 
@@ -34,7 +37,7 @@ const setNextOptions = async (nextConfigPath: string, s3BucketName: string) => {
     cacheHandler: require.resolve(path.join('..', 'cacheHandler', 'index.js'))
   }
 
-  const currentContent = fs.readFileSync(nextConfigPath, 'utf-8')
+  const currentContent = await fs.readFile(nextConfigPath, 'utf-8')
 
   let updatedContent = `module.exports = ${JSON.stringify(updatedConfig, null, 4)};\n`
 
@@ -43,45 +46,63 @@ const setNextOptions = async (nextConfigPath: string, s3BucketName: string) => {
     updatedContent = `export default ${JSON.stringify(updatedConfig, null, 4)};\n`
   }
 
-  fs.writeFileSync(nextConfigPath, updatedContent, 'utf-8')
+  await fs.writeFile(nextConfigPath, updatedContent, 'utf-8')
 
   // Function to revert back to original content of file
-  return () => {
-    fs.writeFileSync(nextConfigPath, currentContent, 'utf-8')
+  return async () => {
+    fs.writeFile(nextConfigPath, currentContent, 'utf-8')
   }
 }
 
-export const buildNext = async (options: BuildOptions) => {
-  const { packager, nextConfigPath, s3BucketName } = options
+const appendRevalidateApi = async (projectPath: string, isAppDir: boolean): Promise<string> => {
+  const routeFolderPath = path.join(projectPath, isAppDir ? 'src/app' : 'src', 'api', 'revalidate')
+  const routePath = path.join(routeFolderPath, 'route.ts')
+  if ((await fs.stat(routeFolderPath)).isDirectory()) {
+    await fs.mkdir(routeFolderPath, { recursive: true })
+  }
+
+  fs.writeFile(routePath, appRouterRevalidate, 'utf-8')
+
+  return routePath
+}
 
+export const buildNext = async (options: BuildOptions): Promise<() => Promise<void>> => {
+  const { packager, nextConfigPath, s3BucketName, projectPath, isAppDir } = options
+
+  const revalidateRoutePath = await appendRevalidateApi(projectPath, isAppDir)
   const clearNextConfig = await setNextOptions(nextConfigPath, s3BucketName)
   childProcess.execSync(packager.buildCommand, { stdio: 'inherit' })
 
-  // Reverts changes to next project
-  return clearNextConfig
+  // Reverts changes to the next project
+  return async () => {
+    await Promise.all([clearNextConfig(), fs.rm(revalidateRoutePath)])
+  }
 }
 
-const copyAssets = (outputPath: string, appPath: string) => {
+const copyAssets = async (outputPath: string, appPath: string) => {
   // Copying static assets (like js, css, images, .etc)
-  fs.cpSync(path.join(appPath, '.next', 'static'), path.join(outputPath, '_next', 'static'), { recursive: true })
-
-  fs.cpSync(path.join(appPath, '.next', 'standalone'), path.join(outputPath, 'server'), {
-    recursive: true
-  })
+  await Promise.all([
+    fs.cp(path.join(appPath, '.next', 'static'), path.join(outputPath, '_next', 'static'), { recursive: true }),
+    fs.cp(path.join(appPath, '.next', 'standalone'), path.join(outputPath, 'server'), {
+      recursive: true
+    })
+  ])
 }
 
 export const buildApp = async (options: BuildAppOptions) => {
   const { projectSettings, outputPath, s3BucketName } = options
 
-  const { packager, nextConfigPath, projectPath } = projectSettings
+  const { packager, nextConfigPath, projectPath, isAppDir } = projectSettings
 
   const cleanNextApp = await buildNext({
     packager,
     nextConfigPath,
-    s3BucketName
+    s3BucketName,
+    isAppDir,
+    projectPath
   })
 
-  copyAssets(outputPath, projectPath)
+  await copyAssets(outputPath, projectPath)
 
   return cleanNextApp
 }

src/cdk/constructs/CheckExpirationLambdaEdge.ts

@@ -1,4 +1,5 @@
 import { Construct } from 'constructs'
+import * as iam from 'aws-cdk-lib/aws-iam'
 import * as cdk from 'aws-cdk-lib'
 import * as lambda from 'aws-cdk-lib/aws-lambda'
 import * as cloudfront from 'aws-cdk-lib/aws-cloudfront'
@@ -9,9 +10,11 @@ import { CacheConfig } from '../../types'
 
 interface CheckExpirationLambdaEdgeProps extends cdk.StackProps {
   renderWorkerQueueUrl: string
+  renderWorkerQueueArn: string
   buildOutputPath: string
   nodejs?: string
   cacheConfig: CacheConfig
+  region: string
 }
 
 const NodeJSEnvironmentMapping: Record<string, lambda.Runtime> = {
@@ -23,7 +26,7 @@ export class CheckExpirationLambdaEdge extends Construct {
   public readonly lambdaEdge: cloudfront.experimental.EdgeFunction
 
   constructor(scope: Construct, id: string, props: CheckExpirationLambdaEdgeProps) {
-    const { nodejs, buildOutputPath, cacheConfig, renderWorkerQueueUrl } = props
+    const { nodejs, buildOutputPath, cacheConfig, renderWorkerQueueUrl, renderWorkerQueueArn, region } = props
     super(scope, id)
 
     const nodeJSEnvironment = NodeJSEnvironmentMapping[nodejs ?? ''] ?? NodeJSEnvironmentMapping['20']
@@ -32,7 +35,8 @@ export class CheckExpirationLambdaEdge extends Construct {
     buildLambda(name, buildOutputPath, {
       define: {
         'process.env.RENDER_QUEUE_URL': JSON.stringify(renderWorkerQueueUrl),
-        'process.env.CACHE_CONFIG': JSON.stringify(cacheConfig)
+        'process.env.CACHE_CONFIG': JSON.stringify(cacheConfig),
+        'process.env.QUEUE_REGION': JSON.stringify(region)
       }
     })
 
@@ -49,6 +53,13 @@ export class CheckExpirationLambdaEdge extends Construct {
       logGroup
     })
 
+    this.lambdaEdge.addToRolePolicy(
+      new iam.PolicyStatement({
+        actions: ['sqs:SendMessage'],
+        resources: [renderWorkerQueueArn]
+      })
+    )
+
     logGroup.grantWrite(this.lambdaEdge)
   }
 }

src/cdk/constructs/RenderServerDistribution.ts

@@ -82,8 +82,8 @@ export class RenderServerDistribution extends Construct {
 
     this.ebInstanceProfileRole.addToPolicy(
       new iam.PolicyStatement({
-        actions: ['s3:Get*', 's3:Put*'],
-        resources: [`${staticS3Bucket.bucketArn}/*`]
+        actions: ['s3:Get*', 's3:Put*', 's3:ListBucket'],
+        resources: [staticS3Bucket.bucketArn, `${staticS3Bucket.bucketArn}/*`]
       })
     )
 

src/cdk/constructs/RenderWorkerDistribution.ts

@@ -95,7 +95,8 @@ export class RenderWorkerDistribution extends Construct {
     this.deadLetterQueue = new sqs.Queue(this, 'DeadLetterQueue', {
       queueName: `${appName}-dead-letter-queue`,
       retentionPeriod: Duration.days(14),
-      removalPolicy: isProduction ? RemovalPolicy.RETAIN : RemovalPolicy.DESTROY
+      removalPolicy: isProduction ? RemovalPolicy.RETAIN : RemovalPolicy.DESTROY,
+      fifo: true
     })
 
     /**
@@ -108,7 +109,8 @@ export class RenderWorkerDistribution extends Construct {
         queue: this.deadLetterQueue,
         maxReceiveCount: 3
       },
-      removalPolicy: isProduction ? RemovalPolicy.RETAIN : RemovalPolicy.DESTROY
+      removalPolicy: isProduction ? RemovalPolicy.RETAIN : RemovalPolicy.DESTROY,
+      fifo: true
     })
 
     /**
@@ -155,8 +157,8 @@ export class RenderWorkerDistribution extends Construct {
      */
     instanceRole.addToPolicy(
       new iam.PolicyStatement({
-        actions: ['s3:Get*', 's3:Put*'],
-        resources: [`${staticS3Bucket.bucketArn}/*`]
+        actions: ['s3:Get*', 's3:Put*', 's3:ListBucket'],
+        resources: [staticS3Bucket.bucketArn, `${staticS3Bucket.bucketArn}/*`]
       })
     )
 
@@ -319,6 +321,7 @@ export class RenderWorkerDistribution extends Construct {
     })
 
     addOutput(this, `${appName}-RenderWorkerQueueUrl`, this.workerQueue.queueUrl)
+    addOutput(this, `${appName}-RenderWorkerQueueArn`, this.workerQueue.queueArn)
     addOutput(this, `${appName}-RenderWorkerApplicationName`, this.application.applicationName!)
     addOutput(this, `${appName}-RenderWorkerEnvironmentName`, this.environment.environmentName!)
     addOutput(this, `${appName}-RenderWorkerVersionsBucketName`, this.versionsBucket.bucketName)

src/cdk/stacks/NextCloudfrontStack.ts

@@ -8,10 +8,11 @@ import { CheckExpirationLambdaEdge } from '../constructs/CheckExpirationLambdaEd
 
 export interface NextCloudfrontStackProps extends StackProps {
   nodejs?: string
-  region?: string
+  region: string
   staticBucketName: string
   renderServerDomain: string
   renderWorkerQueueUrl: string
+  renderWorkerQueueArn: string
   buildOutputPath: string
   cacheConfig: CacheConfig
   imageTTL?: number
@@ -30,6 +31,7 @@ export class NextCloudfrontStack extends Stack {
       staticBucketName,
       renderServerDomain,
       renderWorkerQueueUrl,
+      renderWorkerQueueArn,
       region,
       cacheConfig,
       imageTTL
@@ -48,7 +50,9 @@ export class NextCloudfrontStack extends Stack {
       nodejs,
       renderWorkerQueueUrl,
       buildOutputPath,
-      cacheConfig
+      cacheConfig,
+      renderWorkerQueueArn,
+      region
     })
 
     const staticBucket = s3.Bucket.fromBucketAttributes(this, `${id}-StaticAssetsBucket`, {

src/commands/deploy.ts

@@ -142,6 +142,7 @@ export const deploy = async (config: DeployConfig) => {
         staticBucketName: nextRenderServerStackOutput.StaticBucketName,
         renderServerDomain: nextRenderServerStackOutput.RenderServerDomain,
         renderWorkerQueueUrl: nextRenderServerStackOutput.RenderWorkerQueueUrl,
+        renderWorkerQueueArn: nextRenderServerStackOutput.RenderWorkerQueueArn,
         buildOutputPath: outputPath,
         crossRegionReferences: true,
         region,
@@ -242,6 +243,6 @@ export const deploy = async (config: DeployConfig) => {
     console.error('Failed to deploy:', err)
   } finally {
     cleanOutputFolder()
-    cleanNextApp?.()
+    await cleanNextApp?.()
   }
 }

src/common/project.ts

@@ -13,6 +13,7 @@ export interface ProjectSettings {
   isMonorepo: boolean
   projectPath: string
   nextConfigPath: string
+  isAppDir: boolean
 }
 
 export const findPackager = (appPath: string): ProjectPackager | undefined => {
@@ -29,6 +30,10 @@ export const findNextConfig = (appPath: string): string | undefined => {
   return ['next.config.js', 'next.config.mjs'].find((config) => fs.existsSync(path.join(appPath, config)))
 }
 
+const checkIsAppDir = (appPath: string): boolean => {
+  return fs.existsSync(path.join(appPath, 'src', 'app'))
+}
+
 export const getProjectSettings = (projectPath: string): ProjectSettings | undefined => {
   let currentPath = projectPath
   const nextConfig = findNextConfig(projectPath)
@@ -46,7 +51,8 @@ export const getProjectSettings = (projectPath: string): ProjectSettings | undef
         packager,
         isMonorepo: currentPath !== projectPath,
         projectPath,
-        nextConfigPath: path.join(projectPath, nextConfig)
+        nextConfigPath: path.join(projectPath, nextConfig),
+        isAppDir: checkIsAppDir(projectPath)
       }
     }