diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts
index 600e12062d43..c9be26fa0e87 100644
--- a/packages/next/build/index.ts
+++ b/packages/next/build/index.ts
@@ -1214,6 +1214,8 @@ export default async function build(
             ).createHash('sha256')
 
             cacheHash.update(require('next/package').version)
+            cacheHash.update(hasSsrAmpPages + '')
+            cacheHash.update(ciEnvironment.hasNextSupport + '')
 
             await Promise.all(
               lockFiles.map(async (lockFile) => {
@@ -1242,7 +1244,6 @@ export default async function build(
               processCwd: dir,
               ignore: [
                 '**/next/dist/pages/**/*',
-                '**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*',
                 '**/next/dist/compiled/webpack/(bundle4|bundle5).js',
                 '**/node_modules/webpack5/**/*',
                 '**/next/dist/server/lib/squoosh/**/*.wasm',
@@ -1254,6 +1255,9 @@ export default async function build(
                       '**/node_modules/sharp/**/*',
                     ]
                   : []),
+                ...(!hasSsrAmpPages
+                  ? ['**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*']
+                  : []),
               ],
             }
           )
diff --git a/test/integration/amphtml/pages/amp-ssr.js b/test/integration/amphtml/pages/amp-ssr.js
new file mode 100644
index 000000000000..8cf13df25487
--- /dev/null
+++ b/test/integration/amphtml/pages/amp-ssr.js
@@ -0,0 +1,15 @@
+export const config = { amp: true }
+
+export default function Page() {
+  return (
+    <div>
+      <p id="only-amp">Only AMP for me...</p>
+    </div>
+  )
+}
+
+export function getServerSideProps() {
+  return {
+    props: {},
+  }
+}
diff --git a/test/integration/amphtml/test/index.test.js b/test/integration/amphtml/test/index.test.js
index c1c4ba90b7c3..e0441c76f546 100644
--- a/test/integration/amphtml/test/index.test.js
+++ b/test/integration/amphtml/test/index.test.js
@@ -48,6 +48,17 @@ describe('AMP Usage', () => {
     })
     afterAll(() => stopApp(server))
 
+    it('should have amp optimizer in trace', async () => {
+      const trace = JSON.parse(
+        readFileSync(join(appDir, '.next/next-server.js.nft.json'), 'utf8')
+      )
+      expect(
+        trace.files.some((file) =>
+          file.replace(/\\/g, '/').includes('@ampproject/toolbox-optimizer')
+        )
+      ).toBe(true)
+    })
+
     it('should not contain missing files warning', async () => {
       expect(output).toContain('Compiled successfully')
       expect(output).not.toContain('Could not find files for')