diff --git a/packages/next-plugin-storybook/preset.js b/packages/next-plugin-storybook/preset.js
index 0e83637c8fd4b..d88f476b2152e 100644
--- a/packages/next-plugin-storybook/preset.js
+++ b/packages/next-plugin-storybook/preset.js
@@ -7,7 +7,7 @@ const CWD = process.cwd()
 
 async function webpackFinal(config) {
   const nextConfig = await loadConfig(PHASE_PRODUCTION_BUILD, CWD)
-  const pagesDir = findPagesDir(CWD, !!nextConfig.experimental.appDir)
+  const { pagesDir } = findPagesDir(CWD, !!nextConfig.experimental.appDir)
   const nextWebpackConfig = await getWebpackConfig(CWD, {
     pagesDir,
     entrypoints: {},
diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts
index c3b13bde241e7..fa66f489f6cff 100644
--- a/packages/next/build/index.ts
+++ b/packages/next/build/index.ts
@@ -301,8 +301,8 @@ export default async function build(
       setGlobal('telemetry', telemetry)
 
       const publicDir = path.join(dir, 'public')
-      const hasAppDir = !!config.experimental.appDir
-      const { pagesDir, appDir } = findPagesDir(dir, hasAppDir)
+      const isAppDirEnabled = !!config.experimental.appDir
+      const { pagesDir, appDir } = findPagesDir(dir, isAppDirEnabled)
 
       const hasPublicDir = await fileExists(publicDir)
 
@@ -394,7 +394,7 @@ export default async function build(
                   config.experimental.cpus,
                   config.experimental.workerThreads,
                   telemetry,
-                  hasAppDir
+                  isAppDirEnabled && !!appDir
                 )
               }),
         ])
@@ -1988,7 +1988,7 @@ export default async function build(
         combinedPages.length > 0 ||
         useStatic404 ||
         useDefaultStatic500 ||
-        hasAppDir
+        isAppDirEnabled
       ) {
         const staticGenerationSpan =
           nextBuildSpan.traceChild('static-generation')
diff --git a/packages/next/build/jest/jest.ts b/packages/next/build/jest/jest.ts
index cb4b50887e3a7..be176de4f091f 100644
--- a/packages/next/build/jest/jest.ts
+++ b/packages/next/build/jest/jest.ts
@@ -72,8 +72,8 @@ export default function nextJest(options: { dir?: string } = {}) {
         isEsmProject = packageConfig.type === 'module'
 
         nextConfig = await getConfig(resolvedDir)
-        const hasAppDir = !!nextConfig.experimental.appDir
-        const findPagesDirResult = findPagesDir(resolvedDir, hasAppDir)
+        const isAppDirEnabled = !!nextConfig.experimental.appDir
+        const findPagesDirResult = findPagesDir(resolvedDir, isAppDirEnabled)
         hasServerComponents = !!findPagesDirResult.appDir
         pagesDir = findPagesDirResult.pagesDir
         setUpEnv(resolvedDir, nextConfig)
diff --git a/packages/next/build/webpack-config.ts b/packages/next/build/webpack-config.ts
index c26ba00d51b20..7988eb1374948 100644
--- a/packages/next/build/webpack-config.ts
+++ b/packages/next/build/webpack-config.ts
@@ -563,7 +563,7 @@ export default async function getBaseWebpackConfig(
     rewrites.afterFiles.length > 0 ||
     rewrites.fallback.length > 0
 
-  const hasAppDir = !!config.experimental.appDir
+  const hasAppDir = !!config.experimental.appDir && !!appDir
   const hasConcurrentFeatures = hasReactRoot
   const hasServerComponents = hasAppDir
 
@@ -1587,7 +1587,7 @@ export default async function getBaseWebpackConfig(
             ]
           : []),
         // Alias `next/dynamic` to React.lazy implementation for RSC
-        ...(hasServerComponents && appDir
+        ...(hasServerComponents
           ? [
               {
                 test: codeCondition.test,
diff --git a/packages/next/server/base-server.ts b/packages/next/server/base-server.ts
index 10148095a3499..8c03b09b1c298 100644
--- a/packages/next/server/base-server.ts
+++ b/packages/next/server/base-server.ts
@@ -368,7 +368,8 @@ export default abstract class Server<ServerOptions extends Options = Options> {
     this.buildId = this.getBuildId()
     this.minimalMode = minimalMode || !!process.env.NEXT_PRIVATE_MINIMAL_MODE
 
-    this.hasAppDir = this.getHasAppDir(dev)
+    this.hasAppDir =
+      !!this.nextConfig.experimental.appDir && this.getHasAppDir(dev)
     const serverComponents = this.hasAppDir
     this.serverComponentManifest = serverComponents
       ? this.getServerComponentManifest()
diff --git a/packages/next/server/dev/next-dev-server.ts b/packages/next/server/dev/next-dev-server.ts
index c5a88fce6c4e5..cc34cfb70d123 100644
--- a/packages/next/server/dev/next-dev-server.ts
+++ b/packages/next/server/dev/next-dev-server.ts
@@ -191,7 +191,10 @@ export default class DevServer extends Server {
 
     this.isCustomServer = !options.isNextDevCommand
 
-    const { pagesDir, appDir } = findPagesDir(this.dir, this.hasAppDir)
+    const { pagesDir, appDir } = findPagesDir(
+      this.dir,
+      !!this.nextConfig.experimental.appDir
+    )
     this.pagesDir = pagesDir
     this.appDir = appDir
   }
diff --git a/test/e2e/streaming-ssr/index.test.ts b/test/e2e/streaming-ssr/index.test.ts
index 482346e66ae36..58466bdd5ba07 100644
--- a/test/e2e/streaming-ssr/index.test.ts
+++ b/test/e2e/streaming-ssr/index.test.ts
@@ -22,6 +22,11 @@ describe('react 18 streaming SSR with custom next configs', () => {
   beforeAll(async () => {
     next = await createNext({
       files: {
+        'app/page.js': `
+        export default function Page() {
+          return 'fake-app' /* this should not enable appDir */
+        }
+      `,
         pages: new FileRef(join(__dirname, 'streaming-ssr/pages')),
       },
       nextConfig: require(join(__dirname, 'streaming-ssr/next.config.js')),