diff --git a/packages/next/build/babel/loader/get-config.ts b/packages/next/build/babel/loader/get-config.ts
index 73bdb3df97bcc..22be2e05fb181 100644
--- a/packages/next/build/babel/loader/get-config.ts
+++ b/packages/next/build/babel/loader/get-config.ts
@@ -156,29 +156,10 @@ function getCustomBabelConfig(configFilePath: string) {
     return require(configFilePath)
   }
   throw new Error(
-    'The Next Babel loader does not support MJS or CJS config files.'
+    'The Next.js Babel loader does not support .mjs or .cjs config files.'
   )
 }
 
-function getCustomPresets(presets: any[], customConfig: any) {
-  presets = [...presets, ...customConfig?.presets]
-
-  const hasNextBabelPreset = (customConfig?.presets || [])
-    .filter((preset: any) => {
-      return (
-        preset === 'next/babel' ||
-        (Array.isArray(preset) && preset[0] === 'next/babel')
-      )
-    })
-    .reduce((memo: boolean, presetFound: boolean) => memo || presetFound, false)
-
-  if (!hasNextBabelPreset) {
-    presets.push('next/babel')
-  }
-
-  return presets
-}
-
 /**
  * Generate a new, flat Babel config, ready to be handed to Babel-traverse.
  * This config should have no unresolved overrides, presets, etc.
@@ -192,7 +173,6 @@ function getFreshConfig(
   inputSourceMap?: object | null
 ) {
   let {
-    presets = [],
     isServer,
     pagesDir,
     development,
@@ -200,17 +180,15 @@ function getFreshConfig(
     configFile,
   } = loaderOptions
 
-  let customPlugins = []
+  // Ensures webpack invalidates the cache for this loader when the config file changes
   if (configFile) {
-    const customConfig = getCustomBabelConfig(configFile)
-    presets = getCustomPresets(presets, customConfig)
-    if (customConfig.plugins) {
-      customPlugins = customConfig.plugins
-    }
-  } else {
-    presets = [...presets, 'next/babel']
+    this.addDependency(configFile)
   }
 
+  let customConfig: any = configFile
+    ? getCustomBabelConfig(configFile)
+    : undefined
+
   let options = {
     babelrc: false,
     cloneInputAst: false,
@@ -231,10 +209,28 @@ function getFreshConfig(
 
     plugins: [
       ...getPlugins(loaderOptions, cacheCharacteristics),
-      ...customPlugins,
+      ...(customConfig?.plugins || []),
     ],
 
-    presets,
+    // target can be provided in babelrc
+    target: isServer ? undefined : customConfig?.target,
+    // env can be provided in babelrc
+    env: customConfig?.env,
+
+    presets: (() => {
+      // If presets is defined the user will have next/babel in their babelrc
+      if (customConfig?.presets) {
+        return customConfig.presets
+      }
+
+      // If presets is not defined the user will likely have "env" in their babelrc
+      if (customConfig) {
+        return undefined
+      }
+
+      // If no custom config is provided the default is to use next/babel
+      return ['next/babel']
+    })(),
 
     overrides: loaderOptions.overrides,
 
@@ -261,6 +257,11 @@ function getFreshConfig(
     },
   } as any
 
+  // Babel does strict checks on the config so undefined is not allowed
+  if (typeof options.target === 'undefined') {
+    delete options.target
+  }
+
   Object.defineProperty(options.caller, 'onWarning', {
     enumerable: false,
     writable: false,
@@ -309,8 +310,8 @@ export default function getConfig(
   this: NextJsLoaderContext,
   {
     source,
-    loaderOptions,
     target,
+    loaderOptions,
     filename,
     inputSourceMap,
   }: {
diff --git a/packages/next/build/babel/loader/types.d.ts b/packages/next/build/babel/loader/types.d.ts
index 6b5856bb83dbc..200a4780f3bca 100644
--- a/packages/next/build/babel/loader/types.d.ts
+++ b/packages/next/build/babel/loader/types.d.ts
@@ -11,7 +11,6 @@ export interface NextBabelLoaderOptions {
   isServer: boolean
   development: boolean
   pagesDir: string
-  presets: any[]
   sourceMaps?: any[]
   overrides: any
   caller: any
diff --git a/test/integration/babel-custom/fixtures/babel-env/.babelrc b/test/integration/babel-custom/fixtures/babel-env/.babelrc
new file mode 100644
index 0000000000000..3aa1c2a5c286a
--- /dev/null
+++ b/test/integration/babel-custom/fixtures/babel-env/.babelrc
@@ -0,0 +1,22 @@
+{
+  "env": {
+    "development": {
+      "presets": ["next/babel"]
+    },
+    "production": {
+      "presets": ["next/babel"]
+    },
+    "test": {
+      "presets": [
+        [
+          "next/babel",
+          {
+            "preset-env": {
+              "modules": "commonjs"
+            }
+          }
+        ]
+      ]
+    }
+  }
+}
diff --git a/test/integration/babel-custom/fixtures/babel-env/pages/index.js b/test/integration/babel-custom/fixtures/babel-env/pages/index.js
new file mode 100644
index 0000000000000..69192c68a0ce5
--- /dev/null
+++ b/test/integration/babel-custom/fixtures/babel-env/pages/index.js
@@ -0,0 +1 @@
+export default () => <h1>Hello World</h1>
diff --git a/test/integration/babel-custom/test/index.test.js b/test/integration/babel-custom/test/index.test.js
index 1f224c3915fdc..b502b31aeb75a 100644
--- a/test/integration/babel-custom/test/index.test.js
+++ b/test/integration/babel-custom/test/index.test.js
@@ -6,6 +6,10 @@ import { nextBuild } from 'next-test-utils'
 jest.setTimeout(1000 * 60 * 5)
 
 describe('Babel', () => {
+  it('should allow setting babelrc env', async () => {
+    await nextBuild(join(__dirname, '../fixtures/babel-env'))
+  })
+
   it('should allow setting targets.browsers', async () => {
     await nextBuild(join(__dirname, '../fixtures/targets-browsers'))
   })