diff --git a/packages/cli/builder/src/shared/devServer.ts b/packages/cli/builder/src/shared/devServer.ts
index f56154f91c66..872ba94f02d1 100644
--- a/packages/cli/builder/src/shared/devServer.ts
+++ b/packages/cli/builder/src/shared/devServer.ts
@@ -20,13 +20,16 @@ const defaultDevConfig = {
 export const transformToRsbuildServerOptions = (
   dev: NonNullable<BuilderConfig['dev']>,
   devServer: ToolsDevServerConfig,
+  server?: BuilderConfig['server'],
 ): {
   rsbuildDev: DevConfig;
   rsbuildServer: ServerConfig;
 } => {
   const { host, https, startUrl, beforeStartUrl, ...devConfig } = dev;
 
-  const port = process.env.PORT ? Number(process.env.PORT) : 8080;
+  const port = process.env.PORT
+    ? Number(process.env.PORT)
+    : (server?.port ?? 8080);
   const rsbuildDev: DevConfig = merge(defaultDevConfig, devConfig);
   // setupMiddlewares apply by @modern-js/server
   delete rsbuildDev.setupMiddlewares;
diff --git a/packages/cli/builder/src/shared/parseCommonConfig.ts b/packages/cli/builder/src/shared/parseCommonConfig.ts
index 548261e37f1a..288783de0b43 100644
--- a/packages/cli/builder/src/shared/parseCommonConfig.ts
+++ b/packages/cli/builder/src/shared/parseCommonConfig.ts
@@ -188,6 +188,7 @@ export async function parseCommonConfig(
   const { rsbuildDev, rsbuildServer } = transformToRsbuildServerOptions(
     dev || {},
     devServer || {},
+    builderConfig.server,
   );
 
   rsbuildConfig.server = removeUndefinedKey(rsbuildServer);
diff --git a/packages/cli/builder/src/types.ts b/packages/cli/builder/src/types.ts
index fe4edfd49294..111b29476686 100644
--- a/packages/cli/builder/src/types.ts
+++ b/packages/cli/builder/src/types.ts
@@ -271,6 +271,7 @@ export type BuilderConfig = {
   };
   server?: {
     rsc?: boolean;
+    port?: number;
   };
   performance?: RsbuildConfig['performance'];
   security?: Omit<SecurityConfig, 'sri'>;