diff --git a/packages/histoire-shared/src/types/config.ts b/packages/histoire-shared/src/types/config.ts
index 8fd8048f..3661bf34 100644
--- a/packages/histoire-shared/src/types/config.ts
+++ b/packages/histoire-shared/src/types/config.ts
@@ -128,6 +128,16 @@ export interface HistoireConfig {
      * Only loaded in the browser client.
      */
     browser: string
+  } | {
+    /**
+     * Only loaded while collecting stories in the node server.
+     */
+    server: string
+  } | {
+    /**
+     * Only loaded in the browser client.
+     */
+    browser: string
     /**
      * Only loaded while collecting stories in the node server.
      */
diff --git a/packages/histoire/src/node/vite.ts b/packages/histoire/src/node/vite.ts
index f38e23bd..d475d8ef 100644
--- a/packages/histoire/src/node/vite.ts
+++ b/packages/histoire/src/node/vite.ts
@@ -196,17 +196,18 @@ export async function getViteConfigWithPlugins (isServer: boolean, ctx: Context)
           let file: string
           if (typeof setupFileConfig === 'string') {
             file = setupFileConfig
-          } else if (isServer) {
+          } else if (isServer && 'server' in setupFileConfig) {
             file = setupFileConfig.server
-          } else {
+          } else if ('browser' in setupFileConfig) {
             file = setupFileConfig.browser
           }
-          return this.resolve(resolve(ctx.root, file), importer, {
-            skipSelf: true,
-          })
-        } else {
-          return NOOP_ID
+          if (file) {
+            return this.resolve(resolve(ctx.root, file), importer, {
+              skipSelf: true,
+            })
+          }
         }
+        return NOOP_ID
       }
       if (id.startsWith(CONFIG_ID)) {
         return RESOLVED_CONFIG_ID