feat(openapi): support custom error response format (#1137)

- [x] logic
- [x] docs
- [x] test

Fixes #1124

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Add pluggable error-response customization: custom encoding (handler),
custom schema generation (spec), and custom decoding (client) with clear
fallback behavior.

* **Documentation**
* Revise advanced customization guide to show option-based, selective
customization and clarify default/fallback semantics.

* **Tests**
* Add tests covering custom encode/decode hooks, schema hook invocation,
and fallback scenarios.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: dinwwwh

apps/content/docs/openapi/advanced/customizing-error-response.md

@@ -1,43 +1,96 @@
 ---
 title: Customizing Error Response Format
-description: Learn how to customize the error response format in oRPC OpenAPI handlers to match your application's requirements and improve client compatibility.
+description: Learn how to customize the error response format in oRPC OpenAPI to match your application's requirements and improve client compatibility.
 ---
 
 # Customizing Error Response Format
 
-If you need to customize the error response format to match your application's requirements, you can use [`rootInterceptors`](/docs/rpc-handler#lifecycle) in the handler.
+By default, [OpenAPIHandler](/docs/openapi/openapi-handler), [OpenAPIGenerator](/docs/openapi/openapi-specification), and [OpenAPILink](/docs/openapi/client/openapi-link) share the same error response format. You can customize one, some, or all of them based on your requirements.
 
-::: warning
-Avoid combining this with [Type‑Safe Error Handling](/docs/error-handling#type‐safe-error-handling), as the [OpenAPI Specification Generator](/docs/openapi/openapi-specification) does not yet support custom error response formats.
+::: info
+The examples below use options very close to the default behavior.
 :::
 
-```ts
-import { isORPCErrorJson, isORPCErrorStatus } from '@orpc/client'
+## `OpenAPIHandler`
+
+Use `customErrorResponseBodyEncoder` in [OpenAPIHandler](/docs/openapi/openapi-handler) to customize how an `ORPCError` is formatted in the response.
 
+```ts
 const handler = new OpenAPIHandler(router, {
-  rootInterceptors: [
-    async ({ next }) => {
-      const result = await next()
-
-      if (
-        result.matched
-        && isORPCErrorStatus(result.response.status)
-        && isORPCErrorJson(result.response.body)
-      ) {
-        return {
-          ...result,
-          response: {
-            ...result.response,
-            body: {
-              ...result.response.body,
-              message: 'custom error shape',
-            },
+  customErrorResponseBodyEncoder(error) {
+    return error.toJSON()
+  },
+})
+```
+
+::: info
+Return `null` or `undefined` from `customErrorResponseBodyEncoder` to fallback to the default behavior.
+:::
+
+## `OpenAPIGenerator`
+
+When using [type-safe errors](/docs/error-handling#type‐safe-error-handling), customize the error response format in [OpenAPIGenerator](/docs/openapi/openapi-specification) with `customErrorResponseBodySchema` to match your application's actual error responses.
+
+```ts
+const generator = new OpenAPIGenerator()
+
+const spec = await generator.generate(router, {
+  customErrorResponseBodySchema: (definedErrorDefinitions, status) => {
+    const result: Record<any, any> = {
+      oneOf: [
+        {
+          type: 'object',
+          properties: {
+            defined: { const: false }, // for normal errors
+            code: { type: 'string' },
+            status: { type: 'number' },
+            message: { type: 'string' },
+            data: {},
           },
-        }
-      }
+          required: ['defined', 'code', 'status', 'message'],
+        },
+      ],
+    }
 
-      return result
-    },
-  ],
+    for (const [code, defaultMessage, dataRequired, dataSchema] of definedErrorDefinitions) {
+      result.oneOf.push({
+        type: 'object',
+        properties: {
+          defined: { const: true }, // for typesafe errors
+          code: { const: code },
+          status: { const: status },
+          message: { type: 'string', default: defaultMessage },
+          data: dataSchema,
+        },
+        required: dataRequired ? ['defined', 'code', 'status', 'message', 'data'] : ['defined', 'code', 'status', 'message'],
+      })
+    }
+
+    return result
+  }
+})
+```
+
+::: info
+Return `null` or `undefined` from `customErrorResponseBodySchema` to fallback to the default behavior.
+:::
+
+## `OpenAPILink`
+
+When your backend isn't oRPC or uses a custom error format, you can instruct [OpenAPILink](/docs/openapi/client/openapi-link) how to parse it to an `ORPCError` using the `customErrorResponseBodyDecoder` option.
+
+```ts
+const link = OpenAPILink(contract, {
+  customErrorResponseBodyDecoder: (body, response) => {
+    if (isORPCErrorJson(body)) {
+      return createORPCErrorFromJson(body)
+    }
+
+    return null // default behavior supports any error format
+  }
 })
 ```
+
+::: info
+Return `null` or `undefined` from `customErrorResponseBodyDecoder` to fallback to the default behavior.
+:::

packages/openapi-client/src/adapters/standard/openapi-link-codec.test.ts

@@ -459,5 +459,31 @@ describe('standardOpenapiLinkCodecOptions', () => {
         status: 201,
       }, { context: {}, signal }, ['ping'])).rejects.toThrow('Invalid OpenAPI response format.')
     })
+
+    it('customErrorResponseBodyDecoder', async () => {
+      const error = new ORPCError('TEST')
+      let time = 1
+      const customErrorResponseBodyDecoder = vi.fn(() => {
+        if (time++ === 2) {
+          return null // fallback to default
+        }
+        return error
+      })
+
+      const codec = new StandardOpenapiLinkCodec({ ping: oc }, serializer, {
+        url: 'http://localhost:3000',
+        customErrorResponseBodyDecoder,
+      })
+
+      const response1 = { headers: { 'x-custom': 'value' }, body: async () => 'body', status: 400 }
+      await expect(codec.decode(response1, { context: {} }, ['ping'])).rejects.toSatisfy(e => e === error)
+
+      const response2 = { headers: { 'x-custom': 'value2' }, body: async () => 'body2', status: 405 }
+      await expect(codec.decode(response2, { context: {} }, ['ping'])).rejects.toSatisfy(e => e.status === 405) // default behavior
+
+      expect(customErrorResponseBodyDecoder).toHaveBeenCalledTimes(2)
+      expect(customErrorResponseBodyDecoder).toHaveBeenCalledWith(deserialize.mock.results[0]!.value, response1)
+      expect(customErrorResponseBodyDecoder).toHaveBeenCalledWith(deserialize.mock.results[1]!.value, response2)
+    })
   })
 })

packages/openapi-client/src/adapters/standard/openapi-link-codec.ts

@@ -29,11 +29,22 @@ export interface StandardOpenapiLinkCodecOptions<T extends ClientContext> {
         path: readonly string[],
         input: unknown,
   ]>
+
+  /**
+   * Customize how a response body is decoded into an ORPC error.
+   * Useful when the default decoder cannot fully interpret
+   * your server's error format.
+   *
+   * @remarks
+   * - Return `null | undefined` to fallback to default behavior.
+   */
+  customErrorResponseBodyDecoder?: (deserializedBody: unknown, response: StandardLazyResponse) => ORPCError<any, any> | null | undefined
 }
 
 export class StandardOpenapiLinkCodec<T extends ClientContext> implements StandardLinkCodec<T> {
   private readonly baseUrl: Exclude<StandardOpenapiLinkCodecOptions<T>['url'], undefined>
   private readonly headers: Exclude<StandardOpenapiLinkCodecOptions<T>['headers'], undefined>
+  private readonly customErrorResponseBodyDecoder: StandardOpenapiLinkCodecOptions<T>['customErrorResponseBodyDecoder']
 
   constructor(
     private readonly contract: AnyContractRouter,
@@ -42,6 +53,7 @@ export class StandardOpenapiLinkCodec<T extends ClientContext> implements Standa
   ) {
     this.baseUrl = options.url
     this.headers = options.headers ?? {}
+    this.customErrorResponseBodyDecoder = options.customErrorResponseBodyDecoder
   }
 
   async encode(path: readonly string[], input: unknown, options: ClientOptions<T>): Promise<StandardRequest> {
@@ -216,6 +228,12 @@ export class StandardOpenapiLinkCodec<T extends ClientContext> implements Standa
     })()
 
     if (!isOk) {
+      const error = this.customErrorResponseBodyDecoder?.(deserialized, response)
+
+      if (error !== null && error !== undefined) {
+        throw error
+      }
+
       if (isORPCErrorJson(deserialized)) {
         throw createORPCErrorFromJson(deserialized)
       }

packages/openapi/src/adapters/standard/openapi-codec.test.ts

@@ -264,21 +264,57 @@ describe('standardOpenAPICodec', () => {
     })
   })
 
-  it('.encodeError', async () => {
-    serializer.serialize.mockReturnValueOnce('__serialized__')
+  describe('.encodeError', () => {
+    it('works', async () => {
+      serializer.serialize.mockReturnValueOnce('__serialized__')
 
-    const error = new ORPCError('BAD_GATEWAY', {
-      data: '__data__',
-    })
-    const response = codec.encodeError(error)
+      const error = new ORPCError('BAD_GATEWAY', {
+        data: '__data__',
+      })
+      const response = codec.encodeError(error)
+
+      expect(response).toEqual({
+        status: error.status,
+        headers: {},
+        body: '__serialized__',
+      })
 
-    expect(response).toEqual({
-      status: error.status,
-      headers: {},
-      body: '__serialized__',
+      expect(serializer.serialize).toHaveBeenCalledOnce()
+      expect(serializer.serialize).toHaveBeenCalledWith(error.toJSON(), { outputFormat: 'plain' })
     })
 
-    expect(serializer.serialize).toHaveBeenCalledOnce()
-    expect(serializer.serialize).toHaveBeenCalledWith(error.toJSON(), { outputFormat: 'plain' })
+    it('customErrorResponseBodyEncoder', async () => {
+      let time = 1
+      const customErrorResponseBodyEncoder = vi.fn(() => {
+        if (time++ === 2) {
+          return null // default behavior
+        }
+
+        return '__custom_error_body__'
+      })
+
+      const codec = new StandardOpenAPICodec(serializer, {
+        customErrorResponseBodyEncoder,
+      })
+
+      let time2 = 1
+      serializer.serialize.mockImplementation(() => `__serialized${time2++}__`)
+
+      const error1 = new ORPCError('BAD_GATEWAY', { data: '__data1__' })
+      const response1 = codec.encodeError(error1)
+      expect(response1).toEqual({ status: error1.status, headers: {}, body: '__serialized1__' })
+
+      const error2 = new ORPCError('TEST_2', { data: '__data2__' })
+      const response2 = codec.encodeError(error2)
+      expect(response2).toEqual({ status: error2.status, headers: {}, body: '__serialized2__' })
+
+      expect(customErrorResponseBodyEncoder).toHaveBeenCalledTimes(2)
+      expect(customErrorResponseBodyEncoder).toHaveBeenNthCalledWith(1, error1)
+      expect(customErrorResponseBodyEncoder).toHaveBeenNthCalledWith(2, error2)
+
+      expect(serializer.serialize).toHaveBeenCalledTimes(2)
+      expect(serializer.serialize).toHaveBeenNthCalledWith(1, '__custom_error_body__', { outputFormat: 'plain' })
+      expect(serializer.serialize).toHaveBeenNthCalledWith(2, error2.toJSON(), { outputFormat: 'plain' }) // default behavior
+    })
   })
 })

packages/openapi/src/adapters/standard/openapi-codec.ts

@@ -7,10 +7,27 @@ import { isORPCErrorStatus } from '@orpc/client'
 import { fallbackContractConfig } from '@orpc/contract'
 import { isObject, stringifyJSON } from '@orpc/shared'
 
+export interface StandardOpenAPICodecOptions {
+  /**
+   * Customize how an ORPC error is encoded into a response body.
+   * Use this if your API needs a different error output structure.
+   *
+   * @remarks
+   * - Return `null | undefined` to fallback to default behavior
+   *
+   * @default ((e) => e.toJSON())
+   */
+  customErrorResponseBodyEncoder?: (error: ORPCError<any, any>) => unknown
+}
+
 export class StandardOpenAPICodec implements StandardCodec {
+  private readonly customErrorResponseBodyEncoder: StandardOpenAPICodecOptions['customErrorResponseBodyEncoder']
+
   constructor(
     private readonly serializer: StandardOpenAPISerializer,
+    options: StandardOpenAPICodecOptions = {},
   ) {
+    this.customErrorResponseBodyEncoder = options.customErrorResponseBodyEncoder
   }
 
   async decode(request: StandardLazyRequest, params: StandardParams | undefined, procedure: AnyProcedure): Promise<unknown> {
@@ -88,10 +105,12 @@ export class StandardOpenAPICodec implements StandardCodec {
   }
 
   encodeError(error: ORPCError<any, any>): StandardResponse {
+    const body = this.customErrorResponseBodyEncoder?.(error) ?? error.toJSON()
+
     return {
       status: error.status,
       headers: {},
-      body: this.serializer.serialize(error.toJSON(), { outputFormat: 'plain' }),
+      body: this.serializer.serialize(body, { outputFormat: 'plain' }),
     }
   }
 

packages/openapi/src/adapters/standard/openapi-handler.ts

@@ -1,6 +1,7 @@
 import type { StandardBracketNotationSerializerOptions, StandardOpenAPIJsonSerializerOptions } from '@orpc/openapi-client/standard'
 import type { Context, Router } from '@orpc/server'
 import type { StandardHandlerOptions } from '@orpc/server/standard'
+import type { StandardOpenAPICodecOptions } from './openapi-codec'
 import type { StandardOpenAPIMatcherOptions } from './openapi-matcher'
 import { StandardBracketNotationSerializer, StandardOpenAPIJsonSerializer, StandardOpenAPISerializer } from '@orpc/openapi-client/standard'
 import { StandardHandler } from '@orpc/server/standard'
@@ -9,15 +10,15 @@ import { StandardOpenAPIMatcher } from './openapi-matcher'
 
 export interface StandardOpenAPIHandlerOptions<T extends Context>
   extends StandardHandlerOptions<T>, StandardOpenAPIJsonSerializerOptions,
-  StandardBracketNotationSerializerOptions, StandardOpenAPIMatcherOptions {}
+  StandardBracketNotationSerializerOptions, StandardOpenAPIMatcherOptions, StandardOpenAPICodecOptions {}
 
 export class StandardOpenAPIHandler<T extends Context> extends StandardHandler<T> {
   constructor(router: Router<any, T>, options: NoInfer<StandardOpenAPIHandlerOptions<T>>) {
     const jsonSerializer = new StandardOpenAPIJsonSerializer(options)
     const bracketNotationSerializer = new StandardBracketNotationSerializer(options)
     const serializer = new StandardOpenAPISerializer(jsonSerializer, bracketNotationSerializer)
     const matcher = new StandardOpenAPIMatcher(options)
-    const codec = new StandardOpenAPICodec(serializer)
+    const codec = new StandardOpenAPICodec(serializer, options)
 
     super(router, matcher, codec, options)
   }