Add AWS Bedrock support (#750)

.changeset/ten-beds-kick.md

@@ -0,0 +1,5 @@
+---
+'ai': patch
+---
+
+Add AWS Bedrock support

docs/pages/docs/guides/providers/aws-bedrock.mdx

@@ -0,0 +1,131 @@
+# AWS Bedrock
+
+import { Steps, Callout } from 'nextra-theme-docs';
+
+Vercel AI SDK provides a set of utilities to make it easy to use AWS Bedrocks's API. In this guide, we'll walk through how to use the utilities to create a chat bot.
+
+## Guide: Chat Bot
+
+<Steps>
+
+### Create a Next.js app
+
+Create a Next.js application and install `ai`:
+
+```sh
+pnpm dlx create-next-app my-ai-app
+cd my-ai-app
+pnpm install ai
+```
+
+### Add your AWS Credentials to `.env`
+
+Create a `.env` file in your project root and add your AWS credentials:
+
+```env filename=".env"
+AWS_REGION=YOUR_AWS_REGION
+AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_ACCESS_KEY
+```
+
+### Create a Route Handler
+
+Create a Next.js Route Handler that uses the Edge Runtime to generate a response to a series of messages via AWS Bedrock's API, and returns the response as a streaming text response.
+
+For this example, we'll use the Anthropic model `anthropic.claude-v2` and create a route handler at `app/api/chat/route.ts` that accepts a `POST` request with a `messages` array of strings:
+
+```tsx filename="app/api/chat/route.ts" showLineNumbers
+import {
+  BedrockRuntimeClient,
+  InvokeModelWithResponseStreamCommand,
+} from '@aws-sdk/client-bedrock-runtime';
+import { AWSBedrockAnthropicStream, StreamingTextResponse } from 'ai';
+import { experimental_buildAnthropicPrompt } from 'ai/prompts';
+
+// IMPORTANT! Set the runtime to edge
+export const runtime = 'edge';
+
+export async function POST(req: Request) {
+  // Extract the `prompt` from the body of the request
+  const { messages } = await req.json();
+
+  const bedrockClient = new BedrockRuntimeClient({
+    region: process.env.AWS_REGION ?? '',
+    credentials: {
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? '',
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? '',
+    },
+  });
+
+  // Ask Claude for a streaming chat completion given the prompt
+  const bedrockResponse = await bedrockClient.send(
+    new InvokeModelWithResponseStreamCommand({
+      modelId: 'anthropic.claude-v2',
+      contentType: 'application/json',
+      accept: 'application/json',
+      body: JSON.stringify({
+        prompt: experimental_buildAnthropicPrompt(messages),
+        max_tokens_to_sample: 300,
+      }),
+    }),
+  );
+
+  // Convert the response into a friendly text-stream
+  const stream = AWSBedrockAnthropicStream(bedrockResponse);
+
+  // Respond with the stream
+  return new StreamingTextResponse(stream);
+}
+```
+
+<Callout>
+  Vercel AI SDK provides 2 utility helpers to make the above seamless: First, we
+  pass the streaming `bedrockResponse` we receive from AWS Bedrocks's API to
+  `AWSBedrockAnthropicStream`. This utility class decodes/extracts the text
+  tokens in the response and then re-encodes them properly for simple
+  consumption. We can then pass that new stream directly to
+  [`StreamingTextResponse`](/docs/api-reference/streaming-text-response). This
+  is another utility class that extends the normal Node/Edge Runtime `Response`
+  class with the default headers you probably want (hint: `'Content-Type':
+  'text/plain; charset=utf-8'` is already set for you).
+</Callout>
+
+### Wire up the UI
+
+Create a Client component with a form that we'll use to gather the prompt from the user and then stream back the completion from.
+By default, the [`useChat`](/docs/api-reference#usechat) hook will use the `POST` Route Handler we created above (it defaults to `/api/chat`). You can override this by passing a `api` prop to `useChat({ api: '...'})`.
+
+```tsx filename="app/page.tsx" showLineNumbers
+'use client';
+
+import { useChat } from 'ai/react';
+
+export default function Chat() {
+  const { messages, input, handleInputChange, handleSubmit } = useChat();
+
+  return (
+    <div className="mx-auto w-full max-w-md py-24 flex flex-col stretch">
+      {messages.map(m => (
+        <div key={m.id}>
+          {m.role === 'user' ? 'User: ' : 'AI: '}
+          {m.content}
+        </div>
+      ))}
+
+      <form onSubmit={handleSubmit}>
+        <label>
+          Say something...
+          <input
+            className="fixed w-full max-w-md bottom-0 border border-gray-300 rounded mb-8 shadow-xl p-2"
+            value={input}
+            onChange={handleInputChange}
+          />
+        </label>
+        <button type="submit">Send</button>
+      </form>
+    </div>
+  );
+}
+```
+
+</Steps>

examples/next-anthropic/app/api/chat/route.ts

@@ -1,6 +1,7 @@
 // ./app/api/chat/route.ts
 import Anthropic from '@anthropic-ai/sdk';
 import { AnthropicStream, StreamingTextResponse } from 'ai';
+import { experimental_buildAnthropicPrompt } from 'ai/prompts';
 
 // Create an Anthropic API client (that's edge friendly??)
 const anthropic = new Anthropic({
@@ -10,31 +11,13 @@ const anthropic = new Anthropic({
 // IMPORTANT! Set the runtime to edge
 export const runtime = 'edge';
 
-// Claude has an interesting way of dealing with prompts, so we use a helper function to build one from our request
-// Prompt formatting is discussed briefly at https://docs.anthropic.com/claude/reference/getting-started-with-the-api
-function buildPrompt(
-  messages: { content: string; role: 'system' | 'user' | 'assistant' }[],
-) {
-  return (
-    Anthropic.HUMAN_PROMPT +
-    messages.map(({ content, role }) => {
-      if (role === 'user') {
-        return `Human: ${content}`;
-      } else {
-        return `Assistant: ${content}`;
-      }
-    }) +
-    Anthropic.AI_PROMPT
-  );
-}
-
 export async function POST(req: Request) {
   // Extract the `prompt` from the body of the request
   const { messages } = await req.json();
 
   // Ask Claude for a streaming chat completion given the prompt
   const response = await anthropic.completions.create({
-    prompt: buildPrompt(messages),
+    prompt: experimental_buildAnthropicPrompt(messages),
     model: 'claude-2',
     stream: true,
     max_tokens_to_sample: 300,

examples/next-aws-bedrock/.env.local.example

@@ -0,0 +1,3 @@
+AWS_REGION=YOUR_AWS_REGION
+AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_ACCESS_KEY

examples/next-aws-bedrock/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/next-aws-bedrock/README.md

@@ -0,0 +1,33 @@
+# Vercel AI SDK, Next.js, and AWS Bedrock Example
+
+This example shows how to use the [Vercel AI SDK](https://sdk.vercel.ai/docs) with [Next.js](https://nextjs.org/) and [Amazon Bedrock](https://aws.amazon.com/bedrock/) to create a ChatGPT-like AI-powered streaming chat bot.
+
+## Deploy your own
+
+Deploy the example using [Vercel](https://vercel.com?utm_source=github&utm_medium=readme&utm_campaign=ai-sdk-example):
+
+[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fvercel%2Fai%2Ftree%2Fmain%2Fexamples%2Fnext-aws-bedrock&env=AWS_REGION,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY&envDescription=AWS%20Access%20Information&project-name=vercel-ai-chat-aws-bedrock&repository-name=vercel-ai-chat-aws-bedrock)
+
+## How to use
+
+To run the example locally you need to:
+
+1. Create an [AWS Account](https://portal.aws.amazon.com/billing/signup).
+2. Setup Amazon Bedrock and apply for access to the models that you want to use.
+3. Setup an access key under "Security Credentials".
+4. Set the region of the models, your access key and your secret access key as shown in [the example env file](./.env.local.example) but in a new file called `.env.local`
+5. `pnpm install` to install the required dependencies.
+6. `pnpm dev` to launch the development server.
+7. Go to the browser and try out a chatbot example for one of the following models:
+   - [Anthropic Claude](http://localhost:3000/anthropic): `anthropic.claude-v2`
+   - [Cohere](http://localhost:3000/cohere): `cohere.command-light-text-v14`
+   - [Llama 2](http://localhost:3000/llama2): `meta.llama2-13b-chat-v1`
+
+## Learn More
+
+To learn more about AWS Bedrock, Next.js, and the Vercel AI SDK take a look at the following resources:
+
+- [Vercel AI SDK docs](https://sdk.vercel.ai/docs)
+- [Vercel AI Playground](https://play.vercel.ai)
+- [AWS Bedrock](https://aws.amazon.com/bedrock/)
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.

examples/next-aws-bedrock/app/anthropic/page.tsx

@@ -0,0 +1,46 @@
+'use client';
+
+import { Message } from 'ai';
+import { useChat } from 'ai/react';
+
+export default function Chat() {
+  const { messages, input, handleInputChange, handleSubmit } = useChat({
+    api: './api/chat-anthropic',
+  });
+
+  // Generate a map of message role to text color
+  const roleToColorMap: Record<Message['role'], string> = {
+    system: 'red',
+    user: 'black',
+    function: 'blue',
+    assistant: 'green',
+  };
+
+  return (
+    <div className="flex flex-col w-full max-w-md py-24 mx-auto stretch">
+      {messages.length > 0
+        ? messages.map(m => (
+            <div
+              key={m.id}
+              className="whitespace-pre-wrap"
+              style={{ color: roleToColorMap[m.role] }}
+            >
+              <strong>{`${m.role}: `}</strong>
+              {m.content || JSON.stringify(m.function_call)}
+              <br />
+              <br />
+            </div>
+          ))
+        : null}
+
+      <form onSubmit={handleSubmit}>
+        <input
+          className="fixed bottom-0 w-full max-w-md p-2 mb-8 border border-gray-300 rounded shadow-xl"
+          value={input}
+          placeholder="Say something..."
+          onChange={handleInputChange}
+        />
+      </form>
+    </div>
+  );
+}

examples/next-aws-bedrock/app/api/chat-anthropic/route.ts

@@ -0,0 +1,41 @@
+import {
+  BedrockRuntimeClient,
+  InvokeModelWithResponseStreamCommand,
+} from '@aws-sdk/client-bedrock-runtime';
+import { AWSBedrockAnthropicStream, StreamingTextResponse } from 'ai';
+import { experimental_buildAnthropicPrompt } from 'ai/prompts';
+
+// IMPORTANT! Set the runtime to edge
+export const runtime = 'edge';
+
+export async function POST(req: Request) {
+  // Extract the `prompt` from the body of the request
+  const { messages } = await req.json();
+
+  const bedrockClient = new BedrockRuntimeClient({
+    region: process.env.AWS_REGION ?? '',
+    credentials: {
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? '',
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? '',
+    },
+  });
+
+  // Ask Claude for a streaming chat completion given the prompt
+  const bedrockResponse = await bedrockClient.send(
+    new InvokeModelWithResponseStreamCommand({
+      modelId: 'anthropic.claude-v2',
+      contentType: 'application/json',
+      accept: 'application/json',
+      body: JSON.stringify({
+        prompt: experimental_buildAnthropicPrompt(messages),
+        max_tokens_to_sample: 300,
+      }),
+    }),
+  );
+
+  // Convert the response into a friendly text-stream
+  const stream = AWSBedrockAnthropicStream(bedrockResponse);
+
+  // Respond with the stream
+  return new StreamingTextResponse(stream);
+}

examples/next-aws-bedrock/app/api/chat-cohere/route.ts

@@ -0,0 +1,54 @@
+import {
+  BedrockRuntimeClient,
+  InvokeModelWithResponseStreamCommand,
+} from '@aws-sdk/client-bedrock-runtime';
+import { AWSBedrockCohereStream, StreamingTextResponse } from 'ai';
+
+// IMPORTANT! Set the runtime to edge
+export const runtime = 'edge';
+
+function buildPrompt(
+  messages: { content: string; role: 'system' | 'user' | 'assistant' }[],
+) {
+  return (
+    messages.map(({ content, role }) => {
+      if (role === 'user') {
+        return `Human: ${content}\n`;
+      } else {
+        return `Assistant: ${content}\n`;
+      }
+    }) + 'Assistant:\n'
+  );
+}
+
+export async function POST(req: Request) {
+  // Extract the `prompt` from the body of the request
+  const { messages } = await req.json();
+
+  const bedrockClient = new BedrockRuntimeClient({
+    region: process.env.AWS_REGION ?? '',
+    credentials: {
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? '',
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? '',
+    },
+  });
+
+  // Ask Claude for a streaming chat completion given the prompt
+  const bedrockResponse = await bedrockClient.send(
+    new InvokeModelWithResponseStreamCommand({
+      modelId: 'cohere.command-light-text-v14',
+      contentType: 'application/json',
+      accept: 'application/json',
+      body: JSON.stringify({
+        prompt: buildPrompt(messages),
+        max_tokens: 300,
+      }),
+    }),
+  );
+
+  // Convert the response into a friendly text-stream
+  const stream = AWSBedrockCohereStream(bedrockResponse);
+
+  // Respond with the stream
+  return new StreamingTextResponse(stream);
+}