Skip to content

Commit d39bd61

Browse files
mihaipmihaip
committed
Add Anthropic provider to Monkey
Updates #382
1 parent 20bfdce commit d39bd61

10 files changed

Lines changed: 444 additions & 20 deletions

‎package-lock.json‎

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"private": true,
55
"type": "module",
66
"dependencies": {
7+
"@anthropic-ai/sdk": "^0.55.1",
78
"classnames": "^2.3.2",
89
"file-saver": "^2.0.5",
910
"jszip": "^3.7.0",

‎src/monkey/AnthropicProvider.tsx‎

Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
import Anthropic from "@anthropic-ai/sdk";
2+
import {type Computer, type ComputerAction} from "./Computer";
3+
import {
4+
type ConversationCallbacks,
5+
type Conversation,
6+
type Provider,
7+
} from "./Provider";
8+
import {sleep} from "./util";
9+
10+
export const ANTHROPIC_PROVIDER: Provider = {
11+
id: "anthropic",
12+
label: "Anthropic",
13+
titleLink: (
14+
<a href="https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/computer-use-tool">
15+
Anthropic Computer Use
16+
</a>
17+
),
18+
apiKeyInstructions: (
19+
<>
20+
Create a new API key in your{" "}
21+
<a href="https://console.anthropic.com/settings/keys">
22+
Anthropic Console API keys page
23+
</a>
24+
.
25+
</>
26+
),
27+
createConversation(computer, apiKey) {
28+
return new AnthropicConversation(computer, apiKey);
29+
},
30+
};
31+
32+
export class AnthropicConversation implements Conversation {
33+
private api: Anthropic;
34+
private abortController = new AbortController();
35+
private messages: Anthropic.Beta.Messages.BetaMessageParam[] = [];
36+
37+
constructor(
38+
private computer: Computer,
39+
apiKey?: string
40+
) {
41+
this.api = new Anthropic({
42+
apiKey,
43+
dangerouslyAllowBrowser: true,
44+
});
45+
}
46+
47+
async sendMessage(message: string, callbacks: ConversationCallbacks) {
48+
const {
49+
setWaitingForResponse,
50+
onError,
51+
onAction,
52+
onReasoning,
53+
onAssistantMessage,
54+
onLoopIteration,
55+
} = callbacks;
56+
const {computer} = this;
57+
58+
const tools: Anthropic.Beta.Messages.BetaToolComputerUse20250124[] = [
59+
{
60+
type: "computer_20250124",
61+
name: "computer",
62+
display_width_px: computer.displayWidth,
63+
display_height_px: computer.displayHeight,
64+
display_number: 1,
65+
},
66+
];
67+
68+
this.messages.push({
69+
role: "user",
70+
content: message,
71+
});
72+
73+
while (true) {
74+
let response: Anthropic.Beta.Messages.BetaMessage;
75+
try {
76+
setWaitingForResponse(true);
77+
response = await this.api.beta.messages.create(
78+
{
79+
model: "claude-sonnet-4-20250514",
80+
messages: this.messages,
81+
max_tokens: 6400,
82+
system: computer.instructions,
83+
tools,
84+
betas: ["computer-use-2025-01-24"],
85+
thinking: {
86+
type: "enabled",
87+
budget_tokens: 2014,
88+
},
89+
},
90+
{
91+
signal: this.abortController.signal,
92+
}
93+
);
94+
} catch (error) {
95+
onError(error);
96+
return;
97+
} finally {
98+
setWaitingForResponse(false);
99+
}
100+
101+
let hasToolCalls = false;
102+
const assistantContent: Anthropic.Beta.Messages.BetaMessageParam["content"] =
103+
[];
104+
105+
for (const block of response.content) {
106+
if (block.type === "thinking") {
107+
onReasoning(block.thinking);
108+
assistantContent.push(block);
109+
} else if (block.type === "text") {
110+
if (block.text.trim()) {
111+
onAssistantMessage(block.text);
112+
assistantContent.push(block);
113+
}
114+
} else if (block.type === "tool_use") {
115+
hasToolCalls = true;
116+
assistantContent.push(block);
117+
118+
if (block.name === "computer") {
119+
let action;
120+
try {
121+
action = convertAnthropicActionToComputerAction(
122+
block.input
123+
);
124+
} catch (error) {
125+
onError(error);
126+
return;
127+
}
128+
onAction(action);
129+
130+
const abortSignal = this.abortController.signal;
131+
await computer.handleAction(action);
132+
await sleep(100);
133+
if (abortSignal.aborted) {
134+
return;
135+
}
136+
137+
const newScreenContents =
138+
computer.currentScreenContents();
139+
if (!newScreenContents) {
140+
onError("No screen contents available, stopping.");
141+
return;
142+
}
143+
144+
this.messages.push({
145+
role: "assistant",
146+
content: assistantContent,
147+
});
148+
149+
this.messages.push({
150+
role: "user",
151+
content: [
152+
{
153+
type: "tool_result",
154+
tool_use_id: block.id,
155+
content: [
156+
{
157+
type: "image",
158+
source: {
159+
type: "base64",
160+
media_type: "image/png",
161+
data: newScreenContents.split(
162+
","
163+
)[1],
164+
},
165+
},
166+
],
167+
},
168+
],
169+
});
170+
171+
onLoopIteration?.();
172+
break;
173+
} else {
174+
onError(
175+
new Error(`Unknown tool use type: ${block.name}`)
176+
);
177+
}
178+
} else {
179+
onError(new Error(`Unknown block type: ${block.type}`));
180+
return;
181+
}
182+
}
183+
184+
if (!hasToolCalls) {
185+
if (assistantContent.length > 0) {
186+
this.messages.push({
187+
role: "assistant",
188+
content: assistantContent,
189+
});
190+
}
191+
break;
192+
}
193+
}
194+
}
195+
196+
async stop() {
197+
this.abortController.abort();
198+
this.abortController = new AbortController();
199+
}
200+
}
201+
202+
function convertAnthropicActionToComputerAction(input: any): ComputerAction {
203+
switch (input.action) {
204+
// Basic actions (all versions)
205+
case "screenshot":
206+
return {type: "screenshot"};
207+
case "left_click":
208+
return {
209+
type: "click",
210+
x: input.coordinate[0],
211+
y: input.coordinate[1],
212+
button: "left",
213+
};
214+
case "type":
215+
return {
216+
type: "type",
217+
text: input.text,
218+
};
219+
case "key":
220+
return {
221+
type: "keypress",
222+
keys: input.text.toUpperCase().split("+"),
223+
};
224+
case "mouse_move":
225+
return {
226+
type: "move",
227+
x: input.coordinate[0],
228+
y: input.coordinate[1],
229+
};
230+
231+
// Enhanced actions (computer_20250124)
232+
case "scroll":
233+
return {
234+
type: "scroll",
235+
x: input.coordinate[0],
236+
y: input.coordinate[1],
237+
scroll_x:
238+
input.scroll_direction === "left"
239+
? -(input.scroll_amount ?? 3)
240+
: input.scroll_direction === "right"
241+
? (input.scroll_amount ?? 3)
242+
: 0,
243+
scroll_y:
244+
input.scroll_direction === "up"
245+
? -(input.scroll_amount ?? 3)
246+
: input.scroll_direction === "down"
247+
? (input.scroll_amount ?? 3)
248+
: 0,
249+
};
250+
case "left_click_drag":
251+
return {
252+
type: "drag",
253+
path: [
254+
{x: input.startCoordinate[0], y: input.startCoordinate[1]},
255+
{x: input.endCoordinate[0], y: input.endCoordinate[1]},
256+
],
257+
};
258+
case "right_click":
259+
return {
260+
type: "click",
261+
x: input.coordinate[0],
262+
y: input.coordinate[1],
263+
button: "right",
264+
};
265+
case "middle_click":
266+
return {
267+
type: "click",
268+
x: input.coordinate[0],
269+
y: input.coordinate[1],
270+
button: "wheel",
271+
};
272+
case "double_click":
273+
return {
274+
type: "double_click",
275+
x: input.coordinate[0],
276+
y: input.coordinate[1],
277+
};
278+
case "triple_click":
279+
return {
280+
type: "triple_click",
281+
x: input.coordinate[0],
282+
y: input.coordinate[1],
283+
};
284+
case "left_mouse_down":
285+
return {
286+
type: "mouse_down",
287+
button: "left",
288+
};
289+
case "left_mouse_up":
290+
return {
291+
type: "mouse_up",
292+
button: "left",
293+
};
294+
case "hold_key":
295+
return {
296+
type: "keypress",
297+
keys: input.text.toUpperCase().split("+"),
298+
durationMs: input.duration * 1000,
299+
};
300+
case "wait":
301+
return {type: "wait", durationMs: input.duration * 1000};
302+
303+
default:
304+
throw new Error(`Unknown action type: ${input.action}`);
305+
}
306+
}

0 commit comments

Comments
 (0)