WIP take over

infrastructure/docker/Dockerfile

@@ -112,7 +112,12 @@ RUN install -m 0755 -d /etc/thunderbird/policies && \
       "MoreFromMozilla": false
     },
     "OverrideFirstRunPage": "",
-    "OverridePostUpdatePage": ""
+    "OverridePostUpdatePage": "",
+    "InAppNotification": {
+        "DonationEnabled": false,
+        "SurveyEnabled": false,
+        "MessageEnabled": false
+    }
   }
 }
 EOF

packages/bytebot-agent/package.json

@@ -32,7 +32,7 @@
     "@nestjs/schedule": "^6.0.0",
     "@nestjs/websockets": "^11.1.1",
     "@prisma/client": "^6.6.0",
-    "class-validator": "^0.14.1",
+    "class-validator": "^0.14.2",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1",
     "socket.io": "^4.8.1"
@@ -80,5 +80,8 @@
     ],
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
+  },
+  "engines": {
+    "node": "20"
   }
 }

packages/bytebot-agent/prisma/migrations/20250525204528_add_takeover_state/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "TakeOverState" AS ENUM ('AGENT_CONTROL', 'USER_TAKEOVER', 'USER_CONTROL');
+
+-- AlterTable
+ALTER TABLE "Task" ADD COLUMN     "takeOverState" "TakeOverState" NOT NULL DEFAULT 'AGENT_CONTROL';

packages/bytebot-agent/prisma/schema.prisma

@@ -23,6 +23,12 @@ enum TaskStatus {
   FAILED
 }
 
+enum TakeOverState {
+  AGENT_CONTROL
+  USER_TAKEOVER
+  USER_CONTROL
+}
+
 enum TaskPriority {
   LOW
   MEDIUM
@@ -41,22 +47,23 @@ enum TaskType {
 }
 
 model Task {
-  id          String       @id @default(uuid())
-  description String
-  type        TaskType     @default(IMMEDIATE)
-  status      TaskStatus   @default(PENDING)
-  priority    TaskPriority @default(MEDIUM)
-  createdAt   DateTime     @default(now())
-  createdBy   Role         @default(USER)
-  scheduledFor DateTime?
-  updatedAt   DateTime     @updatedAt
-  executedAt  DateTime?
-  completedAt DateTime?
-  queuedAt    DateTime?
-  error       String?
-  result      Json?
-  messages    Message[]
-  summaries   Summary[]
+  id            String        @id @default(uuid())
+  description   String
+  type          TaskType      @default(IMMEDIATE)
+  status        TaskStatus    @default(PENDING)
+  priority      TaskPriority  @default(MEDIUM)
+  takeOverState TakeOverState @default(AGENT_CONTROL)
+  createdAt     DateTime      @default(now())
+  createdBy     Role          @default(USER)
+  scheduledFor  DateTime?
+  updatedAt     DateTime      @updatedAt
+  executedAt    DateTime?
+  completedAt   DateTime?
+  queuedAt      DateTime?
+  error         String?
+  result        Json?
+  messages      Message[]
+  summaries     Summary[]
 }
 
 model Summary {

packages/bytebot-agent/src/agent/agent.processor.ts

@@ -124,17 +124,14 @@ export class AgentProcessor {
               );
 
               // if the block input type exists, convert it to uppercase and use it as the type
-              const type = block.input.type
-                ? (block.input.type.toUpperCase() as TaskType)
-                : TaskType.IMMEDIATE;
-
-              const priority = block.input.priority
-                ? (block.input.priority.toUpperCase() as TaskPriority)
-                : TaskPriority.MEDIUM;
+              const type = block.input.type?.toUpperCase() as TaskType;
+              const priority =
+                block.input.priority?.toUpperCase() as TaskPriority;
 
               await this.tasksService.create({
                 description: block.input.description,
                 type,
+                createdBy: Role.ASSISTANT,
                 ...(block.input.scheduledFor && {
                   scheduledFor: new Date(block.input.scheduledFor),
                 }),
@@ -161,6 +158,7 @@ export class AgentProcessor {
                 case 'completed': {
                   await this.tasksService.update(taskId, {
                     status: TaskStatus.COMPLETED,
+                    completedAt: new Date(),
                   });
                   break;
                 }

packages/bytebot-agent/src/anthropic/anthropic.constants.ts

@@ -8,6 +8,29 @@ export const DEFAULT_COMPUTER_TOOL_USE_NAME = 'computer_20250124';
 export const AGENT_SYSTEM_PROMPT = `
 You are **Bytebot**, a highly-reliable AI engineer operating a virtual computer whose display measures ${DEFAULT_DISPLAY_SIZE.width} × ${DEFAULT_DISPLAY_SIZE.height} pixels.
 
+The current date is ${new Date().toLocaleDateString()}. The current time is ${new Date().toLocaleTimeString()}. The current timezone is ${Intl.DateTimeFormat().resolvedOptions().timeZone}.
+
+
+────────────────────────
+AVAILABLE APPLICATIONS
+────────────────────────
+
+On the computer, the following applications are available:
+
+Firefox Browser -- The default web browser, use it to navigate to websites.
+Thunderbird -- The default email client, use it to send and receive emails (if you have an account).
+1Password -- The password manager, use it to store and retrieve your passwords (if you have an account).
+Terminal -- The default terminal, use it to run commands.
+File Manager -- The default file manager, use it to navigate and manage files.
+Trash -- The default trash, use it to delete files.
+
+ALL APPLICATIONS ARE GUI BASED, USE THE COMPUTER TOOLS TO INTERACT WITH THEM. ONLY ACCESS THE APPLICATIONS VIA THEIR DESKTOP ICONS.
+
+*Never* use keyboard shortcuts to switch between applications. 
+
+*Never* open the 'Applications' menu from the dock.
+
+
 ────────────────────────
 CORE WORKING PRINCIPLES
 ────────────────────────
@@ -19,11 +42,12 @@ CORE WORKING PRINCIPLES
 3. **Valid Keys Only** - 
    Use **exactly** the identifiers listed in **VALID KEYS** below when supplying \`keys\` to \`computer_type_keys\` or \`computer_press_keys\`. All identifiers come from nut-tree’s \`Key\` enum; they are case-sensitive and contain *no spaces*.
 4. **Verify Every Step** - After each action:  
-   a. \`computer_wait\` just long enough for feedback.  
+   a. \`computer_wait\` for 500ms, or longer if absolutely necessary.
    b. Take another screenshot.  
    c. Confirm the expected state before continuing. If it failed, retry sensibly or abort with \`"status":"failed"\`.
 5. **Efficiency & Clarity** - Combine related key presses; prefer scrolling or dragging over many small moves; minimise unnecessary waits.
 6. **Stay Within Scope** - Do nothing the user didn't request; don't suggest unrelated tasks.
+7. **Security** - If you see a password or other sensitive information (or the user shares it with you), do not repeat it in conversation. When typing a password, use \`computer_type_text\` with \`isSensitive\` set to \`true\`.
 
 ────────────────────────
 TASK LIFECYCLE TEMPLATE
@@ -81,4 +105,4 @@ Remember: **accuracy over speed, clarity over cleverness**.
 Think before each move, keep the desktop clean when you're done, and **always** finish with \`set_task_status\`.
 `;
 
-export const DEFAULT_MODEL = 'claude-sonnet-4-20250514';
+export const DEFAULT_MODEL = 'claude-opus-4-20250514';

packages/bytebot-agent/src/anthropic/anthropic.tools.ts

@@ -209,8 +209,7 @@ export const scrollTool: Anthropic.Tool = {
         },
         required: ['x', 'y'],
         description:
-          'Optional coordinates for where the scroll should occur. Behavior might depend on the OS/application.',
-        nullable: true,
+          'Coordinates for where the scroll should occur. Behavior might depend on the OS/application.',
       },
       direction: {
         type: 'string',
@@ -228,7 +227,7 @@ export const scrollTool: Anthropic.Tool = {
         nullable: true,
       },
     },
-    required: ['direction', 'numScrolls'],
+    required: ['coordinates', 'direction', 'numScrolls'],
   },
 };
 
@@ -297,6 +296,12 @@ export const typeTextTool: Anthropic.Tool = {
           'Optional delay in milliseconds between character presses.',
         nullable: true,
       },
+      isSensitive: {
+        type: 'boolean',
+        description:
+          'Optional flag to indicate if the text contains sensitive information.',
+        nullable: true,
+      },
     },
     required: ['text'],
   },
@@ -367,7 +372,7 @@ export const createTaskTool: Anthropic.Tool = {
       },
       type: {
         type: 'string',
-        enum: ['immediate', 'scheduled'],
+        enum: ['IMMEDIATE', 'SCHEDULED'],
         description: 'The type of the task. Default is immediate.',
       },
       scheduledFor: {
@@ -378,7 +383,7 @@ export const createTaskTool: Anthropic.Tool = {
       },
       priority: {
         type: 'string',
-        enum: ['low', 'medium', 'high', 'urgent'],
+        enum: ['LOW', 'MEDIUM', 'HIGH', 'URGENT'],
         description: 'The priority of the task. Default is medium.',
       },
     },
@@ -401,4 +406,5 @@ export const anthropicTools: Anthropic.Tool[] = [
   screenshotTool,
   cursorPositionTool,
   setTaskStatusTool,
+  createTaskTool,
 ];

packages/bytebot-agent/src/tasks/dto/create-task.dto.ts

@@ -1,5 +1,5 @@
 import { IsDate, IsNotEmpty, IsOptional, IsString } from 'class-validator';
-import { TaskPriority, TaskType } from '@prisma/client';
+import { Role, TaskPriority, TaskType } from '@prisma/client';
 
 export class CreateTaskDto {
   @IsNotEmpty()
@@ -17,4 +17,8 @@ export class CreateTaskDto {
   @IsOptional()
   @IsString()
   priority?: TaskPriority;
+
+  @IsOptional()
+  @IsString()
+  createdBy?: Role;
 }

packages/bytebot-agent/src/tasks/tasks.controller.ts

@@ -67,4 +67,16 @@ export class TasksController {
   ): Promise<Task> {
     return this.tasksService.guideTask(taskId, guideTaskDto);
   }
+
+  @Post(':id/takeover')
+  @HttpCode(HttpStatus.OK)
+  async takeOver(@Param('id') taskId: string): Promise<Task> {
+    return this.tasksService.takeOver(taskId);
+  }
+
+  @Post(':id/resume')
+  @HttpCode(HttpStatus.OK)
+  async resumeControl(@Param('id') taskId: string): Promise<Task> {
+    return this.tasksService.resumeControl(taskId);
+  }
 }