From 830ed5cf490b7f35862986b916e1e9e08b77efc1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 22 May 2026 22:43:51 -0400 Subject: [PATCH 1/3] fix: wire Cmd+K to simulator software keyboard --- AGENTS.md | 8 +- README.md | 8 +- .../run-android-comment-session/action.yml | 2 +- actions/run-ios-comment-session/action.yml | 2 +- cli/DFPrivateSimulatorDisplayBridge.m | 5 + cli/XCWSimctl.h | 4 + cli/XCWSimctl.m | 256 ++++++++++++++++++ cli/native/XCWNativeBridge.h | 2 + cli/native/XCWNativeBridge.m | 28 ++ client/src/api/controls.ts | 29 ++ client/src/app/AppShell.tsx | 208 ++++++++++++-- client/src/features/input/useKeyboardInput.ts | 28 +- .../src/features/simulators/SimulatorMenu.tsx | 97 +++---- .../simulators/SimulatorPickerMenu.tsx | 126 +++++++++ client/src/features/toolbar/Toolbar.tsx | 70 ++--- client/src/styles/components.css | 11 + client/src/styles/layout.css | 48 ++++ client/vite.config.js | 2 +- docs/api/rest.md | 57 ++-- docs/cli/commands.md | 10 +- docs/cli/flags.md | 22 +- docs/extensions/vscode.md | 2 +- docs/guide/daemon.md | 25 +- docs/guide/quick-start.md | 4 +- docs/guide/troubleshooting.md | 4 +- packages/vscode-extension/extension.js | 2 +- packages/vscode-extension/package.json | 2 +- scripts/dev.mjs | 4 +- server/src/api/routes.rs | 72 +++++ server/src/main.rs | 97 +++++-- server/src/native/bridge.rs | 27 ++ server/src/native/ffi.rs | 8 + server/src/service.rs | 56 ++-- server/src/transport/webrtc.rs | 3 + skills/simdeck/SKILL.md | 14 +- 35 files changed, 1112 insertions(+), 231 deletions(-) create mode 100644 client/src/features/simulators/SimulatorPickerMenu.tsx diff --git a/AGENTS.md b/AGENTS.md index aab69475..2a9a478b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,21 +116,21 @@ Run the local daemon: ```sh ./build/simdeck -./build/simdeck daemon start --port 4310 +./build/simdeck daemon start --port 4311 ``` -Running without a subcommand starts a foreground workspace daemon, prints local and LAN HTTP URLs, prints a six-digit pairing code for LAN browsers, and stops when the command exits, when you press `q`, or when you press Ctrl-C. Pass a simulator name or UDID as the only argument to select it by default in the UI. Use `./build/simdeck -d`, `./build/simdeck -k`, and `./build/simdeck -r` as detached start, kill, and restart shortcuts. +Running without a subcommand starts a foreground workspace daemon, prints local and LAN HTTP URLs, prints a six-digit pairing code for LAN browsers, and stops when the command exits, when you press `q`, or when you press Ctrl-C. If the always-on service is active on 4310, running without a subcommand or running `simdeck ui` prints the existing service endpoints instead of starting a project daemon. Pass a simulator name or UDID as the only argument to select it by default in the UI. Use `./build/simdeck -d`, `./build/simdeck -k`, and `./build/simdeck -r` as detached start, kill, and restart shortcuts. Use software H.264 when macOS screen recording starves the hardware encoder: ```sh -./build/simdeck daemon start --port 4310 --video-codec h264-software +./build/simdeck daemon start --port 4311 --video-codec h264-software ``` For LAN access: ```sh -./build/simdeck daemon start --port 4310 --bind 0.0.0.0 --advertise-host 192.168.1.50 +./build/simdeck daemon start --port 4311 --bind 0.0.0.0 --advertise-host 192.168.1.50 ``` Useful direct commands: diff --git a/README.md b/README.md index c8c159c0..952b6321 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,10 @@ routes with the same service token. Normal service restarts preserve that token so paired clients stay connected. Use `simdeck service reset` only when you want to rotate the service token and restart the LaunchAgent. -If port 4310 is already owned by a workspace daemon, the LaunchAgent uses the -next available service-discovery port, up to 4320, instead of stopping that -daemon. +The LaunchAgent service uses port 4310. Project daemons start at port 4311 and +probe upward when that port is busy. When the service is active, `simdeck` and +`simdeck ui` print the existing service endpoints instead of starting a project +daemon; use the `daemon` subcommand when you explicitly want a workspace daemon. CLI commands automatically use the same warm daemon: @@ -179,6 +180,7 @@ simdeck crown --delta 50 simdeck button left-side-button simdeck batch --step "tap --label Continue" --step "type 'hello'" --step "wait-for --label hello" simdeck dismiss-keyboard +simdeck button software-keyboard simdeck home simdeck app-switcher simdeck rotate-left diff --git a/actions/run-android-comment-session/action.yml b/actions/run-android-comment-session/action.yml index cc13fceb..81455861 100644 --- a/actions/run-android-comment-session/action.yml +++ b/actions/run-android-comment-session/action.yml @@ -57,7 +57,7 @@ inputs: simdeck_port: description: Local daemon port on the runner. required: false - default: "4310" + default: "4311" stream_profile: description: SimDeck stream quality profile. required: false diff --git a/actions/run-ios-comment-session/action.yml b/actions/run-ios-comment-session/action.yml index 2e52a00f..d4bb5ba5 100644 --- a/actions/run-ios-comment-session/action.yml +++ b/actions/run-ios-comment-session/action.yml @@ -57,7 +57,7 @@ inputs: simdeck_port: description: Local daemon port on the runner. required: false - default: "4310" + default: "4311" stream_profile: description: SimDeck stream quality profile. required: false diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 65026d1d..883382ed 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -111,6 +111,8 @@ static const uint32_t DFHomeConsumerUsage = 0x65; // Apple Simulator sends Indigo button code 0x191 for Home; 1 is Lock. static const uint32_t DFHomeButtonCode = 0x191; +// Simulator.app dispatches Cmd+K through this private button code. +static const uint32_t DFSoftwareKeyboardButtonCode = 0x3f0; static const NSUInteger DFKeyboardModifierShift = 1 << 0; static const NSUInteger DFKeyboardModifierControl = 1 << 1; static const NSUInteger DFKeyboardModifierOption = 1 << 2; @@ -3610,6 +3612,9 @@ - (BOOL)sendHardwareButtonNamed:(NSString *)buttonName @"side-button": @4, @"side": @4, @"siri": @5, + @"software-keyboard": @(DFSoftwareKeyboardButtonCode), + @"software_keyboard": @(DFSoftwareKeyboardButtonCode), + @"keyboard": @(DFSoftwareKeyboardButtonCode), }; arbitraryButtons = @{ @"power": @[ @(DFConsumerControlUsagePage), @48 ], diff --git a/cli/XCWSimctl.h b/cli/XCWSimctl.h index d8a0932b..902d0c0e 100644 --- a/cli/XCWSimctl.h +++ b/cli/XCWSimctl.h @@ -25,6 +25,10 @@ NS_ASSUME_NONNULL_BEGIN - (nullable NSData *)screenRecordingMP4ForSimulatorUDID:(NSString *)udid durationSeconds:(NSTimeInterval)durationSeconds error:(NSError * _Nullable * _Nullable)error; +- (nullable NSString *)startScreenRecordingForSimulatorUDID:(NSString *)udid + error:(NSError * _Nullable * _Nullable)error; +- (nullable NSData *)stopScreenRecordingWithID:(NSString *)recordingID + error:(NSError * _Nullable * _Nullable)error; - (BOOL)eraseSimulatorWithUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (BOOL)installAppAtPath:(NSString *)appPath simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (BOOL)uninstallBundleID:(NSString *)bundleID simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; diff --git a/cli/XCWSimctl.m b/cli/XCWSimctl.m index efa8e6ea..967cdcd4 100644 --- a/cli/XCWSimctl.m +++ b/cli/XCWSimctl.m @@ -37,6 +37,22 @@ - (BOOL)installIPAAtPath:(NSString *)ipaPath simulatorUDID:(NSString *)udid erro @end +@interface XCWScreenRecordingSession : NSObject + +@property (nonatomic, copy) NSString *identifier; +@property (nonatomic, copy) NSString *udid; +@property (nonatomic, copy) NSString *path; +@property (nonatomic, strong) NSTask *task; +@property (nonatomic, strong) NSMutableData *stdoutData; +@property (nonatomic, strong) NSMutableData *stderrData; +@property (nonatomic, strong) NSFileHandle *stdoutHandle; +@property (nonatomic, strong) NSFileHandle *stderrHandle; + +@end + +@implementation XCWScreenRecordingSession +@end + static NSArray *XCWArrayPayload(id payload, NSString *nestedKey) { if ([payload isKindOfClass:[NSArray class]]) { return payload; @@ -95,6 +111,58 @@ static BOOL XCWWaitForTaskExit(NSTask *task, NSTimeInterval timeoutSeconds) { return NO; } +static NSMutableDictionary *XCWScreenRecordingSessions(void) { + static NSMutableDictionary *sessions = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + sessions = [NSMutableDictionary dictionary]; + }); + return sessions; +} + +static NSString * _Nullable XCWResolveSimctlPath(NSError * _Nullable __autoreleasing *error) { + XCWProcessResult *resolvedSimctl = [XCWProcessRunner runLaunchPath:@"/usr/bin/xcrun" + arguments:@[@"-f", @"simctl"] + inputData:nil + error:error]; + if (resolvedSimctl == nil) { + return nil; + } + NSString *simctlPath = XCWTrimmedString(XCWStringFromData(resolvedSimctl.stdoutData)); + if (resolvedSimctl.terminationStatus != 0 || simctlPath.length == 0) { + if (error != NULL) { + NSString *message = resolvedSimctl.stderrString.length > 0 ? resolvedSimctl.stderrString : @"Unable to locate simctl."; + *error = [XCWSimctl errorWithDescription:message code:34]; + } + return nil; + } + return simctlPath; +} + +static BOOL XCWStopRecordingTask(XCWScreenRecordingSession *session, NSTimeInterval timeoutSeconds) { + NSTask *task = session.task; + if (task.running) { + [task interrupt]; + } + if (!XCWWaitForTaskExit(task, timeoutSeconds) && task.running) { + [task terminate]; + if (!XCWWaitForTaskExit(task, 2.0) && task.running) { + kill(task.processIdentifier, SIGKILL); + XCWWaitForTaskExit(task, 2.0); + } + } + return !task.running; +} + +static void XCWCloseRecordingPipes(XCWScreenRecordingSession *session) { + session.stdoutHandle.readabilityHandler = nil; + session.stderrHandle.readabilityHandler = nil; + if (!session.task.running) { + XCWAppendAvailableData(session.stdoutHandle, session.stdoutData); + XCWAppendAvailableData(session.stderrHandle, session.stderrData); + } +} + static NSString * _Nullable XCWCreateTemporaryDirectory(NSString *prefix, NSError * _Nullable __autoreleasing *error) { NSString *templatePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[NSString stringWithFormat:@"%@-XXXXXX", prefix]]; char *directoryTemplate = strdup(templatePath.fileSystemRepresentation); @@ -859,6 +927,194 @@ - (nullable NSData *)screenRecordingMP4ForSimulatorUDID:(NSString *)udid return nil; } +- (nullable NSString *)startScreenRecordingForSimulatorUDID:(NSString *)udid + error:(NSError * _Nullable __autoreleasing *)error { + if (udid.length == 0) { + if (error != NULL) { + *error = [self.class errorWithDescription:@"Screen recording requires a simulator UDID." code:34]; + } + return nil; + } + + NSMutableDictionary *sessions = XCWScreenRecordingSessions(); + @synchronized(sessions) { + for (XCWScreenRecordingSession *session in sessions.objectEnumerator) { + if ([session.udid isEqualToString:udid]) { + if (error != NULL) { + *error = [self.class errorWithDescription:@"A screen recording is already in progress for this simulator." code:34]; + } + return nil; + } + } + } + + NSString *filename = [NSString stringWithFormat:@"simdeck-%@.mp4", NSUUID.UUID.UUIDString]; + NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent:filename]; + NSString *simctlPath = XCWResolveSimctlPath(error); + if (simctlPath.length == 0) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + return nil; + } + + XCWScreenRecordingSession *session = [[XCWScreenRecordingSession alloc] init]; + session.identifier = NSUUID.UUID.UUIDString; + session.udid = udid; + session.path = path; + session.stdoutData = [NSMutableData data]; + session.stderrData = [NSMutableData data]; + + NSTask *task = [[NSTask alloc] init]; + task.launchPath = simctlPath; + task.arguments = @[@"io", udid, @"recordVideo", @"--codec=h264", @"--force", path]; + task.standardInput = NSFileHandle.fileHandleWithNullDevice; + NSPipe *stdoutPipe = [NSPipe pipe]; + NSPipe *stderrPipe = [NSPipe pipe]; + task.standardOutput = stdoutPipe; + task.standardError = stderrPipe; + session.task = task; + session.stdoutHandle = stdoutPipe.fileHandleForReading; + session.stderrHandle = stderrPipe.fileHandleForReading; + + dispatch_semaphore_t recordingStartedSemaphore = dispatch_semaphore_create(0); + __block BOOL recordingStarted = NO; + __weak XCWScreenRecordingSession *weakSession = session; + + session.stdoutHandle.readabilityHandler = ^(NSFileHandle *handle) { + XCWScreenRecordingSession *strongSession = weakSession; + if (strongSession == nil) { + return; + } + XCWAppendAvailableData(handle, strongSession.stdoutData); + }; + session.stderrHandle.readabilityHandler = ^(NSFileHandle *handle) { + XCWScreenRecordingSession *strongSession = weakSession; + if (strongSession == nil) { + return; + } + @try { + NSData *chunk = [handle availableData]; + if (chunk.length == 0) { + return; + } + BOOL shouldSignal = NO; + @synchronized(strongSession.stderrData) { + [strongSession.stderrData appendData:chunk]; + NSString *text = XCWStringFromData(strongSession.stderrData); + if (!recordingStarted && [text rangeOfString:@"Recording started"].location != NSNotFound) { + recordingStarted = YES; + shouldSignal = YES; + } + } + if (shouldSignal) { + dispatch_semaphore_signal(recordingStartedSemaphore); + } + } @catch (NSException *exception) { + } + }; + + NSError *launchError = nil; + if (![task launchAndReturnError:&launchError]) { + XCWCloseRecordingPipes(session); + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (error != NULL) { + *error = launchError ?: [self.class errorWithDescription:@"Unable to launch simulator screen recording." code:34]; + } + return nil; + } + + NSDate *startDeadline = [NSDate dateWithTimeIntervalSinceNow:10.0]; + BOOL didStart = NO; + while ([startDeadline timeIntervalSinceNow] > 0) { + if (dispatch_semaphore_wait(recordingStartedSemaphore, DISPATCH_TIME_NOW) == 0) { + didStart = YES; + break; + } + if (!task.running) { + break; + } + usleep(10 * 1000); + } + + if (!didStart) { + XCWStopRecordingTask(session, 2.0); + XCWCloseRecordingPipes(session); + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (error != NULL) { + NSString *stderrString = XCWStringFromData(XCWDataSnapshot(session.stderrData)); + NSString *stdoutString = XCWStringFromData(XCWDataSnapshot(session.stdoutData)); + NSString *details = stderrString.length > 0 ? stderrString : stdoutString; + if (details.length == 0) { + details = @"Simulator screen recording did not start before the timeout."; + } + *error = [self.class errorWithDescription:details code:34]; + } + return nil; + } + + @synchronized(sessions) { + sessions[session.identifier] = session; + } + return session.identifier; +} + +- (nullable NSData *)stopScreenRecordingWithID:(NSString *)recordingID + error:(NSError * _Nullable __autoreleasing *)error { + NSString *trimmedID = XCWTrimmedString(recordingID ?: @""); + if (trimmedID.length == 0) { + if (error != NULL) { + *error = [self.class errorWithDescription:@"Screen recording stop requires a recording ID." code:34]; + } + return nil; + } + + NSMutableDictionary *sessions = XCWScreenRecordingSessions(); + XCWScreenRecordingSession *session = nil; + @synchronized(sessions) { + session = sessions[trimmedID]; + if (session != nil) { + [sessions removeObjectForKey:trimmedID]; + } + } + + if (session == nil) { + if (error != NULL) { + *error = [self.class errorWithDescription:@"No active simulator screen recording matched that ID." code:34]; + } + return nil; + } + + XCWStopRecordingTask(session, 10.0); + XCWCloseRecordingPipes(session); + + NSError *readError = nil; + NSData *data = nil; + NSDate *fileDeadline = [NSDate dateWithTimeIntervalSinceNow:2.0]; + do { + data = [NSData dataWithContentsOfFile:session.path options:0 error:&readError]; + if (data.length > 0) { + break; + } + usleep(50 * 1000); + } while ([fileDeadline timeIntervalSinceNow] > 0); + [[NSFileManager defaultManager] removeItemAtPath:session.path error:nil]; + if (data.length > 0) { + return data; + } + + if (error != NULL) { + NSString *stderrString = XCWStringFromData(XCWDataSnapshot(session.stderrData)); + NSString *stdoutString = XCWStringFromData(XCWDataSnapshot(session.stdoutData)); + NSString *details = stderrString.length > 0 ? stderrString : stdoutString; + if (details.length == 0 && readError.localizedDescription.length > 0) { + details = readError.localizedDescription; + } else if (details.length == 0) { + details = @"Simulator screen recording command produced an empty MP4."; + } + *error = [self.class errorWithDescription:details code:34]; + } + return nil; +} + - (BOOL)eraseSimulatorWithUDID:(NSString *)udid error:(NSError * _Nullable __autoreleasing *)error { XCWProcessResult *result = [self.class runSimctl:@[@"erase", udid] error:error]; if (result == nil) { diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index 0fd6bedd..6f7bb765 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -53,6 +53,8 @@ xcw_native_owned_bytes xcw_native_render_chrome_button_png(const char * _Nonnull xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_screenshot_png(const char * _Nonnull udid, bool include_bezel, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_screen_recording_mp4(const char * _Nonnull udid, double duration_seconds, char * _Nullable * _Nullable error_message); +char * _Nullable xcw_native_start_screen_recording(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); +xcw_native_owned_bytes xcw_native_stop_screen_recording(const char * _Nonnull recording_id, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_recent_logs(const char * _Nonnull udid, double seconds, size_t limit, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_accessibility_snapshot(const char * _Nonnull udid, bool has_point, double x, double y, size_t max_depth, char * _Nullable * _Nullable error_message); bool xcw_native_send_touch(const char * _Nonnull udid, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index 61b8c951..69fe0e59 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -562,6 +562,34 @@ xcw_native_owned_bytes xcw_native_screen_recording_mp4(const char *udid, double } } +char *xcw_native_start_screen_recording(const char *udid, char **error_message) { + @autoreleasepool { + XCWSimctl *simctl = [[XCWSimctl alloc] init]; + NSError *error = nil; + NSString *recordingID = [simctl startScreenRecordingForSimulatorUDID:XCWStringFromCString(udid) + error:&error]; + if (recordingID == nil) { + XCWSetErrorMessage(error_message, error); + return NULL; + } + return XCWCopyCString(recordingID); + } +} + +xcw_native_owned_bytes xcw_native_stop_screen_recording(const char *recording_id, char **error_message) { + @autoreleasepool { + XCWSimctl *simctl = [[XCWSimctl alloc] init]; + NSError *error = nil; + NSData *mp4 = [simctl stopScreenRecordingWithID:XCWStringFromCString(recording_id) + error:&error]; + if (mp4 == nil) { + XCWSetErrorMessage(error_message, error); + return (xcw_native_owned_bytes){0}; + } + return XCWOwnedBytesFromData(mp4); + } +} + char *xcw_native_recent_logs(const char *udid, double seconds, size_t limit, char **error_message) { @autoreleasepool { XCWSimctl *simctl = [[XCWSimctl alloc] init]; diff --git a/client/src/api/controls.ts b/client/src/api/controls.ts index 97990270..f46e9748 100644 --- a/client/src/api/controls.ts +++ b/client/src/api/controls.ts @@ -22,12 +22,18 @@ export type ControlMessage = | ({ type: "button" } & ButtonPayload) | ({ type: "crown" } & CrownPayload) | { type: "dismissKeyboard" } + | { type: "toggleSoftwareKeyboard" } | { type: "home" } | { type: "appSwitcher" } | { type: "rotateLeft" } | { type: "rotateRight" } | { type: "toggleAppearance" }; +export interface ScreenRecordingStartResponse { + ok: boolean; + recordingId: string; +} + async function postSimulatorAction( udid: string, action: string, @@ -137,3 +143,26 @@ export function recordSimulatorScreen( }, ); } + +export function startSimulatorScreenRecording( + udid: string, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/screen-recording/start`, + { + method: "POST", + }, + ); +} + +export function stopSimulatorScreenRecording( + udid: string, + recordingId: string, +): Promise { + return fetchSimulatorBlob( + `/api/simulators/${encodeURIComponent(udid)}/screen-recording/${encodeURIComponent(recordingId)}/stop`, + { + method: "POST", + }, + ); +} diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index d135bafb..747bc52d 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -20,9 +20,10 @@ import { captureSimulatorScreenshot, launchSimulatorBundle, openSimulatorUrl, - recordSimulatorScreen, simulatorControlSocketUrl, shutdownSimulator, + startSimulatorScreenRecording, + stopSimulatorScreenRecording, uploadSimulatorApp, type ControlMessage, } from "../api/controls"; @@ -334,7 +335,7 @@ function downloadBlob(blob: Blob, fileName: string) { document.body.appendChild(link); link.click(); link.remove(); - URL.revokeObjectURL(url); + window.setTimeout(() => URL.revokeObjectURL(url), 30_000); } function captureFileBaseName( @@ -345,6 +346,13 @@ function captureFileBaseName( return `SimDeck ${artifact} - ${safeName || simulator.udid}`; } +function formatElapsedRecordingTime(startedAt: number, now: number): string { + const elapsedSeconds = Math.max(0, Math.floor((now - startedAt) / 1000)); + const minutes = Math.floor(elapsedSeconds / 60); + const seconds = elapsedSeconds % 60; + return `${minutes}:${String(seconds).padStart(2, "0")}`; +} + function simulatorDisplaySize( simulator: SimulatorMetadata | null, ): Size | null { @@ -414,6 +422,19 @@ type AppInstallState = { phase: "dragging" | "installing" | "installed"; }; +type CaptureStatus = { + busy: boolean; + label: string; +}; + +type ScreenRecordingState = { + recordingId: string; + simulatorName: string; + startedAt: number; + udid: string; + phase: "recording" | "stopping"; +}; + export interface AppShellProps { apiRoot?: string; fixedSimulatorUDID?: string | null; @@ -472,8 +493,15 @@ export function AppShell({ initialUiState.bundleIDValue ?? "com.apple.Preferences", ); const [menuOpen, setMenuOpen] = useState(false); + const [simulatorMenuOpen, setSimulatorMenuOpen] = useState(false); const [newSimulatorOpen, setNewSimulatorOpen] = useState(false); const [localError, setLocalError] = useState(""); + const [captureStatus, setCaptureStatus] = useState( + null, + ); + const [screenRecording, setScreenRecording] = + useState(null); + const [recordingNow, setRecordingNow] = useState(Date.now()); const [failedStreamUDIDs, setFailedStreamUDIDs] = useState>( () => new Set(), ); @@ -541,9 +569,11 @@ export function AppShell({ useState(null); const menuRef = useRef(null); + const simulatorMenuRef = useRef(null); const appInstallInputRef = useRef(null); const appInstallDragDepthRef = useRef(0); const appInstallStatusTimeoutRef = useRef(0); + const captureStatusTimeoutRef = useRef(0); const outerCanvasRef = useRef(null); const streamCanvasRef = useRef(null); const [outerCanvasElement, setOuterCanvasElement] = @@ -965,6 +995,19 @@ export function AppShell({ isAndroidViewport, ) : ""; + const recordingOverlayLabel = screenRecording + ? screenRecording.phase === "stopping" + ? "Finalizing recording..." + : `Recording ${formatElapsedRecordingTime(screenRecording.startedAt, recordingNow)}` + : ""; + const captureOverlayLabel = appInstallOverlayLabel + ? appInstallOverlayLabel + : recordingOverlayLabel || captureStatus?.label || ""; + const captureOverlayBusy = Boolean( + isInstallingApp || + captureStatus?.busy || + screenRecording?.phase === "stopping", + ); const autoViewportOffsetY = viewMode === "manual" ? 0 : -zoomDockReservedHeight / 2; const screenAspect = screenAspectRatio(effectiveDeviceNaturalSize); @@ -1593,20 +1636,24 @@ export function AppShell({ ]); useEffect(() => { - if (!menuOpen) { + if (!menuOpen && !simulatorMenuOpen) { return; } function handleDocumentPointerDown(event: PointerEvent) { - if (menuRef.current?.contains(event.target as Node)) { - return; + const target = event.target as Node; + if (menuOpen && !menuRef.current?.contains(target)) { + setMenuOpen(false); + } + if (simulatorMenuOpen && !simulatorMenuRef.current?.contains(target)) { + setSimulatorMenuOpen(false); } - setMenuOpen(false); } function handleWindowKeyDown(event: KeyboardEvent) { if (event.key === "Escape") { setMenuOpen(false); + setSimulatorMenuOpen(false); } } @@ -1620,7 +1667,17 @@ export function AppShell({ ); window.removeEventListener("keydown", handleWindowKeyDown); }; - }, [menuOpen]); + }, [menuOpen, simulatorMenuOpen]); + + useEffect(() => { + if (!screenRecording || screenRecording.phase !== "recording") { + return; + } + const interval = window.setInterval(() => { + setRecordingNow(Date.now()); + }, 500); + return () => window.clearInterval(interval); + }, [screenRecording]); useEffect(() => { function handleWindowKeyDown(event: KeyboardEvent) { @@ -1682,6 +1739,9 @@ export function AppShell({ if (appInstallStatusTimeoutRef.current) { clearTimeout(appInstallStatusTimeoutRef.current); } + if (captureStatusTimeoutRef.current) { + clearTimeout(captureStatusTimeoutRef.current); + } }; }, []); @@ -1699,6 +1759,9 @@ export function AppShell({ } sendControl(selectedSimulator.udid, { type: "key", keyCode, modifiers }); }, + onToggleSoftwareKeyboard: () => { + toggleSoftwareKeyboard(); + }, }); const pointerInput = usePointerInput({ @@ -1916,11 +1979,48 @@ export function AppShell({ } } + function toggleSoftwareKeyboard() { + if (!selectedSimulator) { + return; + } + if ( + !sendControl(selectedSimulator.udid, { + type: "toggleSoftwareKeyboard", + }) + ) { + setLocalError("Simulator control stream disconnected."); + } + } + + function setTransientCaptureStatus(label: string, busy: boolean) { + if (captureStatusTimeoutRef.current) { + window.clearTimeout(captureStatusTimeoutRef.current); + captureStatusTimeoutRef.current = 0; + } + setCaptureStatus({ busy, label }); + } + + function clearCaptureStatusLater(label: string, delayMs = 1600) { + if (captureStatusTimeoutRef.current) { + window.clearTimeout(captureStatusTimeoutRef.current); + } + captureStatusTimeoutRef.current = window.setTimeout(() => { + captureStatusTimeoutRef.current = 0; + setCaptureStatus((current) => + current?.label === label ? null : current, + ); + }, delayMs); + } + async function downloadSimulatorScreenshot(withBezel: boolean) { if (!selectedSimulator) { return; } setLocalError(""); + const statusLabel = withBezel + ? "Capturing screenshot with bezel..." + : "Capturing screenshot..."; + setTransientCaptureStatus(statusLabel, true); try { const blob = await captureSimulatorScreenshot(selectedSimulator.udid, { withBezel, @@ -1929,7 +2029,11 @@ export function AppShell({ blob, `${captureFileBaseName(selectedSimulator, "Screenshot")}${withBezel ? " Bezel" : ""}.png`, ); + const successLabel = "Screenshot downloaded"; + setTransientCaptureStatus(successLabel, false); + clearCaptureStatusLater(successLabel); } catch (captureError) { + setCaptureStatus(null); setLocalError( captureError instanceof Error ? captureError.message @@ -1938,18 +2042,68 @@ export function AppShell({ } } - async function downloadSimulatorRecording() { + async function toggleSimulatorRecording() { if (!selectedSimulator) { return; } setLocalError(""); + if (screenRecording) { + if (screenRecording.phase === "stopping") { + return; + } + const recording = screenRecording; + setScreenRecording({ ...recording, phase: "stopping" }); + try { + const blob = await stopSimulatorScreenRecording( + recording.udid, + recording.recordingId, + ); + downloadBlob( + blob, + `${captureFileBaseName( + { + ...selectedSimulator, + name: recording.simulatorName, + udid: recording.udid, + }, + "Recording", + )}.mp4`, + ); + setScreenRecording(null); + const successLabel = "Recording downloaded"; + setTransientCaptureStatus(successLabel, false); + clearCaptureStatusLater(successLabel); + } catch (captureError) { + setScreenRecording(recording); + setLocalError( + captureError instanceof Error + ? captureError.message + : "Recording failed.", + ); + } + return; + } + + if (!selectedSimulator.isBooted) { + setLocalError("Boot the simulator before recording."); + return; + } + setTransientCaptureStatus("Starting recording...", true); try { - const blob = await recordSimulatorScreen(selectedSimulator.udid, 5); - downloadBlob( - blob, - `${captureFileBaseName(selectedSimulator, "Recording")}.mp4`, + const response = await startSimulatorScreenRecording( + selectedSimulator.udid, ); + setCaptureStatus(null); + setRecordingNow(Date.now()); + setScreenRecording({ + phase: "recording", + recordingId: response.recordingId, + simulatorName: selectedSimulator.name, + startedAt: Date.now(), + udid: selectedSimulator.udid, + }); } catch (captureError) { + setCaptureStatus(null); setLocalError( captureError instanceof Error ? captureError.message @@ -2662,8 +2816,10 @@ export function AppShell({ type="file" /> setMenuOpen(false)} + closeSimulatorMenu={() => setSimulatorMenuOpen(false)} debugVisible={debugVisible} error={toolbarError} filteredSimulators={filteredSimulators} @@ -2722,6 +2878,7 @@ export function AppShell({ onOpenBundlePrompt={promptForBundleID} onOpenNewSimulator={() => { setMenuOpen(false); + setSimulatorMenuOpen(false); setNewSimulatorOpen(true); }} onOpenUrlPrompt={promptForURL} @@ -2747,8 +2904,8 @@ export function AppShell({ } setLocalError("Simulator control stream disconnected."); }} - onRecordScreen={() => { - void downloadSimulatorRecording(); + onToggleRecording={() => { + void toggleSimulatorRecording(); }} onStreamEncoderChange={updateStreamEncoder} onStreamFpsChange={updateStreamFps} @@ -2784,15 +2941,28 @@ export function AppShell({ void loadAccessibilityTree(); } }} - onToggleMenu={() => setMenuOpen((current) => !current)} + onToggleMenu={() => { + setSimulatorMenuOpen(false); + setMenuOpen((current) => !current); + }} + onToggleSimulatorMenu={() => { + setMenuOpen(false); + setSimulatorMenuOpen((current) => !current); + }} + onToggleSoftwareKeyboard={toggleSoftwareKeyboard} onToggleTouchOverlay={() => setTouchOverlayVisible((current) => !current) } + recordingActive={screenRecording?.phase === "recording"} + recordingStopping={screenRecording?.phase === "stopping"} remoteStream={remoteStream} search={search} selectedSimulator={selectedSimulator} selectedSimulatorIdentifier={selectedSimulatorDetail} - setSelectedUDID={setSelectedUDID} + setSelectedUDID={(udid) => { + setSelectedUDID(udid); + setSimulatorMenuOpen(false); + }} showBootButton={Boolean( selectedSimulator && !selectedSimulator.isBooted && @@ -2800,6 +2970,8 @@ export function AppShell({ )} streamConfig={streamConfig} streamTransport={streamTransport} + simulatorMenuOpen={simulatorMenuOpen} + simulatorMenuRef={simulatorMenuRef} showStopButton={Boolean( selectedSimulator?.isBooted && !selectedSimulatorTransitionKind, )} @@ -2814,7 +2986,7 @@ export function AppShell({ /> void; + onToggleSoftwareKeyboard?: () => void; } -export function useKeyboardInput({ enabled, onKey }: UseKeyboardInputOptions) { +export function useKeyboardInput({ + enabled, + onKey, + onToggleSoftwareKeyboard, +}: UseKeyboardInputOptions) { const onKeyRef = useRef(onKey); + const onToggleSoftwareKeyboardRef = useRef(onToggleSoftwareKeyboard); useEffect(() => { onKeyRef.current = onKey; }, [onKey]); + useEffect(() => { + onToggleSoftwareKeyboardRef.current = onToggleSoftwareKeyboard; + }, [onToggleSoftwareKeyboard]); + useEffect(() => { if (!enabled) { return; @@ -30,6 +40,12 @@ export function useKeyboardInput({ enabled, onKey }: UseKeyboardInputOptions) { if (isCopyShortcut(event) && hasDocumentSelection()) { return; } + if (isSoftwareKeyboardShortcut(event)) { + event.preventDefault(); + event.stopImmediatePropagation(); + onToggleSoftwareKeyboardRef.current?.(); + return; + } const keyCode = keyCodeForKeyboardEvent(event); if (keyCode == null) { @@ -45,6 +61,16 @@ export function useKeyboardInput({ enabled, onKey }: UseKeyboardInputOptions) { }, [enabled]); } +function isSoftwareKeyboardShortcut(event: KeyboardEvent): boolean { + return ( + event.key.toLowerCase() === "k" && + event.metaKey && + !event.ctrlKey && + !event.altKey && + !event.shiftKey + ); +} + function isCopyShortcut(event: KeyboardEvent): boolean { return ( event.key.toLowerCase() === "c" && diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index 97891020..10e7ef71 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -1,7 +1,4 @@ -import { - MixerHorizontalIcon as MenuIcon, - PlusIcon, -} from "@radix-ui/react-icons"; +import { MixerHorizontalIcon as MenuIcon } from "@radix-ui/react-icons"; import type { RefObject } from "react"; import type { SimulatorMetadata } from "../../api/types"; @@ -13,30 +10,24 @@ import type { StreamTransport, } from "../stream/streamTypes"; import { simulatorHasFixedOrientation } from "./simulatorDisplay"; -import { SimulatorRow } from "./SimulatorRow"; interface SimulatorMenuProps { + captureBusy: boolean; debugVisible: boolean; - filteredSimulators: SimulatorMetadata[]; - hideSimulatorSelection?: boolean; - isLoading: boolean; canInstallApp: boolean; menuOpen: boolean; menuRef: RefObject; onBoot: () => void; onCaptureScreenshot: () => void; onCaptureScreenshotWithBezel: () => void; - onChangeSearch: (value: string) => void; onCloseMenu: () => void; onDismissKeyboard: () => void; onHome: () => void; onInstallAppPrompt: () => void; onOpenAppSwitcher: () => void; onOpenBundlePrompt: () => void; - onOpenNewSimulator: () => void; onOpenUrlPrompt: () => void; onRotateRight: () => void; - onRecordScreen: () => void; onShutdown: () => void; onStreamEncoderChange: (encoder: StreamEncoder) => void; onStreamFpsChange: (fps: StreamFps) => void; @@ -45,11 +36,13 @@ interface SimulatorMenuProps { onToggleAppearance: () => void; onToggleDebug: () => void; onToggleMenu: () => void; + onToggleRecording: () => void; + onToggleSoftwareKeyboard: () => void; onToggleTouchOverlay: () => void; + recordingActive: boolean; + recordingStopping: boolean; remoteStream?: boolean; - search: string; selectedSimulator: SimulatorMetadata | null; - setSelectedUDID: (udid: string) => void; showBootButton: boolean; showStopButton: boolean; streamConfig: StreamConfig; @@ -58,27 +51,22 @@ interface SimulatorMenuProps { } export function SimulatorMenu({ + captureBusy, debugVisible, - filteredSimulators, - hideSimulatorSelection = false, - isLoading, canInstallApp, menuOpen, menuRef, onBoot, onCaptureScreenshot, onCaptureScreenshotWithBezel, - onChangeSearch, onCloseMenu, onDismissKeyboard, onHome, onInstallAppPrompt, onOpenAppSwitcher, onOpenBundlePrompt, - onOpenNewSimulator, onOpenUrlPrompt, onRotateRight, - onRecordScreen, onShutdown, onStreamEncoderChange, onStreamFpsChange, @@ -87,11 +75,13 @@ export function SimulatorMenu({ onToggleAppearance, onToggleDebug, onToggleMenu, + onToggleRecording, + onToggleSoftwareKeyboard, onToggleTouchOverlay, + recordingActive, + recordingStopping, remoteStream = false, - search, selectedSimulator, - setSelectedUDID, showBootButton, showStopButton, streamConfig, @@ -137,49 +127,8 @@ export function SimulatorMenu({ className="menu-popover" onPointerDown={(event) => event.stopPropagation()} > - {!hideSimulatorSelection ? ( - <> - onChangeSearch(event.target.value)} - placeholder="Search simulators..." - value={search} - /> -
- -
-
- {isLoading ?

Loading...

: null} - {!isLoading && filteredSimulators.length === 0 ? ( -

No matches

- ) : null} - {filteredSimulators.map((simulator) => ( - { - setSelectedUDID(simulator.udid); - onCloseMenu(); - }} - simulator={simulator} - /> - ))} -
- - ) : null} {selectedSimulator ? ( <> -
Stream @@ -292,6 +241,7 @@ export function SimulatorMenu({ +
- ) : null} + ) : ( +

No simulator selected

+ )}
) : null}
diff --git a/client/src/features/simulators/SimulatorPickerMenu.tsx b/client/src/features/simulators/SimulatorPickerMenu.tsx new file mode 100644 index 00000000..e29c7da3 --- /dev/null +++ b/client/src/features/simulators/SimulatorPickerMenu.tsx @@ -0,0 +1,126 @@ +import { PlusIcon } from "@radix-ui/react-icons"; +import type { RefObject } from "react"; + +import type { SimulatorMetadata } from "../../api/types"; +import { simulatorRuntimeLabel } from "./simulatorDisplay"; +import { SimulatorRow } from "./SimulatorRow"; + +interface SimulatorPickerMenuProps { + filteredSimulators: SimulatorMetadata[]; + hideSimulatorSelection?: boolean; + isLoading: boolean; + menuOpen: boolean; + menuRef: RefObject; + onChangeSearch: (value: string) => void; + onCloseMenu: () => void; + onOpenNewSimulator: () => void; + onToggleMenu: () => void; + search: string; + selectedSimulator: SimulatorMetadata | null; + selectedSimulatorIdentifier: string; + setSelectedUDID: (udid: string) => void; +} + +export function SimulatorPickerMenu({ + filteredSimulators, + hideSimulatorSelection = false, + isLoading, + menuOpen, + menuRef, + onChangeSearch, + onCloseMenu, + onOpenNewSimulator, + onToggleMenu, + search, + selectedSimulator, + selectedSimulatorIdentifier, + setSelectedUDID, +}: SimulatorPickerMenuProps) { + const content = selectedSimulator ? ( +
+
+ {selectedSimulator.name} + {selectedSimulator.isBooted ? ( + + ) : null} +
+ {selectedSimulatorIdentifier} +
+ ) : ( + + {isLoading ? "Loading..." : "No simulator selected"} + + ); + + if (hideSimulatorSelection) { + return
{content}
; + } + + return ( +
+ + {menuOpen ? ( +
event.stopPropagation()} + > + onChangeSearch(event.target.value)} + placeholder="Search simulators..." + value={search} + /> +
+ +
+
+ {isLoading ?

Loading...

: null} + {!isLoading && filteredSimulators.length === 0 ? ( +

No matches

+ ) : null} + {filteredSimulators.map((simulator) => ( + { + setSelectedUDID(simulator.udid); + onCloseMenu(); + }} + simulator={simulator} + /> + ))} +
+ {selectedSimulator ? ( +
+ {simulatorRuntimeLabel(selectedSimulator)} +
+ ) : null} +
+ ) : null} +
+ ); +} diff --git a/client/src/features/toolbar/Toolbar.tsx b/client/src/features/toolbar/Toolbar.tsx index 769f8f90..7f7c63d6 100644 --- a/client/src/features/toolbar/Toolbar.tsx +++ b/client/src/features/toolbar/Toolbar.tsx @@ -21,6 +21,7 @@ import type { } from "../stream/streamTypes"; import { simulatorHasFixedOrientation } from "../simulators/simulatorDisplay"; import { SimulatorMenu } from "../simulators/SimulatorMenu"; +import { SimulatorPickerMenu } from "../simulators/SimulatorPickerMenu"; interface ToolbarProps { debugVisible: boolean; @@ -43,7 +44,6 @@ interface ToolbarProps { onOpenNewSimulator: () => void; onOpenUrlPrompt: () => void; onRotateRight: () => void; - onRecordScreen: () => void; onShutdown: () => void; onStreamEncoderChange: (encoder: StreamEncoder) => void; onStreamFpsChange: (fps: StreamFps) => void; @@ -54,7 +54,13 @@ interface ToolbarProps { onToggleDevTools: () => void; onToggleHierarchy: () => void; onToggleMenu: () => void; + onToggleRecording: () => void; + onToggleSimulatorMenu: () => void; + onToggleSoftwareKeyboard: () => void; onToggleTouchOverlay: () => void; + captureBusy: boolean; + recordingActive: boolean; + recordingStopping: boolean; remoteStream?: boolean; search: string; selectedSimulator: SimulatorMetadata | null; @@ -68,9 +74,14 @@ interface ToolbarProps { menuOpen: boolean; menuRef: RefObject; closeMenu: () => void; + simulatorMenuOpen: boolean; + simulatorMenuRef: RefObject; + closeSimulatorMenu: () => void; } export function Toolbar({ + captureBusy, + closeSimulatorMenu, closeMenu, debugVisible, devToolsVisible, @@ -94,7 +105,6 @@ export function Toolbar({ onOpenNewSimulator, onOpenUrlPrompt, onRotateRight, - onRecordScreen, onShutdown, onStreamEncoderChange, onStreamFpsChange, @@ -105,7 +115,12 @@ export function Toolbar({ onToggleDevTools, onToggleHierarchy, onToggleMenu, + onToggleRecording, + onToggleSimulatorMenu, + onToggleSoftwareKeyboard, onToggleTouchOverlay, + recordingActive, + recordingStopping, remoteStream = false, search, selectedSimulator, @@ -115,6 +130,8 @@ export function Toolbar({ showStopButton, streamConfig, streamTransport, + simulatorMenuOpen, + simulatorMenuRef, touchOverlayVisible, }: ToolbarProps) { const [errorCopied, setErrorCopied] = useState(false); @@ -152,26 +169,21 @@ export function Toolbar({ - {selectedSimulator ? ( -
-
-
- - {selectedSimulator.name} - - {selectedSimulator.isBooted ? ( - - ) : null} -
- - {selectedSimulatorIdentifier} - -
-
- ) : ( - - {isLoading ? "Loading…" : "No simulator selected"} - - )} +
diff --git a/client/src/styles/components.css b/client/src/styles/components.css index 6476697f..e9af6007 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -278,6 +278,17 @@ z-index: 20; } +.simulator-picker-popover { + width: min(360px, calc(100vw - 24px)); +} + +.simulator-picker-current { + padding: 8px 12px; + border-top: 1px solid var(--border-subtle); + color: var(--text-secondary); + font-size: 11px; +} + .sidebar-search { width: 100%; padding: 8px 12px; diff --git a/client/src/styles/layout.css b/client/src/styles/layout.css index 8d2c0e34..02a53bf4 100644 --- a/client/src/styles/layout.css +++ b/client/src/styles/layout.css @@ -47,6 +47,42 @@ overflow: hidden; } +.simulator-picker-wrap { + flex: 0 1 auto; + min-width: 0; + max-width: min(100%, 360px); +} + +.simulator-picker-wrap > .toolbar-sim-info { + width: auto; +} + +.toolbar-sim-trigger { + appearance: none; + border: 1px solid transparent; + border-radius: 6px; + background: transparent; + color: inherit; + cursor: pointer; + flex: 0 1 auto; + font: inherit; + margin-block: 3px; + max-width: min(360px, 100%); + padding: 2px 8px; + text-align: left; +} + +.toolbar-sim-trigger:hover, +.toolbar-sim-trigger.active { + background: var(--surface-hover); + border-color: var(--border-subtle); +} + +.toolbar-sim-trigger:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} + .toolbar-sim-copy { display: flex; flex-direction: column; @@ -64,6 +100,7 @@ .toolbar-sim-name { font-weight: 600; font-size: 13px; + line-height: 14px; min-width: 0; white-space: nowrap; overflow: hidden; @@ -73,6 +110,7 @@ .toolbar-sim-detail { color: var(--text-secondary); font-size: 12px; + line-height: 13px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; @@ -137,6 +175,16 @@ min-width: 0; } + .simulator-picker-wrap { + flex: 0 1 auto; + max-width: min(52vw, 320px); + } + + .simulator-picker-wrap > .toolbar-sim-info { + flex: 0 1 auto; + width: auto; + } + .toolbar-actions { flex: 0 0 auto; width: auto; diff --git a/client/vite.config.js b/client/vite.config.js index 59c27b27..5dd67ebe 100644 --- a/client/vite.config.js +++ b/client/vite.config.js @@ -7,7 +7,7 @@ export default defineConfig({ host: "127.0.0.1", port: 5173, proxy: { - "/api": "http://127.0.0.1:4310", + "/api": "http://127.0.0.1:4311", }, }, build: { diff --git a/docs/api/rest.md b/docs/api/rest.md index dcf0c9f8..f8149446 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -171,22 +171,23 @@ Response: ## Input -| Method | Path | Body | -| ------ | ----------------------------------------- | -------------------------------------------------------------------- | -| `POST` | `/api/simulators/{udid}/tap` | Selector or coordinate tap | -| `POST` | `/api/simulators/{udid}/touch` | `{ "x": 0.5, "y": 0.5, "phase": "began" }` | -| `POST` | `/api/simulators/{udid}/edge-touch` | `{ "x": 0.5, "y": 0.98, "phase": "began", "edge": "bottom" }` | -| `POST` | `/api/simulators/{udid}/multi-touch` | `{ "x1": 0.35, "y1": 0.5, "x2": 0.65, "y2": 0.5, "phase": "began" }` | -| `POST` | `/api/simulators/{udid}/touch-sequence` | Multiple touch phases | -| `POST` | `/api/simulators/{udid}/key` | `{ "keyCode": 4, "modifiers": 0 }` | -| `POST` | `/api/simulators/{udid}/key-sequence` | `{ "keyCodes": [11,8,15], "delayMs": 5 }` | -| `POST` | `/api/simulators/{udid}/button` | `{ "button": "lock", "durationMs": 50 }` | -| `POST` | `/api/simulators/{udid}/crown` | `{ "delta": 50 }` | -| `POST` | `/api/simulators/{udid}/dismiss-keyboard` | Dismiss the software keyboard | -| `POST` | `/api/simulators/{udid}/home` | Press Home | -| `POST` | `/api/simulators/{udid}/app-switcher` | Open app switcher | -| `POST` | `/api/simulators/{udid}/rotate-left` | Rotate left | -| `POST` | `/api/simulators/{udid}/rotate-right` | Rotate right | +| Method | Path | Body | +| ------ | ------------------------------------------------- | -------------------------------------------------------------------- | +| `POST` | `/api/simulators/{udid}/tap` | Selector or coordinate tap | +| `POST` | `/api/simulators/{udid}/touch` | `{ "x": 0.5, "y": 0.5, "phase": "began" }` | +| `POST` | `/api/simulators/{udid}/edge-touch` | `{ "x": 0.5, "y": 0.98, "phase": "began", "edge": "bottom" }` | +| `POST` | `/api/simulators/{udid}/multi-touch` | `{ "x1": 0.35, "y1": 0.5, "x2": 0.65, "y2": 0.5, "phase": "began" }` | +| `POST` | `/api/simulators/{udid}/touch-sequence` | Multiple touch phases | +| `POST` | `/api/simulators/{udid}/key` | `{ "keyCode": 4, "modifiers": 0 }` | +| `POST` | `/api/simulators/{udid}/key-sequence` | `{ "keyCodes": [11,8,15], "delayMs": 5 }` | +| `POST` | `/api/simulators/{udid}/button` | `{ "button": "lock", "durationMs": 50 }` | +| `POST` | `/api/simulators/{udid}/crown` | `{ "delta": 50 }` | +| `POST` | `/api/simulators/{udid}/dismiss-keyboard` | Dismiss the software keyboard | +| `POST` | `/api/simulators/{udid}/toggle-software-keyboard` | Toggle the software keyboard | +| `POST` | `/api/simulators/{udid}/home` | Press Home | +| `POST` | `/api/simulators/{udid}/app-switcher` | Open app switcher | +| `POST` | `/api/simulators/{udid}/rotate-left` | Rotate left | +| `POST` | `/api/simulators/{udid}/rotate-right` | Rotate right | Touch, edge-touch, and multi-touch coordinates are normalized from `0.0` to `1.0`. @@ -234,17 +235,19 @@ For app-owned `WKWebView` on iOS 16.4 or newer, the app must set `isInspectable ## Evidence And Chrome -| Method | Path | Purpose | -| ------ | ----------------------------------------------- | ---------------------------------------------- | -| `GET` | `/api/simulators/{udid}/screenshot.png` | PNG screenshot, with `?bezel=true` for chrome | -| `POST` | `/api/simulators/{udid}/screen-recording` | MP4 recording with `{ "seconds": 5 }` | -| `GET` | `/api/simulators/{udid}/pasteboard` | Get pasteboard text | -| `POST` | `/api/simulators/{udid}/pasteboard` | Set pasteboard text with `{ "text": "hello" }` | -| `GET` | `/api/simulators/{udid}/logs` | Recent logs | -| `GET` | `/api/simulators/{udid}/chrome-profile` | Screen and chrome geometry | -| `GET` | `/api/simulators/{udid}/chrome.png` | Rendered device chrome PNG | -| `GET` | `/api/simulators/{udid}/chrome-button/{button}` | Rendered button sprite | -| `GET` | `/api/simulators/{udid}/screen-mask.png` | Rendered screen mask PNG | +| Method | Path | Purpose | +| ------ | ------------------------------------------------------------ | ---------------------------------------------- | +| `GET` | `/api/simulators/{udid}/screenshot.png` | PNG screenshot, with `?bezel=true` for chrome | +| `POST` | `/api/simulators/{udid}/screen-recording` | MP4 recording with `{ "seconds": 5 }` | +| `POST` | `/api/simulators/{udid}/screen-recording/start` | Start MP4 recording and return `recordingId` | +| `POST` | `/api/simulators/{udid}/screen-recording/{recordingId}/stop` | Stop recording and return MP4 | +| `GET` | `/api/simulators/{udid}/pasteboard` | Get pasteboard text | +| `POST` | `/api/simulators/{udid}/pasteboard` | Set pasteboard text with `{ "text": "hello" }` | +| `GET` | `/api/simulators/{udid}/logs` | Recent logs | +| `GET` | `/api/simulators/{udid}/chrome-profile` | Screen and chrome geometry | +| `GET` | `/api/simulators/{udid}/chrome.png` | Rendered device chrome PNG | +| `GET` | `/api/simulators/{udid}/chrome-button/{button}` | Rendered button sprite | +| `GET` | `/api/simulators/{udid}/screen-mask.png` | Rendered screen mask PNG | Log query parameters: diff --git a/docs/cli/commands.md b/docs/cli/commands.md index f30a19c4..e5e40738 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -30,9 +30,12 @@ simdeck daemon restart --video-codec software --stream-quality low `simdeck pair` uses the global LaunchAgent-backed service instead of a project-local daemon. It binds the service for LAN access, preserves an existing service token and pairing code when present, detects LAN and Tailscale IPv4 -addresses, and prints a `simdeck://pair` QR for the native iOS app. If the -requested service port is already in use by a workspace daemon, the LaunchAgent -uses the next available port after it. +addresses, and prints a `simdeck://pair` QR for the native iOS app. The service +uses port 4310; workspace daemons start at 4311 and probe upward. + +When the service is active, `simdeck` and `simdeck ui` print the existing +service endpoints instead of launching a project daemon. Use `simdeck daemon +start` or `simdeck daemon restart` when you explicitly want a workspace daemon. `simdeck service restart` also preserves the installed service token so native clients remain paired across service restarts. Use `simdeck service reset` to @@ -123,6 +126,7 @@ simdeck button action simdeck button digital-crown simdeck crown --delta 50 simdeck dismiss-keyboard +simdeck button software-keyboard simdeck home simdeck app-switcher simdeck rotate-left diff --git a/docs/cli/flags.md b/docs/cli/flags.md index 12edc2e5..8c6ea4b8 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -18,17 +18,17 @@ simdeck daemon start --help Used by `simdeck ui`, `daemon start`, `daemon restart`, `service on`, and `service restart`. -| Flag | Default | Notes | -| ---------------------------- | -------------- | ----------------------------------------------------------------------------------- | ------ | ------------ | -| `--port ` | `4310` | HTTP port. LaunchAgent service commands probe up to 4320 when this port is occupied | -| `--bind ` | `127.0.0.1` | Use `0.0.0.0` or `::` for LAN access | -| `--advertise-host ` | detected | Host printed for remote browsers | -| `--client-root ` | bundled client | Static client directory | -| `--video-codec auto | hardware | software` | `auto` | Encoder mode | -| `--stream-quality ` | `full` | `full`, `balanced`, `economy`, `low`, `tiny`, `ci-software`, and related profiles | -| `--local-stream-fps ` | `60` | Local stream frame target | -| `--low-latency` | off | Conservative software H.264 profile | -| `--open` | off | `ui` only | +| Flag | Default | Notes | +| ---------------------------- | -------------------------------------- | --------------------------------------------------------------------------------- | +| `--port ` | `4311` for daemons, `4310` for service | HTTP port. Daemons probe upward when busy | +| `--bind ` | `127.0.0.1` | Use `0.0.0.0` or `::` for LAN access | +| `--advertise-host ` | detected | Host printed for remote browsers | +| `--client-root ` | bundled client | Static client directory | +| `--video-codec ` | `auto` | Encoder mode | +| `--stream-quality ` | `full` | `full`, `balanced`, `economy`, `low`, `tiny`, `ci-software`, and related profiles | +| `--local-stream-fps ` | `60` | Local stream frame target | +| `--low-latency` | off | Conservative software H.264 profile | +| `--open` | off | `ui` only | ## `describe` diff --git a/docs/extensions/vscode.md b/docs/extensions/vscode.md index 2b915d23..15189c3a 100644 --- a/docs/extensions/vscode.md +++ b/docs/extensions/vscode.md @@ -39,7 +39,7 @@ The extension tries the configured server URL first. If it is not reachable and | ------------------------- | ----------------------- | ---------------------------- | | `simdeck.serverUrl` | `http://127.0.0.1:4310` | Preferred daemon URL | | `simdeck.cliPath` | empty | Explicit path to the CLI | -| `simdeck.port` | `4310` | Port for auto-start | +| `simdeck.port` | `4311` | Port for auto-started project daemons | | `simdeck.bindAddress` | `127.0.0.1` | Bind address for auto-start | | `simdeck.autoStartDaemon` | `true` | Start the daemon when needed | diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index 553a3947..4ba5719a 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -42,15 +42,15 @@ This starts or reuses the daemon, then opens the authenticated local URL. `simdeck ui`, `daemon start`, `daemon restart`, and `service restart` use the same core options: -| Flag | Default | Use it when | -| ---------------------------- | -------------- | ------------------------------------------ | ------ | ---------------------------------- | -| `--port ` | `4310` | The default port is busy | -| `--bind ` | `127.0.0.1` | You need LAN access with `0.0.0.0` or `::` | -| `--advertise-host ` | detected | Remote browsers need a specific host or IP | -| `--video-codec auto | hardware | software` | `auto` | You need to force encoder behavior | -| `--stream-quality ` | `full` | You want lower CPU or bandwidth use | -| `--local-stream-fps ` | `60` | You want a different local stream target | -| `--client-root ` | bundled client | You are serving a custom static client | +| Flag | Default | Use it when | +| ---------------------------- | -------------------------------------- | ------------------------------------------ | +| `--port ` | `4311` for daemons, `4310` for service | The default port is busy | +| `--bind ` | `127.0.0.1` | You need LAN access with `0.0.0.0` or `::` | +| `--advertise-host ` | detected | Remote browsers need a specific host or IP | +| `--video-codec ` | `auto` | You need to force encoder behavior | +| `--stream-quality ` | `full` | You want lower CPU or bandwidth use | +| `--local-stream-fps ` | `60` | You want a different local stream target | +| `--client-root ` | bundled client | You are serving a custom static client | Example: @@ -69,9 +69,10 @@ simdeck service reset simdeck service off ``` -When the requested service port is occupied by a workspace daemon, the -LaunchAgent automatically moves to the next available service-discovery port, -up to 4320. Workspace daemons are left running. +The LaunchAgent service uses port 4310. Workspace daemons start at 4311 and +probe upward when the requested daemon port is busy. When the service is active, +`simdeck` and `simdeck ui` report the service endpoints instead of launching a +project daemon. `service on`, `service restart`, and `simdeck pair` preserve the installed service token and pairing code. Use `service reset` when you explicitly want to diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md index 51ef7f9e..497e5205 100644 --- a/docs/guide/quick-start.md +++ b/docs/guide/quick-start.md @@ -11,8 +11,8 @@ SimDeck prints a local browser URL, a LAN URL when one is available, and a pairi ```text SimDeck is ready -Local: http://127.0.0.1:4310 -Network: http://192.168.1.50:4310 +Local: http://127.0.0.1:4311 +Network: http://192.168.1.50:4311 Pair: 123 456 ``` diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 3a32fc6f..64836034 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -23,7 +23,7 @@ simdeck ### Port is already in use ```text -bind HTTP listener on 127.0.0.1:4310 +bind HTTP listener on 127.0.0.1:4311 ``` Use another port: @@ -35,7 +35,7 @@ simdeck ui --port 4320 --open Or find the listener: ```sh -lsof -nP -iTCP:4310 -sTCP:LISTEN +lsof -nP -iTCP:4311 -sTCP:LISTEN ``` If it is an old project daemon: diff --git a/packages/vscode-extension/extension.js b/packages/vscode-extension/extension.js index 63f7acf3..0c9fc2a8 100644 --- a/packages/vscode-extension/extension.js +++ b/packages/vscode-extension/extension.js @@ -75,7 +75,7 @@ function getAutoStartDaemon(config) { async function startProjectDaemon(context) { const config = vscode.workspace.getConfiguration("simdeck"); const cliPath = resolveCliPath(context, config.get("cliPath", "")); - const port = String(config.get("port", 4310)); + const port = String(config.get("port", 4311)); const bindAddress = config.get("bindAddress", "127.0.0.1"); const args = ["ui", "--port", port, "--bind", bindAddress]; diff --git a/packages/vscode-extension/package.json b/packages/vscode-extension/package.json index f4970f09..23724758 100644 --- a/packages/vscode-extension/package.json +++ b/packages/vscode-extension/package.json @@ -53,7 +53,7 @@ }, "simdeck.port": { "type": "number", - "default": 4310, + "default": 4311, "minimum": 1, "maximum": 65535, "description": "Preferred port used when the extension auto-starts the project daemon." diff --git a/scripts/dev.mjs b/scripts/dev.mjs index 30897221..2486a625 100755 --- a/scripts/dev.mjs +++ b/scripts/dev.mjs @@ -7,7 +7,7 @@ import { fileURLToPath } from "node:url"; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const SERVER_BIN = resolve(ROOT, "build/simdeck"); const LOG_PATH = resolve(ROOT, "build/cli.log"); -const SERVER_PORT = "4310"; +const SERVER_PORT = "4311"; function findListeningPids(port) { try { @@ -50,8 +50,8 @@ function isManagedCliProcess(pid) { function stopStaleCliProcesses() { const stalePids = new Set([ - ...findListeningPids(4310), ...findListeningPids(4311), + ...findListeningPids(4312), ]); for (const pid of stalePids) { if (pid === process.pid || !isManagedCliProcess(pid)) { diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index c88e6cd3..0b52f4e9 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -444,6 +444,7 @@ pub(crate) enum ControlMessage { delta: f64, }, DismissKeyboard, + ToggleSoftwareKeyboard, Home, AppSwitcher, RotateLeft, @@ -852,6 +853,14 @@ pub fn router(state: AppState) -> Router { "/api/simulators/{udid}/screen-recording", post(screen_recording), ) + .route( + "/api/simulators/{udid}/screen-recording/start", + post(start_screen_recording), + ) + .route( + "/api/simulators/{udid}/screen-recording/{recording_id}/stop", + post(stop_screen_recording), + ) .route( "/api/simulators/{udid}/toggle-appearance", post(toggle_appearance), @@ -884,6 +893,10 @@ pub fn router(state: AppState) -> Router { "/api/simulators/{udid}/dismiss-keyboard", post(dismiss_keyboard), ) + .route( + "/api/simulators/{udid}/toggle-software-keyboard", + post(toggle_software_keyboard), + ) .route("/api/simulators/{udid}/button", post(press_button)) .route("/api/simulators/{udid}/crown", post(rotate_crown)) .route("/api/simulators/{udid}/home", post(press_home)) @@ -2444,6 +2457,45 @@ async fn screen_recording( Ok((StatusCode::OK, headers, mp4)) } +async fn start_screen_recording( + State(state): State, + Path(udid): Path, +) -> Result, AppError> { + if android::is_android_id(&udid) { + return Err(AppError::bad_request( + "Screen recording is currently supported for iOS simulators only.", + )); + } + let recording_id = + run_bridge_action(state, move |bridge| bridge.start_screen_recording(&udid)).await?; + Ok(Json(json_value!({ + "ok": true, + "recordingId": recording_id, + }))) +} + +async fn stop_screen_recording( + State(state): State, + Path((udid, recording_id)): Path<(String, String)>, +) -> Result<(StatusCode, HeaderMap, Vec), AppError> { + if android::is_android_id(&udid) { + return Err(AppError::bad_request( + "Screen recording is currently supported for iOS simulators only.", + )); + } + let mp4 = run_bridge_action(state, move |bridge| { + bridge.stop_screen_recording(&recording_id) + }) + .await?; + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, "video/mp4".parse().unwrap()); + headers.insert( + header::CACHE_CONTROL, + "no-cache, no-store, must-revalidate".parse().unwrap(), + ); + Ok((StatusCode::OK, headers, mp4)) +} + fn validate_screen_recording_seconds(seconds: Option) -> Result { let seconds = seconds.unwrap_or(5.0); if !seconds.is_finite() || seconds <= 0.0 { @@ -2900,6 +2952,9 @@ async fn run_android_control_message( )), }, ControlMessage::DismissKeyboard => android.dismiss_keyboard(&udid), + ControlMessage::ToggleSoftwareKeyboard => Err(AppError::bad_request( + "Software keyboard toggle is only available for iOS simulators.", + )), ControlMessage::Home => android.press_home(&udid), ControlMessage::AppSwitcher => android.open_app_switcher(&udid), ControlMessage::RotateLeft => android.rotate_left(&udid), @@ -3594,6 +3649,7 @@ pub(crate) async fn run_control_message( } ControlMessage::Crown { delta } => session.rotate_crown(delta), ControlMessage::DismissKeyboard => session.send_key(41, 0), + ControlMessage::ToggleSoftwareKeyboard => session.press_button("software-keyboard", 0), ControlMessage::Home => session.press_home(), ControlMessage::AppSwitcher => session.open_app_switcher(), ControlMessage::RotateLeft => session.rotate_left(), @@ -3789,6 +3845,22 @@ async fn dismiss_keyboard( Ok(json(json_value!({ "ok": true }))) } +async fn toggle_software_keyboard( + State(state): State, + Path(udid): Path, +) -> Result, AppError> { + if android::is_android_id(&udid) { + return Err(AppError::bad_request( + "Software keyboard toggle is only available for iOS simulators.", + )); + } + run_bridge_action(state, move |bridge| { + bridge.press_button(&udid, "software-keyboard", 0) + }) + .await?; + Ok(json(json_value!({ "ok": true }))) +} + async fn press_button( State(state): State, Path(udid): Path, diff --git a/server/src/main.rs b/server/src/main.rs index 091f4bcd..31ff12fb 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -62,6 +62,8 @@ const SERVER_HEALTH_WATCHDOG_PROBE_TIMEOUT: Duration = Duration::from_secs(3); const SERVER_HEALTH_WATCHDOG_STALE_HEARTBEAT: Duration = Duration::from_secs(60); const SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD: usize = 12; const SERVER_HEALTH_WATCHDOG_HTTP_FAILURE_THRESHOLD: usize = 3; +const SERVICE_PORT: u16 = 4310; +const DAEMON_PORT_START: u16 = 4311; #[derive(Parser)] #[command(name = "simdeck")] @@ -84,7 +86,7 @@ struct Cli { #[derive(Subcommand)] enum Command { Ui { - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = DAEMON_PORT_START)] port: u16, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] bind: IpAddr, @@ -106,7 +108,7 @@ enum Command { Pair { #[arg( long, - help = "Defaults to the existing service port, or the next available port near 4310" + help = "Defaults to the existing service port, or 4310 when the service is not installed" )] port: Option, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::UNSPECIFIED))] @@ -140,7 +142,7 @@ enum Command { }, #[command(hide = true)] Serve { - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = SERVICE_PORT)] port: u16, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] bind: IpAddr, @@ -491,7 +493,7 @@ enum Command { #[derive(Subcommand)] enum DaemonCommand { Start { - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = DAEMON_PORT_START)] port: u16, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] bind: IpAddr, @@ -509,7 +511,7 @@ enum DaemonCommand { local_stream_fps: Option, }, Restart { - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = DAEMON_PORT_START)] port: u16, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] bind: IpAddr, @@ -566,7 +568,7 @@ enum StudioCommand { simulator: Option, #[arg(long, default_value = "https://simdeck.djdev.me")] studio_url: String, - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = DAEMON_PORT_START)] port: u16, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] bind: IpAddr, @@ -608,7 +610,7 @@ enum ProviderCommand { max_capacity: u32, #[arg(long, default_value = "iPhone 17 Pro")] simulator_template: String, - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = DAEMON_PORT_START)] port: u16, #[arg(long, value_enum, default_value_t = VideoCodecMode::Software)] video_codec: VideoCodecMode, @@ -624,7 +626,7 @@ enum ProviderCommand { #[derive(Subcommand)] enum ServiceCommand { On { - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = SERVICE_PORT)] port: u16, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] bind: IpAddr, @@ -644,7 +646,7 @@ enum ServiceCommand { access_token: Option, }, Restart { - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = SERVICE_PORT)] port: u16, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] bind: IpAddr, @@ -664,7 +666,7 @@ enum ServiceCommand { access_token: Option, }, Reset { - #[arg(long, default_value_t = 4310)] + #[arg(long, default_value_t = SERVICE_PORT)] port: u16, #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] bind: IpAddr, @@ -852,7 +854,7 @@ struct DaemonMetadata { } fn default_daemon_port() -> u16 { - 4310 + DAEMON_PORT_START } fn default_daemon_bind() -> IpAddr { @@ -1038,7 +1040,7 @@ fn command_service_url_for_udid( impl Default for DaemonLaunchOptions { fn default() -> Self { Self { - port: 4310, + port: DAEMON_PORT_START, bind: IpAddr::V4(Ipv4Addr::LOCALHOST), advertise_host: None, client_root: None, @@ -1498,6 +1500,59 @@ fn print_daemon_start_result(metadata: &DaemonMetadata, started: bool) -> anyhow })) } +fn print_existing_service_endpoints( + result: service::ServiceInstallResult, + selector: Option<&str>, + open: bool, + json: bool, +) -> anyhow::Result<()> { + let target = PairingTarget::from_service(result)?; + let local_url = ui_url("127.0.0.1", target.port, selector); + let addresses: Vec = pairing_addresses(&target) + .into_iter() + .map(|address| PairingAddress { + kind: address.kind, + url: ui_url_from_base(address.url, selector), + }) + .collect(); + + if open { + open_browser(&local_url)?; + } + + if json { + println_json(&serde_json::json!({ + "ok": true, + "target": target.target, + "service": target.service, + "url": local_url, + "started": false, + "serverId": target.server_id, + "pairingCode": target.pairing_code, + "addresses": addresses, + }))?; + return Ok(()); + } + + println!("SimDeck service is already running"); + println!(); + for address in &addresses { + let label = match address.kind { + "local" => "Local:", + "lan" => "LAN:", + "tailscale" => "Tailscale:", + _ => "URL:", + }; + println!("{:>12} {}", label, address.url); + } + println!( + "{:>12} {}", + "Pair:", + format_pairing_code(&target.pairing_code) + ); + Ok(()) +} + #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] struct PairingAddress { @@ -1969,7 +2024,7 @@ fn pair_global_service(options: PairGlobalServiceOptions) -> anyhow::Result<()> } let requested_port = match port { Some(port) => port, - None => service::installed_port()?.unwrap_or(4310), + None => service::installed_port()?.unwrap_or(SERVICE_PORT), }; print_pair_progress(format!("requesting port {requested_port}")); @@ -2025,6 +2080,10 @@ fn pair_global_service(options: PairGlobalServiceOptions) -> anyhow::Result<()> } fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { + if let Some(result) = service::active()? { + return print_existing_service_endpoints(result, selector.as_deref(), false, false); + } + if let Some(metadata) = read_daemon_metadata().ok().flatten() { if daemon_is_healthy(&metadata) { terminate_daemon_metadata(&metadata)?; @@ -2033,7 +2092,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { let project_root = project_root()?; let bind = IpAddr::V4(Ipv4Addr::UNSPECIFIED); - let port = choose_daemon_port_for_bind(4310, bind)?; + let port = choose_daemon_port_for_bind(DAEMON_PORT_START, bind)?; let video_codec = VideoCodecMode::Auto; let low_latency = false; let stream_quality_profile = Some(DEFAULT_LOCAL_STREAM_QUALITY_PROFILE.to_owned()); @@ -2168,7 +2227,10 @@ fn http_url_for_host(host: &str, port: u16) -> String { } fn ui_url(host: &str, port: u16, selector: Option<&str>) -> String { - let mut url = http_url_for_host(host, port); + ui_url_from_base(http_url_for_host(host, port), selector) +} + +fn ui_url_from_base(mut url: String, selector: Option<&str>) -> String { if let Some(selector) = selector.filter(|value| !value.trim().is_empty()) { url.push_str(&format!("/?device={}", percent_encode(selector.trim()))); } @@ -2568,6 +2630,9 @@ fn main() -> anyhow::Result<()> { local_stream_fps, open, } => { + if let Some(result) = service::active()? { + return print_existing_service_endpoints(result, None, open, true); + } let (metadata, started) = ensure_project_daemon_with_status(DaemonLaunchOptions { port, bind, @@ -4292,7 +4357,7 @@ fn describe_ui_snapshot( if source != DescribeUiSource::Auto && source != DescribeUiSource::NativeAx { anyhow::bail!( - "The `{}` hierarchy source requires a running SimDeck daemon. Start it with `simdeck daemon start --port 4310`, or use --source native-ax.", + "The `{}` hierarchy source requires a running SimDeck daemon. Start it with `simdeck daemon start --port 4311`, or use --source native-ax.", source.as_query_value() ); } diff --git a/server/src/native/bridge.rs b/server/src/native/bridge.rs index 4df7ab2f..c695822c 100644 --- a/server/src/native/bridge.rs +++ b/server/src/native/bridge.rs @@ -538,6 +538,33 @@ impl NativeBridge { } } + pub fn start_screen_recording(&self, udid: &str) -> Result { + let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; + let recording_id = unsafe { + let mut error = ptr::null_mut(); + let raw = ffi::xcw_native_start_screen_recording(udid.as_ptr(), &mut error); + string_from_raw(raw, error)? + }; + Ok(recording_id) + } + + pub fn stop_screen_recording(&self, recording_id: &str) -> Result, AppError> { + let recording_id = + CString::new(recording_id).map_err(|e| AppError::bad_request(e.to_string()))?; + unsafe { + let mut error = ptr::null_mut(); + let bytes = ffi::xcw_native_stop_screen_recording(recording_id.as_ptr(), &mut error); + if bytes.data.is_null() { + return Err( + take_error(error).unwrap_or_else(|| AppError::native("Unknown native error.")) + ); + } + let data = std::slice::from_raw_parts(bytes.data, bytes.length).to_vec(); + ffi::xcw_native_free_bytes(bytes); + Ok(data) + } + } + pub fn recent_logs( &self, udid: &str, diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs index 0b3800f7..78b5bd66 100644 --- a/server/src/native/ffi.rs +++ b/server/src/native/ffi.rs @@ -93,6 +93,14 @@ unsafe extern "C" { duration_seconds: f64, error_message: *mut *mut c_char, ) -> xcw_native_owned_bytes; + pub fn xcw_native_start_screen_recording( + udid: *const c_char, + error_message: *mut *mut c_char, + ) -> *mut c_char; + pub fn xcw_native_stop_screen_recording( + recording_id: *const c_char, + error_message: *mut *mut c_char, + ) -> xcw_native_owned_bytes; pub fn xcw_native_recent_logs( udid: *const c_char, seconds: f64, diff --git a/server/src/service.rs b/server/src/service.rs index 457784f0..9edab8ec 100644 --- a/server/src/service.rs +++ b/server/src/service.rs @@ -9,7 +9,6 @@ use std::time::{Duration, Instant}; const SERVICE_LABEL: &str = "org.nativescript.simdeck"; const LEGACY_SERVICE_LABELS: &[&str] = &["dev.nativescript.simdeck"]; -const DEFAULT_SERVICE_DISCOVERY_MAX_PORT: u16 = 4320; #[derive(Clone, Debug)] pub struct ServiceInstallResult { @@ -92,6 +91,32 @@ pub fn installed_port() -> anyhow::Result> { Ok(installed_argument_value("--port")?.and_then(|value| value.parse::().ok())) } +pub fn active() -> anyhow::Result> { + let domain = launchctl_domain()?; + if launchagent_pid(&domain, SERVICE_LABEL).is_none() { + return Ok(None); + } + let Some(arguments) = installed_arguments_for_label(SERVICE_LABEL)? else { + return Ok(None); + }; + let plist_path = plist_path()?; + let log_dir = log_dir()?; + let port = argument_value(&arguments, "--port") + .and_then(|value| value.parse::().ok()) + .unwrap_or(4310); + Ok(Some(ServiceInstallResult { + service: SERVICE_LABEL.to_owned(), + plist_path, + stdout_log: log_dir.join("simdeck.log"), + stderr_log: log_dir.join("simdeck.err.log"), + port, + advertise_host: argument_value(&arguments, "--advertise-host"), + access_token: argument_value(&arguments, "--access-token"), + pairing_code: argument_value(&arguments, "--pairing-code"), + reused: true, + })) +} + fn install(mut options: ServiceOptions) -> anyhow::Result { let plist_path = plist_path()?; let log_dir = log_dir()?; @@ -484,18 +509,11 @@ fn process_exists(pid: u32) -> bool { } fn choose_service_port_for_bind(preferred: u16, bind: IpAddr) -> anyhow::Result { - let start = preferred.max(1024); - let end = if start <= DEFAULT_SERVICE_DISCOVERY_MAX_PORT { - DEFAULT_SERVICE_DISCOVERY_MAX_PORT - } else { - start.saturating_add(10) - }; - for port in start..=end { - if port_available(bind, port) { - return Ok(port); - } + let port = preferred.max(1024); + if port_available(bind, port) { + return Ok(port); } - bail!("No available SimDeck LaunchAgent port between {start} and {end}"); + bail!("SimDeck LaunchAgent port {port} is already in use"); } fn port_available(bind: IpAddr, port: u16) -> bool { @@ -671,19 +689,13 @@ mod tests { } #[test] - fn service_port_selection_skips_occupied_port() { + fn service_port_selection_rejects_occupied_port() { let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("bind test port"); let occupied = listener.local_addr().expect("local addr").port(); - let selected = choose_service_port_for_bind(occupied, IpAddr::V4(Ipv4Addr::LOCALHOST)) - .expect("choose service port"); + let error = choose_service_port_for_bind(occupied, IpAddr::V4(Ipv4Addr::LOCALHOST)) + .expect_err("occupied service port should fail"); - assert!(selected > occupied); - assert!(selected <= occupied.saturating_add(10)); - } - - #[test] - fn default_service_port_search_matches_ios_discovery_window() { - assert_eq!(DEFAULT_SERVICE_DISCOVERY_MAX_PORT, 4320); + assert!(error.to_string().contains("already in use")); } #[test] diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index b0c3f6cb..78674b10 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -943,6 +943,9 @@ async fn run_android_webrtc_control_message( )), }, ControlMessage::DismissKeyboard => state.android.dismiss_keyboard(&udid), + ControlMessage::ToggleSoftwareKeyboard => Err(AppError::bad_request( + "Software keyboard toggle is only available for iOS simulators.", + )), ControlMessage::Home => state.android.press_home(&udid), ControlMessage::AppSwitcher => state.android.open_app_switcher(&udid), ControlMessage::RotateLeft => state.android.rotate_left(&udid), diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index 7b677138..35073047 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -28,18 +28,19 @@ simdeck -r # restarts the daemon simdeck daemon killall # kills all daemons on the machine, use with care ``` -Usually `http://127.0.0.1:4310` or `http://127.0.0.1:4310?device=`. -Port may increment if multiple daemons are running. +Usually `http://127.0.0.1:4311` or `http://127.0.0.1:4311?device=` for +project daemons. The always-on LaunchAgent service stays on +`http://127.0.0.1:4310`. +Daemon ports may increment upward from 4311 if multiple daemons are running. Use `simdeck pair` when a native iOS client needs to pair. It starts or refreshes the global LaunchAgent-backed service, detects LAN and Tailscale IPv4 addresses, and prints a QR with a `simdeck://pair` URL that carries the pairing code plus alternate server addresses. The LaunchAgent service token is stable across `simdeck pair`, `simdeck service on`, and `simdeck service restart`; use `simdeck service reset` only when you -need to rotate the token and restart the service. -If a workspace daemon is already on 4310, the LaunchAgent service moves to the -next available service-discovery port, up to 4320, and leaves the workspace -daemon running. +need to rotate the token and restart the service. When that service is active, +`simdeck` and `simdeck ui` report the service endpoints instead of starting a +project daemon. Always first run `simdeck ui` to open the URL reported by the `simdeck ui` in the in-app browser using Browser Use tool if available. @@ -155,6 +156,7 @@ simdeck key 42 --duration-ms 500 simdeck key-sequence --keycodes h,e,l,l,o --delay-ms 75 simdeck key-combo --modifiers cmd,shift --key z simdeck dismiss-keyboard +simdeck button software-keyboard simdeck button home simdeck button lock --duration-ms 1000 simdeck button side-button From b3b724816029ac151538c728a5e79ad1a70ec421 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 23 May 2026 01:14:06 -0400 Subject: [PATCH 2/3] docs: format VS Code extension docs --- docs/extensions/vscode.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/extensions/vscode.md b/docs/extensions/vscode.md index 15189c3a..0d8f2f25 100644 --- a/docs/extensions/vscode.md +++ b/docs/extensions/vscode.md @@ -35,13 +35,13 @@ The extension tries the configured server URL first. If it is not reachable and ## Settings -| Setting | Default | Purpose | -| ------------------------- | ----------------------- | ---------------------------- | -| `simdeck.serverUrl` | `http://127.0.0.1:4310` | Preferred daemon URL | -| `simdeck.cliPath` | empty | Explicit path to the CLI | +| Setting | Default | Purpose | +| ------------------------- | ----------------------- | ------------------------------------- | +| `simdeck.serverUrl` | `http://127.0.0.1:4310` | Preferred daemon URL | +| `simdeck.cliPath` | empty | Explicit path to the CLI | | `simdeck.port` | `4311` | Port for auto-started project daemons | -| `simdeck.bindAddress` | `127.0.0.1` | Bind address for auto-start | -| `simdeck.autoStartDaemon` | `true` | Start the daemon when needed | +| `simdeck.bindAddress` | `127.0.0.1` | Bind address for auto-start | +| `simdeck.autoStartDaemon` | `true` | Start the daemon when needed | CLI resolution order: From 6a2d848578b16e841d51506c4616c603a7a4a65b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 23 May 2026 01:20:31 -0400 Subject: [PATCH 3/3] fix: add recording symbols to native stubs --- server/native_stubs.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/native_stubs.c b/server/native_stubs.c index 00589074..43984e24 100644 --- a/server/native_stubs.c +++ b/server/native_stubs.c @@ -174,6 +174,18 @@ xcw_native_owned_bytes xcw_native_screen_recording_mp4( return xcw_empty_bytes(error_message); } +char *xcw_native_start_screen_recording(const char *udid, char **error_message) { + (void)udid; + xcw_unsupported(error_message); + return NULL; +} + +xcw_native_owned_bytes xcw_native_stop_screen_recording( + const char *recording_id, char **error_message) { + (void)recording_id; + return xcw_empty_bytes(error_message); +} + char *xcw_native_recent_logs(const char *udid, double seconds, uintptr_t limit, char **error_message) { (void)udid;