From 8df3bc46b9ccad2eb799d771d46bd03618135ba2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 12 May 2026 00:10:11 -0400 Subject: [PATCH 1/4] Add Apple Watch chrome and HID support --- AGENTS.md | 2 +- README.md | 2 + cli/DFPrivateSimulatorDisplayBridge.m | 3 + cli/XCWChromeRenderer.m | 87 ++++++++++++++----- client/src/features/viewport/DeviceChrome.tsx | 18 ++-- client/src/styles/components.css | 6 +- docs/api/rest.md | 4 +- docs/cli/commands.md | 2 + skills/simdeck/SKILL.md | 2 + 9 files changed, 92 insertions(+), 34 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cef353a6..6483b6d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,7 +78,7 @@ Private simulator behavior is implemented locally in: - Accessibility bridge: `cli/XCWAccessibilityBridge.*` The current repo uses the private boot path, private display bridge, and private accessibility translation bridge directly. The browser streams frames from that bridge, injects touch and keyboard events through the same native session layer, inspects accessibility through `AccessibilityPlatformTranslation`, and renders device chrome from `cli/XCWChromeRenderer.*`. -Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, and mute buttons dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. +Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, mute, Apple Watch digital crown, Watch side button, and Watch left-side button dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony/vendor HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. WebKit inspection uses the simulator `webinspectord` Unix socket named `com.apple.webinspectord_sim.socket` and WebKit's binary-plist Remote Inspector selectors. It lists only WebKit content that the runtime exposes as inspectable. For app-owned `WKWebView` on iOS 16.4 and newer, the app must set `isInspectable = true`. ## Build and Run diff --git a/README.md b/README.md index 2093f80f..7b5ede50 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,8 @@ simdeck type --file message.txt simdeck button lock --duration-ms 1000 simdeck button volume-up simdeck button action --duration-ms 1000 +simdeck button digital-crown +simdeck button left-side-button simdeck batch --step "tap --label Continue" --step "type 'hello'" --step "wait-for --label hello" simdeck dismiss-keyboard simdeck home diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 1c90a467..447575a9 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -3415,6 +3415,9 @@ - (BOOL)sendHardwareButtonNamed:(NSString *)buttonName @"volume-down": @[ @(DFConsumerControlUsagePage), @234 ], @"action": @[ @(0x0b), @45 ], @"mute": @[ @(0x0b), @46 ], + @"digital-crown": @[ @(DFConsumerControlUsagePage), @64 ], + @"side-button": @[ @(DFConsumerControlUsagePage), @149 ], + @"left-side-button": @[ @(0xff01), @512 ], }; }); diff --git a/cli/XCWChromeRenderer.m b/cli/XCWChromeRenderer.m index 0926c2a8..dea918ee 100644 --- a/cli/XCWChromeRenderer.m +++ b/cli/XCWChromeRenderer.m @@ -12,6 +12,7 @@ @interface XCWChromeRenderer () + (nullable NSDictionary *)inputNamed:(NSString *)buttonName chromeInfo:(NSDictionary *)chromeInfo error:(NSError * _Nullable __autoreleasing *)error; ++ (NSEdgeInsets)devicePaddingForChromeInfo:(NSDictionary *)chromeInfo; @end @implementation XCWChromeRenderer @@ -665,6 +666,14 @@ + (BOOL)drawInputImagesForChromeInfo:(NSDictionary *)chromeInfo + (CGRect)fullFrameForChromeInfo:(NSDictionary *)chromeInfo chromeSize:(CGSize)chromeSize { + NSEdgeInsets padding = [self devicePaddingForChromeInfo:chromeInfo]; + if (padding.top != 0.0 || padding.left != 0.0 || padding.bottom != 0.0 || padding.right != 0.0) { + return CGRectMake(-padding.left, + -padding.top, + chromeSize.width + padding.left + padding.right, + chromeSize.height + padding.top + padding.bottom); + } + CGRect bounds = CGRectMake(0.0, 0.0, chromeSize.width, chromeSize.height); NSDictionary *json = chromeInfo[@"json"]; NSString *chromePath = chromeInfo[@"chromePath"]; @@ -712,55 +721,85 @@ + (CGRect)inputFrameForInput:(NSDictionary *)input inSize:(CGSize)size offsetName:(NSString *)offsetName { NSDictionary *offsets = [input[@"offsets"] isKindOfClass:[NSDictionary class]] ? input[@"offsets"] : @{}; - NSDictionary *requestedOffset = [offsets[offsetName] isKindOfClass:[NSDictionary class]] ? offsets[offsetName] : nil; NSDictionary *normalOffset = [offsets[@"normal"] isKindOfClass:[NSDictionary class]] ? offsets[@"normal"] : nil; NSDictionary *rolloverOffset = [offsets[@"rollover"] isKindOfClass:[NSDictionary class]] ? offsets[@"rollover"] : nil; - NSDictionary *offset = requestedOffset ?: rolloverOffset ?: normalOffset ?: @{}; - CGFloat offsetX = [self numberValue:offset[@"x"]]; - CGFloat offsetY = [self numberValue:offset[@"y"]]; + NSDictionary *requestedOffset = [offsets[offsetName] isKindOfClass:[NSDictionary class]] ? offsets[offsetName] : nil; + NSDictionary *primaryOffset = normalOffset ?: rolloverOffset ?: requestedOffset ?: @{}; + NSDictionary *secondaryOffset = rolloverOffset ?: normalOffset ?: requestedOffset ?: @{}; + CGFloat normalX = [self numberValue:primaryOffset[@"x"]]; + CGFloat normalY = [self numberValue:primaryOffset[@"y"]]; + CGFloat rolloverX = [self numberValue:secondaryOffset[@"x"]]; + CGFloat rolloverY = [self numberValue:secondaryOffset[@"y"]]; + CGFloat restX = normalX; + CGFloat restY = normalY; NSString *anchor = [input[@"anchor"] isKindOfClass:[NSString class]] ? input[@"anchor"] : @""; NSString *align = [input[@"align"] isKindOfClass:[NSString class]] ? input[@"align"] : @""; - CGFloat x = offsetX - (assetSize.width / 2.0); - CGFloat y = offsetY; + if ([offsetName isEqualToString:@"rollover"]) { + restX = rolloverX; + restY = rolloverY; + } else if ([anchor isEqualToString:@"left"]) { + restX = rolloverX; + restY = rolloverY; + } else if ([anchor isEqualToString:@"right"] || + [anchor isEqualToString:@"top"] || + [anchor isEqualToString:@"bottom"]) { + restX = (2.0 * normalX) - rolloverX; + restY = (2.0 * normalY) - rolloverY; + } + + CGFloat x = restX - (assetSize.width / 2.0); + CGFloat y = restY; if ([anchor isEqualToString:@"left"]) { - x = offsetX - (assetSize.width / 2.0); + x = restX - (assetSize.width / 2.0); } else if ([anchor isEqualToString:@"right"]) { - x = size.width + offsetX - (assetSize.width / 2.0); + x = size.width + restX; + y = restY; } else if ([anchor isEqualToString:@"top"]) { - y = offsetY; + if ([align isEqualToString:@"trailing"]) { + x = size.width + restX - assetSize.width; + } else { + x = restX; + } + y = restY - assetSize.height; } else if ([anchor isEqualToString:@"bottom"]) { - y = size.height + offsetY; + if ([align isEqualToString:@"trailing"]) { + x = size.width + restX - assetSize.width; + } else { + x = restX; + } + y = size.height + restY; } if ([anchor isEqualToString:@"left"] || [anchor isEqualToString:@"right"]) { if ([align isEqualToString:@"center"]) { - y = (size.height - assetSize.height) / 2.0 + offsetY; + y = (size.height - assetSize.height) / 2.0 + restY; } else if ([align isEqualToString:@"trailing"]) { - y = size.height - assetSize.height + offsetY; + y = size.height - assetSize.height + restY; } } else if ([anchor isEqualToString:@"top"] || [anchor isEqualToString:@"bottom"]) { - CGFloat baseX = 0.0; if ([align isEqualToString:@"center"]) { - baseX = size.width / 2.0; - } else if ([align isEqualToString:@"trailing"]) { - baseX = size.width; - } - x = baseX + offsetX - (assetSize.width / 2.0); - if ([align isEqualToString:@"center"]) { - x = (size.width / 2.0) + offsetX - (assetSize.width / 2.0); - } else if ([align isEqualToString:@"trailing"]) { - x = size.width + offsetX - (assetSize.width / 2.0); + x = (size.width / 2.0) + restX - (assetSize.width / 2.0); } } else if ([align isEqualToString:@"center"]) { - x = (size.width - assetSize.width) / 2.0 + offsetX; + x = (size.width - assetSize.width) / 2.0 + restX; } else if ([align isEqualToString:@"trailing"]) { - x = size.width - assetSize.width + offsetX; + x = size.width - assetSize.width + restX; } return CGRectMake(x, y, assetSize.width, assetSize.height); } ++ (NSEdgeInsets)devicePaddingForChromeInfo:(NSDictionary *)chromeInfo { + NSDictionary *json = chromeInfo[@"json"]; + NSDictionary *images = [json[@"images"] isKindOfClass:[NSDictionary class]] ? json[@"images"] : @{}; + NSDictionary *padding = [images[@"devicePadding"] isKindOfClass:[NSDictionary class]] ? images[@"devicePadding"] : @{}; + return NSEdgeInsetsMake([self numberValue:padding[@"top"]], + [self numberValue:padding[@"left"]], + [self numberValue:padding[@"bottom"]], + [self numberValue:padding[@"right"]]); +} + + (NSArray *> *)buttonProfilesForChromeInfo:(NSDictionary *)chromeInfo chromeSize:(CGSize)chromeSize chromeOffset:(CGPoint)chromeOffset { diff --git a/client/src/features/viewport/DeviceChrome.tsx b/client/src/features/viewport/DeviceChrome.tsx index 41f96325..09fb80ed 100644 --- a/client/src/features/viewport/DeviceChrome.tsx +++ b/client/src/features/viewport/DeviceChrome.tsx @@ -201,11 +201,13 @@ export function DeviceChrome({ const CHROME_BUTTON_WIRE_NAMES: Record = { action: "action", + "digital-crown": "digital-crown", home: "home", + "left-side-button": "left-side-button", lock: "power", mute: "mute", power: "power", - "side-button": "power", + "side-button": "side-button", "volume-down": "volume-down", "volume-up": "volume-up", }; @@ -303,16 +305,20 @@ function ChromeButtonHitTarget({ left: `${(button.x / totalWidth) * 100}%`, top: `${(button.y / totalHeight) * 100}%`, width: `${(button.width / totalWidth) * 100}%`, - "--button-rest-x": `${(rolloverDelta.x / Math.max(button.width, 1)) * 100}%`, - "--button-rest-y": `${(rolloverDelta.y / Math.max(button.height, 1)) * 100}%`, - "--button-hover-x": `${((rolloverDelta.x * 2) / Math.max(button.width, 1)) * 100}%`, - "--button-hover-y": `${((rolloverDelta.y * 2) / Math.max(button.height, 1)) * 100}%`, + "--button-rest-x": "0%", + "--button-rest-y": "0%", + "--button-hover-x": `${(rolloverDelta.x / Math.max(button.width, 1)) * 100}%`, + "--button-hover-y": `${(rolloverDelta.y / Math.max(button.height, 1)) * 100}%`, + "--button-pressed-x": `${((-rolloverDelta.x) / Math.max(button.width, 1)) * 100}%`, + "--button-pressed-y": `${((-rolloverDelta.y) / Math.max(button.height, 1)) * 100}%`, } as CSSProperties & Record< | "--button-rest-x" | "--button-rest-y" | "--button-hover-x" - | "--button-hover-y", + | "--button-hover-y" + | "--button-pressed-x" + | "--button-pressed-y", string >; diff --git a/client/src/styles/components.css b/client/src/styles/components.css index 8f073478..16e8ecae 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -1575,7 +1575,11 @@ .device-chrome-button:active, .device-chrome-button.is-pressed { - transform: translate3d(var(--button-rest-x, 0), var(--button-rest-y, 0), 0); + transform: translate3d( + var(--button-pressed-x, var(--button-rest-x, 0)), + var(--button-pressed-y, var(--button-rest-y, 0)), + 0 + ); } .pan-enabled .device-bezel, diff --git a/docs/api/rest.md b/docs/api/rest.md index 30dd39b8..2470c132 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -382,8 +382,8 @@ Content-Type: application/json Supported button names match the CLI and chrome controls: `home`, `lock`, `power`, `side-button`, `volume-up`, `volume-down`, `action`, `mute`, -`app-switcher`, `siri`, and `apple-pay`. `durationMs` defaults to `0` and is -used for press-and-hold interactions. +`digital-crown`, `left-side-button`, `app-switcher`, `siri`, and `apple-pay`. +`durationMs` defaults to `0` and is used for press-and-hold interactions. For live chrome interactions, send explicit button edges instead of a completed press: diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 101aac04..77a4d7ee 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -238,6 +238,8 @@ simdeck type --file message.txt simdeck button lock --duration-ms 1000 simdeck button volume-up simdeck button action --duration-ms 1000 +simdeck button digital-crown +simdeck button left-side-button simdeck dismiss-keyboard simdeck home simdeck app-switcher diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index cb6bfedd..b042a6df 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -146,6 +146,8 @@ simdeck button volume-up simdeck button volume-down simdeck button action --duration-ms 1000 simdeck button mute +simdeck button digital-crown +simdeck button left-side-button simdeck button siri simdeck button apple-pay simdeck home From bf632997326d41b6cbd59221087c463d85ee4e44 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 12 May 2026 11:28:59 -0400 Subject: [PATCH 2/4] Improve Apple Watch chrome input --- AGENTS.md | 2 +- README.md | 1 + cli/DFPrivateSimulatorDisplayBridge.h | 2 + cli/DFPrivateSimulatorDisplayBridge.m | 75 +++++++++++++++++++ cli/XCWPrivateSimulatorSession.h | 2 + cli/XCWPrivateSimulatorSession.m | 5 ++ cli/native/XCWNativeBridge.h | 2 + cli/native/XCWNativeBridge.m | 27 +++++++ cli/native/XCWNativeSession.h | 2 + cli/native/XCWNativeSession.m | 5 ++ client/src/api/controls.ts | 7 ++ client/src/api/types.ts | 4 + client/src/app/AppShell.tsx | 34 ++++++++- client/src/features/viewport/DeviceChrome.tsx | 4 +- client/src/styles/components.css | 2 +- client/src/styles/tokens.css | 6 +- docs/api/rest.md | 14 ++++ docs/cli/commands.md | 1 + server/src/api/routes.rs | 34 +++++++++ server/src/main.rs | 38 ++++++++++ server/src/native/bridge.rs | 27 +++++++ server/src/native/ffi.rs | 10 +++ server/src/simulators/session.rs | 4 + skills/simdeck/SKILL.md | 1 + 24 files changed, 302 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6483b6d2..bf2c3289 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,7 +78,7 @@ Private simulator behavior is implemented locally in: - Accessibility bridge: `cli/XCWAccessibilityBridge.*` The current repo uses the private boot path, private display bridge, and private accessibility translation bridge directly. The browser streams frames from that bridge, injects touch and keyboard events through the same native session layer, inspects accessibility through `AccessibilityPlatformTranslation`, and renders device chrome from `cli/XCWChromeRenderer.*`. -Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, mute, Apple Watch digital crown, Watch side button, and Watch left-side button dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony/vendor HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. +Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, mute, Apple Watch digital crown, Watch side button, and Watch left-side button dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony/vendor HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. Apple Watch Digital Crown rotation dispatches through `IndigoHIDMessageForScrollEvent` with the same digitizer target as touch input. WebKit inspection uses the simulator `webinspectord` Unix socket named `com.apple.webinspectord_sim.socket` and WebKit's binary-plist Remote Inspector selectors. It lists only WebKit content that the runtime exposes as inspectable. For app-owned `WKWebView` on iOS 16.4 and newer, the app must set `isInspectable = true`. ## Build and Run diff --git a/README.md b/README.md index 7b5ede50..bafd891b 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ simdeck button lock --duration-ms 1000 simdeck button volume-up simdeck button action --duration-ms 1000 simdeck button digital-crown +simdeck crown --delta 50 simdeck button left-side-button simdeck batch --step "tap --label Continue" --step "type 'hello'" --step "wait-for --label hello" simdeck dismiss-keyboard diff --git a/cli/DFPrivateSimulatorDisplayBridge.h b/cli/DFPrivateSimulatorDisplayBridge.h index 5c99a66c..9ef01862 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.h +++ b/cli/DFPrivateSimulatorDisplayBridge.h @@ -85,6 +85,8 @@ NS_SWIFT_NAME(PrivateSimulatorDisplayBridge) usagePage:(nullable NSNumber *)usagePage usage:(nullable NSNumber *)usage error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(sendHardwareButton(named:pressed:usagePage:usage:)); +- (BOOL)rotateDigitalCrownByDelta:(double)delta + error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(rotateDigitalCrown(delta:)); - (BOOL)rotateRight:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(rotateRight()); - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(rotateLeft()); diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 447575a9..ce1b0a44 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -38,6 +38,7 @@ typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardNSEventFn)(NSEvent *event); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForButtonFn)(uint32_t buttonCode, uint32_t operation, uint32_t target); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForHIDArbitraryFn)(uint32_t target, uint32_t page, uint32_t usage, uint32_t operation); +typedef IndigoHIDMessage *(*DFIndigoHIDMessageForScrollEventFn)(uint32_t target, double deltaX, double deltaY, double momentumPhase); typedef IndigoHIDMessage *(*DFIndigoHIDServiceMessageFn)(void); #pragma pack(push, 4) @@ -1431,6 +1432,29 @@ static void DFWarmIndigoHIDServices(id hidClient) { return message; } +static IndigoHIDMessage *DFCreateScrollHIDMessage(uint32_t target, double deltaX, double deltaY, NSError **error) { + DFIndigoHIDMessageForScrollEventFn scrollMessage = (DFIndigoHIDMessageForScrollEventFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForScrollEvent"); + if (scrollMessage == NULL) { + if (error != NULL) { + *error = DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + @"SimulatorKit did not expose IndigoHIDMessageForScrollEvent." + ); + } + return NULL; + } + + IndigoHIDMessage *message = scrollMessage(target, deltaX, deltaY, 0); + if (message == NULL && error != NULL) { + *error = DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + [NSString stringWithFormat:@"SimulatorKit could not construct scroll HID for delta %.3f.", deltaY] + ); + } + + return message; +} + static BOOL DFCallSwiftUnitAngleMeasurementGetterByFunction(id selfObject, void *function, DFUnitAngleMeasurement *measurement) { if (selfObject == nil || function == NULL || measurement == NULL) { return NO; @@ -3497,6 +3521,57 @@ - (BOOL)sendHardwareButtonNamed:(NSString *)buttonName return success; } +- (BOOL)rotateDigitalCrownByDelta:(double)delta + error:(NSError * _Nullable __autoreleasing *)error { + if (!isfinite(delta)) { + if (error != NULL) { + *error = DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + @"Digital Crown delta must be finite." + ); + } + return NO; + } + + __block BOOL success = NO; + __block NSError *dispatchError = nil; + + dispatch_block_t work = ^{ + if (self->_hidClient == nil) { + dispatchError = DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + @"SimulatorKit did not provide a headless HID client for Digital Crown rotation." + ); + return; + } + + NSError *messageError = nil; + IndigoHIDMessage *message = DFCreateScrollHIDMessage(DFIndigoTouchTarget, 0, delta, &messageError); + if (message == NULL || !DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) { + dispatchError = messageError; + return; + } + + DFLog(@"Sending Digital Crown rotation delta=%.3f", delta); + success = YES; + }; + + if (dispatch_get_specific(DFPrivateSimulatorCallbackQueueKey) != NULL) { + work(); + } else { + dispatch_sync(_callbackQueue, work); + } + + if (!success && error != NULL) { + *error = dispatchError ?: DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + @"SimulatorKit rejected Digital Crown rotation." + ); + } + + return success; +} + - (BOOL)rotateRight:(NSError * _Nullable __autoreleasing *)error { return [self rotateByDegrees:90.0 error:error]; } diff --git a/cli/XCWPrivateSimulatorSession.h b/cli/XCWPrivateSimulatorSession.h index a12eab63..346531c6 100644 --- a/cli/XCWPrivateSimulatorSession.h +++ b/cli/XCWPrivateSimulatorSession.h @@ -66,6 +66,8 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData, usagePage:(nullable NSNumber *)usagePage usage:(nullable NSNumber *)usage error:(NSError * _Nullable * _Nullable)error; +- (BOOL)rotateDigitalCrownByDelta:(double)delta + error:(NSError * _Nullable * _Nullable)error; - (BOOL)openAppSwitcher:(NSError * _Nullable * _Nullable)error; - (BOOL)rotateRight:(NSError * _Nullable * _Nullable)error; - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error; diff --git a/cli/XCWPrivateSimulatorSession.m b/cli/XCWPrivateSimulatorSession.m index e5bd5136..fe177c1a 100644 --- a/cli/XCWPrivateSimulatorSession.m +++ b/cli/XCWPrivateSimulatorSession.m @@ -361,6 +361,11 @@ - (BOOL)sendHardwareButtonNamed:(NSString *)buttonName error:error]; } +- (BOOL)rotateDigitalCrownByDelta:(double)delta + error:(NSError * _Nullable __autoreleasing *)error { + return [_displayBridge rotateDigitalCrownByDelta:delta error:error]; +} + - (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error { return [_displayBridge openAppSwitcher:error]; } diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index ff4ac7f1..526dbd38 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -53,6 +53,7 @@ bool xcw_native_press_home(const char * _Nonnull udid, char * _Nullable * _Nulla bool xcw_native_open_app_switcher(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); bool xcw_native_press_button(const char * _Nonnull udid, const char * _Nonnull button_name, uint32_t duration_ms, char * _Nullable * _Nullable error_message); bool xcw_native_send_button(const char * _Nonnull udid, const char * _Nonnull button_name, bool pressed, bool has_usage, uint32_t usage_page, uint32_t usage, char * _Nullable * _Nullable error_message); +bool xcw_native_rotate_crown(const char * _Nonnull udid, double delta, char * _Nullable * _Nullable error_message); bool xcw_native_rotate_right(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); bool xcw_native_rotate_left(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); bool xcw_native_erase_simulator(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); @@ -84,6 +85,7 @@ bool xcw_native_session_send_key(void * _Nonnull handle, uint16_t key_code, uint bool xcw_native_session_press_home(void * _Nonnull handle, char * _Nullable * _Nullable error_message); bool xcw_native_session_press_button(void * _Nonnull handle, const char * _Nonnull button_name, uint32_t duration_ms, char * _Nullable * _Nullable error_message); bool xcw_native_session_send_button(void * _Nonnull handle, const char * _Nonnull button_name, bool pressed, bool has_usage, uint32_t usage_page, uint32_t usage, char * _Nullable * _Nullable error_message); +bool xcw_native_session_rotate_crown(void * _Nonnull handle, double delta, char * _Nullable * _Nullable error_message); bool xcw_native_session_open_app_switcher(void * _Nonnull handle, char * _Nullable * _Nullable error_message); bool xcw_native_session_rotate_right(void * _Nonnull handle, char * _Nullable * _Nullable error_message); bool xcw_native_session_rotate_left(void * _Nonnull handle, char * _Nullable * _Nullable error_message); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index e93241e7..d36630d7 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -761,6 +761,22 @@ bool xcw_native_send_button(const char *udid, const char *button_name, bool pres } } +bool xcw_native_rotate_crown(const char *udid, double delta, char **error_message) { + @autoreleasepool { + DFPrivateSimulatorDisplayBridge *bridge = XCWInputBridgeForUDID(udid, error_message); + if (bridge == nil) { + return false; + } + NSError *error = nil; + BOOL ok = [bridge rotateDigitalCrownByDelta:delta error:&error]; + [bridge disconnect]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + bool xcw_native_rotate_right(const char *udid, char **error_message) { @autoreleasepool { DFPrivateSimulatorDisplayBridge *bridge = XCWInputBridgeForUDID(udid, error_message); @@ -1032,6 +1048,17 @@ bool xcw_native_session_send_button(void *handle, const char *button_name, bool } } +bool xcw_native_session_rotate_crown(void *handle, double delta, char **error_message) { + @autoreleasepool { + NSError *error = nil; + BOOL ok = [XCWNativeSessionFromHandle(handle) rotateDigitalCrownByDelta:delta error:&error]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + bool xcw_native_session_open_app_switcher(void *handle, char **error_message) { @autoreleasepool { NSError *error = nil; diff --git a/cli/native/XCWNativeSession.h b/cli/native/XCWNativeSession.h index 0018af53..76550fd2 100644 --- a/cli/native/XCWNativeSession.h +++ b/cli/native/XCWNativeSession.h @@ -44,6 +44,8 @@ NS_ASSUME_NONNULL_BEGIN usagePage:(nullable NSNumber *)usagePage usage:(nullable NSNumber *)usage error:(NSError * _Nullable * _Nullable)error; +- (BOOL)rotateDigitalCrownByDelta:(double)delta + error:(NSError * _Nullable * _Nullable)error; - (BOOL)openAppSwitcher:(NSError * _Nullable * _Nullable)error; - (BOOL)rotateRight:(NSError * _Nullable * _Nullable)error; - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error; diff --git a/cli/native/XCWNativeSession.m b/cli/native/XCWNativeSession.m index 9140729b..f99ff5e4 100644 --- a/cli/native/XCWNativeSession.m +++ b/cli/native/XCWNativeSession.m @@ -171,6 +171,11 @@ - (BOOL)sendHardwareButtonNamed:(NSString *)buttonName error:error]; } +- (BOOL)rotateDigitalCrownByDelta:(double)delta + error:(NSError * _Nullable __autoreleasing *)error { + return [self.session rotateDigitalCrownByDelta:delta error:error]; +} + - (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error { return [self.session openAppSwitcher:error]; } diff --git a/client/src/api/controls.ts b/client/src/api/controls.ts index 1abc0c1f..7affba7b 100644 --- a/client/src/api/controls.ts +++ b/client/src/api/controls.ts @@ -2,6 +2,7 @@ import { accessTokenFromLocation, apiRequest } from "./client"; import { apiUrl } from "./config"; import type { ButtonPayload, + CrownPayload, EdgeTouchPayload, KeyPayload, LaunchPayload, @@ -18,6 +19,7 @@ export type ControlMessage = | ({ type: "multiTouch" } & MultiTouchPayload) | ({ type: "key" } & KeyPayload) | ({ type: "button" } & ButtonPayload) + | ({ type: "crown" } & CrownPayload) | { type: "dismissKeyboard" } | { type: "home" } | { type: "appSwitcher" } @@ -30,6 +32,7 @@ async function postSimulatorAction( action: string, payload?: | ButtonPayload + | CrownPayload | KeyPayload | LaunchPayload | OpenUrlPayload @@ -99,6 +102,10 @@ export function pressSimulatorButton(udid: string, payload: ButtonPayload) { return postSimulatorAction(udid, "button", payload); } +export function rotateDigitalCrown(udid: string, payload: CrownPayload) { + return postSimulatorAction(udid, "crown", payload); +} + export function openAppSwitcher(udid: string) { return postSimulatorAction(udid, "app-switcher"); } diff --git a/client/src/api/types.ts b/client/src/api/types.ts index fe0533ce..05684a78 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -314,6 +314,10 @@ export interface ButtonPayload { usage?: number; } +export interface CrownPayload { + delta: number; +} + export interface LaunchPayload { bundleId: string; } diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 78b29066..ddbdbc89 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -23,6 +23,7 @@ import { openSimulatorUrl, pressHome, pressSimulatorButton, + rotateDigitalCrown, rotateRight, simulatorControlSocketUrl, shutdownSimulator, @@ -750,11 +751,18 @@ export function AppShell({ const chromeHasInteractiveButtons = Boolean( viewportChromeProfile?.buttons?.length, ); + const chromeHasCrown = Boolean( + viewportChromeProfile?.buttons?.some( + (button) => + button.type?.toLowerCase() === "crown" || + button.name.toLowerCase() === "digital-crown", + ), + ); const chromeUrl = selectedSimulator ? buildChromeUrl( selectedSimulator.udid, streamStamp, - !chromeHasInteractiveButtons, + !chromeHasInteractiveButtons || chromeHasCrown, ) : ""; const chromeButtonUrl = useCallback( @@ -1664,6 +1672,11 @@ export function AppShell({ return; } + if (chromeHasCrown && selectedSimulator.isBooted) { + sendCrownRotation(deltaY); + return; + } + setViewMode("manual"); setPan((currentPan) => clampPan( @@ -1836,6 +1849,25 @@ export function AppShell({ } } + function sendCrownRotation(delta: number) { + if (!selectedSimulator || !Number.isFinite(delta) || delta === 0) { + return; + } + setAccessibilitySelectedId(""); + setAccessibilityHoveredId(null); + if ( + !sendControl(selectedSimulator.udid, { + type: "crown", + delta, + }) + ) { + void runAction( + () => rotateDigitalCrown(selectedSimulator.udid, { delta }), + false, + ); + } + } + function prepareSimulatorInput() { setMenuOpen(false); setAccessibilitySelectedId(""); diff --git a/client/src/features/viewport/DeviceChrome.tsx b/client/src/features/viewport/DeviceChrome.tsx index 09fb80ed..9f9c88a6 100644 --- a/client/src/features/viewport/DeviceChrome.tsx +++ b/client/src/features/viewport/DeviceChrome.tsx @@ -309,8 +309,8 @@ function ChromeButtonHitTarget({ "--button-rest-y": "0%", "--button-hover-x": `${(rolloverDelta.x / Math.max(button.width, 1)) * 100}%`, "--button-hover-y": `${(rolloverDelta.y / Math.max(button.height, 1)) * 100}%`, - "--button-pressed-x": `${((-rolloverDelta.x) / Math.max(button.width, 1)) * 100}%`, - "--button-pressed-y": `${((-rolloverDelta.y) / Math.max(button.height, 1)) * 100}%`, + "--button-pressed-x": `${(-rolloverDelta.x / Math.max(button.width, 1)) * 100}%`, + "--button-pressed-y": `${(-rolloverDelta.y / Math.max(button.height, 1)) * 100}%`, } as CSSProperties & Record< | "--button-rest-x" diff --git a/client/src/styles/components.css b/client/src/styles/components.css index 16e8ecae..70bb6b13 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -142,7 +142,7 @@ padding: 6px; border: 1px solid var(--border); border-radius: 10px; - background: color-mix(in srgb, var(--surface) 94%, transparent); + background: var(--zoom-toolbar-bg); box-shadow: 0 12px 24px rgba(0, 0, 0, 0.2); backdrop-filter: blur(10px); } diff --git a/client/src/styles/tokens.css b/client/src/styles/tokens.css index 9b1e7c19..c8fd552a 100644 --- a/client/src/styles/tokens.css +++ b/client/src/styles/tokens.css @@ -1,9 +1,10 @@ :root { color-scheme: light dark; - --bg: #1e1e1e; + --bg: #181818; --surface: #252526; --surface-hover: #2d2d2e; - --toolbar-bg: var(--surface); + --toolbar-bg: #181818; + --zoom-toolbar-bg: #2d2d2d; --toolbar-shadow: none; --border: #3c3c3c; --border-subtle: #333333; @@ -28,6 +29,7 @@ --surface: #ffffff; --surface-hover: #f5f5f5; --toolbar-bg: #ffffff; + --zoom-toolbar-bg: #ffffff; --toolbar-shadow: 0 1px 6px rgba(0, 0, 0, 0.08); --border: #e3e3e3; --border-subtle: #eeeeee; diff --git a/docs/api/rest.md b/docs/api/rest.md index 2470c132..c5c0a428 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -396,6 +396,20 @@ press: may also pass `usagePage` and `usage` from the device profile when an exact HID usage is available. +### `POST /api/simulators/{udid}/crown` + +Rotates the Apple Watch Digital Crown using scroll delta semantics: + +```http +POST /api/simulators/{udid}/crown +Content-Type: application/json + +{ "delta": 50 } +``` + +The browser UI sends this automatically when scrolling over a Watch screen or +crown chrome. + ### `POST /api/simulators/{udid}/home` Presses the home button: diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 77a4d7ee..1175911e 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -239,6 +239,7 @@ simdeck button lock --duration-ms 1000 simdeck button volume-up simdeck button action --duration-ms 1000 simdeck button digital-crown +simdeck crown --delta 50 simdeck button left-side-button simdeck dismiss-keyboard simdeck home diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 9d767552..62c3ceca 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -301,6 +301,9 @@ pub(crate) enum ControlMessage { usage_page: Option, usage: Option, }, + Crown { + delta: f64, + }, DismissKeyboard, Home, AppSwitcher, @@ -328,6 +331,11 @@ struct ButtonPayload { usage: Option, } +#[derive(Deserialize)] +struct CrownPayload { + delta: f64, +} + #[derive(Deserialize)] struct ChromePngQuery { buttons: Option, @@ -449,6 +457,9 @@ enum BatchStep { button: String, duration_ms: Option, }, + Crown { + delta: f64, + }, Launch { bundle_id: String, }, @@ -602,6 +613,7 @@ pub fn router(state: AppState) -> Router { post(dismiss_keyboard), ) .route("/api/simulators/{udid}/button", post(press_button)) + .route("/api/simulators/{udid}/crown", post(rotate_crown)) .route("/api/simulators/{udid}/home", post(press_home)) .route( "/api/simulators/{udid}/app-switcher", @@ -2482,6 +2494,7 @@ pub(crate) async fn run_control_message( session.press_button(&button, duration_ms.unwrap_or(0)) } } + ControlMessage::Crown { delta } => session.rotate_crown(delta), ControlMessage::DismissKeyboard => session.send_key(41, 0), ControlMessage::Home => session.press_home(), ControlMessage::AppSwitcher => session.open_app_switcher(), @@ -2625,6 +2638,23 @@ async fn press_button( Ok(json(json_value!({ "ok": true }))) } +async fn rotate_crown( + State(state): State, + Path(udid): Path, + Json(payload): Json, +) -> Result, AppError> { + if !payload.delta.is_finite() { + return Err(AppError::bad_request( + "Request body must include finite `delta`.", + )); + } + run_bridge_action(state, move |bridge| { + bridge.rotate_crown(&udid, payload.delta) + }) + .await?; + Ok(json(json_value!({ "ok": true }))) +} + async fn press_home( State(state): State, Path(udid): Path, @@ -3429,6 +3459,10 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul .await?; Ok(json_value!({ "action": "button" })) } + BatchStep::Crown { delta } => { + run_bridge_action(state, move |bridge| bridge.rotate_crown(&udid, delta)).await?; + Ok(json_value!({ "action": "crown" })) + } BatchStep::Launch { bundle_id } => { if android::is_android_id(&udid) { run_android_action(state, move |android| { diff --git a/server/src/main.rs b/server/src/main.rs index 523c23fc..9f50b3ff 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -355,6 +355,11 @@ enum Command { #[arg(long, default_value_t = 0)] duration_ms: u32, }, + Crown { + udid: String, + #[arg(long, default_value_t = 50.0)] + delta: f64, + }, Batch { udid: String, #[arg(long = "step")] @@ -1305,6 +1310,7 @@ fn is_known_command(value: &str) -> bool { | "key-combo" | "type" | "button" + | "crown" | "batch" | "dismiss-keyboard" | "home" @@ -2715,6 +2721,17 @@ fn main() -> anyhow::Result<()> { )?; Ok(()) } + Command::Crown { udid, delta } => { + if let Some(server_url) = service_url.as_deref() { + service_crown(server_url, &udid, delta)?; + } else { + bridge.rotate_crown(&udid, delta)?; + } + println_json( + &serde_json::json!({ "ok": true, "udid": udid, "action": "crown", "delta": delta }), + )?; + Ok(()) + } Command::Batch { udid, steps, @@ -3471,6 +3488,15 @@ fn service_button( ) } +fn service_crown(server_url: &str, udid: &str, delta: f64) -> anyhow::Result<()> { + service_post_ok( + server_url, + udid, + "crown", + &serde_json::json!({ "delta": delta }), + ) +} + fn service_post_ok(server_url: &str, udid: &str, action: &str, body: &Value) -> anyhow::Result<()> { let path = format!("/api/simulators/{}/{}", url_path_component(udid), action); let deadline = Instant::now() + Duration::from_secs(45); @@ -4593,6 +4619,10 @@ fn batch_line_to_json_step(line: &str) -> anyhow::Result { "button": tokens.get(1).map(String::as_str).unwrap_or(""), "durationMs": args.value("duration-ms").and_then(|value| value.parse::().ok()).unwrap_or(0), }), + "crown" => serde_json::json!({ + "action": "crown", + "delta": args.value("delta").and_then(|value| value.parse::().ok()).unwrap_or(50.0), + }), "home" => serde_json::json!({ "action": "home" }), "dismiss-keyboard" => serde_json::json!({ "action": "dismissKeyboard" }), "app-switcher" => serde_json::json!({ "action": "appSwitcher" }), @@ -4845,6 +4875,14 @@ fn run_batch_step( bridge.press_button(udid, button, 0)?; Ok("button") } + "crown" => { + let delta = tokens + .get(1) + .and_then(|value| value.parse::().ok()) + .unwrap_or(50.0); + bridge.rotate_crown(udid, delta)?; + Ok("crown") + } "key" => { let key = tokens.get(1).ok_or_else(|| { crate::error::AppError::bad_request("key requires a keycode or key name.") diff --git a/server/src/native/bridge.rs b/server/src/native/bridge.rs index 0cf186f1..0de5d2db 100644 --- a/server/src/native/bridge.rs +++ b/server/src/native/bridge.rs @@ -473,6 +473,20 @@ impl NativeBridge { } } + pub fn rotate_crown(&self, udid: &str, delta: f64) -> Result<(), AppError> { + if !delta.is_finite() { + return Err(AppError::bad_request("Digital Crown delta must be finite.")); + } + let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; + unsafe { + let mut error = ptr::null_mut(); + bool_result( + ffi::xcw_native_rotate_crown(udid.as_ptr(), delta, &mut error), + error, + ) + } + } + pub fn rotate_right(&self, udid: &str) -> Result<(), AppError> { let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; unsafe { @@ -899,6 +913,19 @@ impl NativeSession { } } + pub fn rotate_crown(&self, delta: f64) -> Result<(), AppError> { + if !delta.is_finite() { + return Err(AppError::bad_request("Digital Crown delta must be finite.")); + } + unsafe { + let mut error = ptr::null_mut(); + bool_result( + ffi::xcw_native_session_rotate_crown(self.handle, delta, &mut error), + error, + ) + } + } + pub fn open_app_switcher(&self) -> Result<(), AppError> { unsafe { let mut error = ptr::null_mut(); diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs index 9d6b2df1..803b0e26 100644 --- a/server/src/native/ffi.rs +++ b/server/src/native/ffi.rs @@ -124,6 +124,11 @@ unsafe extern "C" { usage: u32, error_message: *mut *mut c_char, ) -> bool; + pub fn xcw_native_rotate_crown( + udid: *const c_char, + delta: f64, + error_message: *mut *mut c_char, + ) -> bool; pub fn xcw_native_rotate_right(udid: *const c_char, error_message: *mut *mut c_char) -> bool; pub fn xcw_native_rotate_left(udid: *const c_char, error_message: *mut *mut c_char) -> bool; pub fn xcw_native_erase_simulator(udid: *const c_char, error_message: *mut *mut c_char) @@ -255,6 +260,11 @@ unsafe extern "C" { usage: u32, error_message: *mut *mut c_char, ) -> bool; + pub fn xcw_native_session_rotate_crown( + handle: *mut c_void, + delta: f64, + error_message: *mut *mut c_char, + ) -> bool; pub fn xcw_native_session_open_app_switcher( handle: *mut c_void, error_message: *mut *mut c_char, diff --git a/server/src/simulators/session.rs b/server/src/simulators/session.rs index dea84b4d..4748fb63 100644 --- a/server/src/simulators/session.rs +++ b/server/src/simulators/session.rs @@ -279,6 +279,10 @@ impl SimulatorSession { .send_button(button, pressed, usage_page, usage) } + pub fn rotate_crown(&self, delta: f64) -> Result<(), AppError> { + self.inner.native.rotate_crown(delta) + } + pub fn open_app_switcher(&self) -> Result<(), AppError> { self.inner.native.open_app_switcher() } diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index b042a6df..877a3b19 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -147,6 +147,7 @@ simdeck button volume-down simdeck button action --duration-ms 1000 simdeck button mute simdeck button digital-crown +simdeck crown --delta 50 simdeck button left-side-button simdeck button siri simdeck button apple-pay From be67599a7c004cb14b8c377f935113b52f7f9dc7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 12 May 2026 11:42:54 -0400 Subject: [PATCH 3/4] Handle crown control in Android paths --- server/native_stubs.c | 14 ++++++++++++++ server/src/api/routes.rs | 3 +++ server/src/transport/webrtc.rs | 3 +++ 3 files changed, 20 insertions(+) diff --git a/server/native_stubs.c b/server/native_stubs.c index b838229a..03b239d8 100644 --- a/server/native_stubs.c +++ b/server/native_stubs.c @@ -212,6 +212,13 @@ bool xcw_native_send_button(const char *udid, const char *button_name, return xcw_unsupported(error_message); } +bool xcw_native_rotate_crown(const char *udid, double delta, + char **error_message) { + (void)udid; + (void)delta; + return xcw_unsupported(error_message); +} + bool xcw_native_rotate_right(const char *udid, char **error_message) { (void)udid; return xcw_unsupported(error_message); @@ -421,6 +428,13 @@ bool xcw_native_session_send_button(void *handle, const char *button_name, return xcw_unsupported(error_message); } +bool xcw_native_session_rotate_crown(void *handle, double delta, + char **error_message) { + (void)handle; + (void)delta; + return xcw_unsupported(error_message); +} + bool xcw_native_session_open_app_switcher(void *handle, char **error_message) { (void)handle; return xcw_unsupported(error_message); diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 62c3ceca..46318972 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -1894,6 +1894,9 @@ async fn run_android_control_message( ControlMessage::AppSwitcher => android.open_app_switcher(&udid), ControlMessage::RotateLeft => android.rotate_left(&udid), ControlMessage::RotateRight => android.rotate_right(&udid), + ControlMessage::Crown { .. } => Err(AppError::bad_request( + "Digital Crown rotation is only available for Apple Watch simulators.", + )), ControlMessage::ToggleAppearance => android.toggle_appearance(&udid), ControlMessage::Touch { .. } | ControlMessage::EdgeTouch { .. } diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 1aac81e6..0858d8c3 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -891,6 +891,9 @@ async fn run_android_webrtc_control_message( ControlMessage::AppSwitcher => state.android.open_app_switcher(&udid), ControlMessage::RotateLeft => state.android.rotate_left(&udid), ControlMessage::RotateRight => state.android.rotate_right(&udid), + ControlMessage::Crown { .. } => Err(AppError::bad_request( + "Digital Crown rotation is only available for Apple Watch simulators.", + )), ControlMessage::ToggleAppearance => state.android.toggle_appearance(&udid), ControlMessage::Touch { .. } | ControlMessage::EdgeTouch { .. } From 977e971e1d88a034c870545894cec6b98ce42820 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 12 May 2026 12:02:59 -0400 Subject: [PATCH 4/4] Harden simulator integration CI paths --- scripts/integration/js-api.mjs | 2 +- server/src/android.rs | 31 +++++++++++++++++++++++++++---- server/src/api/routes.rs | 15 +++++++++++++-- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/scripts/integration/js-api.mjs b/scripts/integration/js-api.mjs index ad915f97..2cacee79 100644 --- a/scripts/integration/js-api.mjs +++ b/scripts/integration/js-api.mjs @@ -168,7 +168,7 @@ async function main() { await session.waitFor( simulatorUDID, { id: "fixture.continue" }, - { source: "native-ax", maxDepth: 3, timeoutMs: 5_000, pollMs: 250 }, + { source: "native-ax", maxDepth: 3, timeoutMs: 20_000, pollMs: 250 }, ); }); await measuredStep("JS tree describe", async () => { diff --git a/server/src/android.rs b/server/src/android.rs index 168865ff..3f738308 100644 --- a/server/src/android.rs +++ b/server/src/android.rs @@ -21,6 +21,8 @@ const ANDROID_TOUCH_SWIPE_THRESHOLD: f64 = 0.025; const ANDROID_TOUCH_MIN_DURATION_MS: u128 = 80; const ANDROID_TOUCH_MAX_DURATION_MS: u128 = 1500; const ANDROID_COMMAND_TIMEOUT: Duration = Duration::from_secs(30); +const ANDROID_UIAUTOMATOR_DUMP_ATTEMPTS: usize = 10; +const ANDROID_UIAUTOMATOR_DUMP_RETRY_DELAY: Duration = Duration::from_millis(250); const RUNNING_EMULATOR_CACHE_TTL: Duration = Duration::from_secs(2); const AVD_GRPC_PORT_CACHE_TTL: Duration = Duration::from_secs(60); const SCREEN_SIZE_CACHE_TTL: Duration = Duration::from_secs(1); @@ -775,21 +777,42 @@ impl AndroidBridge { max_depth: Option, ) -> Result { let serial = self.serial_for_id(id)?; + let max_depth = max_depth.unwrap_or(80).min(80); + let mut last_error = None; + for attempt in 1..=ANDROID_UIAUTOMATOR_DUMP_ATTEMPTS { + match self.android_accessibility_tree_for_serial(&serial, max_depth) { + Ok(tree) => return Ok(tree), + Err(error) => last_error = Some(error), + } + if attempt < ANDROID_UIAUTOMATOR_DUMP_ATTEMPTS { + thread::sleep(ANDROID_UIAUTOMATOR_DUMP_RETRY_DELAY); + } + } + + Err(last_error.unwrap_or_else(|| { + AppError::native("Unable to capture Android UIAutomator hierarchy.") + })) + } + + fn android_accessibility_tree_for_serial( + &self, + serial: &str, + max_depth: usize, + ) -> Result { let raw = self.run_adb_shell( - &serial, + serial, "uiautomator dump /sdcard/simdeck_ui.xml >/dev/null && cat /sdcard/simdeck_ui.xml", )?; let xml = extract_xml(&raw); let document = roxmltree::Document::parse(xml).map_err(|error| { AppError::native(format!("Unable to parse UIAutomator XML: {error}")) })?; - let mut roots = Vec::new(); let root = document.root_element(); - let max_depth = max_depth.unwrap_or(80).min(80); + let (width, height) = self.screen_size_for_serial(serial)?; + let mut roots = Vec::new(); for child in root.children().filter(|node| node.has_tag_name("node")) { roots.push(android_node_value(child, 0, max_depth)); } - let (width, height) = self.screen_size_for_serial(&serial)?; if roots.is_empty() { roots.push(json!({ "type": "screen", diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 46318972..721e4d01 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -43,6 +43,7 @@ use tracing::Level; const SIMULATOR_INVENTORY_CACHE_TTL: Duration = Duration::from_secs(5); const SIMULATOR_INVENTORY_TIMEOUT: Duration = Duration::from_secs(8); +const SIMULATOR_INVENTORY_FORCE_REFRESH_TIMEOUT: Duration = Duration::from_secs(90); const H264_WS_MAGIC: &[u8; 4] = b"SDH1"; const H264_WS_HEADER_LEN: usize = 40; const H264_WS_FLAG_KEYFRAME: u8 = 1 << 0; @@ -5412,15 +5413,25 @@ async fn list_simulators_cached( } } + let inventory_timeout = if force_refresh { + SIMULATOR_INVENTORY_FORCE_REFRESH_TIMEOUT + } else { + SIMULATOR_INVENTORY_TIMEOUT + }; + let simulators = match timeout( - SIMULATOR_INVENTORY_TIMEOUT, + inventory_timeout, run_bridge_action(state.clone(), |bridge| bridge.list_simulators()), ) .await { Ok(result) => result?, Err(_) => { - tracing::warn!("Timed out listing iOS simulators; returning cached inventory."); + tracing::warn!( + timeout_seconds = inventory_timeout.as_secs(), + force_refresh, + "Timed out listing iOS simulators; returning cached inventory." + ); let guard = state.simulator_inventory.inner.lock().await; return Ok(guard.simulators.clone().unwrap_or_default()); }