diff --git a/Cargo.lock b/Cargo.lock index a5df95936..ffe2ed192 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,25 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -32,6 +51,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.0", + "objc2", +] + [[package]] name = "downcast-rs" version = "1.2.1" @@ -49,6 +78,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "itoa" version = "1.0.18" @@ -63,9 +98,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "linkcode-pty" @@ -82,6 +117,11 @@ dependencies = [ name = "linkcode-sim" version = "0.1.0" dependencies = [ + "block2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-foundation", "serde", "serde_json", ] @@ -110,6 +150,57 @@ dependencies = [ "libc", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", + "objc2-exception-helper", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.0", + "block2", + "dispatch2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-exception-helper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a1c5fbb72d7735b076bb47b578523aedc40f3c439bea6dfd595c089d79d98a" +dependencies = [ + "cc", +] + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + [[package]] name = "portable-pty" version = "0.9.0" @@ -219,6 +310,12 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "syn" version = "2.0.118" diff --git a/crates/linkcode-sim/Cargo.toml b/crates/linkcode-sim/Cargo.toml index a5a7f6d83..21978159c 100644 --- a/crates/linkcode-sim/Cargo.toml +++ b/crates/linkcode-sim/Cargo.toml @@ -14,5 +14,15 @@ name = "linkcode-sim" path = "src/main.rs" [dependencies] +libc = "0.2.189" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" + +# The private-framework layer (CoreSimulator/SimulatorKit via objc2/block2) is macOS-only and its +# use is all behind `#[cfg(target_os = "macos")]`. Scope these deps to macOS so a Linux CI build +# doesn't compile objc2's Objective-C exception helper, which needs an ObjC toolchain (cc1obj). +[target.'cfg(target_os = "macos")'.dependencies] +block2 = "0.6.2" +objc2 = { version = "0.6.4", features = ["exception"] } +objc2-core-foundation = "0.3.2" +objc2-foundation = "0.3.2" diff --git a/crates/linkcode-sim/NOTICE b/crates/linkcode-sim/NOTICE new file mode 100644 index 000000000..e652a63d9 --- /dev/null +++ b/crates/linkcode-sim/NOTICE @@ -0,0 +1,15 @@ +linkcode-sim + +The private-framework interactive layer (framebuffer capture and HID injection under +`src/private/`) ports recipes from baguette, an iOS Simulator control tool: + + baguette — https://github.com/tddworks/baguette + Copyright (c) tddworks + Licensed under the Apache License, Version 2.0 + +The undocumented CoreSimulator / SimulatorKit call sequences reproduced here — the IOHIDEvent +digitizer tap recipe for iOS 26, the framebuffer descriptor enumeration and screen-callback +registration, and the SimulatorKit framework path resolution — were derived from baguette's +implementation. They are reimplemented in Rust; no baguette source is included verbatim. + +A copy of the Apache License 2.0 is available at https://www.apache.org/licenses/LICENSE-2.0. diff --git a/crates/linkcode-sim/PROTOCOL.md b/crates/linkcode-sim/PROTOCOL.md index 8cc63412b..ac2a2b348 100644 --- a/crates/linkcode-sim/PROTOCOL.md +++ b/crates/linkcode-sim/PROTOCOL.md @@ -28,6 +28,7 @@ Sidecar to daemon: | ---: | --- | --- | | `0x81` | `RESULT` | JSON [`Result`](#result) | | `0x82` | `SCREENSHOT` | binary [`Screenshot body`](#screenshot-body) | +| `0x83` | `STREAM_FRAME` | binary [`Stream frame body`](#stream-frame-body) (unsolicited, while a stream runs) | Unknown frame types are ignored. A malformed `REQUEST` fails only that request — the sidecar replies with an `invalidRequest` `RESULT` for its `requestId` and keeps serving. If the `requestId` itself cannot be recovered, the sidecar logs to stderr and the daemon's pending request is reclaimed by its own timeout. @@ -58,6 +59,24 @@ Unknown frame types are ignored. A malformed `REQUEST` fails only that request Per-op deadlines are enforced sidecar-side (`boot` 180s, `install` 120s, `screenshot` 30s, others 60s); a child that outlives its deadline is killed and reported as `timeout`. +`probe` additionally reports `interactive: bool` — whether this host can drive simulators through the private-API path below (macOS with SimulatorKit). The daemon gates the interactive ops on it. + +### Ops (P1 — private API: HID injection + framebuffer streaming, macOS only) + +Off macOS, or when SimulatorKit is unavailable, every P1 op fails with `xcodeMissing`. + +| `type` | Params | Result | +| --- | --- | --- | +| `tap` | `udid`, `x`, `y` (normalised 0..1) | `{}` | +| `swipe` | `udid`, `x0`, `y0`, `x1`, `y1`, `durationMs?` | `{}` | +| `button` | `udid`, `button` (`home`/`lock`) | `{}` | +| `streamStart` | `udid`, `fps?` (60), `quality?` (0.6), `scale?` (1.0) | `{ streaming, fps, scale }` — JPEG frames then arrive on `STREAM_FRAME`s | +| `streamStop` | `udid` | `{}` | + +`scale` (0.1..1.0) downscales each frame before JPEG encode: at native resolution the encode bounds the frame rate near ~55 fps, so `scale` below 1.0 both lifts the achievable rate toward the display's 60 Hz and cuts bandwidth (e.g. `0.5` ≈ one third the bytes). `tap`/`swipe` stay in normalized 0..1 coordinates, so a downscaled stream needs no coordinate adjustment. + +Input (`tap`/`swipe`/`button`) runs in the sidecar's main process via a per-udid warmed HID client and is stable. Framebuffer streaming runs in a **crash-isolated worker subprocess**: the sidecar spawns it, reads its frames, and respawns it on the intermittent hard crashes of the private framebuffer path. If the worker crash-loops and gives up, the stream degrades to `simctl io screenshot` frames — slower, but frames never stop and the sidecar never crashes. + ## Result Success: @@ -92,6 +111,16 @@ A successful `screenshot` responds with raw image bytes instead of JSON, so capt A failed `screenshot` responds with a normal `RESULT` error. Exactly one of the two arrives per request. +## Stream frame body + +While a `streamStart` stream runs, the sidecar pushes unsolicited `STREAM_FRAME`s (raw JPEG bytes, no base64): + +```text +[u16 little-endian udid_length][udid UTF-8 bytes][image bytes] +``` + +Frames stop after `streamStop` (or when the daemon closes the pipe). The daemon routes them by `udid`. + ## Lifecycle expectations - One sidecar process serves many concurrent requests; the daemon lazily starts it on first use. diff --git a/crates/linkcode-sim/README.md b/crates/linkcode-sim/README.md index 4815453de..0ab4e0582 100644 --- a/crates/linkcode-sim/README.md +++ b/crates/linkcode-sim/README.md @@ -1,6 +1,6 @@ # linkcode-sim -The Link Code iOS Simulator host: one long-lived process the daemon spawns to drive Apple's iOS Simulator. P0 wraps the public `xcrun simctl` CLI only (lifecycle, install/launch, screenshot); the streaming/HID phase is CODE-396. The stdio wire contract is [`PROTOCOL.md`](./PROTOCOL.md); the TypeScript client is `@linkcode/sim` (`packages/host/sim`). +The Link Code iOS Simulator host: one long-lived process the daemon spawns to drive Apple's iOS Simulator. P0 wraps the public `xcrun simctl` CLI (lifecycle, install/launch, screenshot); P1 adds live framebuffer streaming (JPEG `STREAM_FRAME`s) and touch/button injection through CoreSimulator/SimulatorKit private frameworks, in a crash-isolated capture worker. The stdio wire contract is [`PROTOCOL.md`](./PROTOCOL.md); the TypeScript client is `@linkcode/sim` (`packages/host/sim`). macOS-only at runtime (it needs Xcode's simulator tooling); it compiles and answers `probe` with a structured `xcodeMissing` error everywhere else, so workspace-wide `cargo test` stays green on any platform. @@ -12,3 +12,24 @@ cargo test -p linkcode-sim --test device_loop -- --ignored # full boot→instal ``` Release binaries are staged by `apps/desktop/scripts/stage-sidecar.mts` into `apps/desktop/sidecar/${arch}` (macOS only) and shipped via electron-builder `extraResources`. + +## Benchmark + +`bench-encode` times the JPEG encode hot path — the per-frame CoreGraphics/ImageIO cost that bounds the framebuffer stream, since a single reader thread does the encode. `1000 / avg_ms` is the sustainable frame-rate ceiling. It needs no simulator: it encodes a synthetic BGRA frame across a resolution/quality sweep. + +```sh +cargo build -p linkcode-sim --release +./target/release/linkcode-sim bench-encode [iters] # default 120 iters/config +``` + +On 2026-07-22 (macOS arm64), 150 iters/config: + +| Resolution | Quality | avg ms | p95 ms | Size | fps (avg) | fps (peak) | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 1206×2622 (native) | 0.60 | 6.49 | 8.13 | 705K | 154 | 182 | +| 904×1966 | 0.60 | 3.45 | 3.98 | 396K | 290 | 332 | +| 603×1311 | 0.60 | 1.50 | 1.73 | 177K | 665 | 730 | +| 402×874 | 0.60 | 0.77 | 0.95 | 79K | 1294 | 1441 | +| 1206×2622 | 0.30 | 6.39 | 7.74 | 705K | 156 | 181 | + +Takeaways: the native-resolution encode ceiling (~154 fps) leaves ~2.5× headroom over 60 fps; cost is set by pixel count (the DCT), not JPEG quality (0.3 ≈ 0.6); downscaling is the real lever (half resolution ≈ 4× cheaper). The synthetic frame is high-entropy, so the numbers are conservative — a mostly-flat real screen encodes a touch faster. Use it to catch encode regressions and size the fps/resolution budget, not as an absolute. diff --git a/crates/linkcode-sim/src/capture.rs b/crates/linkcode-sim/src/capture.rs new file mode 100644 index 000000000..943dc82c1 --- /dev/null +++ b/crates/linkcode-sim/src/capture.rs @@ -0,0 +1,330 @@ +//! Crash-isolated framebuffer streaming. +//! +//! The steady-state framebuffer path (see [`crate::private::screen`]) is stable, but the private +//! frameworks can still abort hard in ways that are not catchable in-process — most of all a cold +//! CoreSimulator XPC connection racing class registration ("Attempt to use unknown class", a +//! `SIGABRT`). Rather than let that take down the sidecar (which also serves the simctl lifecycle +//! RPCs), the capture runs in a disposable **worker subprocess**: the sidecar spawns +//! `linkcode-sim capture-worker `, reads length-prefixed JPEG frames from its +//! stdout, and respawns it with backoff if it dies. Input injection and lifecycle RPCs stay in the +//! parent and are never affected by a capture crash. +//! +//! This is the standard isolation pattern for fragile native code (GPU/plugin sandboxes): contain +//! the crash, supervise, recover — defense in depth around a path that is now stable in steady state. + +use std::io::{self, Read, Write}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +/// One framebuffer JPEG frame. +pub type Frame = Arc>; + +/// A drift-free, precise frame pacer (macOS). It advances an absolute deadline by a fixed interval +/// each tick and waits to it with `mach_wait_until`, whose accuracy is well under a millisecond. +/// `thread::sleep` on macOS coalesces timers and overshoots by several ms — enough that even a +/// deadline loop built on it caps 60 fps near 55. `mach_wait_until` does not busy-spin, so locking to +/// the target rate costs no CPU beyond encoding the frames themselves. Times are in mach ticks. +#[cfg(target_os = "macos")] +pub(crate) struct FrameClock { + interval_ticks: u64, + next: u64, +} + +#[cfg(target_os = "macos")] +impl FrameClock { + pub(crate) fn new(fps: u32) -> FrameClock { + let interval_ns = 1_000_000_000u64 / u64::from(fps.max(1)); + FrameClock { + interval_ticks: mach_time::ns_to_ticks(interval_ns), + next: mach_time::now(), + } + } + + /// Wait until the next frame deadline. If a stall put us more than a full frame behind, resync to + /// now instead of emitting a catch-up burst of duplicate frames. + pub(crate) fn tick(&mut self) { + self.next = self.next.wrapping_add(self.interval_ticks); + let now = mach_time::now(); + if self.next > now { + mach_time::wait_until(self.next); + } else if now - self.next > self.interval_ticks { + self.next = now; + } + } +} + +/// Minimal `mach_absolute_time` timebase wrappers for precise pacing (see [`FrameClock`]). +#[cfg(target_os = "macos")] +mod mach_time { + use std::sync::OnceLock; + + #[repr(C)] + struct Timebase { + numer: u32, + denom: u32, + } + + unsafe extern "C" { + fn mach_absolute_time() -> u64; + fn mach_wait_until(deadline: u64) -> i32; + fn mach_timebase_info(info: *mut Timebase) -> i32; + } + + /// `(numer, denom)`: mach ticks → nanoseconds is `ticks * numer / denom`. Constant per boot. + fn timebase() -> (u64, u64) { + static TB: OnceLock<(u64, u64)> = OnceLock::new(); + *TB.get_or_init(|| { + let mut tb = Timebase { numer: 0, denom: 0 }; + // SAFETY: fills a valid out-param; returns KERN_SUCCESS with a nonzero denom on macOS. + unsafe { mach_timebase_info(&mut tb) }; + (u64::from(tb.numer.max(1)), u64::from(tb.denom.max(1))) + }) + } + + pub(super) fn now() -> u64 { + // SAFETY: reads the monotonic mach clock; no preconditions. + unsafe { mach_absolute_time() } + } + + pub(super) fn ns_to_ticks(ns: u64) -> u64 { + let (numer, denom) = timebase(); + (u128::from(ns) * u128::from(denom) / u128::from(numer)) as u64 + } + + pub(super) fn wait_until(deadline_ticks: u64) { + // SAFETY: blocks the calling thread until the absolute mach-time deadline. + unsafe { mach_wait_until(deadline_ticks) }; + } +} + +/// Largest accepted worker frame (a JPEG at simulator resolution is well under this). +const MAX_WORKER_FRAME: usize = 32 * 1024 * 1024; +/// Backoff between worker respawns after a crash. +const RESPAWN_BACKOFF: Duration = Duration::from_millis(500); +/// A worker that lived at least this long resets the crash-loop counter. +const HEALTHY_AFTER: Duration = Duration::from_secs(5); +/// Consecutive fast crashes before the stream gives up and reports unavailable. +const MAX_FAST_CRASHES: u32 = 6; + +/// Encoder/pacing parameters for a capture stream, threaded from `streamStart` down to the worker +/// (and across the process boundary as CLI args). Clamping happens at the boundary, not here. +#[derive(Clone, Copy)] +pub struct StreamParams { + pub fps: u32, + pub quality: f64, + /// Downscale factor applied before JPEG encode (0..1; 1.0 = native resolution). + pub scale: f64, +} + +/// A supervised framebuffer stream for one device. Holds the latest delivered frame; the worker +/// subprocess is spawned, read, and respawned by a background manager thread. +pub struct CaptureStream { + latest: Arc>>, + stopped: Arc, + /// Set true once the manager gives up after a crash loop, so callers can degrade. + dead: Arc, + /// The live worker child's pid (0 = none). Non-zero only while the manager holds the un-reaped + /// `Child`, so the pid can't be reused — making a kill-on-drop safe to target it. + worker_pid: Arc, + manager: Option>, +} + +impl CaptureStream { + /// Start streaming device `udid` with `params` by supervising a capture worker. + pub fn start(udid: String, params: StreamParams) -> CaptureStream { + let latest = Arc::new(Mutex::new(None)); + let stopped = Arc::new(AtomicBool::new(false)); + let dead = Arc::new(AtomicBool::new(false)); + let worker_pid = Arc::new(AtomicU32::new(0)); + let manager = thread::spawn({ + let latest = Arc::clone(&latest); + let stopped = Arc::clone(&stopped); + let dead = Arc::clone(&dead); + let worker_pid = Arc::clone(&worker_pid); + move || supervise(&udid, params, &latest, &stopped, &dead, &worker_pid) + }); + CaptureStream { + latest, + stopped, + dead, + worker_pid, + manager: Some(manager), + } + } + + /// The most recently delivered frame, or `None` before the first frame (or once dead). + pub fn latest(&self) -> Option { + self.latest + .lock() + .expect("capture stream mutex poisoned") + .clone() + } + + /// Whether the worker crash-looped and the stream gave up. + pub fn is_dead(&self) -> bool { + self.dead.load(Ordering::Relaxed) + } +} + +impl Drop for CaptureStream { + fn drop(&mut self) { + self.stopped.store(true, Ordering::Relaxed); + // The manager may be parked in a blocking `read_exact` on a worker that opened its + // framebuffer but never wrote a frame (a boot/shutdown transition); `stopped` alone can't + // wake that read, so kill the worker to close its stdout and let the read return. The pid is + // non-zero only while the manager holds the un-reaped child, so it can't have been reused. + let pid = self.worker_pid.load(Ordering::Relaxed); + if pid != 0 { + // SAFETY: best-effort SIGKILL to our capture-worker child (or a not-yet-reaped zombie of + // it); no other process can hold this pid while `worker_pid` is set. + unsafe { libc::kill(pid as libc::pid_t, libc::SIGKILL) }; + } + if let Some(manager) = self.manager.take() { + let _ = manager.join(); + } + } +} + +/// Manager loop: (re)spawn the worker, pump its frames, back off and retry on crash, give up after +/// too many fast crashes. +fn supervise( + udid: &str, + params: StreamParams, + latest: &Arc>>, + stopped: &Arc, + dead: &Arc, + worker_pid: &Arc, +) { + let mut fast_crashes = 0u32; + while !stopped.load(Ordering::Relaxed) { + let started = Instant::now(); + match spawn_worker(udid, params) { + Ok(mut child) => { + // Publish the pid before reading so a concurrent drop can kill a stuck worker; clear + // it only after `wait()` reaps it, keeping the pid unreusable while it is set. + worker_pid.store(child.id(), Ordering::Relaxed); + // A drop that set `stopped` between the spawn and the store above may have read pid 0 + // and not killed us; kill the child ourselves so the pump/wait below can't block on + // a worker that never writes. + if stopped.load(Ordering::Relaxed) { + // SAFETY: our own just-spawned, un-reaped child pid. + unsafe { libc::kill(child.id() as libc::pid_t, libc::SIGKILL) }; + } + pump_worker(&mut child, latest, stopped); + let _ = child.wait(); + worker_pid.store(0, Ordering::Relaxed); + } + Err(err) => { + eprintln!("sim capture: failed to spawn worker: {err}"); + } + } + if stopped.load(Ordering::Relaxed) { + break; + } + // A worker that ran a while then died (device shut down, transient crash) is not a loop. + fast_crashes = if started.elapsed() < HEALTHY_AFTER { + fast_crashes + 1 + } else { + 0 + }; + if fast_crashes >= MAX_FAST_CRASHES { + eprintln!("sim capture: worker keeps crashing; giving up on {udid}"); + dead.store(true, Ordering::Relaxed); + break; + } + thread::sleep(RESPAWN_BACKOFF); + } +} + +fn spawn_worker(udid: &str, params: StreamParams) -> io::Result { + let exe = std::env::current_exe()?; + Command::new(exe) + .arg("capture-worker") + .arg(udid) + .arg(format!("{}", params.quality)) + .arg(format!("{}", params.fps)) + .arg(format!("{}", params.scale)) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() +} + +/// Read length-prefixed JPEG frames from the worker until EOF (worker exit/crash) or stop. +fn pump_worker(child: &mut Child, latest: &Arc>>, stopped: &Arc) { + let Some(stdout) = child.stdout.take() else { + return; + }; + let mut reader = io::BufReader::new(stdout); + let mut header = [0u8; 4]; + while !stopped.load(Ordering::Relaxed) { + if reader.read_exact(&mut header).is_err() { + break; // worker gone + } + let len = u32::from_le_bytes(header) as usize; + if len == 0 || len > MAX_WORKER_FRAME { + break; // corrupt stream + } + let mut frame = vec![0u8; len]; + if reader.read_exact(&mut frame).is_err() { + break; + } + *latest.lock().expect("capture stream mutex poisoned") = Some(Arc::new(frame)); + } +} + +/// The worker entry point (`linkcode-sim capture-worker `): open the +/// device's framebuffer and stream length-prefixed JPEG frames to stdout at `fps`, each downscaled by +/// `scale`. Exits non-zero on any failure; the parent respawns. A hard `SIGABRT` from the private +/// path also just ends this process. +#[cfg(target_os = "macos")] +pub fn run_worker() -> ! { + use crate::private::{Screen, SimDevice}; + + let mut args = std::env::args().skip(2); + let udid = args.next().unwrap_or_default(); + let quality: f64 = args.next().and_then(|a| a.parse().ok()).unwrap_or(0.6); + let fps: u32 = args + .next() + .and_then(|a| a.parse().ok()) + .unwrap_or(60) + .clamp(1, 60); + let scale: f64 = args + .next() + .and_then(|a| a.parse::().ok()) + .unwrap_or(1.0) + .clamp(0.1, 1.0); + + let Some(device) = SimDevice::resolve(&udid) else { + eprintln!("sim capture-worker: device {udid} not found"); + std::process::exit(2); + }; + // Warm the device's CoreSimulator XPC connection before enumerating framebuffer ports: the HID + // client init establishes it and registers the proxy classes, which otherwise race the cold + // connection and abort framebuffer open with "unknown class". + let _warm = crate::private::Input::warm(&device); + let Some(screen) = Screen::open(&device) else { + eprintln!("sim capture-worker: no framebuffer for {udid}"); + std::process::exit(3); + }; + + let mut clock = FrameClock::new(fps); + let mut stdout = io::stdout().lock(); + loop { + if let Some(jpeg) = screen.capture_jpeg(quality, scale) { + let len = u32::try_from(jpeg.len()).unwrap_or(0); + if len == 0 + || stdout.write_all(&len.to_le_bytes()).is_err() + || stdout.write_all(&jpeg).is_err() + || stdout.flush().is_err() + { + break; // parent closed the pipe + } + } + clock.tick(); + } + std::process::exit(0); +} diff --git a/crates/linkcode-sim/src/interactive.rs b/crates/linkcode-sim/src/interactive.rs new file mode 100644 index 000000000..a6f6c505a --- /dev/null +++ b/crates/linkcode-sim/src/interactive.rs @@ -0,0 +1,299 @@ +//! The interactive (private-API) op handlers: HID injection and framebuffer streaming. +//! +//! Off macOS, and on a macOS host whose Xcode lacks SimulatorKit, every op fails with a stable +//! `unsupported` error so the daemon gates the capability. On macOS the ops resolve a per-udid +//! warmed HID client (`Input`) and drive taps/swipes/buttons, or start/stop a crash-isolated +//! [`CaptureStream`](crate::capture::CaptureStream) whose frames are pushed to the daemon as +//! `STREAM_FRAME`s. + +use std::sync::mpsc::Sender; + +use serde_json::{Value, json}; + +use crate::OutMsg; +use crate::rpc::{ButtonKind, ErrorCode, OpError}; + +fn unsupported() -> OpError { + OpError::new( + ErrorCode::XcodeMissing, + "interactive simulator control is unavailable on this host", + ) +} + +#[cfg(target_os = "macos")] +pub use imp::{available, button, stream_start, stream_stop, swipe, tap}; + +#[cfg(not(target_os = "macos"))] +mod stubs { + use super::*; + + pub fn available() -> bool { + false + } + pub fn tap(_udid: &str, _x: f64, _y: f64) -> Result { + Err(unsupported()) + } + pub fn swipe( + _udid: &str, + _x0: f64, + _y0: f64, + _x1: f64, + _y1: f64, + _duration_ms: u64, + ) -> Result { + Err(unsupported()) + } + pub fn button(_udid: &str, _button: ButtonKind) -> Result { + Err(unsupported()) + } + pub fn stream_start( + _udid: &str, + _fps: u32, + _quality: f64, + _scale: f64, + _tx: &Sender, + ) -> Result { + Err(unsupported()) + } + pub fn stream_stop(_udid: &str) -> Result { + Err(unsupported()) + } +} + +#[cfg(not(target_os = "macos"))] +pub use stubs::{available, button, stream_start, stream_stop, swipe, tap}; + +#[cfg(target_os = "macos")] +mod imp { + use std::collections::HashMap; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex, OnceLock}; + use std::thread; + use std::time::{Duration, Instant}; + + use super::*; + use crate::capture::{CaptureStream, Frame, FrameClock}; + use crate::private::{self, Button, Input, SimDevice}; + use crate::proto::{STREAM_FRAME, encode_stream_frame}; + + /// A live-but-silent private worker (framebuffer registration produced no callbacks) is treated + /// as unusable after this long with no new frame, and the pusher degrades to simctl. + const SILENT_FALLBACK_AFTER: Duration = Duration::from_secs(3); + + /// Warmed HID clients and running streams, keyed by udid. Warming a client is expensive, so it + /// is cached; a stream is one crash-isolated worker plus a pusher thread. + struct Registry { + inputs: HashMap>, + streams: HashMap, + } + + struct StreamHandle { + // Held so the CaptureStream (and its worker) stay alive until the handle is removed; the + // pusher thread holds the other Arc and stops when `stop` is set. + #[allow(dead_code, reason = "ownership hold that keeps the stream alive")] + stream: Arc, + stop: Arc, + pusher: Option>, + } + + fn registry() -> &'static Mutex { + static REGISTRY: OnceLock> = OnceLock::new(); + REGISTRY.get_or_init(|| { + Mutex::new(Registry { + inputs: HashMap::new(), + streams: HashMap::new(), + }) + }) + } + + pub fn available() -> bool { + private::interactive_available() + } + + /// Resolve (and cache) a warmed HID client for `udid`. + fn input_for(udid: &str) -> Result, OpError> { + let mut reg = registry().lock().expect("interactive registry poisoned"); + if let Some(input) = reg.inputs.get(udid) { + return Ok(Arc::clone(input)); + } + let device = SimDevice::resolve(udid).ok_or_else(|| { + OpError::new(ErrorCode::SimctlFailed, format!("device {udid} not found")) + })?; + let input = Input::warm(&device).ok_or_else(unsupported).map(Arc::new)?; + reg.inputs.insert(udid.to_owned(), Arc::clone(&input)); + Ok(input) + } + + pub fn tap(udid: &str, x: f64, y: f64) -> Result { + if input_for(udid)?.tap(x, y, Duration::from_millis(80)) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "tap failed")) + } + } + + pub fn swipe( + udid: &str, + x0: f64, + y0: f64, + x1: f64, + y1: f64, + duration_ms: u64, + ) -> Result { + let duration = if duration_ms == 0 { + Duration::from_millis(250) + } else { + Duration::from_millis(duration_ms) + }; + let steps = 10u32; + let step = duration / (steps + 2); + if input_for(udid)?.swipe(x0, y0, x1, y1, steps, step) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "swipe failed")) + } + } + + pub fn button(udid: &str, button: ButtonKind) -> Result { + let button = match button { + ButtonKind::Home => Button::Home, + ButtonKind::Lock => Button::Lock, + }; + if input_for(udid)?.button(button, Duration::from_millis(80)) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "button press failed")) + } + } + + pub fn stream_start( + udid: &str, + fps: u32, + quality: f64, + scale: f64, + tx: &Sender, + ) -> Result { + if !available() { + return Err(unsupported()); + } + // Warming the HID connection first stabilizes the framebuffer worker's cold open. + let _ = input_for(udid); + let fps = fps.clamp(1, 60); + let scale = scale.clamp(0.1, 1.0); + let mut reg = registry().lock().expect("interactive registry poisoned"); + if reg.streams.contains_key(udid) { + // An idempotent retry keeps the documented success shape (PROTOCOL.md), with + // `alreadyStreaming` only as an extra flag, so a caller validating the result still sees + // `streaming`/`fps`/`scale`. + return Ok( + json!({ "streaming": true, "fps": fps, "scale": scale, "alreadyStreaming": true }), + ); + } + let stream = Arc::new(CaptureStream::start( + udid.to_owned(), + crate::capture::StreamParams { + fps, + quality: quality.clamp(0.1, 1.0), + scale, + }, + )); + let stop = Arc::new(AtomicBool::new(false)); + let pusher = thread::spawn({ + let stream = Arc::clone(&stream); + let stop = Arc::clone(&stop); + let tx = tx.clone(); + let udid = udid.to_owned(); + move || push_frames(&udid, fps, &stream, &stop, &tx) + }); + reg.streams.insert( + udid.to_owned(), + StreamHandle { + stream, + stop, + pusher: Some(pusher), + }, + ); + Ok(json!({ "streaming": true, "fps": fps, "scale": scale })) + } + + pub fn stream_stop(udid: &str) -> Result { + let handle = registry() + .lock() + .expect("interactive registry poisoned") + .streams + .remove(udid); + if let Some(mut handle) = handle { + handle.stop.store(true, Ordering::Relaxed); + if let Some(pusher) = handle.pusher.take() { + let _ = pusher.join(); + } + } + Ok(json!({})) + } + + /// Push framebuffer frames to the daemon at `fps`. Prefers the fast private capture stream; if + /// its crash-isolated worker gives up (the private API is unusable on this host/state), it + /// degrades to `simctl io screenshot` — slower, but frames never stop and the sidecar never + /// crashes. De-duplicates the private frames by identity so a static screen doesn't flood. + fn push_frames( + udid: &str, + fps: u32, + stream: &CaptureStream, + stop: &AtomicBool, + tx: &Sender, + ) { + // Poll the private stream on a drift-free clock so a locked worker fps reaches the wire + // without the per-frame sleep overshoot that would otherwise sample it below target. + let mut clock = FrameClock::new(fps); + // simctl screenshots cost ~200-400ms, so poll them well below the private fps. + let fallback_interval = Duration::from_millis(500); + // Retain the last-sent frame (not a raw pointer): comparing addresses alone risks ABA — a + // freed frame's address reused by a new one would read as "already sent". + let mut last: Option = None; + // If the worker stays alive but never delivers (framebuffer registration produced no + // callbacks), fall back to simctl instead of sending nothing forever. + let mut last_progress = Instant::now(); + while !stop.load(Ordering::Relaxed) { + // Prefer a fresh private frame; sending one marks the worker as producing. + if let Some(frame) = stream.latest() + && last.as_ref().is_none_or(|prev| !Arc::ptr_eq(prev, &frame)) + { + last = Some(Arc::clone(&frame)); + last_progress = Instant::now(); + if let Ok(body) = encode_stream_frame(udid, &frame) + && tx + .send(OutMsg::Frame { + type_byte: STREAM_FRAME, + body, + }) + .is_err() + { + break; // daemon gone + } + clock.tick(); + continue; + } + // No fresh private frame: if the worker gave up, or has been silent past the timeout, + // degrade to a public simctl screenshot so frames never stop. + if stream.is_dead() || last_progress.elapsed() >= SILENT_FALLBACK_AFTER { + let tick = Instant::now(); + if let Ok(jpeg) = crate::simctl::screenshot(udid, crate::rpc::ImageFormat::Jpeg) + && let Ok(body) = encode_stream_frame(udid, &jpeg) + && tx + .send(OutMsg::Frame { + type_byte: STREAM_FRAME, + body, + }) + .is_err() + { + break; // daemon gone + } + if let Some(rest) = fallback_interval.checked_sub(tick.elapsed()) { + thread::sleep(rest); + } + } else { + clock.tick(); + } + } + } +} diff --git a/crates/linkcode-sim/src/main.rs b/crates/linkcode-sim/src/main.rs index db4c66cd5..843f3ca6c 100644 --- a/crates/linkcode-sim/src/main.rs +++ b/crates/linkcode-sim/src/main.rs @@ -4,6 +4,18 @@ //! protocol (see [`proto`]). Requests arrive on stdin; results go to stdout. Each request runs //! on its own thread so a slow boot never blocks a screenshot. +// The framebuffer-capture and HID surface is macOS-only (its consumers are behind +// `#[cfg(target_os = "macos")]`), so on other targets the streaming machinery, its imports, and the +// diag-dispatch locals are unreachable — don't fail the `-D warnings` lint on that expected slack. +#![cfg_attr( + not(target_os = "macos"), + allow(dead_code, unused_imports, unused_variables) +)] + +mod capture; +mod interactive; +#[cfg(target_os = "macos")] +mod private; mod proto; mod rpc; mod simctl; @@ -23,7 +35,7 @@ use crate::proto::{ use crate::rpc::{ErrorCode, Op, OpError, Request, RequestIdOnly, error_body, success_body}; /// A frame bound for the daemon, or the sentinel that tells the writer thread to stop. -enum OutMsg { +pub(crate) enum OutMsg { Frame { type_byte: u8, body: Vec }, Stop, } @@ -83,6 +95,27 @@ impl InflightGate { } fn main() { + let subcommand = std::env::args().nth(1); + // The crash-isolated framebuffer capture worker (spawned by the sidecar itself). + #[cfg(target_os = "macos")] + if subcommand.as_deref() == Some("capture-worker") { + capture::run_worker(); + } + // Hidden diagnostic path to exercise the private-framework layer against a real booted device: + // `linkcode-sim diag-interactive `. + #[cfg(target_os = "macos")] + if subcommand.as_deref() == Some("diag-interactive") { + diag_interactive(); + return; + } + // Benchmark the JPEG encode ceiling (the capture stream's single-thread frame-rate bound): + // `linkcode-sim bench-encode [iters]`. No simulator needed. + #[cfg(target_os = "macos")] + if subcommand.as_deref() == Some("bench-encode") { + bench_encode(); + return; + } + let (tx, rx) = channel::(); // Sole stdout owner: serializes frames from every request thread. @@ -163,7 +196,7 @@ fn main() { fn serve(request: Request, tx: &Sender) { let request_id = request.request_id; let outcome = match request.op { - Op::Probe => simctl::probe(), + Op::Probe => probe_with_capabilities(), Op::List => simctl::list(), Op::Boot { udid } => simctl::boot(&udid), Op::Shutdown { udid } => simctl::shutdown(&udid), @@ -196,6 +229,23 @@ fn serve(request: Request, tx: &Sender) { Err(e) => Err(e), } } + Op::Tap { udid, x, y } => interactive::tap(&udid, x, y), + Op::Swipe { + udid, + x0, + y0, + x1, + y1, + duration_ms, + } => interactive::swipe(&udid, x0, y0, x1, y1, duration_ms), + Op::Button { udid, button } => interactive::button(&udid, button), + Op::StreamStart { + udid, + fps, + quality, + scale, + } => interactive::stream_start(&udid, fps, quality, scale, tx), + Op::StreamStop { udid } => interactive::stream_stop(&udid), }; match outcome { Ok(result) => send(tx, RESULT, success_body(&request_id, result)), @@ -203,6 +253,15 @@ fn serve(request: Request, tx: &Sender) { } } +/// The probe result augmented with the interactive (private-API framebuffer + HID) capability bit. +fn probe_with_capabilities() -> Result { + let mut result = simctl::probe()?; + if let Some(object) = result.as_object_mut() { + object.insert("interactive".to_owned(), interactive::available().into()); + } + Ok(result) +} + fn send(tx: &Sender, type_byte: u8, body: Vec) { let _ = tx.send(OutMsg::Frame { type_byte, body }); } @@ -210,3 +269,100 @@ fn send(tx: &Sender, type_byte: u8, body: Vec) { fn send_error(tx: &Sender, request_id: &str, error: &OpError) { send(tx, RESULT, error_body(request_id, error)); } + +/// Diagnostic entry (macOS only): drive the SUPERVISED capture stream (crash-isolated worker) plus +/// input injection against a real device. `linkcode-sim diag-interactive [x] [y]`. +/// Verifies the stream survives worker crashes and keeps delivering frames. +#[cfg(target_os = "macos")] +fn diag_interactive() { + use std::time::Duration; + let udid = std::env::args() + .nth(2) + .expect("usage: diag-interactive "); + let out = std::env::args() + .nth(3) + .unwrap_or_else(|| "diag.jpg".to_owned()); + let x: f64 = std::env::args() + .nth(4) + .and_then(|a| a.parse().ok()) + .unwrap_or(0.5); + let y: f64 = std::env::args() + .nth(5) + .and_then(|a| a.parse().ok()) + .unwrap_or(0.5); + eprintln!( + "interactive available: {}", + private::interactive_available() + ); + + let device = private::SimDevice::resolve(&udid).expect("device not found"); + let input = private::Input::warm(&device).expect("HID warm failed"); + let stream = capture::CaptureStream::start( + udid.clone(), + capture::StreamParams { + fps: 12, + quality: 0.6, + scale: 1.0, + }, + ); + eprintln!("supervised capture stream started; driving input for ~5s"); + + let start = std::time::Instant::now(); + let mut frames = 0u32; + let mut last_len = 0usize; + while start.elapsed() < Duration::from_secs(5) && !stream.is_dead() { + input.tap(x, y, Duration::from_millis(30)); + if let Some(frame) = stream.latest() { + frames += 1; + last_len = frame.len(); + std::fs::write(&out, &**frame).expect("write jpeg"); + } + std::thread::sleep(Duration::from_millis(80)); + } + eprintln!( + "stream dead={} frames-seen={frames} last={last_len} bytes; wrote {out}", + stream.is_dead() + ); +} + +/// Benchmark entry (macOS only): time the JPEG encode across a resolution/quality sweep and print the +/// implied max frame rate. `linkcode-sim bench-encode [iters]` (default 120). The encode runs on one +/// reader thread, so `1000 / avg_ms` is the sustainable stream ceiling; the capture memcpy runs on a +/// separate thread and is not the bound. +#[cfg(target_os = "macos")] +fn bench_encode() { + let iters: u32 = std::env::args() + .nth(2) + .and_then(|a| a.parse().ok()) + .unwrap_or(120); + // Native iPhone 17 Pro framebuffer is 1206×2622; sweep it and progressive downscales at the + // default stream quality, then the same full res at a lower quality to show quality is not the + // bound. + let configs = [ + (1206usize, 2622usize, 0.6f64), + (904, 1966, 0.6), + (603, 1311, 0.6), + (402, 874, 0.6), + (1206, 2622, 0.3), + ]; + println!( + "encode bench — {iters} iters/config (single reader thread = stream fps ceiling)\n{:>11} {:>5} {:>8} {:>8} {:>7} {:>9} {:>8}", + "resolution", "q", "avg ms", "p95 ms", "size", "fps(avg)", "fps(peak)" + ); + for (w, h, q) in configs { + match private::bench_encode(w, h, q, iters) { + Some(b) => println!( + "{:>4}x{:<6} {:>5.2} {:>8.2} {:>8.2} {:>6}K {:>9.1} {:>8.1}", + b.width, + b.height, + b.quality, + b.avg_ms, + b.p95_ms, + b.out_kib, + b.fps(), + b.peak_fps() + ), + None => println!("{w}x{h} q{q}: encode failed"), + } + } +} diff --git a/crates/linkcode-sim/src/private/debug.rs b/crates/linkcode-sim/src/private/debug.rs new file mode 100644 index 000000000..8ae499979 --- /dev/null +++ b/crates/linkcode-sim/src/private/debug.rs @@ -0,0 +1,24 @@ +//! Env-gated diagnostic logging for the private-framework layer. +//! +//! The private paths cross undocumented CoreSimulator/SimulatorKit surfaces, so keep a way to trace +//! them without shipping noise. Set `LINKCODE_SIM_DEBUG=1` to print to stderr; off by default. + +use std::sync::OnceLock; + +pub fn enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + std::env::var("LINKCODE_SIM_DEBUG").is_ok_and(|v| v != "0" && !v.is_empty()) + }) +} + +/// Print a diagnostic line to stderr when `LINKCODE_SIM_DEBUG` is set. +macro_rules! dbg_log { + ($($arg:tt)*) => { + if $crate::private::debug::enabled() { + eprintln!("[sim] {}", format!($($arg)*)); + } + }; +} + +pub(crate) use dbg_log; diff --git a/crates/linkcode-sim/src/private/device.rs b/crates/linkcode-sim/src/private/device.rs new file mode 100644 index 000000000..5c3c33b23 --- /dev/null +++ b/crates/linkcode-sim/src/private/device.rs @@ -0,0 +1,117 @@ +//! Resolving a CoreSimulator `SimDevice` object by udid. +//! +//! Recipe ported from baguette's `CoreSimulators.swift`: `SimServiceContext` → default device set → +//! `availableDevices` → match `UDID.UUIDString`. Everything goes through the ObjC runtime; there is +//! no bridging header. + +use std::ptr; + +use objc2::rc::Retained; +use objc2::runtime::{AnyClass, AnyObject, Sel}; +use objc2::{msg_send, sel}; +use objc2_foundation::NSString; + +use super::framework; + +/// A resolved `SimDevice`, retained for the lifetime of a stream/input session. +pub struct SimDevice { + pub object: Retained, +} + +// SAFETY: SimDevice objects are thread-safe for the read/HID operations we perform; the sidecar +// serializes writes per device through the mux anyway. +unsafe impl Send for SimDevice {} + +impl SimDevice { + /// Resolve the booted-or-not `SimDevice` for `udid`, or `None` if CoreSimulator can't be reached + /// or no device matches. + pub fn resolve(udid: &str) -> Option { + framework::load(); + let set = default_device_set()?; + let devices = available_devices(&set); + for device in devices { + if device_udid(&device).as_deref() == Some(udid) { + return Some(SimDevice { object: device }); + } + } + None + } + + pub fn object_ptr(&self) -> *mut AnyObject { + Retained::as_ptr(&self.object).cast_mut() + } +} + +fn default_device_set() -> Option> { + let ctx = shared_service_context()?; + let sel = sel!(defaultDeviceSetWithError:); + if !responds_to(&ctx, sel) { + return None; + } + let mut err: *mut AnyObject = ptr::null_mut(); + // SAFETY: dynamic message to a method with an `(NSError**)` out-param; we pass a valid slot and + // ignore the error object. The returned set is autoreleased — retain it to hold across the call. + let set: *mut AnyObject = unsafe { msg_send![&*ctx, defaultDeviceSetWithError: &mut err] }; + retain(set) +} + +fn shared_service_context() -> Option> { + let cls = AnyClass::get(c"SimServiceContext")?; + let dev = framework::developer_dir(); + let dir = NSString::from_str(&dev.to_string_lossy()); + let mut err: *mut AnyObject = ptr::null_mut(); + // SAFETY: class method with a developer-dir NSString and an `(NSError**)` out-param. + let ctx: *mut AnyObject = + unsafe { msg_send![cls, sharedServiceContextForDeveloperDir: &*dir, error: &mut err] }; + retain(ctx) +} + +fn available_devices(set: &Retained) -> Vec> { + let key = NSString::from_str("availableDevices"); + // SAFETY: KVC read returning an NSArray (or nil). + let array: *mut AnyObject = unsafe { msg_send![&**set, valueForKey: &*key] }; + if array.is_null() { + return Vec::new(); + } + // SAFETY: array is an NSArray*; count + objectAtIndex: are its standard accessors. + let count: usize = unsafe { msg_send![array, count] }; + let mut out = Vec::with_capacity(count); + for index in 0..count { + // SAFETY: index < count. + let device: *mut AnyObject = unsafe { msg_send![array, objectAtIndex: index] }; + if let Some(device) = retain(device) { + out.push(device); + } + } + out +} + +fn device_udid(device: &Retained) -> Option { + let key = NSString::from_str("UDID"); + // SAFETY: KVC read returning an NSUUID (or nil). + let uuid: *mut AnyObject = unsafe { msg_send![&**device, valueForKey: &*key] }; + if uuid.is_null() { + return None; + } + // SAFETY: uuid is an NSUUID*; UUIDString returns an NSString. + let string: *mut NSString = unsafe { msg_send![uuid, UUIDString] }; + if string.is_null() { + return None; + } + // SAFETY: string is a valid NSString for the duration of this read. + Some(unsafe { (*string).to_string() }) +} + +fn responds_to(object: &Retained, sel: Sel) -> bool { + // SAFETY: respondsToSelector: is defined on NSObject. + unsafe { msg_send![&**object, respondsToSelector: sel] } +} + +/// Retain a +0 (autoreleased/borrowed) object pointer into an owned `Retained`, or `None` if null. +fn retain(ptr: *mut AnyObject) -> Option> { + if ptr.is_null() { + return None; + } + // SAFETY: ptr is a live ObjC object we did not create; retaining it takes shared ownership. + unsafe { Retained::retain(ptr) } +} diff --git a/crates/linkcode-sim/src/private/framework.rs b/crates/linkcode-sim/src/private/framework.rs new file mode 100644 index 000000000..16bec1c87 --- /dev/null +++ b/crates/linkcode-sim/src/private/framework.rs @@ -0,0 +1,168 @@ +//! Locating and loading the private simulator frameworks. +//! +//! Recipe ported from baguette (Apache-2.0; see crate NOTICE): CoreSimulator lives in a stable +//! system path; SimulatorKit moved from `Contents/Developer/Library/PrivateFrameworks` (Xcode ≤26) +//! up to `Contents/SharedFrameworks` (Xcode 27), so both layouts are probed oldest-first. The IOKit +//! `IOHIDEvent*` symbols are in the dyld shared cache and reachable via `RTLD_DEFAULT` once anything +//! is loaded. + +use std::ffi::{CString, c_void}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; + +const CORE_SIMULATOR: &str = + "/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator"; +const SIMKIT_SUFFIX: &str = "SimulatorKit.framework/SimulatorKit"; + +/// A dlopen handle to SimulatorKit. `None` means no Xcode on this machine carries SimulatorKit — +/// the caller degrades to the public simctl path. +pub struct Frameworks { + pub simulator_kit: *mut c_void, +} + +// SAFETY: the handle is a process-lifetime dlopen result, only ever read to resolve symbols. +unsafe impl Send for Frameworks {} +unsafe impl Sync for Frameworks {} + +static FRAMEWORKS: OnceLock> = OnceLock::new(); + +/// Load CoreSimulator + SimulatorKit once. Returns `None` when SimulatorKit cannot be found, which +/// the sidecar reports as a non-interactive capability rather than failing. +pub fn load() -> Option<&'static Frameworks> { + FRAMEWORKS.get_or_init(load_once).as_ref() +} + +fn load_once() -> Option { + // CoreSimulator is also loaded by simctl-adjacent paths, but load it here so its classes + // (SimServiceContext, SimDevice) are registered before we message them. Best-effort. + dlopen(CORE_SIMULATOR); + + let kit_path = simulator_kit_path(&developer_dir())?; + let simulator_kit = dlopen(&kit_path.to_string_lossy())?; + Some(Frameworks { simulator_kit }) +} + +/// Resolve a developer directory that actually contains SimulatorKit: `xcode-select -p` first, then +/// a scan of `/Applications/Xcode*.app`, else the canonical default so a later error names a path +/// the user recognizes. +pub fn developer_dir() -> PathBuf { + if let Some(dir) = xcode_select_dir() + && simulator_kit_path(&dir).is_some() + { + return dir; + } + if let Some(dir) = scan_applications() { + return dir; + } + xcode_select_dir() + .unwrap_or_else(|| PathBuf::from("/Applications/Xcode.app/Contents/Developer")) +} + +/// The candidate SimulatorKit paths for a developer dir, in probe order (Xcode ≤26 then 27). +pub fn simulator_kit_candidates(developer_dir: &Path) -> [PathBuf; 2] { + let contents = developer_dir.parent().unwrap_or(developer_dir); + [ + developer_dir.join(format!("Library/PrivateFrameworks/{SIMKIT_SUFFIX}")), + contents.join(format!("SharedFrameworks/{SIMKIT_SUFFIX}")), + ] +} + +/// The first candidate that exists, or `None` when this Xcode carries no SimulatorKit. +pub fn simulator_kit_path(developer_dir: &Path) -> Option { + simulator_kit_candidates(developer_dir) + .into_iter() + .find(|path| path.exists()) +} + +fn xcode_select_dir() -> Option { + let output = Command::new("/usr/bin/xcode-select") + .arg("-p") + .output() + .ok()?; + if !output.status.success() { + return None; + } + let dir = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if dir.is_empty() { + None + } else { + Some(PathBuf::from(dir)) + } +} + +fn scan_applications() -> Option { + let canonical = PathBuf::from("/Applications/Xcode.app/Contents/Developer"); + if simulator_kit_path(&canonical).is_some() { + return Some(canonical); + } + let mut entries: Vec<_> = std::fs::read_dir("/Applications") + .ok()? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name()) + .filter_map(|name| name.into_string().ok()) + .filter(|name| name.starts_with("Xcode") && name.ends_with(".app") && name != "Xcode.app") + .collect(); + entries.sort(); + for app in entries { + let dir = PathBuf::from(format!("/Applications/{app}/Contents/Developer")); + if simulator_kit_path(&dir).is_some() { + return Some(dir); + } + } + None +} + +/// `dlopen(path, RTLD_NOW | RTLD_GLOBAL)`; `None` on failure. +fn dlopen(path: &str) -> Option<*mut c_void> { + let c_path = CString::new(path).ok()?; + // SAFETY: c_path is a valid NUL-terminated C string kept alive across the call. + let handle = unsafe { libc::dlopen(c_path.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL) }; + if handle.is_null() { None } else { Some(handle) } +} + +/// Resolve a symbol from a specific dlopen handle. +/// +/// # Safety +/// The returned pointer is only valid to transmute to the symbol's true ABI; the caller asserts the +/// signature. `handle` must be a live dlopen handle. +pub unsafe fn dlsym(handle: *mut c_void, name: &str) -> Option<*mut c_void> { + let c_name = CString::new(name).ok()?; + // SAFETY: forwarded to the caller's contract; c_name outlives the call. + let sym = unsafe { libc::dlsym(handle, c_name.as_ptr()) }; + if sym.is_null() { None } else { Some(sym) } +} + +/// Resolve a symbol from the dyld shared cache (`RTLD_DEFAULT`) — used for the public `IOHIDEvent*` +/// creators that are not in SimulatorKit itself. +/// +/// # Safety +/// Same contract as [`dlsym`]. +pub unsafe fn dlsym_default(name: &str) -> Option<*mut c_void> { + let c_name = CString::new(name).ok()?; + // RTLD_DEFAULT is the special handle (void*)-2 on Darwin. + let rtld_default = (-2isize) as *mut c_void; + // SAFETY: forwarded to the caller's contract; c_name outlives the call. + let sym = unsafe { libc::dlsym(rtld_default, c_name.as_ptr()) }; + if sym.is_null() { None } else { Some(sym) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn candidates_cover_both_xcode_layouts() { + let dir = Path::new("/Applications/Xcode.app/Contents/Developer"); + let candidates = simulator_kit_candidates(dir); + assert!( + candidates[0].ends_with( + "Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit" + ) + ); + assert!( + candidates[1] + .ends_with("Contents/SharedFrameworks/SimulatorKit.framework/SimulatorKit") + ); + } +} diff --git a/crates/linkcode-sim/src/private/input.rs b/crates/linkcode-sim/src/private/input.rs new file mode 100644 index 000000000..4335a6f71 --- /dev/null +++ b/crates/linkcode-sim/src/private/input.rs @@ -0,0 +1,387 @@ +//! Touch and button injection into a booted simulator. +//! +//! Recipe ported from baguette's `IndigoHIDInput.swift` + `IOHIDDigitizerDispatch.swift` +//! (Apache-2.0; see crate NOTICE). On iOS 26 a tap CANNOT go through the 9-arg +//! `IndigoHIDMessageForMouseNSEvent` — it is misread as a Home gesture or dropped. The working path +//! builds a real `IOHIDEvent` digitizer parent+finger pair, runs it through +//! `IndigoHIDMessageForTrackpadEventFromHIDEventRef`, then patches the two byte slots the wrapper +//! leaves uninitialised (the `0x32` touch-target tag and the edge bitmask). Buttons (home/lock) use +//! the legacy `IndigoHIDMessageForButton`, which SpringBoard still honors on Face ID devices. + +use std::ffi::c_void; +use std::ptr; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::thread::sleep; +use std::time::Duration; + +use objc2::msg_send; +use objc2::rc::Retained; +use objc2::runtime::{AnyClass, AnyObject}; + +use super::device::SimDevice; +use super::framework; + +// IOHIDEvent creators — public IOKit symbols in the dyld shared cache. ABIs transcribed verbatim +// from the baguette Swift typedefs. `bool` is C `_Bool` (1 byte); the AArch64/x86-64 ABIs place the +// integer- and float-class args from the declared order, so no manual register juggling is needed. +type CreateDigitizerFn = unsafe extern "C" fn( + *const c_void, // allocator (null) + u64, // timestamp + u32, // transducer type (2 = finger) + u32, // index + u32, // identifier + u32, // event mask + u32, // button mask + f64, // x + f64, // y + f64, // z + f64, // tip pressure + f64, // barrel pressure + bool, // range + bool, // touch + u32, // options +) -> *mut c_void; // IOHIDEventRef (+1, Create rule) + +type CreateFingerFn = unsafe extern "C" fn( + *const c_void, // allocator (null) + u64, // timestamp + u32, // index + u32, // identifier + u32, // event mask + f64, // x + f64, // y + f64, // z + f64, // tip pressure + f64, // twist + bool, // range + bool, // touch + u32, // options +) -> *mut c_void; // IOHIDEventRef (+1) + +type AppendEventFn = unsafe extern "C" fn(*mut c_void, *mut c_void, u32); +type TrackpadWrapFn = unsafe extern "C" fn(*const c_void) -> *mut c_void; +type ButtonFn = unsafe extern "C" fn(u32, u32, u32) -> *mut c_void; +type ServiceFn = unsafe extern "C" fn() -> *mut c_void; + +unsafe extern "C" { + fn CFRelease(cf: *const c_void); + fn malloc_size(ptr: *const c_void) -> usize; + fn mach_absolute_time() -> u64; +} + +const TRANSDUCER_FINGER: u32 = 2; +const TOUCH_TARGET: u32 = 0x32; +/// `IndigoHIDMessageForButton` arg0 for the legacy home/lock path; 3rd arg is the digitizer routing +/// target (0x33), not a timestamp. +const HOME_ARG0: u32 = 0x0; +const LOCK_ARG0: u32 = 0x1; +const LEGACY_BUTTON_TARGET: u32 = 0x33; + +/// Which hardware button to press. +#[derive(Clone, Copy)] +pub enum Button { + Home, + Lock, +} + +/// Touch phase for a streamed gesture. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Phase { + Down, + Move, + Up, +} + +impl Phase { + fn event_mask(self) -> u32 { + match self { + // Range | Touch | Position for a sustained touch; Touch | Position for the lift. + Self::Down | Self::Move => 0x07, + Self::Up => 0x06, + } + } + fn range(self) -> bool { + self != Self::Up + } + fn touch(self) -> bool { + self != Self::Up + } +} + +struct Symbols { + create_digitizer: CreateDigitizerFn, + create_finger: CreateFingerFn, + append_event: AppendEventFn, + trackpad_wrap: TrackpadWrapFn, + button: ButtonFn, + create_pointer_service: Option, + create_mouse_service: Option, +} + +/// A warmed HID client bound to one device, plus the resolved private symbols. Created lazily; a +/// resolution failure means this host cannot inject input (the caller degrades to view-only). +pub struct Input { + client: Retained, + symbols: Symbols, + touch_counter: AtomicU32, +} + +// SAFETY: the sidecar serializes input per device; the HID client and C fn pointers are only touched +// under that serialization. +unsafe impl Send for Input {} +unsafe impl Sync for Input {} + +impl Input { + /// Resolve symbols, create + warm the `SimDeviceLegacyHIDClient` for `device`. `None` when + /// SimulatorKit or the HID client cannot be reached. + pub fn warm(device: &SimDevice) -> Option { + let symbols = resolve_symbols()?; + let client = make_hid_client(device)?; + let input = Input { + client, + symbols, + touch_counter: AtomicU32::new(0), + }; + input.warm_services(); + Some(input) + } + + /// Single-finger tap at a normalised (0..1) point, holding for `hold`. + pub fn tap(&self, x: f64, y: f64, hold: Duration) -> bool { + let id = self.next_identifier(); + if !self.send_touch(x, y, id, Phase::Down) { + return false; + } + sleep(hold.max(Duration::from_millis(20))); + self.send_touch(x, y, id, Phase::Up) + } + + /// Continuous swipe between two normalised points over `steps` interpolated moves. + pub fn swipe(&self, x0: f64, y0: f64, x1: f64, y1: f64, steps: u32, step: Duration) -> bool { + let steps = steps.max(2); + let id = self.next_identifier(); + if !self.send_touch(x0, y0, id, Phase::Down) { + return false; + } + let mut ok = 0u32; + for i in 1..=steps { + sleep(step); + let t = f64::from(i) / f64::from(steps); + let x = x0 + (x1 - x0) * t; + let y = y0 + (y1 - y0) * t; + if self.send_touch(x, y, id, Phase::Move) { + ok += 1; + } + } + sleep(step); + self.send_touch(x1, y1, id, Phase::Up) && ok >= steps / 2 + } + + /// Press a hardware button (home/lock) for `hold`. + pub fn button(&self, button: Button, hold: Duration) -> bool { + let arg0 = match button { + Button::Home => HOME_ARG0, + Button::Lock => LOCK_ARG0, + }; + // SAFETY: button fn resolved in `resolve_symbols`; direction 1=down, 2=up (0 crashes + // backboardd). Each returns a freshly malloc'd message consumed by send (freeWhenDone). + let down = unsafe { (self.symbols.button)(arg0, 1, LEGACY_BUTTON_TARGET) }; + if down.is_null() { + return false; + } + self.send_message(down); + sleep(hold.max(Duration::from_millis(20))); + let up = unsafe { (self.symbols.button)(arg0, 2, LEGACY_BUTTON_TARGET) }; + if up.is_null() { + return false; + } + self.send_message(up); + true + } + + fn next_identifier(&self) -> u32 { + let id = self + .touch_counter + .fetch_add(1, Ordering::Relaxed) + .wrapping_add(1); + if id == 0 { 1 } else { id } + } + + /// Build one digitizer parent+finger event, wrap it, patch the target/edge byte slots, and send. + fn send_touch(&self, x: f64, y: f64, identifier: u32, phase: Phase) -> bool { + let x = clamp01(x); + let y = clamp01(y); + let mask = phase.event_mask(); + let range = phase.range(); + let touch = phase.touch(); + let now = unsafe { mach_absolute_time() }; + + // SAFETY: creators resolved in resolve_symbols; args match the transcribed ABIs. Returns are + // +1 IOHIDEventRefs we own and release below. + let parent = unsafe { + (self.symbols.create_digitizer)( + ptr::null(), + now, + TRANSDUCER_FINGER, + 0, + identifier, + mask, + 0, + x, + y, + 0.0, + 0.0, + 0.0, + range, + touch, + 0, + ) + }; + if parent.is_null() { + return false; + } + let finger = unsafe { + (self.symbols.create_finger)( + ptr::null(), + now, + 0, + identifier, + mask, + x, + y, + 0.0, + 0.0, + 0.0, + range, + touch, + 0, + ) + }; + if !finger.is_null() { + // SAFETY: append copies the child into the parent; both stay valid here. + unsafe { (self.symbols.append_event)(parent, finger, 0) }; + } + // SAFETY: wrapper reads the parent event and returns a malloc'd Indigo message (or null). + let message = unsafe { (self.symbols.trackpad_wrap)(parent.cast_const()) }; + // The wrapper has copied out what it needs; release the events regardless of outcome. + unsafe { + if !finger.is_null() { + CFRelease(finger.cast_const()); + } + CFRelease(parent.cast_const()); + } + if message.is_null() { + return false; + } + patch_message(message); + self.send_message(message); + true + } + + fn warm_services(&self) { + for create in [ + self.symbols.create_pointer_service, + self.symbols.create_mouse_service, + ] + .into_iter() + .flatten() + { + // SAFETY: service creators take no args and return a malloc'd message consumed by send. + let message = unsafe { create() }; + if !message.is_null() { + self.send_message(message); + sleep(Duration::from_millis(20)); + } + } + } + + /// Dispatch a malloc'd Indigo message; `freeWhenDone: true` hands ownership to the client. + fn send_message(&self, message: *mut c_void) { + let null_obj: *mut AnyObject = ptr::null_mut(); + // SAFETY: the client implements sendWithMessage:freeWhenDone:completionQueue:completion:; + // message is a live malloc'd buffer the client frees. + unsafe { + let _: () = msg_send![ + &*self.client, + sendWithMessage: message, + freeWhenDone: true, + completionQueue: null_obj, + completion: null_obj, + ]; + } + } +} + +/// Patch the two byte slots the trackpad wrapper leaves uninitialised: the touch-target routing tag +/// (offset 0x6c, and 0x10c on the larger layout) and the edge bitmask (0x3a/0x3b, 0xda/0xdb). We +/// only ever inject interior touches, so the edge bits stay zero. +fn patch_message(message: *mut c_void) { + // SAFETY: message is a malloc'd buffer at least 0x70 bytes; malloc_size gates the larger writes. + unsafe { + message + .byte_add(0x6c) + .cast::() + .write_unaligned(TOUCH_TARGET); + let size = malloc_size(message.cast_const()); + if size >= 0x110 { + message + .byte_add(0x10c) + .cast::() + .write_unaligned(TOUCH_TARGET); + } + message.byte_add(0x3a).cast::().write_unaligned(0); + message.byte_add(0x3b).cast::().write_unaligned(0); + if size >= 0xdc { + message.byte_add(0xda).cast::().write_unaligned(0); + message.byte_add(0xdb).cast::().write_unaligned(0); + } + } +} + +fn make_hid_client(device: &SimDevice) -> Option> { + let cls = AnyClass::get(c"_TtC12SimulatorKit24SimDeviceLegacyHIDClient")?; + let mut err: *mut AnyObject = ptr::null_mut(); + // SAFETY: alloc + initWithDevice:error: on the resolved class; the device object outlives the + // call. init returns a +1 object we own. + let client: *mut AnyObject = unsafe { + let allocated: *mut AnyObject = msg_send![cls, alloc]; + msg_send![allocated, initWithDevice: device.object_ptr(), error: &mut err] + }; + if client.is_null() { + return None; + } + // SAFETY: init already returned +1; adopt it without an extra retain. + unsafe { Retained::from_raw(client) } +} + +fn resolve_symbols() -> Option { + let frameworks = framework::load()?; + let kit = frameworks.simulator_kit; + // SAFETY: each dlsym result is transmuted to the ABI its symbol name denotes; the fn pointers + // are only invoked with matching argument types above. + unsafe { + let create_digitizer = framework::dlsym_default("IOHIDEventCreateDigitizerEvent")?; + let create_finger = framework::dlsym_default("IOHIDEventCreateDigitizerFingerEvent")?; + let append_event = framework::dlsym_default("IOHIDEventAppendEvent")?; + let trackpad_wrap = + framework::dlsym(kit, "IndigoHIDMessageForTrackpadEventFromHIDEventRef")?; + let button = framework::dlsym(kit, "IndigoHIDMessageForButton")?; + Some(Symbols { + create_digitizer: std::mem::transmute::<*mut c_void, CreateDigitizerFn>( + create_digitizer, + ), + create_finger: std::mem::transmute::<*mut c_void, CreateFingerFn>(create_finger), + append_event: std::mem::transmute::<*mut c_void, AppendEventFn>(append_event), + trackpad_wrap: std::mem::transmute::<*mut c_void, TrackpadWrapFn>(trackpad_wrap), + button: std::mem::transmute::<*mut c_void, ButtonFn>(button), + create_pointer_service: framework::dlsym(kit, "IndigoHIDMessageToCreatePointerService") + .map(|p| std::mem::transmute::<*mut c_void, ServiceFn>(p)), + create_mouse_service: framework::dlsym(kit, "IndigoHIDMessageToCreateMouseService") + .map(|p| std::mem::transmute::<*mut c_void, ServiceFn>(p)), + }) + } +} + +fn clamp01(v: f64) -> f64 { + v.clamp(0.0, 1.0) +} diff --git a/crates/linkcode-sim/src/private/mod.rs b/crates/linkcode-sim/src/private/mod.rs new file mode 100644 index 000000000..8bcea54e0 --- /dev/null +++ b/crates/linkcode-sim/src/private/mod.rs @@ -0,0 +1,21 @@ +//! Private-framework layer: live framebuffer capture and HID injection via CoreSimulator + +//! SimulatorKit. macOS-only; recipes ported from baguette (Apache-2.0, see the crate NOTICE). +//! +//! Everything here is best-effort and degrades: [`interactive_available`] reports whether the +//! frameworks resolved, and the sidecar falls back to public `simctl` (screenshot polling, no touch) +//! when they don't — so an Xcode without SimulatorKit is view-only, not broken. + +pub(crate) mod debug; +mod device; +mod framework; +mod input; +mod screen; + +pub use device::SimDevice; +pub use input::{Button, Input}; +pub use screen::{Screen, bench_encode}; + +/// Whether this host can drive simulators interactively (private frameworks resolved). +pub fn interactive_available() -> bool { + framework::load().is_some() +} diff --git a/crates/linkcode-sim/src/private/screen.rs b/crates/linkcode-sim/src/private/screen.rs new file mode 100644 index 000000000..61894a4d1 --- /dev/null +++ b/crates/linkcode-sim/src/private/screen.rs @@ -0,0 +1,940 @@ +//! Capturing the simulator framebuffer as JPEG. +//! +//! Uses CoreSimulator's `SimDisplayIOSurfaceRenderable` screen: `device.io` → `deviceIOPorts` → the +//! `com.apple.framebuffer.display` descriptor(s), then +//! `registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:`. +//! Registering those callbacks is what makes CoreSimulator composite the framebuffer headlessly (no +//! Simulator.app viewer needed) — `framebufferSurface` stays nil until delivery is active. +//! +//! The delivery model on the Xcode 26 `SimStreamProcessor` era is: `surfacesChangedCallback` fires +//! once at activation (and again only on a resize/realloc) with the live `IOSurface`; `frameCallback` +//! fires per composited frame; and the display surface then updates in place. So the surfaces +//! callback retains the delivered surface, and the frame callback (on the same serial queue, +//! synchronized with frame production) locks-and-copies it into an owned buffer; the reader thread +//! then does the CoreGraphics/ImageIO encode off the queue. The hot per-frame path therefore touches +//! only local IOSurface C calls — it never re-messages the CoreSimulator XPC proxy, which is what +//! made the earlier "poll `framebufferSurface` every frame" recipe abort intermittently inside the +//! proxy machinery. +//! +//! Two things this path must get right or it crashes hard: the callback blocks must be captureless +//! and reach state through a process global (`CURRENT_SINK`), because ROCK's XPC delivery does not +//! preserve a pointer captured in the block body; and the ImageIO options dictionary must use the +//! standard CFType callbacks, because ImageIO deep-copies and validates it. The path is still driven +//! only from the crash-isolated worker subprocess ([`crate::capture`]), never in the sidecar's main +//! process, so any residual hard abort (e.g. a cold-connect class race) is contained. + +use std::ffi::{CString, c_ulong, c_void}; +use std::ptr; +use std::sync::Mutex; +use std::sync::atomic::{AtomicPtr, Ordering}; + +use objc2::rc::Retained; +use objc2::runtime::{AnyObject, Sel}; +use objc2::sel; +use objc2_foundation::{NSString, NSUUID}; + +use super::debug::dbg_log; +use super::device::SimDevice; + +// Hand-rolled ObjC blocks for the three screen callbacks. block2's `RcBlock` crashes the +// CoreSimulator callback path; a hand-rolled global block with a signature is what the API accepts. +// The blocks are captureless: ROCK's XPC delivery does not preserve a pointer captured in the block +// body (it reads back null in the callback), so the callbacks reach state through [`CURRENT_SINK`] +// instead. All three share one layout so a single builder and invoke-pointer cast serve them. +#[repr(C)] +struct BlockDescriptor { + reserved: c_ulong, + size: c_ulong, + /// Objective-C type encoding of the block signature; required with `BLOCK_HAS_SIGNATURE`. + /// CoreSimulator reads it, so a signature-less block makes registration throw. + signature: *const i8, +} + +// SAFETY: descriptors are immutable; each signature points at a static NUL-terminated string. +unsafe impl Sync for BlockDescriptor {} + +#[repr(C)] +struct CallbackBlock { + isa: *const c_void, + flags: i32, + reserved: i32, + /// One of the three invoke fns below, stored type-erased; the descriptor's signature tells the + /// runtime the real ABI to call it with. + invoke: *const c_void, + descriptor: *const BlockDescriptor, +} + +/// The active stream's sink, reached by the callbacks. The blocks are handed to a CoreSimulator +/// `ROCKRemoteProxy` and delivered back through `ROCKInvocation`/XPC, which does NOT preserve a +/// pointer captured in the block body — a captured `sink` field reads back null in the callback and +/// segfaults. So the sink is reached through this process-global pointer instead, read from the +/// callback's own address space. `Screen::open` runs once per (worker) process, so a single global is +/// exactly right; it is set at registration and cleared on drop. +static CURRENT_SINK: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// The active sink, or `None` if no stream is registered. +fn current_sink() -> Option<&'static FrameSink> { + let ptr = CURRENT_SINK.load(Ordering::Acquire); + // SAFETY: when non-null, `ptr` is the boxed sink owned by the live `Screen` that set it; it is + // cleared to null in `Screen::drop` before the box is freed, so a non-null read is valid. + unsafe { ptr.as_ref() } +} + +unsafe extern "C" { + static _NSConcreteGlobalBlock: c_void; +} + +const BLOCK_IS_GLOBAL: i32 = 1 << 28; +const BLOCK_HAS_SIGNATURE: i32 = 1 << 30; +/// `void (^)(void)` — the per-frame callback (we copy on the reader, so this is a no-op). +const FRAME_SIGNATURE: &[u8] = b"v8@?0\0"; +/// `void (^)(IOSurface *unmasked, IOSurface *masked)` — the surfaces-changed callback. +const SURFACES_SIGNATURE: &[u8] = b"v24@?0@8@16\0"; +/// `void (^)(id)` — the properties-changed callback (no-op). +const PROPS_SIGNATURE: &[u8] = b"v16@?0@8\0"; + +static FRAME_DESCRIPTOR: BlockDescriptor = block_descriptor(FRAME_SIGNATURE); +static SURFACES_DESCRIPTOR: BlockDescriptor = block_descriptor(SURFACES_SIGNATURE); +static PROPS_DESCRIPTOR: BlockDescriptor = block_descriptor(PROPS_SIGNATURE); + +const fn block_descriptor(signature: &'static [u8]) -> BlockDescriptor { + BlockDescriptor { + reserved: 0, + size: size_of::() as c_ulong, + signature: signature.as_ptr().cast::(), + } +} + +/// Per-composited-frame callback: copy the current surface into an owned frame. Runs on +/// CoreSimulator's serial delivery queue, synchronized with frame production, so the surface backing +/// is valid here — it is NOT valid to retain and read later on another thread (recycled buffer). Only +/// the memcpy happens here; the reader thread does the CoreGraphics/ImageIO encode. +unsafe extern "C" fn frame_invoke(_block: *mut CallbackBlock) { + if let Some(sink) = current_sink() { + sink.grab_current(); + } +} + +/// Surfaces-changed callback: retain the delivered unmasked (full, bezel-free) surface as the current +/// framebuffer; ignore the masked one (rounded-corner alpha we don't want). Also copies it once right +/// away — the frame callback only fires on *new* composited frames, so a static screen (no motion +/// after boot) would otherwise never yield a first frame. +unsafe extern "C" fn surfaces_invoke( + _block: *mut CallbackBlock, + unmasked: *mut c_void, + _masked: *mut c_void, +) { + if let Some(sink) = current_sink() { + sink.set_current(unmasked); + sink.grab_current(); + } +} + +/// Properties-changed callback: no-op. +unsafe extern "C" fn props_invoke(_block: *mut CallbackBlock, _props: *mut c_void) {} + +/// A raw BGRA frame copied out of an `IOSurface`. Owned pixels, so it outlives the surface lock and +/// can be encoded off the callback queue. +struct RawFrame { + width: usize, + height: usize, + stride: usize, + pixels: Vec, +} + +/// A CFRetained `IOSurface`, released on drop. The surfaces callback delivers the surface borrowed +/// (valid only for that call), so we retain it to hold across later frame callbacks — the way +/// `SimDisplayView` retains the surface it renders each display frame. Its content updates in place. +struct Held(*mut c_void); + +// SAFETY: a retained IOSurface is safe to reference until released; it is only locked/copied on the +// serial delivery queue. +unsafe impl Send for Held {} + +impl Held { + fn new(surface: *mut c_void) -> Held { + // SAFETY: surface is a live IOSurface (CFType) for the duration of the delivering callback; + // retain to hold it past that call. + unsafe { CFRetain(surface.cast_const()) }; + Held(surface) + } +} + +impl Drop for Held { + fn drop(&mut self) { + // SAFETY: balances the CFRetain in `new`. + unsafe { CFRelease(self.0.cast_const()) }; + } +} + +/// Shared state between the callbacks (CoreSimulator's serial delivery queue) and the capture reader +/// (the sidecar's request thread). `current` is the retained live surface, swapped by the surfaces +/// callback and copied by the frame callback — both on the same serial queue. `latest` is the owned +/// copy the reader encodes. +struct FrameSink { + current: Mutex>, + latest: Mutex>>, +} + +// SAFETY: both fields are mutex-guarded; the surface is only locked/copied on the delivery queue. +unsafe impl Send for FrameSink {} +unsafe impl Sync for FrameSink {} + +impl FrameSink { + /// Adopt a freshly delivered surface as current, retaining it and releasing the previous one. + fn set_current(&self, surface: *mut c_void) { + if surface.is_null() { + return; + } + *self.current.lock().expect("frame sink mutex poisoned") = Some(Held::new(surface)); + } + + /// Copy the current surface into `latest`. Called from the frame callback on the delivery queue, + /// where the retained surface's backing is live and coherent with the just-composited frame. + fn grab_current(&self) { + let frame = { + let guard = self.current.lock().expect("frame sink mutex poisoned"); + let Some(held) = guard.as_ref() else { + return; + }; + copy_surface(held.0) + }; + if let Some(frame) = frame { + *self.latest.lock().expect("frame sink mutex poisoned") = + Some(std::sync::Arc::new(frame)); + } + } + + fn latest(&self) -> Option> { + self.latest + .lock() + .expect("frame sink mutex poisoned") + .clone() + } +} + +/// Copy the pixels of a locked BGRA surface into an owned [`RawFrame`]. Minimal surface access: read +/// dimensions, lock read-only, memcpy row by row (surfaces can be padded), unlock. +fn copy_surface(surface: *mut c_void) -> Option { + if surface.is_null() { + return None; + } + // SAFETY: surface is a live IOSurface; read-only lock, bounded row copies, matching unlock. + unsafe { + const READ_ONLY: u32 = 1; + if IOSurfaceLock(surface, READ_ONLY, ptr::null_mut()) != 0 { + return None; + } + let width = IOSurfaceGetWidth(surface); + let height = IOSurfaceGetHeight(surface); + let stride = IOSurfaceGetBytesPerRow(surface); + let base = IOSurfaceGetBaseAddress(surface); + let frame = if base.is_null() || width == 0 || height == 0 || stride < width * 4 { + None + } else { + let mut pixels = vec![0u8; stride * height]; + std::ptr::copy_nonoverlapping(base.cast::(), pixels.as_mut_ptr(), stride * height); + Some(RawFrame { + width, + height, + stride, + pixels, + }) + }; + IOSurfaceUnlock(surface, READ_ONLY, ptr::null_mut()); + frame + } +} + +/// Build a leaked, captureless callback block with `invoke`/`descriptor`. It carries no context — +/// the callbacks reach the sink through [`CURRENT_SINK`], since a captured pointer does not survive +/// ROCK's XPC delivery. +fn callback_block(invoke: *const c_void, descriptor: *const BlockDescriptor) -> *mut c_void { + let block = Box::new(CallbackBlock { + isa: (&raw const _NSConcreteGlobalBlock).cast::(), + flags: BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE, + reserved: 0, + invoke, + descriptor, + }); + // Leak: CoreSimulator retains the block for the whole registration. + (Box::into_raw(block) as *mut c_void).cast() +} + +// The framebuffer ports are CoreSimulator XPC proxies (`ROCKRemoteProxy`) that answer via +// `forwardInvocation:` — the messaged selectors are not in their class method lists, so objc2's +// `msg_send!` (which verifies against the method list) rejects them. Raw `objc_msgSend` uses the +// runtime's real dispatch, which honors forwarding. +unsafe extern "C" { + fn objc_msgSend(); +} + +/// Send a no-argument, object-returning message via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live object that handles `sel` (directly or by forwarding); `sel` must name +/// a zero-argument, object-returning method. +unsafe fn send_obj(receiver: *mut AnyObject, sel: Sel) -> *mut AnyObject { + // SAFETY: transmute the runtime dispatcher to this call's concrete ABI, per the fn contract. + let f: unsafe extern "C" fn(*mut AnyObject, Sel) -> *mut AnyObject = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel) } +} + +/// Send `respondsToSelector:` via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live object. +unsafe fn responds_to(receiver: *mut AnyObject, target: Sel) -> bool { + // SAFETY: respondsToSelector: takes a SEL and returns BOOL; transmute to that ABI. + let f: unsafe extern "C" fn(*mut AnyObject, Sel, Sel) -> bool = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel!(respondsToSelector:), target) } +} + +/// Send `objectAtIndex:` via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live NSArray and `index` within bounds. +unsafe fn object_at_index(receiver: *mut AnyObject, index: usize) -> *mut AnyObject { + // SAFETY: objectAtIndex: takes NSUInteger, returns id; transmute to that ABI. + let f: unsafe extern "C" fn(*mut AnyObject, Sel, usize) -> *mut AnyObject = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel!(objectAtIndex:), index) } +} + +/// Send `count` via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live NSArray. +unsafe fn array_count(receiver: *mut AnyObject) -> usize { + // SAFETY: count returns NSUInteger; transmute to that ABI. + let f: unsafe extern "C" fn(*mut AnyObject, Sel) -> usize = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel!(count)) } +} + +/// Send `valueForKey:` via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live object and `key` a live NSString. +unsafe fn value_for_key_raw(receiver: *mut AnyObject, key: *mut AnyObject) -> *mut AnyObject { + // SAFETY: valueForKey: takes id, returns id; transmute to that ABI. + let f: unsafe extern "C" fn(*mut AnyObject, Sel, *mut AnyObject) -> *mut AnyObject = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel!(valueForKey:), key) } +} + +// IOSurface (public framework). +#[link(name = "IOSurface", kind = "framework")] +unsafe extern "C" { + fn IOSurfaceLock(surface: *mut c_void, options: u32, seed: *mut u32) -> i32; + fn IOSurfaceUnlock(surface: *mut c_void, options: u32, seed: *mut u32) -> i32; + fn IOSurfaceGetBaseAddress(surface: *mut c_void) -> *mut c_void; + fn IOSurfaceGetWidth(surface: *mut c_void) -> usize; + fn IOSurfaceGetHeight(surface: *mut c_void) -> usize; + fn IOSurfaceGetBytesPerRow(surface: *mut c_void) -> usize; +} + +// CoreGraphics (public framework). +#[link(name = "CoreGraphics", kind = "framework")] +unsafe extern "C" { + fn CGColorSpaceCreateDeviceRGB() -> *mut c_void; + fn CGColorSpaceRelease(space: *mut c_void); + fn CGBitmapContextCreate( + data: *mut c_void, + width: usize, + height: usize, + bits_per_component: usize, + bytes_per_row: usize, + space: *mut c_void, + bitmap_info: u32, + ) -> *mut c_void; + fn CGBitmapContextCreateImage(context: *mut c_void) -> *mut c_void; + fn CGContextRelease(context: *mut c_void); + fn CGContextDrawImage(context: *mut c_void, rect: CGRect, image: *mut c_void); + fn CGContextSetInterpolationQuality(context: *mut c_void, quality: i32); + fn CGImageRelease(image: *mut c_void); +} + +#[repr(C)] +struct CGPoint { + x: f64, + y: f64, +} +#[repr(C)] +struct CGSize { + width: f64, + height: f64, +} +#[repr(C)] +struct CGRect { + origin: CGPoint, + size: CGSize, +} + +/// `kCGInterpolationHigh` — best downscale quality (Lanczos-ish), worth it since the encode of the +/// smaller image is what we saved time on. +const KCG_INTERPOLATION_HIGH: i32 = 3; + +// ImageIO (public framework). +#[link(name = "ImageIO", kind = "framework")] +unsafe extern "C" { + fn CGImageDestinationCreateWithData( + data: *mut c_void, + type_: *const c_void, + count: usize, + options: *const c_void, + ) -> *mut c_void; + fn CGImageDestinationAddImage(dest: *mut c_void, image: *mut c_void, properties: *const c_void); + fn CGImageDestinationFinalize(dest: *mut c_void) -> bool; + static kCGImageDestinationLossyCompressionQuality: *const c_void; +} + +// CoreFoundation (public framework). +#[link(name = "CoreFoundation", kind = "framework")] +unsafe extern "C" { + fn CFRetain(cf: *const c_void) -> *const c_void; + fn CFRelease(cf: *const c_void); + fn CFDataCreateMutable(allocator: *const c_void, capacity: isize) -> *mut c_void; + fn CFDataGetLength(data: *const c_void) -> isize; + fn CFDataGetBytePtr(data: *const c_void) -> *const u8; + fn CFNumberCreate( + allocator: *const c_void, + type_: isize, + value: *const c_void, + ) -> *const c_void; + fn CFStringCreateWithCString( + allocator: *const c_void, + cstr: *const i8, + encoding: u32, + ) -> *const c_void; + fn CFDictionaryCreate( + allocator: *const c_void, + keys: *const *const c_void, + values: *const *const c_void, + count: isize, + key_callbacks: *const c_void, + value_callbacks: *const c_void, + ) -> *const c_void; + static kCFTypeDictionaryKeyCallBacks: c_void; + static kCFTypeDictionaryValueCallBacks: c_void; +} + +// dispatch (public): a serial queue for CoreSimulator to deliver the screen callbacks on. +unsafe extern "C" { + fn dispatch_queue_create(label: *const i8, attr: *const c_void) -> *mut c_void; +} + +const KCG_ALPHA_PREMULTIPLIED_FIRST: u32 = 2; +const KCG_BYTE_ORDER_32_LITTLE: u32 = 2 << 12; +const KCF_NUMBER_DOUBLE_TYPE: isize = 13; +const KCF_STRING_ENCODING_UTF8: u32 = 0x0800_0100; + +/// How long [`Screen::open`] waits for the first surface to be delivered before giving up. +const ACTIVATION_TIMEOUT_MS: u64 = 2000; + +/// The framebuffer of one booted device, held for a streaming session. Registration makes +/// CoreSimulator composite the framebuffer headlessly and push the display surface into the sink; the +/// reader copies that held surface on demand. The registration UUID is retained to unregister on drop. +pub struct Screen { + /// Every `com.apple.framebuffer.display` descriptor we registered on, retained. A device exposes + /// several (main LCD plus secondary/overlay planes); we register on all and hold the surface each + /// delivers. + descriptors: Vec>, + uuid: Retained, + /// The surfaces callback stashes the current surface here; the reader copies it. Boxed so the + /// callback's captured raw pointer stays valid, and dropped only after callbacks are unregistered. + sink: Box, + /// The serial dispatch queue CoreSimulator delivers callbacks on; retained for the registration. + queue: *mut c_void, +} + +// SAFETY: the descriptors are messaged read-only, the sink is internally synchronized, and the queue +// is only handed to CoreSimulator and released on drop. +unsafe impl Send for Screen {} + +impl Drop for Screen { + fn drop(&mut self) { + // Stop the callbacks from reaching the sink, then unregister. Cleared before the boxed sink is + // freed so an in-flight callback either sees the live sink or (after this) sees null. + CURRENT_SINK.store(ptr::null_mut(), Ordering::Release); + // Unregister so no further callback can fire. + let uuid = Retained::as_ptr(&self.uuid).cast_mut().cast::(); + for descriptor in &self.descriptors { + let descriptor = Retained::as_ptr(descriptor).cast_mut(); + if unsafe { responds_to(descriptor, sel!(unregisterScreenCallbacksWithUUID:)) } { + // SAFETY: unregister takes the UUID we registered with. + let f: unsafe extern "C" fn(*mut AnyObject, Sel, *mut AnyObject) = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(descriptor, sel!(unregisterScreenCallbacksWithUUID:), uuid) }; + } + } + if !self.queue.is_null() { + // SAFETY: the queue was created +1 by dispatch_queue_create and handed to CoreSimulator, + // which retained it for the registration we just tore down; release our reference. + unsafe { CFRelease(self.queue.cast_const()) }; + } + } +} + +impl Screen { + /// Open the framebuffer of a booted device and start surface delivery. `None` when the device has + /// no framebuffer (not booted), the IO plumbing is unavailable, no descriptor supports the screen + /// callbacks, or no surface is delivered within the activation timeout (the caller degrades to + /// simctl). + pub fn open(device: &SimDevice) -> Option { + // The framebuffer probe messages CoreSimulator XPC proxies whose forwarding autoreleases + // reply objects; hold a pool for the whole probe, and retain the descriptors (they survive + // the drain) before returning. + objc2::rc::autoreleasepool(|_| { + let io = io_client(device)?; + // Populate the port list lazily. + // SAFETY: io is a live SimDeviceIOClient that handles updateIOPorts (returns void). + unsafe { send_obj(Retained::as_ptr(&io).cast_mut(), sel!(updateIOPorts)) }; + let ports = value_for_key(&io, "deviceIOPorts")?; + let ports_ptr = Retained::as_ptr(&ports).cast_mut(); + // SAFETY: deviceIOPorts is an NSArray. + let count = unsafe { array_count(ports_ptr) }; + dbg_log!("open: {count} io ports"); + let mut descriptors = Vec::new(); + for index in 0..count { + // SAFETY: index < count. + let port = unsafe { object_at_index(ports_ptr, index) }; + if port.is_null() { + continue; + } + if let Some(descriptor) = framebuffer_descriptor(port, index) { + descriptors.push(descriptor); + } + } + if descriptors.is_empty() { + return None; + } + let screen = register(descriptors)?; + // Wait for delivery to activate (the surfaces callback fires ~once at activation, and the + // proxy also exposes `framebufferSurface` once live). If neither yields a surface, the + // connection is dead/degraded — report None so the caller degrades to simctl. + if screen.await_first_surface() { + Some(screen) + } else { + dbg_log!("open: no surface within activation timeout"); + None + } + }) + } + + /// Wait for the first frame to be copied out by the frame callback, up to + /// [`ACTIVATION_TIMEOUT_MS`]. A copied frame (not just a delivered surface pointer) proves the + /// whole delivery path is live. + fn await_first_surface(&self) -> bool { + for _ in 0..(ACTIVATION_TIMEOUT_MS / 10) { + if self.sink.latest().is_some() { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + false + } + + /// The latest copied framebuffer encoded as JPEG at `quality` (0..1), downscaled by `scale` + /// (0..1; 1.0 = native), or `None` if no frame has been copied yet. The frame callback did the + /// surface copy on the delivery queue; this reader thread only encodes the owned pixels — the + /// CoreSimulator proxy is never messaged on this path. + pub fn capture_jpeg(&self, quality: f64, scale: f64) -> Option> { + let frame = self.sink.latest()?; + encode_bgra_jpeg(&frame, quality.clamp(0.1, 1.0), scale.clamp(0.1, 1.0)) + } +} + +/// Encode an owned BGRA [`RawFrame`] to JPEG bytes via CoreGraphics + ImageIO, optionally downscaled +/// by `scale`. Runs on the reader thread from owned pixels — no live surface involved. Downscaling +/// (CoreGraphics resample into a smaller context) cuts the JPEG encode cost roughly by `scale²`, +/// which is the lever for a higher locked frame rate and lower bandwidth. +fn encode_bgra_jpeg(frame: &RawFrame, quality: f64, scale: f64) -> Option> { + // SAFETY: pixels back a `width×height`, `stride`-padded BGRA buffer; every CF/CG object created + // here is released on all paths. + unsafe { + let color_space = CGColorSpaceCreateDeviceRGB(); + let bitmap_info = KCG_ALPHA_PREMULTIPLIED_FIRST | KCG_BYTE_ORDER_32_LITTLE; + let context = CGBitmapContextCreate( + frame.pixels.as_ptr().cast_mut().cast::(), + frame.width, + frame.height, + 8, + frame.stride, + color_space, + bitmap_info, + ); + if context.is_null() { + CGColorSpaceRelease(color_space); + return None; + } + let image = CGBitmapContextCreateImage(context); + CGContextRelease(context); + if image.is_null() { + CGColorSpaceRelease(color_space); + return None; + } + // Optionally resample into a smaller context before encoding. + let scaled = if scale < 0.999 { + downscale_image( + image, + frame.width, + frame.height, + scale, + color_space, + bitmap_info, + ) + } else { + ptr::null_mut() + }; + CGColorSpaceRelease(color_space); + let target = if scaled.is_null() { image } else { scaled }; + let bytes = encode_cgimage_jpeg(target, quality); + if !scaled.is_null() { + CGImageRelease(scaled); + } + CGImageRelease(image); + bytes + } +} + +/// Draw `image` into a `scale`-sized context and return the resampled `CGImage` (caller releases), or +/// null on failure. `color_space`/`bitmap_info` must match the source so colors are preserved. +/// +/// # Safety +/// `image` must be a live CGImageRef and `color_space` a live CGColorSpaceRef. +unsafe fn downscale_image( + image: *mut c_void, + width: usize, + height: usize, + scale: f64, + color_space: *mut c_void, + bitmap_info: u32, +) -> *mut c_void { + unsafe { + let w = (((width as f64) * scale).round() as usize).max(1); + let h = (((height as f64) * scale).round() as usize).max(1); + // bytes_per_row 0 lets CoreGraphics pick an aligned stride for the allocation it owns. + let ctx = CGBitmapContextCreate(ptr::null_mut(), w, h, 8, 0, color_space, bitmap_info); + if ctx.is_null() { + return ptr::null_mut(); + } + CGContextSetInterpolationQuality(ctx, KCG_INTERPOLATION_HIGH); + let rect = CGRect { + origin: CGPoint { x: 0.0, y: 0.0 }, + size: CGSize { + width: w as f64, + height: h as f64, + }, + }; + CGContextDrawImage(ctx, rect, image); + let scaled = CGBitmapContextCreateImage(ctx); + CGContextRelease(ctx); + scaled + } +} + +/// The measured cost of encoding one framebuffer at a given resolution/quality, with the frame rate +/// it implies (a single reader thread does the encode, so `1000 / avg_ms` is the sustainable ceiling). +pub struct EncodeBench { + pub width: usize, + pub height: usize, + pub quality: f64, + pub avg_ms: f64, + pub best_ms: f64, + pub p95_ms: f64, + pub out_kib: usize, +} + +impl EncodeBench { + /// Sustainable frames per second at the average encode cost. + pub fn fps(&self) -> f64 { + if self.avg_ms > 0.0 { + 1000.0 / self.avg_ms + } else { + 0.0 + } + } + + /// Best-case frames per second (fastest encode observed). + pub fn peak_fps(&self) -> f64 { + if self.best_ms > 0.0 { + 1000.0 / self.best_ms + } else { + 0.0 + } + } +} + +/// Benchmark the JPEG encode hot path (the capture pipeline's single-thread ceiling) on a synthetic +/// BGRA frame of `width`×`height`, timing `iters` encodes at `quality` after a warmup. No simulator +/// is needed — this isolates the CoreGraphics/ImageIO cost that bounds the stream's frame rate. +pub fn bench_encode(width: usize, height: usize, quality: f64, iters: u32) -> Option { + let stride = width * 4; + // A gradient plus a per-pixel wobble so the JPEG has realistic entropy (a flat frame would + // compress trivially and understate the encode cost). + let mut pixels = vec![0u8; stride * height]; + for y in 0..height { + let row = &mut pixels[y * stride..y * stride + width * 4]; + for x in 0..width { + let p = &mut row[x * 4..x * 4 + 4]; + p[0] = (x ^ y) as u8; // B + p[1] = (x.wrapping_add(y)) as u8; // G + p[2] = (x.wrapping_mul(3) ^ y) as u8; // R + p[3] = 0xFF; // A + } + } + let frame = RawFrame { + width, + height, + stride, + pixels, + }; + let q = quality.clamp(0.1, 1.0); + + // Warm up: the first encode pays one-time ImageIO/CoreGraphics setup. The sweep sizes the frame + // directly, so encode at native scale here. + let mut out_len = 0usize; + for _ in 0..3 { + out_len = encode_bgra_jpeg(&frame, q, 1.0)?.len(); + } + + let mut samples = Vec::with_capacity(iters as usize); + for _ in 0..iters { + let start = std::time::Instant::now(); + let jpeg = encode_bgra_jpeg(&frame, q, 1.0)?; + samples.push(start.elapsed().as_secs_f64() * 1000.0); + out_len = jpeg.len(); + } + samples.sort_by(|a, b| a.partial_cmp(b).expect("no NaN durations")); + let avg_ms = samples.iter().sum::() / samples.len() as f64; + let best_ms = samples[0]; + let p95_ms = samples[(samples.len() * 95 / 100).min(samples.len() - 1)]; + + Some(EncodeBench { + width, + height, + quality: q, + avg_ms, + best_ms, + p95_ms, + out_kib: out_len / 1024, + }) +} + +/// Write a `CGImage` to JPEG via ImageIO at `quality`. +/// +/// # Safety +/// `image` must be a live CGImageRef. +unsafe fn encode_cgimage_jpeg(image: *mut c_void, quality: f64) -> Option> { + unsafe { + let data = CFDataCreateMutable(ptr::null(), 0); + if data.is_null() { + return None; + } + let jpeg_uti = CString::new("public.jpeg").ok()?; + let type_ = + CFStringCreateWithCString(ptr::null(), jpeg_uti.as_ptr(), KCF_STRING_ENCODING_UTF8); + let options = quality_dictionary(quality); + let dest = CGImageDestinationCreateWithData(data, type_, 1, options); + if !type_.is_null() { + CFRelease(type_); + } + if !options.is_null() { + CFRelease(options); + } + if dest.is_null() { + CFRelease(data.cast_const()); + return None; + } + CGImageDestinationAddImage(dest, image, ptr::null()); + let ok = CGImageDestinationFinalize(dest); + CFRelease(dest.cast_const()); + let out = if ok { + let len = CFDataGetLength(data.cast_const()); + let bytes = CFDataGetBytePtr(data.cast_const()); + if len > 0 && !bytes.is_null() { + Some(std::slice::from_raw_parts(bytes, len as usize).to_vec()) + } else { + None + } + } else { + None + }; + CFRelease(data.cast_const()); + out + } +} + +/// A one-entry `{ kCGImageDestinationLossyCompressionQuality: quality }` dictionary, or null on +/// failure. Uses the standard CFType key/value callbacks so the dict properly retains its CF +/// members — ImageIO deep-copies and validates the properties dict, which crashes on a dict built +/// with null callbacks (its members are then treated as opaque, non-CF pointers). +/// +/// # Safety +/// Caller releases the returned dictionary. +unsafe fn quality_dictionary(quality: f64) -> *const c_void { + unsafe { + let number = CFNumberCreate( + ptr::null(), + KCF_NUMBER_DOUBLE_TYPE, + (&raw const quality).cast(), + ); + if number.is_null() { + return ptr::null(); + } + let keys = [kCGImageDestinationLossyCompressionQuality]; + let values = [number]; + let dict = CFDictionaryCreate( + ptr::null(), + keys.as_ptr(), + values.as_ptr(), + 1, + (&raw const kCFTypeDictionaryKeyCallBacks).cast(), + (&raw const kCFTypeDictionaryValueCallBacks).cast(), + ); + // The dict retained the number under the CFType callbacks; drop our reference. + CFRelease(number); + dict + } +} + +/// Register the three screen callbacks on every framebuffer descriptor so CoreSimulator composites +/// and delivers the display surface (no Simulator.app viewer needed). Returns the assembled `Screen`, +/// or `None` if no descriptor supports the registration. +fn register(descriptors: Vec>) -> Option { + let uuid = NSUUID::new(); + let label = c"ai.linkcode.sim.framebuffer"; + // SAFETY: dispatch_queue_create with a static label and null (serial) attr returns a +1 queue. + let queue = unsafe { dispatch_queue_create(label.as_ptr(), ptr::null()) }; + // The sink is boxed (stable address) and owned by the Screen; the callbacks reach it through + // CURRENT_SINK. Publish it before registering so an immediately-delivered callback finds it. + let sink = Box::new(FrameSink { + current: Mutex::new(None), + latest: Mutex::new(None), + }); + CURRENT_SINK.store((&*sink as *const FrameSink).cast_mut(), Ordering::Release); + let register_sel = sel!(registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:); + let mut registered = 0; + for descriptor in &descriptors { + let descriptor_ptr = Retained::as_ptr(descriptor).cast_mut(); + if !unsafe { responds_to(descriptor_ptr, register_sel) } { + continue; + } + let frame = callback_block(frame_invoke as *const c_void, &raw const FRAME_DESCRIPTOR); + let surfaces = callback_block( + surfaces_invoke as *const c_void, + &raw const SURFACES_DESCRIPTOR, + ); + let props = callback_block(props_invoke as *const c_void, &raw const PROPS_DESCRIPTOR); + // SAFETY: the 5-object-argument registration selector. UUID + queue stay alive in the Screen; + // the blocks are leaked captureless globals; the sink is reached via CURRENT_SINK. + unsafe { + let f: unsafe extern "C" fn( + *mut AnyObject, + Sel, + *mut AnyObject, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + ) = std::mem::transmute(objc_msgSend as *const c_void); + f( + descriptor_ptr, + register_sel, + Retained::as_ptr(&uuid).cast_mut().cast::(), + queue, + frame, + surfaces, + props, + ); + } + registered += 1; + } + dbg_log!("registered {registered} framebuffer descriptors"); + if registered == 0 { + CURRENT_SINK.store(ptr::null_mut(), Ordering::Release); + if !queue.is_null() { + // SAFETY: nothing adopted the queue; release the +1 from dispatch_queue_create. + unsafe { CFRelease(queue.cast_const()) }; + } + return None; + } + Some(Screen { + descriptors, + uuid, + sink, + queue, + }) +} + +/// Probe one IO port: if it is the `com.apple.framebuffer.display` port, retain and return its +/// descriptor. +fn framebuffer_descriptor(port: *mut AnyObject, index: usize) -> Option> { + if !unsafe { responds_to(port, sel!(portIdentifier)) } { + return None; + } + let pid = unsafe { send_obj(port, sel!(portIdentifier)) }; + let identifier = if pid.is_null() { + String::new() + } else { + nsstring_utf8(pid) + }; + dbg_log!("port {index}: identifier={identifier:?}"); + if identifier != "com.apple.framebuffer.display" { + return None; + } + if !unsafe { responds_to(port, sel!(descriptor)) } { + return None; + } + let descriptor = unsafe { send_obj(port, sel!(descriptor)) }; + if descriptor.is_null() || !unsafe { responds_to(descriptor, sel!(framebufferSurface)) } { + return None; + } + // SAFETY: descriptor is a live +0 object; retain to hold across the session. + unsafe { Retained::retain(descriptor) } +} + +fn io_client(device: &SimDevice) -> Option> { + // SAFETY: `io` returns the device's SimDeviceIOClient (+0) or nil. + let io = unsafe { send_obj(device.object_ptr(), sel!(io)) }; + if io.is_null() { + return None; + } + // SAFETY: io is a live +0 object; retain to hold across the session. + unsafe { Retained::retain(io) } +} + +fn value_for_key(object: &Retained, key: &str) -> Option> { + let key = NSString::from_str(key); + let key_ptr = (&*key as *const NSString).cast_mut().cast::(); + // SAFETY: KVC read via raw dispatch. + let value = unsafe { value_for_key_raw(Retained::as_ptr(object).cast_mut(), key_ptr) }; + if value.is_null() { + return None; + } + // SAFETY: value is a live +0 object. + unsafe { Retained::retain(value) } +} + +/// Read an NSString-typed object's bytes via raw `UTF8String`. The port identifier is an NSString; +/// reading it through objc2's `NSString::to_string` would re-message via the verifying macro and can +/// trip over CoreSimulator's string proxies, so use raw dispatch and guard with respondsToSelector:. +fn nsstring_utf8(object: *mut AnyObject) -> String { + if object.is_null() || !unsafe { responds_to(object, sel!(UTF8String)) } { + return String::new(); + } + // SAFETY: UTF8String returns a const char* valid until the autorelease pool drains. + let ptr = unsafe { + let f: unsafe extern "C" fn(*mut AnyObject, Sel) -> *const i8 = + std::mem::transmute(objc_msgSend as *const c_void); + f(object, sel!(UTF8String)) + }; + if ptr.is_null() { + return String::new(); + } + // SAFETY: ptr is a NUL-terminated C string from UTF8String. + unsafe { std::ffi::CStr::from_ptr(ptr).to_string_lossy().into_owned() } +} diff --git a/crates/linkcode-sim/src/proto.rs b/crates/linkcode-sim/src/proto.rs index edbdac235..5ffd5c660 100644 --- a/crates/linkcode-sim/src/proto.rs +++ b/crates/linkcode-sim/src/proto.rs @@ -18,6 +18,24 @@ pub const REQUEST: u8 = 0x01; // Sidecar → daemon. pub const RESULT: u8 = 0x81; pub const SCREENSHOT: u8 = 0x82; +/// A streamed framebuffer frame: `[u16 LE udid_len][udid][jpeg]` (unsolicited, while a stream runs). +pub const STREAM_FRAME: u8 = 0x83; + +/// Encode a `STREAM_FRAME` body: `[u16 LE udid_len][udid][jpeg bytes]`. +pub fn encode_stream_frame(udid: &str, jpeg: &[u8]) -> io::Result> { + let id = udid.as_bytes(); + if id.is_empty() || id.len() > MAX_REQUEST_ID_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid udid length", + )); + } + let mut out = Vec::with_capacity(2 + id.len() + jpeg.len()); + out.extend_from_slice(&(id.len() as u16).to_le_bytes()); + out.extend_from_slice(id); + out.extend_from_slice(jpeg); + Ok(out) +} /// Read one frame. Returns `Ok(None)` on a clean end-of-stream (the daemon closed the pipe). pub fn read_frame(reader: &mut impl Read) -> io::Result)>> { diff --git a/crates/linkcode-sim/src/rpc.rs b/crates/linkcode-sim/src/rpc.rs index 84cbf343f..cd1484529 100644 --- a/crates/linkcode-sim/src/rpc.rs +++ b/crates/linkcode-sim/src/rpc.rs @@ -53,6 +53,50 @@ pub enum Op { #[serde(default)] format: ImageFormat, }, + /// Single-finger tap at a normalised (0..1) point (private API; P1). + Tap { udid: String, x: f64, y: f64 }, + /// Swipe between two normalised (0..1) points over `duration_ms` (private API; P1). + Swipe { + udid: String, + x0: f64, + y0: f64, + x1: f64, + y1: f64, + #[serde(default)] + duration_ms: u64, + }, + /// Press a hardware button (private API; P1). + Button { udid: String, button: ButtonKind }, + /// Start streaming the device framebuffer as JPEG `FRAME`s at `fps` (private API; P1). + StreamStart { + udid: String, + #[serde(default = "default_fps")] + fps: u32, + #[serde(default = "default_quality")] + quality: f64, + #[serde(default = "default_scale")] + scale: f64, + }, + /// Stop a running framebuffer stream. + StreamStop { udid: String }, +} + +fn default_fps() -> u32 { + 60 +} +fn default_quality() -> f64 { + 0.6 +} +fn default_scale() -> f64 { + 1.0 +} + +/// Hardware buttons exposable over the wire (extended as the private HID layer grows). +#[derive(Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ButtonKind { + Home, + Lock, } /// Screenshot encodings supported by `simctl io screenshot --type`.