[SR] Capture gestures/motion events

sentry-android-replay/api/sentry-android-replay.api

@@ -37,7 +37,7 @@ public final class io/sentry/android/replay/ReplayCache : java/io/Closeable {
 	public final fun rotate (J)V
 }
 
-public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, java/io/Closeable {
+public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/TouchRecorderCallback, java/io/Closeable {
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)V
 	public synthetic fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
@@ -49,6 +49,7 @@ public final class io/sentry/android/replay/ReplayIntegration : android/content/
 	public fun onLowMemory ()V
 	public fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V
 	public fun onScreenshotRecorded (Ljava/io/File;J)V
+	public fun onTouchEvent (Landroid/view/MotionEvent;)V
 	public fun pause ()V
 	public fun register (Lio/sentry/IHub;Lio/sentry/SentryOptions;)V
 	public fun resume ()V
@@ -89,6 +90,40 @@ public final class io/sentry/android/replay/ScreenshotRecorderConfig$Companion {
 	public final fun from (Landroid/content/Context;Lio/sentry/SentryReplayOptions;)Lio/sentry/android/replay/ScreenshotRecorderConfig;
 }
 
+public abstract interface class io/sentry/android/replay/TouchRecorderCallback {
+	public abstract fun onTouchEvent (Landroid/view/MotionEvent;)V
+}
+
+public class io/sentry/android/replay/util/FixedWindowCallback : android/view/Window$Callback {
+	public final field delegate Landroid/view/Window$Callback;
+	public fun <init> (Landroid/view/Window$Callback;)V
+	public fun dispatchGenericMotionEvent (Landroid/view/MotionEvent;)Z
+	public fun dispatchKeyEvent (Landroid/view/KeyEvent;)Z
+	public fun dispatchKeyShortcutEvent (Landroid/view/KeyEvent;)Z
+	public fun dispatchPopulateAccessibilityEvent (Landroid/view/accessibility/AccessibilityEvent;)Z
+	public fun dispatchTouchEvent (Landroid/view/MotionEvent;)Z
+	public fun dispatchTrackballEvent (Landroid/view/MotionEvent;)Z
+	public fun onActionModeFinished (Landroid/view/ActionMode;)V
+	public fun onActionModeStarted (Landroid/view/ActionMode;)V
+	public fun onAttachedToWindow ()V
+	public fun onContentChanged ()V
+	public fun onCreatePanelMenu (ILandroid/view/Menu;)Z
+	public fun onCreatePanelView (I)Landroid/view/View;
+	public fun onDetachedFromWindow ()V
+	public fun onMenuItemSelected (ILandroid/view/MenuItem;)Z
+	public fun onMenuOpened (ILandroid/view/Menu;)Z
+	public fun onPanelClosed (ILandroid/view/Menu;)V
+	public fun onPointerCaptureChanged (Z)V
+	public fun onPreparePanel (ILandroid/view/View;Landroid/view/Menu;)Z
+	public fun onProvideKeyboardShortcuts (Ljava/util/List;Landroid/view/Menu;I)V
+	public fun onSearchRequested ()Z
+	public fun onSearchRequested (Landroid/view/SearchEvent;)Z
+	public fun onWindowAttributesChanged (Landroid/view/WindowManager$LayoutParams;)V
+	public fun onWindowFocusChanged (Z)V
+	public fun onWindowStartingActionMode (Landroid/view/ActionMode$Callback;)Landroid/view/ActionMode;
+	public fun onWindowStartingActionMode (Landroid/view/ActionMode$Callback;I)Landroid/view/ActionMode;
+}
+
 public abstract interface class io/sentry/android/replay/video/SimpleFrameMuxer {
 	public abstract fun getVideoTime ()J
 	public abstract fun isStarted ()Z

sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt

@@ -5,6 +5,7 @@ import android.content.Context
 import android.content.res.Configuration
 import android.graphics.Bitmap
 import android.os.Build
+import android.view.MotionEvent
 import io.sentry.Hint
 import io.sentry.IHub
 import io.sentry.Integration
@@ -32,7 +33,7 @@ public class ReplayIntegration(
     private val recorderProvider: (() -> Recorder)? = null,
     private val recorderConfigProvider: ((configChanged: Boolean) -> ScreenshotRecorderConfig)? = null,
     private val replayCacheProvider: ((replayId: SentryId) -> ReplayCache)? = null
-) : Integration, Closeable, ScreenshotRecorderCallback, ReplayController, ComponentCallbacks {
+) : Integration, Closeable, ScreenshotRecorderCallback, TouchRecorderCallback, ReplayController, ComponentCallbacks {
 
     // needed for the Java's call site
     constructor(context: Context, dateProvider: ICurrentDateProvider) : this(
@@ -72,7 +73,7 @@ public class ReplayIntegration(
         }
 
         this.hub = hub
-        recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this)
+        recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, this)
         isEnabled.set(true)
 
         try {
@@ -220,4 +221,8 @@ public class ReplayIntegration(
     }
 
     override fun onLowMemory() = Unit
+
+    override fun onTouchEvent(event: MotionEvent) {
+        captureStrategy?.onTouchEvent(event)
+    }
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt

@@ -1,8 +1,13 @@
 package io.sentry.android.replay
 
 import android.annotation.TargetApi
+import android.view.MotionEvent
 import android.view.View
+import android.view.Window
+import io.sentry.SentryLevel.DEBUG
+import io.sentry.SentryLevel.ERROR
 import io.sentry.SentryOptions
+import io.sentry.android.replay.util.FixedWindowCallback
 import io.sentry.android.replay.util.gracefullyShutdown
 import io.sentry.android.replay.util.scheduleAtFixedRateSafely
 import java.lang.ref.WeakReference
@@ -16,7 +21,8 @@ import kotlin.LazyThreadSafetyMode.NONE
 @TargetApi(26)
 internal class WindowRecorder(
     private val options: SentryOptions,
-    private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null
+    private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null,
+    private val touchRecorderCallback: TouchRecorderCallback? = null
 ) : Recorder {
 
     internal companion object {
@@ -39,7 +45,11 @@ internal class WindowRecorder(
         if (added) {
             rootViews.add(WeakReference(root))
             recorder?.bind(root)
+
+            root.startGestureTracking()
         } else {
+            root.stopGestureTracking()
+
             recorder?.unbind(root)
             rootViews.removeAll { it.get() == root }
 
@@ -86,6 +96,60 @@ internal class WindowRecorder(
         isRecording.set(false)
     }
 
+    override fun close() {
+        stop()
+        capturer.gracefullyShutdown(options)
+    }
+
+    private fun View.startGestureTracking() {
+        val window = phoneWindow
+        if (window == null) {
+            options.logger.log(DEBUG, "Window is invalid, not tracking gestures")
+            return
+        }
+
+        if (touchRecorderCallback == null) {
+            options.logger.log(DEBUG, "TouchRecorderCallback is null, not tracking gestures")
+            return
+        }
+
+        val delegate = window.callback
+        window.callback = SentryReplayGestureRecorder(options, touchRecorderCallback, delegate)
+    }
+
+    private fun View.stopGestureTracking() {
+        val window = phoneWindow
+        if (window == null) {
+            options.logger.log(DEBUG, "Window was null in stopGestureTracking")
+            return
+        }
+
+        if (window.callback is SentryReplayGestureRecorder) {
+            val delegate = (window.callback as SentryReplayGestureRecorder).delegate
+            window.callback = delegate
+        }
+    }
+
+    private class SentryReplayGestureRecorder(
+        private val options: SentryOptions,
+        private val touchRecorderCallback: TouchRecorderCallback?,
+        delegate: Window.Callback?
+    ) : FixedWindowCallback(delegate) {
+        override fun dispatchTouchEvent(event: MotionEvent?): Boolean {
+            if (event != null) {
+                val copy: MotionEvent = MotionEvent.obtainNoHistory(event)
+                try {
+                    touchRecorderCallback?.onTouchEvent(copy)
+                } catch (e: Throwable) {
+                    options.logger.log(ERROR, "Error dispatching touch event", e)
+                } finally {
+                    copy.recycle()
+                }
+            }
+            return super.dispatchTouchEvent(event)
+        }
+    }
+
     private class RecorderExecutorServiceThreadFactory : ThreadFactory {
         private var cnt = 0
         override fun newThread(r: Runnable): Thread {
@@ -94,9 +158,8 @@ internal class WindowRecorder(
             return ret
         }
     }
+}
 
-    override fun close() {
-        stop()
-        capturer.gracefullyShutdown(options)
-    }
+public interface TouchRecorderCallback {
+    fun onTouchEvent(event: MotionEvent)
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt

@@ -1,5 +1,6 @@
 package io.sentry.android.replay.capture
 
+import android.view.MotionEvent
 import io.sentry.Breadcrumb
 import io.sentry.DateUtils
 import io.sentry.Hint
@@ -17,13 +18,19 @@ import io.sentry.android.replay.util.submitSafely
 import io.sentry.protocol.SentryId
 import io.sentry.rrweb.RRWebBreadcrumbEvent
 import io.sentry.rrweb.RRWebEvent
+import io.sentry.rrweb.RRWebIncrementalSnapshotEvent
+import io.sentry.rrweb.RRWebInteractionEvent
+import io.sentry.rrweb.RRWebInteractionEvent.InteractionType
+import io.sentry.rrweb.RRWebInteractionMoveEvent
+import io.sentry.rrweb.RRWebInteractionMoveEvent.Position
 import io.sentry.rrweb.RRWebMetaEvent
 import io.sentry.rrweb.RRWebSpanEvent
 import io.sentry.rrweb.RRWebVideoEvent
 import io.sentry.transport.ICurrentDateProvider
 import io.sentry.util.FileUtils
 import java.io.File
 import java.util.Date
+import java.util.LinkedList
 import java.util.concurrent.Executors
 import java.util.concurrent.ScheduledExecutorService
 import java.util.concurrent.ThreadFactory
@@ -51,6 +58,10 @@ internal abstract class BaseCaptureStrategy(
             "http.response_content_length",
             "http.request_content_length"
         )
+
+        // rrweb values
+        private const val TOUCH_MOVE_DEBOUNCE_THRESHOLD = 50
+        private const val CAPTURE_MOVE_EVENT_THRESHOLD = 500
     }
 
     protected var cache: ReplayCache? = null
@@ -60,6 +71,12 @@ internal abstract class BaseCaptureStrategy(
     override val currentSegment = AtomicInteger(0)
     override val replayCacheDir: File? get() = cache?.replayCacheDir
 
+    protected val currentEvents = LinkedList<RRWebEvent>()
+    private val currentEventsLock = Any()
+    private val currentPositions = mutableListOf<Position>()
+    private var touchMoveBaseline = 0L
+    private var lastCapturedMoveEvent = 0L
+
     protected val replayExecutor: ScheduledExecutorService by lazy {
         executor ?: Executors.newSingleThreadScheduledExecutor(ReplayExecutorServiceThreadFactory())
     }
@@ -249,6 +266,12 @@ internal abstract class BaseCaptureStrategy(
             }
         }
 
+        rotateCurrentEvents(endTimestamp.time) { event ->
+            if (event.timestamp >= segmentTimestamp.time) {
+                recordingPayload += event
+            }
+        }
+
         val recording = ReplayRecording().apply {
             this.segmentId = segmentId
             payload = recordingPayload.sortedBy { it.timestamp }
@@ -265,10 +288,33 @@ internal abstract class BaseCaptureStrategy(
         this.recorderConfig = recorderConfig
     }
 
+    override fun onTouchEvent(event: MotionEvent) {
+        val rrwebEvent = event.toRRWebIncrementalSnapshotEvent()
+        if (rrwebEvent != null) {
+            synchronized(currentEventsLock) {
+                currentEvents += rrwebEvent
+            }
+        }
+    }
+
     override fun close() {
         replayExecutor.gracefullyShutdown(options)
     }
 
+    protected fun rotateCurrentEvents(
+        until: Long,
+        callback: ((RRWebEvent) -> Unit)? = null
+    ) {
+        synchronized(currentEventsLock) {
+            var event = currentEvents.peek()
+            while (event != null && event.timestamp <= until) {
+                callback?.invoke(event)
+                currentEvents.remove()
+                event = currentEvents.peek()
+            }
+        }
+    }
+
     private class ReplayExecutorServiceThreadFactory : ThreadFactory {
         private var cnt = 0
         override fun newThread(r: Runnable): Thread {
@@ -335,4 +381,63 @@ internal abstract class BaseCaptureStrategy(
             data = breadcrumbData
         }
     }
+
+    private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): RRWebIncrementalSnapshotEvent? {
+        val event = this
+        return when (val action = event.actionMasked) {
+            MotionEvent.ACTION_MOVE -> {
+                // we only throttle move events as those can be overwhelming
+                val now = dateProvider.currentTimeMillis
+                if (lastCapturedMoveEvent != 0L && lastCapturedMoveEvent + TOUCH_MOVE_DEBOUNCE_THRESHOLD > now) {
+                    return null
+                }
+                lastCapturedMoveEvent = now
+
+                // idk why but rrweb does it like dis
+                if (touchMoveBaseline == 0L) {
+                    touchMoveBaseline = now
+                }
+
+                currentPositions += Position().apply {
+                    x = event.x * recorderConfig.scaleFactorX
+                    y = event.y * recorderConfig.scaleFactorY
+                    id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
+                    timeOffset = now - touchMoveBaseline
+                }
+
+                val totalOffset = now - touchMoveBaseline
+                return if (totalOffset > CAPTURE_MOVE_EVENT_THRESHOLD) {
+                    RRWebInteractionMoveEvent().apply {
+                        timestamp = now
+                        positions = currentPositions.map { pos ->
+                            pos.timeOffset -= totalOffset
+                            pos
+                        }
+                    }.also {
+                        currentPositions.clear()
+                        touchMoveBaseline = 0L
+                    }
+                } else {
+                    null
+                }
+            }
+
+            MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
+                RRWebInteractionEvent().apply {
+                    timestamp = dateProvider.currentTimeMillis
+                    x = event.x * recorderConfig.scaleFactorX
+                    y = event.y * recorderConfig.scaleFactorY
+                    id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
+                    interactionType = when (action) {
+                        MotionEvent.ACTION_UP -> InteractionType.TouchEnd
+                        MotionEvent.ACTION_DOWN -> InteractionType.TouchStart
+                        MotionEvent.ACTION_CANCEL -> InteractionType.TouchCancel
+                        else -> InteractionType.TouchMove_Departed // should not happen
+                    }
+                }
+            }
+
+            else -> null
+        }
+    }
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BufferCaptureStrategy.kt

@@ -1,5 +1,6 @@
 package io.sentry.android.replay.capture
 
+import android.view.MotionEvent
 import io.sentry.DateUtils
 import io.sentry.Hint
 import io.sentry.IHub
@@ -171,4 +172,10 @@ internal class BufferCaptureStrategy(
         captureStrategy.start(segmentId = currentSegment.get(), replayId = currentReplayId.get(), cleanupOldReplays = false)
         return captureStrategy
     }
+
+    override fun onTouchEvent(event: MotionEvent) {
+        super.onTouchEvent(event)
+        val bufferLimit = dateProvider.currentTimeMillis - options.experimental.sessionReplay.errorReplayDuration
+        rotateCurrentEvents(bufferLimit)
+    }
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/capture/CaptureStrategy.kt

@@ -1,5 +1,6 @@
 package io.sentry.android.replay.capture
 
+import android.view.MotionEvent
 import io.sentry.Hint
 import io.sentry.android.replay.ReplayCache
 import io.sentry.android.replay.ScreenshotRecorderConfig
@@ -27,6 +28,8 @@ internal interface CaptureStrategy {
 
     fun onConfigurationChanged(recorderConfig: ScreenshotRecorderConfig)
 
+    fun onTouchEvent(event: MotionEvent)
+
     fun convert(): CaptureStrategy
 
     fun close()