[SR] Capture gestures/motion events

sentry-android-replay/api/sentry-android-replay.api

@@ -37,7 +37,7 @@ public final class io/sentry/android/replay/ReplayCache : java/io/Closeable {
 	public final fun rotate (J)V
 }
 
-public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, java/io/Closeable {
+public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/TouchRecorderCallback, java/io/Closeable {
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)V
 	public synthetic fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
@@ -49,6 +49,7 @@ public final class io/sentry/android/replay/ReplayIntegration : android/content/
 	public fun onLowMemory ()V
 	public fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V
 	public fun onScreenshotRecorded (Ljava/io/File;J)V
+	public fun onTouchEvent (Landroid/view/MotionEvent;)V
 	public fun pause ()V
 	public fun register (Lio/sentry/IHub;Lio/sentry/SentryOptions;)V
 	public fun resume ()V
@@ -89,6 +90,40 @@ public final class io/sentry/android/replay/ScreenshotRecorderConfig$Companion {
 	public final fun from (Landroid/content/Context;Lio/sentry/SentryReplayOptions;)Lio/sentry/android/replay/ScreenshotRecorderConfig;
 }
 
+public abstract interface class io/sentry/android/replay/TouchRecorderCallback {
+	public abstract fun onTouchEvent (Landroid/view/MotionEvent;)V
+}
+
+public class io/sentry/android/replay/util/FixedWindowCallback : android/view/Window$Callback {
+	public final field delegate Landroid/view/Window$Callback;
+	public fun <init> (Landroid/view/Window$Callback;)V
+	public fun dispatchGenericMotionEvent (Landroid/view/MotionEvent;)Z
+	public fun dispatchKeyEvent (Landroid/view/KeyEvent;)Z
+	public fun dispatchKeyShortcutEvent (Landroid/view/KeyEvent;)Z
+	public fun dispatchPopulateAccessibilityEvent (Landroid/view/accessibility/AccessibilityEvent;)Z
+	public fun dispatchTouchEvent (Landroid/view/MotionEvent;)Z
+	public fun dispatchTrackballEvent (Landroid/view/MotionEvent;)Z
+	public fun onActionModeFinished (Landroid/view/ActionMode;)V
+	public fun onActionModeStarted (Landroid/view/ActionMode;)V
+	public fun onAttachedToWindow ()V
+	public fun onContentChanged ()V
+	public fun onCreatePanelMenu (ILandroid/view/Menu;)Z
+	public fun onCreatePanelView (I)Landroid/view/View;
+	public fun onDetachedFromWindow ()V
+	public fun onMenuItemSelected (ILandroid/view/MenuItem;)Z
+	public fun onMenuOpened (ILandroid/view/Menu;)Z
+	public fun onPanelClosed (ILandroid/view/Menu;)V
+	public fun onPointerCaptureChanged (Z)V
+	public fun onPreparePanel (ILandroid/view/View;Landroid/view/Menu;)Z
+	public fun onProvideKeyboardShortcuts (Ljava/util/List;Landroid/view/Menu;I)V
+	public fun onSearchRequested ()Z
+	public fun onSearchRequested (Landroid/view/SearchEvent;)Z
+	public fun onWindowAttributesChanged (Landroid/view/WindowManager$LayoutParams;)V
+	public fun onWindowFocusChanged (Z)V
+	public fun onWindowStartingActionMode (Landroid/view/ActionMode$Callback;)Landroid/view/ActionMode;
+	public fun onWindowStartingActionMode (Landroid/view/ActionMode$Callback;I)Landroid/view/ActionMode;
+}
+
 public abstract interface class io/sentry/android/replay/video/SimpleFrameMuxer {
 	public abstract fun getVideoTime ()J
 	public abstract fun isStarted ()Z

sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt

@@ -5,6 +5,7 @@ import android.content.Context
 import android.content.res.Configuration
 import android.graphics.Bitmap
 import android.os.Build
+import android.view.MotionEvent
 import io.sentry.Hint
 import io.sentry.IHub
 import io.sentry.Integration
@@ -32,7 +33,7 @@ public class ReplayIntegration(
     private val recorderProvider: (() -> Recorder)? = null,
     private val recorderConfigProvider: ((configChanged: Boolean) -> ScreenshotRecorderConfig)? = null,
     private val replayCacheProvider: ((replayId: SentryId) -> ReplayCache)? = null
-) : Integration, Closeable, ScreenshotRecorderCallback, ReplayController, ComponentCallbacks {
+) : Integration, Closeable, ScreenshotRecorderCallback, TouchRecorderCallback, ReplayController, ComponentCallbacks {
 
     // needed for the Java's call site
     constructor(context: Context, dateProvider: ICurrentDateProvider) : this(
@@ -72,7 +73,7 @@ public class ReplayIntegration(
         }
 
         this.hub = hub
-        recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this)
+        recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, this)
         isEnabled.set(true)
 
         try {
@@ -220,4 +221,8 @@ public class ReplayIntegration(
     }
 
     override fun onLowMemory() = Unit
+
+    override fun onTouchEvent(event: MotionEvent) {
+        captureStrategy?.onTouchEvent(event)
+    }
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt

@@ -1,8 +1,13 @@
 package io.sentry.android.replay
 
 import android.annotation.TargetApi
+import android.view.MotionEvent
 import android.view.View
+import android.view.Window
+import io.sentry.SentryLevel.DEBUG
+import io.sentry.SentryLevel.ERROR
 import io.sentry.SentryOptions
+import io.sentry.android.replay.util.FixedWindowCallback
 import io.sentry.android.replay.util.gracefullyShutdown
 import io.sentry.android.replay.util.scheduleAtFixedRateSafely
 import java.lang.ref.WeakReference
@@ -16,7 +21,8 @@ import kotlin.LazyThreadSafetyMode.NONE
 @TargetApi(26)
 internal class WindowRecorder(
     private val options: SentryOptions,
-    private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null
+    private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null,
+    private val touchRecorderCallback: TouchRecorderCallback? = null
 ) : Recorder {
 
     internal companion object {
@@ -39,7 +45,11 @@ internal class WindowRecorder(
         if (added) {
             rootViews.add(WeakReference(root))
             recorder?.bind(root)
+
+            root.startGestureTracking()
         } else {
+            root.stopGestureTracking()
+
             recorder?.unbind(root)
             rootViews.removeAll { it.get() == root }
 
@@ -86,6 +96,60 @@ internal class WindowRecorder(
         isRecording.set(false)
     }
 
+    override fun close() {
+        stop()
+        capturer.gracefullyShutdown(options)
+    }
+
+    private fun View.startGestureTracking() {
+        val window = phoneWindow
+        if (window == null) {
+            options.logger.log(DEBUG, "Window is invalid, not tracking gestures")
+            return
+        }
+
+        if (touchRecorderCallback == null) {
+            options.logger.log(DEBUG, "TouchRecorderCallback is null, not tracking gestures")
+            return
+        }
+
+        val delegate = window.callback
+        window.callback = SentryReplayGestureRecorder(options, touchRecorderCallback, delegate)
+    }
+
+    private fun View.stopGestureTracking() {
+        val window = phoneWindow
+        if (window == null) {
+            options.logger.log(DEBUG, "Window was null in stopGestureTracking")
+            return
+        }
+
+        if (window.callback is SentryReplayGestureRecorder) {
+            val delegate = (window.callback as SentryReplayGestureRecorder).delegate
+            window.callback = delegate
+        }
+    }
+
+    private class SentryReplayGestureRecorder(
+        private val options: SentryOptions,
+        private val touchRecorderCallback: TouchRecorderCallback?,
+        delegate: Window.Callback?
+    ) : FixedWindowCallback(delegate) {
+        override fun dispatchTouchEvent(event: MotionEvent?): Boolean {
+            if (event != null) {
+                val copy: MotionEvent = MotionEvent.obtain(event)
+                try {
+                    touchRecorderCallback?.onTouchEvent(copy)
+                } catch (e: Throwable) {
+                    options.logger.log(ERROR, "Error dispatching touch event", e)
+                } finally {
+                    copy.recycle()
+                }
+            }
+            return super.dispatchTouchEvent(event)
+        }
+    }
+
     private class RecorderExecutorServiceThreadFactory : ThreadFactory {
         private var cnt = 0
         override fun newThread(r: Runnable): Thread {
@@ -94,9 +158,8 @@ internal class WindowRecorder(
             return ret
         }
     }
+}
 
-    override fun close() {
-        stop()
-        capturer.gracefullyShutdown(options)
-    }
+public interface TouchRecorderCallback {
+    fun onTouchEvent(event: MotionEvent)
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt

@@ -1,5 +1,6 @@
 package io.sentry.android.replay.capture
 
+import android.view.MotionEvent
 import io.sentry.Breadcrumb
 import io.sentry.DateUtils
 import io.sentry.Hint
@@ -17,6 +18,11 @@ import io.sentry.android.replay.util.submitSafely
 import io.sentry.protocol.SentryId
 import io.sentry.rrweb.RRWebBreadcrumbEvent
 import io.sentry.rrweb.RRWebEvent
+import io.sentry.rrweb.RRWebIncrementalSnapshotEvent
+import io.sentry.rrweb.RRWebInteractionEvent
+import io.sentry.rrweb.RRWebInteractionEvent.InteractionType
+import io.sentry.rrweb.RRWebInteractionMoveEvent
+import io.sentry.rrweb.RRWebInteractionMoveEvent.Position
 import io.sentry.rrweb.RRWebMetaEvent
 import io.sentry.rrweb.RRWebSpanEvent
 import io.sentry.rrweb.RRWebVideoEvent
@@ -42,6 +48,7 @@ internal abstract class BaseCaptureStrategy(
 
     internal companion object {
         private const val TAG = "CaptureStrategy"
+        private const val DEBOUNCE_TIMEOUT = 200
         private val snakecasePattern = "_[a-z]".toRegex()
         private val supportedNetworkData = setOf(
             "status_code",
@@ -59,6 +66,8 @@ internal abstract class BaseCaptureStrategy(
     override val currentReplayId = AtomicReference(SentryId.EMPTY_ID)
     override val currentSegment = AtomicInteger(0)
     override val replayCacheDir: File? get() = cache?.replayCacheDir
+    private val currentEvents = mutableListOf<RRWebEvent>()
+    private val lastExecutionTime = AtomicLong(0)
 
     protected val replayExecutor: ScheduledExecutorService by lazy {
         executor ?: Executors.newSingleThreadScheduledExecutor(ReplayExecutorServiceThreadFactory())
@@ -248,6 +257,12 @@ internal abstract class BaseCaptureStrategy(
                 }
             }
         }
+        currentEvents.removeAll {
+            if (it.timestamp > segmentTimestamp.time && it.timestamp < endTimestamp.time) {
+                recordingPayload += it
+            }
+            it.timestamp < endTimestamp.time
+        }
 
         val recording = ReplayRecording().apply {
             this.segmentId = segmentId
@@ -265,6 +280,13 @@ internal abstract class BaseCaptureStrategy(
         this.recorderConfig = recorderConfig
     }
 
+    override fun onTouchEvent(event: MotionEvent) {
+        val rrwebEvent = event.toRRWebIncrementalSnapshotEvent()
+        if (rrwebEvent != null) {
+            currentEvents += rrwebEvent
+        }
+    }
+
     override fun close() {
         replayExecutor.gracefullyShutdown(options)
     }
@@ -335,4 +357,45 @@ internal abstract class BaseCaptureStrategy(
             data = breadcrumbData
         }
     }
+
+    private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): RRWebIncrementalSnapshotEvent? {
+        val event = this
+        return when (val action = event.actionMasked) {
+            MotionEvent.ACTION_MOVE -> {
+                // we only throttle move events as those can be overwhelming
+                val now = dateProvider.getCurrentTimeMillis()
+                if (lastExecutionTime.get() != 0L && lastExecutionTime.get() + DEBOUNCE_TIMEOUT > now) {
+                    return null
+                }
+                lastExecutionTime.set(now)
+
+                RRWebInteractionMoveEvent().apply {
+                    timestamp = dateProvider.currentTimeMillis
+                    positions = listOf(
+                        Position().apply {
+                            x = event.x * recorderConfig.scaleFactorX
+                            y = event.y * recorderConfig.scaleFactorY
+                            id = event.getPointerId(event.actionIndex)
+                            timeOffset = 0 // TODO: is this needed?
+                        }
+                    ) // TODO: support multiple pointers
+                }
+            }
+            MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
+                RRWebInteractionEvent().apply {
+                    timestamp = dateProvider.currentTimeMillis
+                    x = event.x * recorderConfig.scaleFactorX
+                    y = event.y * recorderConfig.scaleFactorY
+                    id = event.getPointerId(event.actionIndex)
+                    interactionType = when (action) {
+                        MotionEvent.ACTION_UP -> InteractionType.TouchEnd
+                        MotionEvent.ACTION_DOWN -> InteractionType.TouchStart
+                        MotionEvent.ACTION_CANCEL -> InteractionType.TouchCancel
+                        else -> InteractionType.TouchMove_Departed // should not happen
+                    }
+                }
+            }
+            else -> null
+        }
+    }
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/capture/CaptureStrategy.kt

@@ -1,5 +1,6 @@
 package io.sentry.android.replay.capture
 
+import android.view.MotionEvent
 import io.sentry.Hint
 import io.sentry.android.replay.ReplayCache
 import io.sentry.android.replay.ScreenshotRecorderConfig
@@ -27,6 +28,8 @@ internal interface CaptureStrategy {
 
     fun onConfigurationChanged(recorderConfig: ScreenshotRecorderConfig)
 
+    fun onTouchEvent(event: MotionEvent)
+
     fun convert(): CaptureStrategy
 
     fun close()