Merge pull request #1854 from DataDog/jmoskovich/rum-2709/record-reso…

…urces RUM-2709: Implement resource capture during traversal
DataDog · Feb 21, 2024 · 1e65536 · 1e65536
2 parents 077adb0 + 79cacc9
commit 1e65536
Show file tree

Hide file tree

Showing 22 changed files with 620 additions and 216 deletions.
diff --git a/detekt_custom.yml b/detekt_custom.yml
@@ -590,6 +590,7 @@ datadog:
       - "java.lang.Thread.getDefaultUncaughtExceptionHandler()"
       - "java.lang.Thread.interrupted()"
       - "java.lang.Thread.setDefaultUncaughtExceptionHandler(java.lang.Thread.UncaughtExceptionHandler?)"
+      - "java.util.Collections.synchronizedSet(kotlin.collections.MutableSet?)"
       - "java.util.Queue.isEmpty()"
       - "java.util.Queue.isNotEmpty()"
       - "java.util.Queue.poll()"

diff --git a/...-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/SessionReplayPrivacy.kt b/...-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/SessionReplayPrivacy.kt
@@ -19,6 +19,7 @@ import android.widget.TextView
 import android.widget.Toolbar
 import androidx.appcompat.widget.SwitchCompat
 import com.datadog.android.api.InternalLogger
+import com.datadog.android.sessionreplay.internal.async.RecordedDataQueueHandler
 import com.datadog.android.sessionreplay.internal.recorder.base64.Base64LRUCache
 import com.datadog.android.sessionreplay.internal.recorder.base64.Base64Serializer
 import com.datadog.android.sessionreplay.internal.recorder.base64.BitmapPool
@@ -77,8 +78,12 @@ enum class SessionReplayPrivacy {
     MASK_USER_INPUT;
 
     @Suppress("LongMethod")
-    internal fun mappers(internalLogger: InternalLogger): List<MapperTypeWrapper> {
-        val base64Serializer = buildBase64Serializer()
+    internal fun mappers(
+        internalLogger: InternalLogger,
+        applicationId: String,
+        recordedDataQueueHandler: RecordedDataQueueHandler
+    ): List<MapperTypeWrapper> {
+        val base64Serializer = buildBase64Serializer(applicationId, recordedDataQueueHandler)
         val imageWireframeHelper = ImageWireframeHelper(logger = internalLogger, base64Serializer = base64Serializer)
         val uniqueIdentifierGenerator = UniqueIdentifierGenerator
 
@@ -167,11 +172,17 @@ enum class SessionReplayPrivacy {
         return mappersList
     }
 
-    private fun buildBase64Serializer(): Base64Serializer {
+    private fun buildBase64Serializer(
+        applicationId: String,
+        recordedDataQueueHandler:
+        RecordedDataQueueHandler
+    ): Base64Serializer {
         val bitmapPool = BitmapPool()
         val base64LRUCache = Base64LRUCache()
 
         val builder = Base64Serializer.Builder(
+            applicationId = applicationId,
+            recordedDataQueueHandler = recordedDataQueueHandler,
             bitmapPool = bitmapPool,
             base64LRUCache = base64LRUCache
         )

diff --git a/...session-replay/src/main/kotlin/com/datadog/android/sessionreplay/SessionReplayRecorder.kt b/...session-replay/src/main/kotlin/com/datadog/android/sessionreplay/SessionReplayRecorder.kt
@@ -77,6 +77,8 @@ internal class SessionReplayRecorder : OnWindowRefreshedCallback, Recorder {
             MutationResolver(internalLogger)
         )
 
+        val applicationId = rumContextProvider.getRumContext().applicationId
+
         this.appContext = appContext
         this.rumContextProvider = rumContextProvider
         this.privacy = privacy
@@ -91,10 +93,17 @@ internal class SessionReplayRecorder : OnWindowRefreshedCallback, Recorder {
             timeProvider = timeProvider,
             internalLogger = internalLogger
         )
+
         this.viewOnDrawInterceptor = ViewOnDrawInterceptor(
             recordedDataQueueHandler = recordedDataQueueHandler,
             SnapshotProducer(
-                TreeViewTraversal(customMappers + privacy.mappers(internalLogger)),
+                TreeViewTraversal(
+                customMappers + privacy.mappers(
+                    internalLogger,
+                    applicationId,
+                    recordedDataQueueHandler
+                )
+                ),
                 ComposedOptionSelectorDetector(
                     customOptionSelectorDetectors + DefaultOptionSelectorDetector()
                 )

diff --git a/...ion-replay/src/main/kotlin/com/datadog/android/sessionreplay/internal/ResourcesFeature.kt b/...ion-replay/src/main/kotlin/com/datadog/android/sessionreplay/internal/ResourcesFeature.kt
@@ -11,7 +11,7 @@ import com.datadog.android.api.feature.FeatureSdkCore
 import com.datadog.android.api.feature.StorageBackedFeature
 import com.datadog.android.api.net.RequestFactory
 import com.datadog.android.api.storage.FeatureStorageConfiguration
-import com.datadog.android.sessionreplay.internal.net.ResourceRequestFactory
+import com.datadog.android.sessionreplay.internal.net.ResourcesRequestFactory
 import com.datadog.android.sessionreplay.internal.storage.NoOpResourcesWriter
 import com.datadog.android.sessionreplay.internal.storage.ResourcesWriter
 import com.datadog.android.sessionreplay.internal.storage.SessionReplayResourcesWriter
@@ -32,7 +32,7 @@ internal class ResourcesFeature(
 
     override val name: String = SESSION_REPLAY_RESOURCES_FEATURE_NAME
 
-    override val requestFactory: RequestFactory = ResourceRequestFactory(
+    override val requestFactory: RequestFactory = ResourcesRequestFactory(
         customEndpointUrl = customEndpointUrl,
         internalLogger = sdkCore.internalLogger
     )
@@ -68,6 +68,6 @@ internal class ResourcesFeature(
             )
 
         internal const val SESSION_REPLAY_RESOURCES_FEATURE_NAME = "session-replay-resources"
-        internal const val RESOURCES_ENDPOINT_ENABLED = false
+        internal const val RESOURCE_ENDPOINT_FEATURE_FLAG = false
     }
 }
diff --git a/...replay/src/main/kotlin/com/datadog/android/sessionreplay/internal/SessionReplayFeature.kt b/...replay/src/main/kotlin/com/datadog/android/sessionreplay/internal/SessionReplayFeature.kt
@@ -22,7 +22,7 @@ import com.datadog.android.sessionreplay.NoOpRecorder
 import com.datadog.android.sessionreplay.Recorder
 import com.datadog.android.sessionreplay.SessionReplayPrivacy
 import com.datadog.android.sessionreplay.SessionReplayRecorder
-import com.datadog.android.sessionreplay.internal.ResourcesFeature.Companion.RESOURCES_ENDPOINT_ENABLED
+import com.datadog.android.sessionreplay.internal.ResourcesFeature.Companion.RESOURCE_ENDPOINT_FEATURE_FLAG
 import com.datadog.android.sessionreplay.internal.net.BatchesToSegmentsMapper
 import com.datadog.android.sessionreplay.internal.net.SegmentRequestFactory
 import com.datadog.android.sessionreplay.internal.recorder.OptionSelectorDetector
@@ -100,7 +100,7 @@ internal class SessionReplayFeature(
         this.appContext = appContext
         sdkCore.setEventReceiver(SESSION_REPLAY_FEATURE_NAME, this)
 
-        val resourcesWriter = if (RESOURCES_ENDPOINT_ENABLED) {
+        val resourcesWriter = if (RESOURCE_ENDPOINT_FEATURE_FLAG) {
             val resourcesFeature = registerResourceFeature(sdkCore)
             resourcesFeature.dataWriter
         } else {

diff --git a/...play/src/main/kotlin/com/datadog/android/sessionreplay/internal/async/DataQueueHandler.kt b/...play/src/main/kotlin/com/datadog/android/sessionreplay/internal/async/DataQueueHandler.kt
@@ -0,0 +1,24 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2016-Present Datadog, Inc.
+ */
+
+package com.datadog.android.sessionreplay.internal.async
+
+import com.datadog.android.sessionreplay.internal.recorder.SystemInformation
+import com.datadog.android.sessionreplay.model.MobileSegment
+
+internal interface DataQueueHandler {
+    fun addResourceItem(
+        identifier: String,
+        applicationId: String,
+        resourceData: ByteArray
+    ): ResourceRecordedDataQueueItem?
+    fun addTouchEventItem(
+        pointerInteractions: List<MobileSegment.MobileRecord>
+    ): TouchEventRecordedDataQueueItem?
+    fun addSnapshotItem(systemInformation: SystemInformation): SnapshotRecordedDataQueueItem?
+    fun tryToConsumeItems()
+    fun clearAndStopProcessingQueue()
+}
diff --git a/.../src/main/kotlin/com/datadog/android/sessionreplay/internal/async/NoopDataQueueHandler.kt b/.../src/main/kotlin/com/datadog/android/sessionreplay/internal/async/NoopDataQueueHandler.kt
@@ -0,0 +1,34 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2016-Present Datadog, Inc.
+ */
+
+package com.datadog.android.sessionreplay.internal.async
+
+import com.datadog.android.sessionreplay.internal.recorder.SystemInformation
+import com.datadog.android.sessionreplay.model.MobileSegment
+
+internal class NoopDataQueueHandler : DataQueueHandler {
+    override fun addResourceItem(
+        identifier: String,
+        applicationId: String,
+        resourceData: ByteArray
+    ): ResourceRecordedDataQueueItem? = null
+
+    override fun addTouchEventItem(
+        pointerInteractions: List<MobileSegment.MobileRecord>
+    ): TouchEventRecordedDataQueueItem? = null
+
+    override fun addSnapshotItem(
+        systemInformation: SystemInformation
+    ): SnapshotRecordedDataQueueItem? = null
+
+    override fun tryToConsumeItems() {
+        // noop
+    }
+
+    override fun clearAndStopProcessingQueue() {
+        // noop
+    }
+}
diff --git a/.../main/kotlin/com/datadog/android/sessionreplay/internal/async/RecordedDataQueueHandler.kt b/.../main/kotlin/com/datadog/android/sessionreplay/internal/async/RecordedDataQueueHandler.kt
@@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit
  * The items are added to the queue from the main thread and processed on a background thread.
  */
 @Suppress("TooManyFunctions")
-internal class RecordedDataQueueHandler {
+internal class RecordedDataQueueHandler : DataQueueHandler {
     private var executorService: ExecutorService
     private var processor: RecordedDataProcessor
     private var rumContextDataHandler: RumContextDataHandler
@@ -85,13 +85,34 @@ internal class RecordedDataQueueHandler {
     }
 
     @Synchronized
-    internal fun clearAndStopProcessingQueue() {
+    override fun clearAndStopProcessingQueue() {
         recordedDataQueue.clear()
         executorService.shutdown()
     }
 
     @MainThread
-    internal fun addTouchEventItem(
+    override fun addResourceItem(
+        identifier: String,
+        applicationId: String,
+        resourceData: ByteArray
+    ): ResourceRecordedDataQueueItem? {
+        val rumContextData = rumContextDataHandler.createRumContextData()
+            ?: return null
+
+        val item = ResourceRecordedDataQueueItem(
+            recordedQueuedItemContext = rumContextData,
+            identifier = identifier,
+            applicationId = applicationId,
+            resourceData = resourceData
+        )
+
+        insertIntoRecordedDataQueue(item)
+
+        return item
+    }
+
+    @MainThread
+    override fun addTouchEventItem(
         pointerInteractions: List<MobileSegment.MobileRecord>
     ): TouchEventRecordedDataQueueItem? {
         val rumContextData = rumContextDataHandler.createRumContextData()
@@ -108,7 +129,7 @@ internal class RecordedDataQueueHandler {
     }
 
     @MainThread
-    internal fun addSnapshotItem(
+    override fun addSnapshotItem(
         systemInformation: SystemInformation
     ): SnapshotRecordedDataQueueItem? {
         val rumContextData = rumContextDataHandler.createRumContextData()
@@ -132,7 +153,7 @@ internal class RecordedDataQueueHandler {
      * If neither of the previous conditions occurs, the loop breaks.
      */
     @MainThread
-    internal fun tryToConsumeItems() {
+    override fun tryToConsumeItems() {
         // no need to create a thread if the queue is empty
         if (recordedDataQueue.isEmpty()) {
             return
@@ -200,6 +221,9 @@ internal class RecordedDataQueueHandler {
 
             is TouchEventRecordedDataQueueItem ->
                 processTouchEvent(nextItem)
+
+            is ResourceRecordedDataQueueItem ->
+                processResourceEvent(nextItem)
         }
     }
 
@@ -208,6 +232,11 @@ internal class RecordedDataQueueHandler {
         processor.processScreenSnapshots(item)
     }
 
+    @WorkerThread
+    private fun processResourceEvent(item: ResourceRecordedDataQueueItem) {
+        processor.processResources(item)
+    }
+
     @WorkerThread
     private fun processTouchEvent(item: TouchEventRecordedDataQueueItem) {
         processor.processTouchEventsRecords(item)

diff --git a/.../main/kotlin/com/datadog/android/sessionreplay/internal/net/ResourceRequestBodyFactory.kt b/.../main/kotlin/com/datadog/android/sessionreplay/internal/net/ResourceRequestBodyFactory.kt
@@ -10,7 +10,9 @@ import androidx.annotation.VisibleForTesting
 import com.datadog.android.api.InternalLogger
 import com.datadog.android.api.storage.RawBatchEvent
 import com.datadog.android.sessionreplay.internal.processor.EnrichedResource.Companion.APPLICATION_ID_KEY
+import com.datadog.android.sessionreplay.internal.processor.EnrichedResource.Companion.APPLICATION_KEY
 import com.datadog.android.sessionreplay.internal.processor.EnrichedResource.Companion.FILENAME_KEY
+import com.datadog.android.sessionreplay.internal.processor.EnrichedResource.Companion.ID_KEY
 import com.datadog.android.sessionreplay.internal.utils.MiscUtils
 import com.google.gson.JsonObject
 import okhttp3.MediaType.Companion.toMediaTypeOrNull
@@ -122,13 +124,15 @@ internal class ResourceRequestBodyFactory(
     }
 
     private fun addApplicationIdSection(builder: MultipartBody.Builder, applicationId: String) {
-        val data = JsonObject()
-        data.addProperty(APPLICATION_ID_KEY, applicationId)
-        data.addProperty(TYPE_KEY, TYPE_RESOURCE)
+        val applicationIdOuter = JsonObject()
+        val applicationIdInner = JsonObject()
+        applicationIdInner.addProperty(ID_KEY, applicationId)
+        applicationIdOuter.add(APPLICATION_KEY, applicationIdInner)
+        applicationIdOuter.addProperty(TYPE_KEY, TYPE_RESOURCE)
 
         @Suppress("TooGenericExceptionCaught")
         val body = try {
-            data.toString().toRequestBody(CONTENT_TYPE_APPLICATION)
+            applicationIdOuter.toString().toRequestBody(CONTENT_TYPE_APPLICATION)
         } catch (e: ArrayIndexOutOfBoundsException) {
             // we have data, so should not be able to throw this
             internalLogger.log(
@@ -150,7 +154,7 @@ internal class ResourceRequestBodyFactory(
 
         if (body != null) {
             builder.addFormDataPart(
-                name = NAME_RESOURCE,
+                name = NAME_EVENT,
                 filename = FILENAME_BLOB,
                 body = body
             )
@@ -196,7 +200,7 @@ internal class ResourceRequestBodyFactory(
         internal const val TYPE_KEY = "type"
         internal const val TYPE_RESOURCE = "resource"
         internal const val NAME_IMAGE = "image"
-        internal const val NAME_RESOURCE = "resource"
+        internal const val NAME_EVENT = "event"
         internal const val FILENAME_BLOB = "blob"
 
         internal const val MULTIPLE_APPLICATION_ID_ERROR =

diff --git a/...ay/internal/net/ResourceRequestFactory.kt → ...y/internal/net/ResourcesRequestFactory.kt b/...ay/internal/net/ResourceRequestFactory.kt → ...y/internal/net/ResourcesRequestFactory.kt
@@ -18,7 +18,7 @@ import java.io.IOException
 import java.util.Locale
 import java.util.UUID
 
-internal class ResourceRequestFactory(
+internal class ResourcesRequestFactory(
     internal val customEndpointUrl: String?,
     private val internalLogger: InternalLogger,
     private val resourceRequestBodyFactory: ResourceRequestBodyFactory =

diff --git a/.../src/main/kotlin/com/datadog/android/sessionreplay/internal/processor/EnrichedResource.kt b/.../src/main/kotlin/com/datadog/android/sessionreplay/internal/processor/EnrichedResource.kt
@@ -32,7 +32,9 @@ internal data class EnrichedResource(
     }
 
     internal companion object {
-        internal const val APPLICATION_ID_KEY = "application_id"
+        internal const val APPLICATION_ID_KEY = "applicationId"
+        internal const val APPLICATION_KEY = "application"
+        internal const val ID_KEY = "id"
         internal const val FILENAME_KEY = "filename"
     }
 }