Closes #6314: Support fetching / downloading data URIs

components/browser/engine-gecko-beta/src/main/java/mozilla/components/browser/engine/gecko/fetch/GeckoViewFetchClient.kt

@@ -11,6 +11,7 @@ import mozilla.components.concept.fetch.Headers
 import mozilla.components.concept.fetch.MutableHeaders
 import mozilla.components.concept.fetch.Request
 import mozilla.components.concept.fetch.Response
+import mozilla.components.concept.fetch.isDataUri
 
 import org.mozilla.geckoview.GeckoRuntime
 import org.mozilla.geckoview.GeckoWebExecutor
@@ -39,6 +40,10 @@ class GeckoViewFetchClient(
 
     @Throws(IOException::class)
     override fun fetch(request: Request): Response {
+        if (request.isDataUri()) {
+            return fetchDataUri(request)
+        }
+
         val webRequest = request.toWebRequest(defaultHeaders)
 
         val readTimeOut = request.readTimeout ?: maxReadTimeOut

components/browser/engine-gecko-nightly/src/androidTest/java/mozilla/components/browser/engine/gecko/fetch/geckoview/GeckoViewFetchTestCases.kt

@@ -130,4 +130,10 @@ class GeckoViewFetchTestCases : mozilla.components.tooling.fetch.tests.FetchTest
     override fun getThrowsIOExceptionWhenHostNotReachable() {
         super.getThrowsIOExceptionWhenHostNotReachable()
     }
+
+    @Test
+    @UiThreadTest
+    override fun getDataUri() {
+        super.getDataUri()
+    }
 }

components/browser/engine-gecko-nightly/src/main/java/mozilla/components/browser/engine/gecko/fetch/GeckoViewFetchClient.kt

@@ -11,6 +11,7 @@ import mozilla.components.concept.fetch.Headers
 import mozilla.components.concept.fetch.MutableHeaders
 import mozilla.components.concept.fetch.Request
 import mozilla.components.concept.fetch.Response
+import mozilla.components.concept.fetch.isDataUri
 
 import org.mozilla.geckoview.GeckoRuntime
 import org.mozilla.geckoview.GeckoWebExecutor
@@ -39,6 +40,10 @@ class GeckoViewFetchClient(
 
     @Throws(IOException::class)
     override fun fetch(request: Request): Response {
+        if (request.isDataUri()) {
+            return fetchDataUri(request)
+        }
+
         val webRequest = request.toWebRequest(defaultHeaders)
 
         val readTimeOut = request.readTimeout ?: maxReadTimeOut

components/browser/engine-gecko/src/main/java/mozilla/components/browser/engine/gecko/fetch/GeckoViewFetchClient.kt

@@ -11,6 +11,7 @@ import mozilla.components.concept.fetch.Headers
 import mozilla.components.concept.fetch.MutableHeaders
 import mozilla.components.concept.fetch.Request
 import mozilla.components.concept.fetch.Response
+import mozilla.components.concept.fetch.isDataUri
 
 import org.mozilla.geckoview.GeckoRuntime
 import org.mozilla.geckoview.GeckoWebExecutor
@@ -39,6 +40,10 @@ class GeckoViewFetchClient(
 
     @Throws(IOException::class)
     override fun fetch(request: Request): Response {
+        if (request.isDataUri()) {
+            return fetchDataUri(request)
+        }
+
         val webRequest = request.toWebRequest(defaultHeaders)
 
         val readTimeOut = request.readTimeout ?: maxReadTimeOut

components/concept/fetch/src/main/java/mozilla/components/concept/fetch/Client.kt

@@ -4,6 +4,10 @@
 
 package mozilla.components.concept.fetch
 
+import android.util.Base64
+import mozilla.components.concept.fetch.Response.Companion.CONTENT_LENGTH_HEADER
+import mozilla.components.concept.fetch.Response.Companion.CONTENT_TYPE_HEADER
+import java.io.ByteArrayInputStream
 import java.io.IOException
 
 /**
@@ -40,6 +44,44 @@ abstract class Client {
     @Throws(IOException::class)
     abstract fun fetch(request: Request): Response
 
+    /**
+     * Generates a [Response] by decoding a base64 encoded data URI.
+     *
+     * @param request The [Request] for the data URI.
+     * @return The generated [Response] including the decoded bytes as body.
+     */
+    @Suppress("ThrowsCount", "TooGenericExceptionCaught")
+    protected fun fetchDataUri(request: Request): Response {
+        if (!request.isDataUri()) {
+            throw IOException("Not a data URI")
+        }
+
+        val dataUri = request.url
+        if (!dataUri.contains(DATA_URI_BASE64_EXT)) {
+            throw IOException("Data URI must be base64 encoded")
+        }
+
+        return try {
+            val contentType = dataUri.substringAfter(DATA_URI_SCHEME).substringBefore(DATA_URI_BASE64_EXT)
+            val bytes = Base64.decode(dataUri.substring(dataUri.lastIndexOf(',') + 1), Base64.DEFAULT)
+            val headers = MutableHeaders().apply {
+                set(CONTENT_LENGTH_HEADER, bytes.size.toString())
+                if (contentType.isNotEmpty()) {
+                    set(CONTENT_TYPE_HEADER, contentType)
+                }
+            }
+
+            Response(
+                dataUri,
+                Response.SUCCESS,
+                headers,
+                Response.Body(ByteArrayInputStream(bytes), contentType)
+            )
+        } catch (e: Exception) {
+            throw IOException("Failed to decode data URI")
+        }
+    }
+
     /**
      * List of default headers that should be added to every request unless overridden by the headers in the request.
      */
@@ -61,4 +103,9 @@ abstract class Client {
         // We expect all clients to support and use keep-alive by default.
         "Connection" to "keep-alive"
     )
+
+    companion object {
+        const val DATA_URI_BASE64_EXT = ";base64"
+        const val DATA_URI_SCHEME = "data:"
+    }
 }

components/concept/fetch/src/main/java/mozilla/components/concept/fetch/Request.kt

@@ -149,3 +149,8 @@ data class Request(
         OMIT
     }
 }
+
+/**
+ * Checks whether or not the request is for a data URI.
+ */
+fun Request.isDataUri() = url.startsWith("data:")

components/concept/fetch/src/main/java/mozilla/components/concept/fetch/Response.kt

@@ -128,6 +128,9 @@ data class Response(
     companion object {
         val SUCCESS_STATUS_RANGE = 200..299
         val CLIENT_ERROR_STATUS_RANGE = 400..499
+        const val SUCCESS = 200
+        const val CONTENT_TYPE_HEADER = "Content-Type"
+        const val CONTENT_LENGTH_HEADER = "Content-Length"
     }
 }
 

components/feature/downloads/src/main/java/mozilla/components/feature/downloads/manager/FetchDownloadManager.kt

@@ -55,8 +55,8 @@ class FetchDownloadManager<T : AbstractFetchDownloadService>(
      * @return the id reference of the scheduled download.
      */
     override fun download(download: DownloadState, cookie: String): Long? {
-        if (!download.isScheme(listOf("http", "https"))) {
-            // We are ignoring everything that is not http or https. This is a limitation of
+        if (!download.isScheme(listOf("http", "https", "data"))) {
+            // We are ignoring everything that is not http(s) or data. This is a limitation of
             // GeckoView: https://bugzilla.mozilla.org/show_bug.cgi?id=1501735 and
             // https://bugzilla.mozilla.org/show_bug.cgi?id=1432949
             return null

components/lib/fetch-httpurlconnection/build.gradle

@@ -27,7 +27,8 @@ dependencies {
 
     implementation project(':concept-fetch')
 
-    testImplementation Dependencies.testing_junit
+    testImplementation Dependencies.androidx_test_core
+    testImplementation Dependencies.androidx_test_junit
     testImplementation Dependencies.testing_robolectric
     testImplementation Dependencies.testing_mockito
 

components/lib/fetch-httpurlconnection/src/main/java/mozilla/components/lib/fetch/httpurlconnection/HttpURLConnectionClient.kt

@@ -9,6 +9,7 @@ import mozilla.components.concept.fetch.Headers
 import mozilla.components.concept.fetch.MutableHeaders
 import mozilla.components.concept.fetch.Request
 import mozilla.components.concept.fetch.Response
+import mozilla.components.concept.fetch.isDataUri
 import mozilla.components.lib.fetch.httpurlconnection.HttpURLConnectionClient.Companion.getOrCreateCookieManager
 import java.io.FileNotFoundException
 import java.io.IOException
@@ -25,6 +26,10 @@ import java.util.zip.GZIPInputStream
 class HttpURLConnectionClient : Client() {
     @Throws(IOException::class)
     override fun fetch(request: Request): Response {
+        if (request.isDataUri()) {
+            return fetchDataUri(request)
+        }
+
         val connection = (URL(request.url).openConnection() as HttpURLConnection)
 
         connection.setupWith(request)

components/lib/fetch-httpurlconnection/src/test/java/mozilla/components/lib/fetch/httpurlconnection/HttpUrlConnectionFetchTestCases.kt

@@ -4,14 +4,17 @@
 
 package mozilla.components.lib.fetch.httpurlconnection
 
+import androidx.test.ext.junit.runners.AndroidJUnit4
 import mozilla.components.concept.fetch.Client
 import mozilla.components.concept.fetch.Request
 import org.junit.Assert.assertFalse
 import org.junit.Assert.assertTrue
 import org.junit.Test
+import org.junit.runner.RunWith
 import java.net.HttpURLConnection
 import java.net.URL
 
+@RunWith(AndroidJUnit4::class)
 class HttpUrlConnectionFetchTestCases : mozilla.components.tooling.fetch.tests.FetchTestCases() {
     override fun createNewClient(): Client = HttpURLConnectionClient()
 

components/lib/fetch-httpurlconnection/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker

@@ -0,0 +1,2 @@
+mock-maker-inline
+// This allows mocking final classes (classes are final by default in Kotlin)

components/lib/fetch-okhttp/src/main/java/mozilla/components/lib/fetch/okhttp/OkHttpClient.kt

@@ -10,6 +10,7 @@ import mozilla.components.concept.fetch.Headers
 import mozilla.components.concept.fetch.MutableHeaders
 import mozilla.components.concept.fetch.Request
 import mozilla.components.concept.fetch.Response
+import mozilla.components.concept.fetch.isDataUri
 import mozilla.components.lib.fetch.okhttp.OkHttpClient.Companion.CACHE_MAX_SIZE
 import mozilla.components.lib.fetch.okhttp.OkHttpClient.Companion.getOrCreateCookieManager
 import okhttp3.Cache
@@ -30,6 +31,10 @@ class OkHttpClient(
     private val context: Context? = null
 ) : Client() {
     override fun fetch(request: Request): Response {
+        if (request.isDataUri()) {
+            return fetchDataUri(request)
+        }
+
         val requestClient = client.rebuildFor(request, context)
 
         val requestBuilder = createRequestBuilderWithBody(request)

components/tooling/fetch-tests/src/main/java/mozilla/components/tooling/fetch/tests/FetchTestCases.kt

@@ -491,6 +491,25 @@ abstract class FetchTestCases {
         }
     }
 
+    @Test
+    open fun getDataUri() {
+        val client = createNewClient()
+        val response = client.fetch(Request(url = "data:text/plain;charset=utf-8;base64,SGVsbG8sIFdvcmxkIQ=="))
+        assertEquals("13", response.headers["Content-Length"])
+        assertEquals("text/plain;charset=utf-8", response.headers["Content-Type"])
+        assertEquals("Hello, World!", response.body.string())
+
+        val responseNoCharset = client.fetch(Request(url = "data:text/plain;base64,SGVsbG8sIFdvcmxkIQ=="))
+        assertEquals("13", responseNoCharset.headers["Content-Length"])
+        assertEquals("text/plain", responseNoCharset.headers["Content-Type"])
+        assertEquals("Hello, World!", responseNoCharset.body.string())
+
+        val responseNoContentType = client.fetch(Request(url = "data:;base64,SGVsbG8sIFdvcmxkIQ=="))
+        assertEquals("13", responseNoContentType.headers["Content-Length"])
+        assertNull(responseNoContentType.headers["Content-Type"])
+        assertEquals("Hello, World!", responseNoContentType.body.string())
+    }
+
     private inline fun withServerResponding(
         vararg responses: MockResponse,
         crossinline block: MockWebServer.(Client) -> Unit