Google Cloud Storage: remove download limit #2084

Remove 8MB download limit on GCS files.
akka · Jan 14, 2020 · bc2d777 · bc2d777
1 parent 993cbae
commit bc2d777
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 2 deletions.
diff --git a/...storage/src/main/scala/akka/stream/alpakka/googlecloud/storage/impl/GCStorageStream.scala b/...storage/src/main/scala/akka/stream/alpakka/googlecloud/storage/impl/GCStorageStream.scala
@@ -231,7 +231,7 @@ import scala.util.control.NonFatal
           )
         ).mapAsync(parallelism)(entityForSuccessOption)
           .map {
-            _.map(_.dataBytes.mapMaterializedValue(_ => NotUsed))
+            _.map(_.withoutSizeLimit.dataBytes.mapMaterializedValue(_ => NotUsed))
           }
 
       }

diff --git a/google-cloud-storage/src/test/resources/application.conf b/google-cloud-storage/src/test/resources/application.conf
@@ -4,6 +4,9 @@ akka {
   loglevel = "INFO"
 }
 
+// Extra small client parsing limit to allow testing for payloads that are over this limit
+akka.http.client.parsing.max-content-length = 10000
+
 //#settings
 
 privateKey ="""-----BEGIN PRIVATE KEY-----

diff --git a/google-cloud-storage/src/test/scala/docs/scaladsl/GCStorageSourceSpec.scala b/google-cloud-storage/src/test/scala/docs/scaladsl/GCStorageSourceSpec.scala
@@ -285,7 +285,8 @@ class GCStorageSourceSpec
     }
 
     "download file when file exists" in {
-      val fileContent = "Google storage file content"
+      // Ensure file content is above the size limit set by akka.http.client.parsing.max-content-length
+      val fileContent = "Google storage file content" + ("x" * 10000)
       val fileContentGeneration = "Google storage file content (archived)"
 
       mockFileDownload(fileContent)