Skip to content

Commit

Permalink
RUMM-3267: Remove mapping file size check
Browse files Browse the repository at this point in the history
  • Loading branch information
@0xnm
0xnm committed May 9, 2023
1 parent bb41295 commit 06a2950
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 163 deletions.
2 changes: 2 additions & 0 deletions ...droid-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/DdExtensionConfiguration.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ open class DdExtensionConfiguration(
*/
var mappingFilePath: String? = null

// TODO RUMM-0000 Deprecate and then remove this API once all DCs support large file upload
/**
* Short aliases to use for package prefixes. Allows to replace, for example,
* androidx.appcompat with something shorter, reducing the size of the mapping file. Key is
Expand All @@ -74,6 +75,7 @@ open class DdExtensionConfiguration(
*/
var mappingFilePackageAliases: Map<String, String> = emptyMap()

// TODO RUMM-0000 Deprecate and then remove this API once all DCs support large file upload
/**
* This property removes indents from each line of the mapping file, reducing its size.
*
Expand Down
37 changes: 12 additions & 25 deletions ...ndroid-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/OkHttpUploader.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import org.json.JSONTokener
import java.io.File
import java.io.IOException
import java.net.HttpURLConnection
import java.util.Locale
import java.util.concurrent.TimeUnit

internal class OkHttpUploader : Uploader {
Expand Down Expand Up @@ -55,7 +56,7 @@ internal class OkHttpUploader : Uploader {
useGzip: Boolean
) {
LOGGER.info("Uploading mapping file for $identifier (site=${site.domain}):\n")
val body = createBody(site, identifier, mappingFile, repositoryFile, repositoryInfo)
val body = createBody(identifier, mappingFile, repositoryFile, repositoryInfo)

val requestBuilder = Request.Builder()
.url(site.uploadEndpoint())
Expand Down Expand Up @@ -91,25 +92,12 @@ internal class OkHttpUploader : Uploader {
// region Internal

private fun createBody(
site: DatadogSite,
identifier: DdAppIdentifier,
mappingFile: File,
repositoryFile: File?,
repositoryInfo: RepositoryInfo?
): MultipartBody {
val mappingFileBody = mappingFile.asRequestBody(MEDIA_TYPE_TXT)
// have to use name, because comparison of enum vs spied/mocked enum directly will always fail
val sizeLimit = if (site.name == DatadogSite.US1.name) {
MAX_MAP_FILE_SIZE_IN_BYTES_US1
} else {
MAX_MAP_FILE_SIZE_IN_BYTES
}
if (mappingFileBody.contentLength() > sizeLimit) {
throw MaxSizeExceededException(
MAX_MAP_SIZE_EXCEEDED_ERROR_FORMAT
.format(mappingFile.absolutePath, sizeLimit / MEGABYTE_IN_BYTES)
)
}

val eventJson = JSONObject()
eventJson.put("version", identifier.version)
Expand Down Expand Up @@ -169,6 +157,10 @@ internal class OkHttpUploader : Uploader {
"Unable to upload mapping file for $identifier because of a request timeout; " +
"check your network connection"
)
statusCode == HttpURLConnection.HTTP_ENTITY_TOO_LARGE ->
throw MaxSizeExceededException(
MAX_MAP_SIZE_EXCEEDED_ERROR.format(Locale.US, identifier)
)
statusCode >= HttpURLConnection.HTTP_BAD_REQUEST -> {
if (statusCode == HttpURLConnection.HTTP_BAD_REQUEST &&
validateApiKey(site, apiKey) == false
Expand Down Expand Up @@ -256,7 +248,7 @@ internal class OkHttpUploader : Uploader {
companion object {

// TODO add a plugin to automatically sync this with the `MavenConfig` value
internal const val VERSION = "1.8.0"
internal const val VERSION = "1.9.0"

internal const val HEADER_API_KEY = "DD-API-KEY"
internal const val HEADER_EVP_ORIGIN = "DD-EVP-ORIGIN"
Expand All @@ -270,19 +262,14 @@ internal class OkHttpUploader : Uploader {
internal const val KEY_JVM_MAPPING = "jvm_mapping"
internal const val KEY_REPOSITORY = "repository"

// 60s is currently only for US1, others have 45s. But anyway server can drop connection
// by itself.
internal val NETWORK_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(60)
internal val MEDIA_TYPE_TXT = "text/plain".toMediaTypeOrNull()
internal val MEDIA_TYPE_JSON = "application/json".toMediaTypeOrNull()
internal const val MEGABYTE_IN_BYTES = 1024 * 1024L
internal const val MAX_MAP_FILE_SIZE_IN_BYTES = 50L * MEGABYTE_IN_BYTES // 50 MB

// temporary until all DCs have a common limit
internal const val MAX_MAP_FILE_SIZE_IN_BYTES_US1 = 100L * MEGABYTE_IN_BYTES // 100 MB
internal const val MAX_MAP_SIZE_EXCEEDED_ERROR_FORMAT =
"The proguard mapping file at: [%s] size exceeded the maximum %s MB size. " +
"This task cannot be performed."

internal const val MAX_MAP_SIZE_EXCEEDED_ERROR =
"Unable to upload mapping file for %s because mapping file is too" +
" large; please refer to documentation regarding the limits"

internal val successfulCodes = arrayOf(
HttpURLConnection.HTTP_OK,
HttpURLConnection.HTTP_CREATED,
Expand Down
144 changes: 6 additions & 138 deletions ...id-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/internal/OkHttpUploaderTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ import org.mockito.Mock
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.junit.jupiter.MockitoSettings
import org.mockito.kotlin.doReturn
import org.mockito.kotlin.spy
import org.mockito.kotlin.whenever
import org.mockito.quality.Strictness
import java.io.File
Expand Down Expand Up @@ -626,56 +625,17 @@ internal class OkHttpUploaderTest {
}

@Test
fun `M throw a MaxSizeExceededException W upload() { mappingFile size exceeded 50 MB, non-US1}`(
forge: Forge
) {
fun `M throw a MaxSizeExceededException W upload() { response 413 }`() {
// GIVEN
val fakeFileSize =
forge.aLong(min = OkHttpUploader.MAX_MAP_FILE_SIZE_IN_BYTES + 1)
val fakeTooLargeMappingFile = spy(File(tempDir, forge.anAlphabeticalString()))
doReturn(fakeFileSize)
.whenever(fakeTooLargeMappingFile).length()
mockUploadResponse = MockResponse()
.setResponseCode(HttpURLConnection.HTTP_ENTITY_TOO_LARGE)
.setBody("{}")

// THEN
val exception = assertThrows<MaxSizeExceededException> {
testedUploader.upload(
mockSite,
fakeTooLargeMappingFile,
fakeRepositoryFile,
fakeApiKey,
fakeIdentifier,
fakeRepositoryInfo,
useGzip = true
)
}
assertThat(exception.message).isEqualTo(
OkHttpUploader.MAX_MAP_SIZE_EXCEEDED_ERROR_FORMAT.format(
Locale.US,
fakeTooLargeMappingFile.absolutePath,
OkHttpUploader.MAX_MAP_FILE_SIZE_IN_BYTES / OkHttpUploader.MEGABYTE_IN_BYTES
)
)
assertThat(mockWebServer.requestCount).isEqualTo(0)
}

@Test
fun `M throw a MaxSizeExceededException W upload() { mappingFile size exceeded 100 MB, US1}`(
forge: Forge
) {
// GIVEN
val fakeFileSize =
forge.aLong(min = OkHttpUploader.MAX_MAP_FILE_SIZE_IN_BYTES_US1 + 1)
val fakeTooLargeMappingFile = spy(File(tempDir, forge.anAlphabeticalString()))
doReturn(fakeFileSize)
.whenever(fakeTooLargeMappingFile).length()
val fakeSite = spy(DatadogSite.US1)
whenever(fakeSite.uploadEndpoint()) doReturn fakeUploadUrl

// THEN
val exception = assertThrows<MaxSizeExceededException> {
testedUploader.upload(
fakeSite,
fakeTooLargeMappingFile,
fakeMappingFile,
fakeRepositoryFile,
fakeApiKey,
fakeIdentifier,
Expand All @@ -684,100 +644,8 @@ internal class OkHttpUploaderTest {
)
}
assertThat(exception.message).isEqualTo(
OkHttpUploader.MAX_MAP_SIZE_EXCEEDED_ERROR_FORMAT.format(
Locale.US,
fakeTooLargeMappingFile.absolutePath,
OkHttpUploader.MAX_MAP_FILE_SIZE_IN_BYTES_US1 / OkHttpUploader.MEGABYTE_IN_BYTES
)
)
assertThat(mockWebServer.requestCount).isEqualTo(0)
}

@Test
fun `𝕄 upload proper request π•Ž upload() { mappingFile size smaller than 50 MB, non-US1 }`(
forge: Forge
) {
// Given
val fakeFileSize =
forge.aLong(min = 1, max = OkHttpUploader.MAX_MAP_FILE_SIZE_IN_BYTES)
val fakeTooLargeMappingFile = spy(File(tempDir, forge.anAlphabeticalString()))
doReturn(fakeFileSize)
.whenever(fakeTooLargeMappingFile).length()
mockUploadResponse = MockResponse()
.setResponseCode(HttpURLConnection.HTTP_OK)
.setBody("{}")

// When
testedUploader.upload(
mockSite,
fakeMappingFile,
fakeRepositoryFile,
fakeApiKey,
fakeIdentifier,
fakeRepositoryInfo,
useGzip = true
)

// Then
assertThat(mockWebServer.requestCount).isEqualTo(1)
}

@Test
fun `𝕄 upload proper request π•Ž upload() { mappingFile size smaller than 100 MB, US1 }`(
forge: Forge
) {
// Given
val fakeFileSize =
forge.aLong(min = 1, max = OkHttpUploader.MAX_MAP_FILE_SIZE_IN_BYTES_US1)
val fakeTooLargeMappingFile = spy(File(tempDir, forge.anAlphabeticalString()))
doReturn(fakeFileSize)
.whenever(fakeTooLargeMappingFile).length()
mockUploadResponse = MockResponse()
.setResponseCode(HttpURLConnection.HTTP_OK)
.setBody("{}")
val fakeSite = spy(DatadogSite.US1)
whenever(fakeSite.uploadEndpoint()) doReturn fakeUploadUrl

// When
testedUploader.upload(
fakeSite,
fakeMappingFile,
fakeRepositoryFile,
fakeApiKey,
fakeIdentifier,
fakeRepositoryInfo,
useGzip = true
OkHttpUploader.MAX_MAP_SIZE_EXCEEDED_ERROR.format(Locale.US, fakeIdentifier)
)

// Then
assertThat(mockWebServer.requestCount).isEqualTo(1)
}

@Test
fun `𝕄 upload proper request π•Ž upload() { mappingFile size is 50 MB }`(
forge: Forge
) {
// Given
val fakeTooLargeMappingFile = spy(File(tempDir, forge.anAlphabeticalString()))
doReturn(OkHttpUploader.MAX_MAP_FILE_SIZE_IN_BYTES)
.whenever(fakeTooLargeMappingFile).length()
mockUploadResponse = MockResponse()
.setResponseCode(HttpURLConnection.HTTP_OK)
.setBody("{}")

// When
testedUploader.upload(
mockSite,
fakeMappingFile,
fakeRepositoryFile,
fakeApiKey,
fakeIdentifier,
fakeRepositoryInfo,
useGzip = true
)

// Then
assertThat(mockWebServer.requestCount).isEqualTo(1)
}

inner class MockDispatcher : Dispatcher() {
Expand Down

0 comments on commit 06a2950

Please sign in to comment.