diff --git a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/UpdaterHtmlClient.kt b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/UpdaterHtmlClient.kt index 3b615c3d..85899888 100644 --- a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/UpdaterHtmlClient.kt +++ b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/UpdaterHtmlClient.kt @@ -17,8 +17,14 @@ import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine +data class UrlRequest( + val url: String, + val lastModified: String? = null +) + interface UpdaterHtmlClient { suspend fun get(url: String): String? + suspend fun getFullResponse(request: UrlRequest): HttpResponse? } object UpdaterHtmlClientFactory { @@ -31,9 +37,20 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient { @JvmStatic private val LOGGER = LoggerFactory.getLogger(this::class.java) private val TOKEN: String? = GithubAuth.readToken() + + + fun extractBody(response: HttpResponse?): String? { + if (response == null) { + return null + } + val writer = StringWriter() + IOUtils.copy(response.entity.content, writer, Charset.defaultCharset()) + return writer.toString() + } + } - class ResponseHandler(val client: DefaultUpdaterHtmlClient, val continuation: Continuation) : FutureCallback { + class ResponseHandler(val client: DefaultUpdaterHtmlClient, private val continuation: Continuation, val request: UrlRequest?) : FutureCallback { override fun cancelled() { continuation.resumeWithException(Exception("cancelled")) } @@ -45,13 +62,13 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient { continuation.resumeWithException(Exception("No response body")) } isARedirect(response) -> { - client.getData(URL(response.getFirstHeader("location").value), continuation) + client.getData(UrlRequest(response.getFirstHeader("location").value, request?.lastModified), continuation) } response.statusLine.statusCode == 404 -> { continuation.resumeWithException(NotFoundException()) } - response.statusLine.statusCode == 200 -> { - continuation.resume(extractBody(response)) + response.statusLine.statusCode == 200 || response.statusLine.statusCode == 304 -> { + continuation.resume(response) } else -> { continuation.resumeWithException(Exception("Unexpected response ${response.statusLine.statusCode}")) @@ -62,12 +79,6 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient { } } - private fun extractBody(response: HttpResponse): String { - val writer = StringWriter() - IOUtils.copy(response.entity.content, writer, Charset.defaultCharset()) - return writer.toString() - } - private fun isARedirect(response: HttpResponse): Boolean { return response.statusLine.statusCode == 307 || response.statusLine.statusCode == 301 || @@ -84,10 +95,15 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient { } } - private fun getData(url: URL, continuation: Continuation) { + private fun getData(urlRequest: UrlRequest, continuation: Continuation) { try { + val url = URL(urlRequest.url) val request = HttpGet(url.toURI()) + if (urlRequest.lastModified != null) { + request.addHeader("If-Modified-Since", urlRequest.lastModified) + } + if (url.host.endsWith("github.com") && TOKEN != null) { request.setHeader("Authorization", "token $TOKEN") } @@ -99,27 +115,27 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient { HttpClientFactory.getHttpClient() } - client.execute(request, ResponseHandler(this, continuation)) + client.execute(request, ResponseHandler(this, continuation, urlRequest)) } catch (e: Exception) { continuation.resumeWith(Result.failure(e)) } } - override suspend fun get(url: String): String? { + override suspend fun getFullResponse(request: UrlRequest): HttpResponse? { //Retry up to 10 times for (retryCount in 1..10) { try { - LOGGER.info("Getting $url") - val body: String = suspendCoroutine { continuation -> - getData(URL(url), continuation) + LOGGER.info("Getting ${request.url} ${request.lastModified}") + val response: HttpResponse = suspendCoroutine { continuation -> + getData(request, continuation) } - LOGGER.info("Got $url") - return body + LOGGER.info("Got ${request.url}") + return response } catch (e: NotFoundException) { return null } catch (e: Exception) { - LOGGER.error("Failed to read data retrying $retryCount $url", e) + LOGGER.error("Failed to read data retrying $retryCount ${request.url}", e) delay(1000) } } @@ -127,6 +143,10 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient { return null } + override suspend fun get(url: String): String? { + return extractBody(getFullResponse(UrlRequest(url))) + } + class NotFoundException : Throwable() } diff --git a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CacheDbEntry.kt b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CacheDbEntry.kt index a04c551c..a11d5dc3 100644 --- a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CacheDbEntry.kt +++ b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CacheDbEntry.kt @@ -2,5 +2,6 @@ package net.adoptopenjdk.api.v3.dataSources.mongo class CacheDbEntry( val url: String, + val lastModified: String? = null, val data: String? ) diff --git a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CachedGithubHtmlClient.kt b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CachedGithubHtmlClient.kt index 07fd28ae..08184fb3 100644 --- a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CachedGithubHtmlClient.kt +++ b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CachedGithubHtmlClient.kt @@ -6,7 +6,9 @@ import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.async import kotlinx.coroutines.delay import kotlinx.coroutines.launch +import net.adoptopenjdk.api.v3.dataSources.DefaultUpdaterHtmlClient import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClientFactory +import net.adoptopenjdk.api.v3.dataSources.UrlRequest import org.slf4j.LoggerFactory import java.util.concurrent.Executors import java.util.concurrent.LinkedBlockingQueue @@ -21,7 +23,8 @@ object CachedGithubHtmlClient { private val internalDbStore = InternalDbStoreFactory.get() //List of urls to be refreshed in the background - private val workList = LinkedBlockingQueue() + private val workList = LinkedBlockingQueue() + init { //Do refresh in the background @@ -31,9 +34,9 @@ object CachedGithubHtmlClient { suspend fun getUrl(url: String): String? { val cachedEntry = internalDbStore.getCachedWebpage(url) return if (cachedEntry == null) { - get(url) + get(UrlRequest(url)) } else { - workList.offer(url) + workList.offer(UrlRequest(url, cachedEntry.lastModified)) cachedEntry.data } } @@ -41,26 +44,37 @@ object CachedGithubHtmlClient { private fun cacheRefreshDaemonThread(): suspend CoroutineScope.() -> Unit { return { while (true) { - val url = workList.take() + val request = workList.take() async { - LOGGER.info("Enqueuing $url") - return@async get(url) + LOGGER.info("Enqueuing ${request.url} ${request.lastModified}") + return@async get(request) }.await() } } } - private suspend fun get(url: String): String? { + private suspend fun get(request: UrlRequest): String? { //Retry up to 10 times for (retryCount in 1..10) { try { - LOGGER.info("Getting $url") - val body = UpdaterHtmlClientFactory.client.get(url) - internalDbStore.putCachedWebpage(url, body) - LOGGER.info("Got $url") + LOGGER.info("Getting ${request.url} ${request.lastModified}") + val response = UpdaterHtmlClientFactory.client.getFullResponse(request) + + + if (response?.statusLine?.statusCode == 304) { + //asset has not updated + return null + } + + val body = DefaultUpdaterHtmlClient.extractBody(response) + + val lastModified = response?.getFirstHeader("Last-Modified")?.value + + internalDbStore.putCachedWebpage(request.url, lastModified, body) + LOGGER.info("Got ${request.url}") return body } catch (e: Exception) { - LOGGER.error("Failed to read data retrying $retryCount $url", e) + LOGGER.error("Failed to read data retrying $retryCount ${request.url}", e) delay(1000) } } diff --git a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/InternalDbStore.kt b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/InternalDbStore.kt index 12d2dc11..ea15715a 100644 --- a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/InternalDbStore.kt +++ b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/InternalDbStore.kt @@ -36,11 +36,11 @@ class InternalDbStore : MongoInterface(MongoClientFactory.get()) { } - suspend fun putCachedWebpage(url: String, data: String?) { + suspend fun putCachedWebpage(url: String, lastModified: String?, data: String?) { GlobalScope.launch { webCache.updateOne( Document("url", url), - CacheDbEntry(url, data), + CacheDbEntry(url, lastModified, data), UpdateOptions().upsert(true), false) } diff --git a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/mapping/adopt/AdoptReleaseMapper.kt b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/mapping/adopt/AdoptReleaseMapper.kt index a9d228b2..78f28800 100644 --- a/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/mapping/adopt/AdoptReleaseMapper.kt +++ b/adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/mapping/adopt/AdoptReleaseMapper.kt @@ -80,8 +80,8 @@ object AdoptReleaseMapper : ReleaseMapper() { parseVersionInfo(release, release_name) } .ifEmpty { throw Exception("Failed to parse version $release_name") } - .first() - + .sorted() + .last() } private fun parseVersionInfo(release: GHRelease, release_name: String): List { diff --git a/adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/AdoptMetadataVersionParsingTest.kt b/adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/AdoptMetadataVersionParsingTest.kt index 42aa4d03..189040ca 100644 --- a/adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/AdoptMetadataVersionParsingTest.kt +++ b/adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/AdoptMetadataVersionParsingTest.kt @@ -1,11 +1,19 @@ package net.adoptopenjdk.api +import io.mockk.every +import io.mockk.mockk import kotlinx.coroutines.runBlocking import net.adoptopenjdk.api.v3.JsonMapper import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClient import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClientFactory +import net.adoptopenjdk.api.v3.dataSources.UrlRequest import net.adoptopenjdk.api.v3.dataSources.github.graphql.models.GHRelease import net.adoptopenjdk.api.v3.mapping.adopt.AdoptReleaseMapper +import org.apache.http.HttpEntity +import org.apache.http.HttpResponse +import org.apache.http.ProtocolVersion +import org.apache.http.message.BasicHeader +import org.apache.http.message.BasicStatusLine import org.junit.jupiter.api.Test import kotlin.test.assertEquals @@ -43,6 +51,16 @@ class AdoptMetadataVersionParsingTest { } """.trimIndent() } + + override suspend fun getFullResponse(request: UrlRequest): HttpResponse? { + val metadataResponse = mockk() + val entity = mockk() + every { entity.content } returns get(request.url)?.byteInputStream() + every { metadataResponse.statusLine } returns BasicStatusLine(ProtocolVersion("", 1, 1), 200, "") + every { metadataResponse.entity } returns entity + every { metadataResponse.getFirstHeader("Last-Modified") } returns BasicHeader("Last-Modified", "") + return metadataResponse + } } diff --git a/adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/BaseTest.kt b/adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/BaseTest.kt index 8d711791..9a729644 100644 --- a/adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/BaseTest.kt +++ b/adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/BaseTest.kt @@ -6,7 +6,9 @@ import de.flapdoodle.embed.mongo.config.MongodConfigBuilder import de.flapdoodle.embed.mongo.config.Net import de.flapdoodle.embed.mongo.distribution.Version import de.flapdoodle.embed.process.runtime.Network +import io.mockk.every import io.mockk.junit5.MockKExtension +import io.mockk.mockk import kotlinx.coroutines.runBlocking import net.adoptopenjdk.api.v3.AdoptReposBuilder import net.adoptopenjdk.api.v3.AdoptRepository @@ -16,6 +18,7 @@ import net.adoptopenjdk.api.v3.dataSources.ApiPersistenceFactory import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClient import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClientFactory import net.adoptopenjdk.api.v3.dataSources.UpdaterJsonMapper +import net.adoptopenjdk.api.v3.dataSources.UrlRequest import net.adoptopenjdk.api.v3.dataSources.github.graphql.models.PageInfo import net.adoptopenjdk.api.v3.dataSources.github.graphql.models.summary.GHReleaseSummary import net.adoptopenjdk.api.v3.dataSources.github.graphql.models.summary.GHReleasesSummary @@ -24,6 +27,11 @@ import net.adoptopenjdk.api.v3.dataSources.models.AdoptRepos import net.adoptopenjdk.api.v3.dataSources.models.FeatureRelease import net.adoptopenjdk.api.v3.dataSources.persitence.mongo.MongoClientFactory import net.adoptopenjdk.api.v3.models.Release +import org.apache.http.HttpEntity +import org.apache.http.HttpResponse +import org.apache.http.ProtocolVersion +import org.apache.http.message.BasicHeader +import org.apache.http.message.BasicStatusLine import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.extension.ExtendWith @@ -49,6 +57,17 @@ abstract class BaseTest { return null } + override suspend fun getFullResponse(request: UrlRequest): HttpResponse? { + val metadataResponse = mockk() + + val entity = mockk() + every { entity.content } returns get(request.url)?.byteInputStream() + every { metadataResponse.statusLine } returns BasicStatusLine(ProtocolVersion("", 1, 1), 200, "") + every { metadataResponse.entity } returns entity + every { metadataResponse.getFirstHeader("Last-Modified") } returns BasicHeader("Last-Modified", "") + return metadataResponse + } + } }