Merge in latest updates from staging

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/UpdaterHtmlClient.kt

@@ -17,8 +17,14 @@ import kotlin.coroutines.resumeWithException
 import kotlin.coroutines.suspendCoroutine
 
 
+data class UrlRequest(
+        val url: String,
+        val lastModified: String? = null
+)
+
 interface UpdaterHtmlClient {
     suspend fun get(url: String): String?
+    suspend fun getFullResponse(request: UrlRequest): HttpResponse?
 }
 
 object UpdaterHtmlClientFactory {
@@ -31,9 +37,20 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
         @JvmStatic
         private val LOGGER = LoggerFactory.getLogger(this::class.java)
         private val TOKEN: String? = GithubAuth.readToken()
+
+
+        fun extractBody(response: HttpResponse?): String? {
+            if (response == null) {
+                return null
+            }
+            val writer = StringWriter()
+            IOUtils.copy(response.entity.content, writer, Charset.defaultCharset())
+            return writer.toString()
+        }
+
     }
 
-    class ResponseHandler(val client: DefaultUpdaterHtmlClient, val continuation: Continuation<String>) : FutureCallback<HttpResponse> {
+    class ResponseHandler(val client: DefaultUpdaterHtmlClient, private val continuation: Continuation<HttpResponse>, val request: UrlRequest?) : FutureCallback<HttpResponse> {
         override fun cancelled() {
             continuation.resumeWithException(Exception("cancelled"))
         }
@@ -45,13 +62,13 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
                         continuation.resumeWithException(Exception("No response body"))
                     }
                     isARedirect(response) -> {
-                        client.getData(URL(response.getFirstHeader("location").value), continuation)
+                        client.getData(UrlRequest(response.getFirstHeader("location").value, request?.lastModified), continuation)
                     }
                     response.statusLine.statusCode == 404 -> {
                         continuation.resumeWithException(NotFoundException())
                     }
-                    response.statusLine.statusCode == 200 -> {
-                        continuation.resume(extractBody(response))
+                    response.statusLine.statusCode == 200 || response.statusLine.statusCode == 304 -> {
+                        continuation.resume(response)
                     }
                     else -> {
                         continuation.resumeWithException(Exception("Unexpected response ${response.statusLine.statusCode}"))
@@ -62,12 +79,6 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
             }
         }
 
-        private fun extractBody(response: HttpResponse): String {
-            val writer = StringWriter()
-            IOUtils.copy(response.entity.content, writer, Charset.defaultCharset())
-            return writer.toString()
-        }
-
         private fun isARedirect(response: HttpResponse): Boolean {
             return response.statusLine.statusCode == 307 ||
                     response.statusLine.statusCode == 301 ||
@@ -84,10 +95,15 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
         }
     }
 
-    private fun getData(url: URL, continuation: Continuation<String>) {
+    private fun getData(urlRequest: UrlRequest, continuation: Continuation<HttpResponse>) {
         try {
+            val url = URL(urlRequest.url)
             val request = HttpGet(url.toURI())
 
+            if (urlRequest.lastModified != null) {
+                request.addHeader("If-Modified-Since", urlRequest.lastModified)
+            }
+
             if (url.host.endsWith("github.com") && TOKEN != null) {
                 request.setHeader("Authorization", "token $TOKEN")
             }
@@ -99,34 +115,38 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
                         HttpClientFactory.getHttpClient()
                     }
 
-            client.execute(request, ResponseHandler(this, continuation))
+            client.execute(request, ResponseHandler(this, continuation, urlRequest))
         } catch (e: Exception) {
             continuation.resumeWith(Result.failure(e))
         }
     }
 
 
-    override suspend fun get(url: String): String? {
+    override suspend fun getFullResponse(request: UrlRequest): HttpResponse? {
         //Retry up to 10 times
         for (retryCount in 1..10) {
             try {
-                LOGGER.info("Getting $url")
-                val body: String = suspendCoroutine { continuation ->
-                    getData(URL(url), continuation)
+                LOGGER.info("Getting ${request.url} ${request.lastModified}")
+                val response: HttpResponse = suspendCoroutine { continuation ->
+                    getData(request, continuation)
                 }
-                LOGGER.info("Got $url")
-                return body
+                LOGGER.info("Got  ${request.url}")
+                return response
             } catch (e: NotFoundException) {
                 return null
             } catch (e: Exception) {
-                LOGGER.error("Failed to read data retrying $retryCount $url", e)
+                LOGGER.error("Failed to read data retrying $retryCount ${request.url}", e)
                 delay(1000)
             }
         }
 
         return null
     }
 
+    override suspend fun get(url: String): String? {
+        return extractBody(getFullResponse(UrlRequest(url)))
+    }
+
     class NotFoundException : Throwable()
 }
 

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CacheDbEntry.kt

@@ -2,5 +2,6 @@ package net.adoptopenjdk.api.v3.dataSources.mongo
 
 class CacheDbEntry(
         val url: String,
+        val lastModified: String? = null,
         val data: String?
 )

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/CachedGithubHtmlClient.kt

@@ -6,7 +6,9 @@ import kotlinx.coroutines.asCoroutineDispatcher
 import kotlinx.coroutines.async
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
+import net.adoptopenjdk.api.v3.dataSources.DefaultUpdaterHtmlClient
 import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClientFactory
+import net.adoptopenjdk.api.v3.dataSources.UrlRequest
 import org.slf4j.LoggerFactory
 import java.util.concurrent.Executors
 import java.util.concurrent.LinkedBlockingQueue
@@ -21,7 +23,8 @@ object CachedGithubHtmlClient {
     private val internalDbStore = InternalDbStoreFactory.get()
 
     //List of urls to be refreshed in the background
-    private val workList = LinkedBlockingQueue<String>()
+    private val workList = LinkedBlockingQueue<UrlRequest>()
+
 
     init {
         //Do refresh in the background
@@ -31,36 +34,47 @@ object CachedGithubHtmlClient {
     suspend fun getUrl(url: String): String? {
         val cachedEntry = internalDbStore.getCachedWebpage(url)
         return if (cachedEntry == null) {
-            get(url)
+            get(UrlRequest(url))
         } else {
-            workList.offer(url)
+            workList.offer(UrlRequest(url, cachedEntry.lastModified))
             cachedEntry.data
         }
     }
 
     private fun cacheRefreshDaemonThread(): suspend CoroutineScope.() -> Unit {
         return {
             while (true) {
-                val url = workList.take()
+                val request = workList.take()
                 async {
-                    LOGGER.info("Enqueuing $url")
-                    return@async get(url)
+                    LOGGER.info("Enqueuing ${request.url} ${request.lastModified}")
+                    return@async get(request)
                 }.await()
             }
         }
     }
 
-    private suspend fun get(url: String): String? {
+    private suspend fun get(request: UrlRequest): String? {
         //Retry up to 10 times
         for (retryCount in 1..10) {
             try {
-                LOGGER.info("Getting $url")
-                val body = UpdaterHtmlClientFactory.client.get(url)
-                internalDbStore.putCachedWebpage(url, body)
-                LOGGER.info("Got $url")
+                LOGGER.info("Getting  ${request.url} ${request.lastModified}")
+                val response = UpdaterHtmlClientFactory.client.getFullResponse(request)
+
+
+                if (response?.statusLine?.statusCode == 304) {
+                    //asset has not updated
+                    return null
+                }
+
+                val body = DefaultUpdaterHtmlClient.extractBody(response)
+
+                val lastModified = response?.getFirstHeader("Last-Modified")?.value
+
+                internalDbStore.putCachedWebpage(request.url, lastModified, body)
+                LOGGER.info("Got ${request.url}")
                 return body
             } catch (e: Exception) {
-                LOGGER.error("Failed to read data retrying $retryCount $url", e)
+                LOGGER.error("Failed to read data retrying $retryCount ${request.url}", e)
                 delay(1000)
             }
         }

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/mongo/InternalDbStore.kt

@@ -36,11 +36,11 @@ class InternalDbStore : MongoInterface(MongoClientFactory.get()) {
     }
 
 
-    suspend fun putCachedWebpage(url: String, data: String?) {
+    suspend fun putCachedWebpage(url: String, lastModified: String?, data: String?) {
         GlobalScope.launch {
             webCache.updateOne(
                     Document("url", url),
-                    CacheDbEntry(url, data),
+                    CacheDbEntry(url, lastModified, data),
                     UpdateOptions().upsert(true),
                     false)
         }

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/mapping/adopt/AdoptReleaseMapper.kt

@@ -80,8 +80,8 @@ object AdoptReleaseMapper : ReleaseMapper() {
                     parseVersionInfo(release, release_name)
                 }
                 .ifEmpty { throw Exception("Failed to parse version $release_name") }
-                .first()
-
+                .sorted()
+                .last()
     }
 
     private fun parseVersionInfo(release: GHRelease, release_name: String): List<VersionData> {

adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/AdoptMetadataVersionParsingTest.kt

@@ -1,11 +1,19 @@
 package net.adoptopenjdk.api
 
+import io.mockk.every
+import io.mockk.mockk
 import kotlinx.coroutines.runBlocking
 import net.adoptopenjdk.api.v3.JsonMapper
 import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClient
 import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClientFactory
+import net.adoptopenjdk.api.v3.dataSources.UrlRequest
 import net.adoptopenjdk.api.v3.dataSources.github.graphql.models.GHRelease
 import net.adoptopenjdk.api.v3.mapping.adopt.AdoptReleaseMapper
+import org.apache.http.HttpEntity
+import org.apache.http.HttpResponse
+import org.apache.http.ProtocolVersion
+import org.apache.http.message.BasicHeader
+import org.apache.http.message.BasicStatusLine
 import org.junit.jupiter.api.Test
 import kotlin.test.assertEquals
 
@@ -43,6 +51,16 @@ class AdoptMetadataVersionParsingTest {
                         }
                     """.trimIndent()
                 }
+
+                override suspend fun getFullResponse(request: UrlRequest): HttpResponse? {
+                    val metadataResponse = mockk<HttpResponse>()
+                    val entity = mockk<HttpEntity>()
+                    every { entity.content } returns get(request.url)?.byteInputStream()
+                    every { metadataResponse.statusLine } returns BasicStatusLine(ProtocolVersion("", 1, 1), 200, "")
+                    every { metadataResponse.entity } returns entity
+                    every { metadataResponse.getFirstHeader("Last-Modified") } returns BasicHeader("Last-Modified", "")
+                    return metadataResponse
+                }
             }
 
 

adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/BaseTest.kt

@@ -6,7 +6,9 @@ import de.flapdoodle.embed.mongo.config.MongodConfigBuilder
 import de.flapdoodle.embed.mongo.config.Net
 import de.flapdoodle.embed.mongo.distribution.Version
 import de.flapdoodle.embed.process.runtime.Network
+import io.mockk.every
 import io.mockk.junit5.MockKExtension
+import io.mockk.mockk
 import kotlinx.coroutines.runBlocking
 import net.adoptopenjdk.api.v3.AdoptReposBuilder
 import net.adoptopenjdk.api.v3.AdoptRepository
@@ -16,6 +18,7 @@ import net.adoptopenjdk.api.v3.dataSources.ApiPersistenceFactory
 import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClient
 import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClientFactory
 import net.adoptopenjdk.api.v3.dataSources.UpdaterJsonMapper
+import net.adoptopenjdk.api.v3.dataSources.UrlRequest
 import net.adoptopenjdk.api.v3.dataSources.github.graphql.models.PageInfo
 import net.adoptopenjdk.api.v3.dataSources.github.graphql.models.summary.GHReleaseSummary
 import net.adoptopenjdk.api.v3.dataSources.github.graphql.models.summary.GHReleasesSummary
@@ -24,6 +27,11 @@ import net.adoptopenjdk.api.v3.dataSources.models.AdoptRepos
 import net.adoptopenjdk.api.v3.dataSources.models.FeatureRelease
 import net.adoptopenjdk.api.v3.dataSources.persitence.mongo.MongoClientFactory
 import net.adoptopenjdk.api.v3.models.Release
+import org.apache.http.HttpEntity
+import org.apache.http.HttpResponse
+import org.apache.http.ProtocolVersion
+import org.apache.http.message.BasicHeader
+import org.apache.http.message.BasicStatusLine
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.extension.ExtendWith
@@ -49,6 +57,17 @@ abstract class BaseTest {
                     return null
                 }
 
+                override suspend fun getFullResponse(request: UrlRequest): HttpResponse? {
+                    val metadataResponse = mockk<HttpResponse>()
+
+                    val entity = mockk<HttpEntity>()
+                    every { entity.content } returns get(request.url)?.byteInputStream()
+                    every { metadataResponse.statusLine } returns BasicStatusLine(ProtocolVersion("", 1, 1), 200, "")
+                    every { metadataResponse.entity } returns entity
+                    every { metadataResponse.getFirstHeader("Last-Modified") } returns BasicHeader("Last-Modified", "")
+                    return metadataResponse
+                }
+
             }
         }