Bring #124, #114 and #111 from staging into master

adoptopenjdk-api-v3-persistance/src/main/kotlin/net/adoptopenjdk/api/v3/DownloadStatsInterface.kt

@@ -1,7 +1,8 @@
-package net.adoptopenjdk.api.v3;
+package net.adoptopenjdk.api.v3
 
 import net.adoptopenjdk.api.v3.dataSources.APIDataStore
 import net.adoptopenjdk.api.v3.dataSources.ApiPersistenceFactory
+import net.adoptopenjdk.api.v3.dataSources.persitence.ApiPersistence
 import net.adoptopenjdk.api.v3.models.DbStatsEntry
 import net.adoptopenjdk.api.v3.models.DownloadDiff
 import net.adoptopenjdk.api.v3.models.DownloadStats
@@ -18,10 +19,17 @@ class StatEntry(
         val count: Long
 )
 
-class DownloadStatsInterface {
-    private val dataStore = ApiPersistenceFactory.get()
+class DownloadStatsInterface(
+        private val dataStore: ApiPersistence = ApiPersistenceFactory.get()
+) {
 
-    suspend fun getTrackingStats(days: Int?, from: ZonedDateTime?, to: ZonedDateTime?, source: StatsSource?, featureVersion: Int?, dockerRepo: String?): List<DownloadDiff> {
+    suspend fun getTrackingStats(
+            days: Int? = null,
+            from: ZonedDateTime? = null,
+            to: ZonedDateTime? = null,
+            source: StatsSource? = null,
+            featureVersion: Int? = null,
+            dockerRepo: String? = null): List<DownloadDiff> {
 
         //need +1 as for a diff you need num days +1 from db
         val daysSince = (days ?: 30) + 1
@@ -84,15 +92,33 @@ class DownloadStatsInterface {
         return sumDailyStats(
                 dataStore
                         .getGithubStats(start, end)
+                        .groupBy { it.date.toLocalDate() }
+                        .flatMap { grouped ->
+                            grouped.value
+                                    .groupBy { it.feature_version }
+                                    .map { featureVersionsForDay ->
+                                        featureVersionsForDay.value.maxBy { it.date }!!
+                                    }
+                        }
                         .filter { featureVersion == null || it.feature_version == featureVersion }
+                        .sortedBy { it.date }
         )
     }
 
     private suspend fun getDockerDownloadStatsByDate(start: ZonedDateTime, end: ZonedDateTime, dockerRepo: String?): List<StatEntry> {
         return sumDailyStats(
                 dataStore
                         .getDockerStats(start, end)
+                        .groupBy { it.date.toLocalDate() }
+                        .flatMap { grouped ->
+                            grouped.value
+                                    .groupBy { it.repo }
+                                    .map { repoForDay ->
+                                        repoForDay.value.maxBy { it.date }!!
+                                    }
+                        }
                         .filter { dockerRepo == null || it.repo == dockerRepo }
+                        .sortedBy { it.date }
         )
     }
 

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/HttpClientFactory.kt

@@ -3,6 +3,7 @@ package net.adoptopenjdk.api.v3
 import org.apache.http.HttpRequest
 import org.apache.http.HttpResponse
 import org.apache.http.client.RedirectStrategy
+import org.apache.http.client.config.RequestConfig
 import org.apache.http.client.methods.HttpUriRequest
 import org.apache.http.impl.NoConnectionReuseStrategy
 import org.apache.http.impl.nio.client.HttpAsyncClients
@@ -13,10 +14,19 @@ object HttpClientFactory {
     private val client: HttpAsyncClient
     private val noRedirect: HttpAsyncClient
 
+    val REQUEST_CONFIG = RequestConfig
+            .copy(RequestConfig.DEFAULT)
+            .setConnectTimeout(5000)
+            .setSocketTimeout(5000)
+            .setConnectionRequestTimeout(5000)
+            .build()!!
+
     init {
+
         val client = HttpAsyncClients.custom()
                 .setConnectionReuseStrategy(NoConnectionReuseStrategy())
                 .disableCookieManagement()
+                .setDefaultRequestConfig(REQUEST_CONFIG)
                 .build()
         client.start()
         this.client = client
@@ -33,6 +43,7 @@ object HttpClientFactory {
 
                 })
                 .setConnectionReuseStrategy(NoConnectionReuseStrategy())
+                .setDefaultRequestConfig(REQUEST_CONFIG)
                 .build()
         noRedirect.start()
         this.noRedirect = noRedirect

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/dataSources/UpdaterHtmlClient.kt

@@ -1,11 +1,12 @@
 package net.adoptopenjdk.api.v3.dataSources
 
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.withTimeout
 import net.adoptopenjdk.api.v3.HttpClientFactory
 import net.adoptopenjdk.api.v3.dataSources.github.GithubAuth
 import org.apache.commons.io.IOUtils
 import org.apache.http.HttpResponse
-import org.apache.http.client.methods.HttpGet
+import org.apache.http.client.methods.RequestBuilder
 import org.apache.http.concurrent.FutureCallback
 import org.slf4j.LoggerFactory
 import java.io.StringWriter
@@ -37,7 +38,7 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
         @JvmStatic
         private val LOGGER = LoggerFactory.getLogger(this::class.java)
         private val TOKEN: String? = GithubAuth.readToken()
-
+        private const val REQUEST_TIMEOUT = 12000L
 
         fun extractBody(response: HttpResponse?): String? {
             if (response == null) {
@@ -98,7 +99,10 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
     private fun getData(urlRequest: UrlRequest, continuation: Continuation<HttpResponse>) {
         try {
             val url = URL(urlRequest.url)
-            val request = HttpGet(url.toURI())
+            val request = RequestBuilder
+                    .get(url.toURI())
+                    .setConfig(HttpClientFactory.REQUEST_CONFIG)
+                    .build()
 
             if (urlRequest.lastModified != null) {
                 request.addHeader("If-Modified-Since", urlRequest.lastModified)
@@ -116,6 +120,7 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
                     }
 
             client.execute(request, ResponseHandler(this, continuation, urlRequest))
+
         } catch (e: Exception) {
             continuation.resumeWith(Result.failure(e))
         }
@@ -127,8 +132,10 @@ class DefaultUpdaterHtmlClient : UpdaterHtmlClient {
         for (retryCount in 1..10) {
             try {
                 LOGGER.info("Getting ${request.url} ${request.lastModified}")
-                val response: HttpResponse = suspendCoroutine { continuation ->
-                    getData(request, continuation)
+                val response: HttpResponse = withTimeout(REQUEST_TIMEOUT) {
+                    suspendCoroutine<HttpResponse> { continuation ->
+                        getData(request, continuation)
+                    }
                 }
                 LOGGER.info("Got  ${request.url}")
                 return response

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/stats/DockerStatsInterface.kt

@@ -67,7 +67,7 @@ class DockerStatsInterface {
 
     private fun getStatsForUrl(url: String): JsonObject {
         return runBlocking {
-            val stats = UpdaterHtmlClientFactory.client.get(url);
+            val stats = UpdaterHtmlClientFactory.client.get(url)
             if (stats == null) {
                 throw Exception("Stats not returned")
             }

adoptopenjdk-api-v3-updater/src/main/kotlin/net/adoptopenjdk/api/v3/stats/StatsInterface.kt

@@ -1,13 +1,13 @@
 package net.adoptopenjdk.api.v3.stats
 
-import kotlinx.coroutines.runBlocking
 import net.adoptopenjdk.api.v3.DownloadStatsInterface
 import net.adoptopenjdk.api.v3.dataSources.ApiPersistenceFactory
 import net.adoptopenjdk.api.v3.dataSources.models.AdoptRepos
 import net.adoptopenjdk.api.v3.dataSources.persitence.ApiPersistence
 import net.adoptopenjdk.api.v3.models.StatsSource
 import org.slf4j.LoggerFactory
 import java.time.ZoneOffset
+import java.time.ZonedDateTime
 
 class StatsInterface {
 
@@ -30,17 +30,36 @@ class StatsInterface {
     }
 
     private suspend fun removeBadDownloadStats() {
-        val tracking = downloadStatsInterface.getTrackingStats(10, null, null, StatsSource.all, null, null)
+        val tracking = downloadStatsInterface.getTrackingStats(days = 10, source = StatsSource.all)
         tracking
                 .filter { it.daily <= 0 }
                 .forEach { entry ->
-                    val start = entry.date.toLocalDate().atStartOfDay()
-                    val end = entry.date.toLocalDate().plusDays(1).atStartOfDay()
-                    LOGGER.info("Removing bad stats between $start $end")
-                    runBlocking {
-                        database.removeStatsBetween(start.atZone(ZoneOffset.UTC), end.atZone(ZoneOffset.UTC))
-                    }
+                    val start = entry.date.toLocalDate().atStartOfDay().atZone(ZoneOffset.UTC)
+                    val end = entry.date.toLocalDate().plusDays(1).atStartOfDay().atZone(ZoneOffset.UTC)
+
+                    printStatDebugInfo(start, end)
+                    database.removeStatsBetween(start, end)
                 }
     }
 
+    private suspend fun printStatDebugInfo(start: ZonedDateTime, end: ZonedDateTime) {
+        LOGGER.info("Removing bad stats between $start $end")
+        printStats(start, end)
+        LOGGER.info("Day before: $start $end")
+        printStats(start.minusDays(1), end.minusDays(1))
+    }
+
+    private suspend fun printStats(start: ZonedDateTime, end: ZonedDateTime) {
+        database
+                .getGithubStats(start, end)
+                .forEach { stat ->
+                    LOGGER.info("github stat: ${stat.feature_version} ${stat.downloads}")
+                }
+
+        database
+                .getDockerStats(start, end)
+                .forEach { stat ->
+                    LOGGER.info("docker stat: ${stat.repo} ${stat.pulls}")
+                }
+    }
 }

adoptopenjdk-api-v3-updater/src/main/resources/logback.xml

@@ -1,21 +1,19 @@
 <configuration>
+    <logger name="org.apache.http.client.protocol.ResponseProcessCookies" level="ERROR"/>
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>%date %level [%thread] %logger{10} [%file:%line]%n%msg%n</pattern>
         </encoder>
     </appender>
 
-
     <appender name="FILE" class="ch.qos.logback.core.FileAppender">
         <file>/tmp/updater.log</file>
-
         <encoder>
             <pattern>%date %level [%thread] %logger{10} [%file:%line]%n%msg%n</pattern>
         </encoder>
     </appender>
 
-
     <root level="info">
         <appender-ref ref="STDOUT"/>
         <appender-ref ref="FILE"/>

adoptopenjdk-api-v3-updater/src/test/kotlin/net/adoptopenjdk/api/DockerStatsInterfaceTest.kt

@@ -1,13 +1,22 @@
 package net.adoptopenjdk.api
 
+import io.mockk.coEvery
+import io.mockk.mockk
 import kotlinx.coroutines.runBlocking
+import net.adoptopenjdk.api.v3.DownloadStatsInterface
 import net.adoptopenjdk.api.v3.dataSources.ApiPersistenceFactory
 import net.adoptopenjdk.api.v3.dataSources.DefaultUpdaterHtmlClient
 import net.adoptopenjdk.api.v3.dataSources.UpdaterHtmlClientFactory
+import net.adoptopenjdk.api.v3.dataSources.persitence.ApiPersistence
+import net.adoptopenjdk.api.v3.models.DockerDownloadStatsDbEntry
+import net.adoptopenjdk.api.v3.models.GithubDownloadStatsDbEntry
+import net.adoptopenjdk.api.v3.models.StatsSource
 import net.adoptopenjdk.api.v3.stats.DockerStatsInterface
 import org.junit.Assert
 import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.Test
+import java.time.ZonedDateTime
+import kotlin.test.assertEquals
 
 
 class DockerStatsInterfaceTest {
@@ -30,5 +39,42 @@ class DockerStatsInterfaceTest {
             Assert.assertTrue(stats.size > 0)
         }
     }
+
+
+    @Test
+    fun onlyLastStatEntryPerDayIsRead() {
+        runBlocking {
+            val apiPersistanceMock = mockk<ApiPersistence>()
+            coEvery { apiPersistanceMock.getGithubStats(any<ZonedDateTime>(), any<ZonedDateTime>()) } returns listOf(
+                    GithubDownloadStatsDbEntry(ZonedDateTime.now().minusMinutes(10), 100, 8),
+                    GithubDownloadStatsDbEntry(ZonedDateTime.now().minusMinutes(20), 90, 8),
+                    GithubDownloadStatsDbEntry(ZonedDateTime.now().minusDays(1), 80, 8),
+                    GithubDownloadStatsDbEntry(ZonedDateTime.now().minusMinutes(15), 20, 9)
+            )
+
+            coEvery { apiPersistanceMock.getDockerStats(any<ZonedDateTime>(), any<ZonedDateTime>()) } returns listOf(
+                    DockerDownloadStatsDbEntry(ZonedDateTime.now().minusMinutes(10), 100, "a-stats-repo"),
+                    DockerDownloadStatsDbEntry(ZonedDateTime.now().minusMinutes(20), 90, "a-stats-repo"),
+                    DockerDownloadStatsDbEntry(ZonedDateTime.now().minusDays(1), 80, "a-stats-repo"),
+                    DockerDownloadStatsDbEntry(ZonedDateTime.now().minusMinutes(15), 20, "a-different-stats-repo")
+            )
+
+            val downloadStatsInterface = DownloadStatsInterface(apiPersistanceMock)
+
+            var stats = downloadStatsInterface.getTrackingStats(10, source = StatsSource.github, featureVersion = 8)
+            assertEquals(100, stats[0].total)
+            stats = downloadStatsInterface.getTrackingStats(10, source = StatsSource.github)
+            assertEquals(120, stats[0].total)
+
+            stats = downloadStatsInterface.getTrackingStats(10, source = StatsSource.dockerhub, dockerRepo = "a-stats-repo")
+            assertEquals(100, stats[0].total)
+            stats = downloadStatsInterface.getTrackingStats(10, source = StatsSource.dockerhub)
+            assertEquals(120, stats[0].total)
+
+            stats = downloadStatsInterface.getTrackingStats(10)
+            assertEquals(240, stats[0].total)
+
+        }
+    }
 }