wip

common/src/main/java/de/codesourcery/versiontracker/common/IVersionProvider.java

@@ -103,7 +103,9 @@ public Statistics createCopy() {
      * This method must be <b>thread-safe</b>.
      *
      * @param info
-     * @param force
+     * @param force whether to fetch version information even if the Maven indexer XML indicates we already have the latest data.
+     *              This is mostly useful to work around a bug in previous versions that would not properly retrieve release dates
+     *              for all available versions.
      * @return
      * @throws IOException
      */

common/src/main/java/de/codesourcery/versiontracker/common/server/BackgroundUpdater.java

@@ -237,7 +237,7 @@ public void doUpdate(VersionInfo info) {
                     }
                     try
                     {
-                        provider.update(info, xxxx );
+                        provider.update(info, false );
                     }
                     finally
                     {

common/src/main/java/de/codesourcery/versiontracker/common/server/MavenCentralVersionProvider.java

@@ -197,7 +197,7 @@ public static void main(String[] args) throws IOException
         VersionInfo data = new VersionInfo();
         data.artifact = test;
         long start = System.currentTimeMillis();
-        final UpdateResult result = new MavenCentralVersionProvider().update( data, xxxx );
+        final UpdateResult result = new MavenCentralVersionProvider().update( data, false );
         long end = System.currentTimeMillis();
         System.out.println("TIME: "+(end-start)+" ms");
         System.out.println("RESULT: "+result);

common/src/main/java/de/codesourcery/versiontracker/common/server/VersionTracker.java

@@ -191,7 +191,7 @@ private void updateArtifactFromServer(Artifact artifact,
 
                         try 
                         {
-                            versionProvider.update( newInfo, xxxx );
+                            versionProvider.update( newInfo, true );
                         }
                         catch (Exception e) 
                         {

common/src/test/java/de/codesourcery/versiontracker/common/server/APIImplTest.java

@@ -34,6 +34,7 @@
 import java.util.Set;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.easymock.EasyMock.anyBoolean;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
@@ -67,11 +68,9 @@ public void testDataUnavailableFromStorage() throws Exception
 
         // mock version provider
         final IVersionProvider versionProvider = createMock( IVersionProvider.class );
-        expect( versionProvider.update( isA( VersionInfo.class ), xxxx ) ).andAnswer( () -> {
+        expect( versionProvider.update( isA( VersionInfo.class ), anyBoolean() ) ).andAnswer( () -> {
 
             final VersionInfo versionInfo = getCurrentArgument( 0 );
-            final Set<String> additionalVersionsToFetchReleaseDatesFor = getCurrentArgument( 1 );
-            assertThat( additionalVersionsToFetchReleaseDatesFor ).containsOnly( "1.0.0" );
 
             assertThat( versionInfo.artifact.artifactId ).isEqualTo( art1.artifactId );
             assertThat( versionInfo.artifact.groupId ).isEqualTo( art1.groupId );
@@ -143,7 +142,6 @@ protected IVersionTracker createVersionTracker(SharedLockCache lockCache) {
             // verify assertions
             assertThat( response.artifacts ).hasSize( 1 );
             final ArtifactResponse resp1 = response.artifacts.get( 0 );
-            resp1.updateAvailable = ArtifactResponse.UpdateAvailable.YES;
             assertThat( resp1.artifact ).isEqualTo( art1 );
             assertThat( resp1.latestVersion ).isEqualTo( latestRelease );
             assertThat( resp1.currentVersion.versionString ).isEqualTo( art1.version );

common/src/test/java/de/codesourcery/versiontracker/common/server/MavenCentralVersionProviderTest.java

@@ -21,9 +21,13 @@
 import de.codesourcery.versiontracker.common.IVersionProvider;
 import de.codesourcery.versiontracker.common.Version;
 import de.codesourcery.versiontracker.common.VersionInfo;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
@@ -73,7 +77,6 @@ private void doTest(WireMockRuntimeInfo webServer, boolean scrapeAdditionalRelea
     <release>3.12.0</release>
     <versions>
       <version>3.11</version>
-      <version>3.11.1</version>
       <version>3.12.0</version>
     </versions>
     <lastUpdated>20210301214036</lastUpdated>
@@ -83,28 +86,25 @@ private void doTest(WireMockRuntimeInfo webServer, boolean scrapeAdditionalRelea
         final String metaDataURL = "/"+MavenCentralVersionProvider.metaDataPath( info.artifact );
         stubFor(get( metaDataURL ).willReturn( ok( metadata ) ) );
 
-
         final ZonedDateTime releaseDate1 = date( "2021-07-11 11:12" );
         final ZonedDateTime releaseDate2 = date( "2021-07-12 12:13" );
 
-        final String jsonResponse = """
-{"responseHeader":{"status":0,"QTime":3,"params":{"q":"g:org.apache.commons AND a:commons-lang3 AND v: 3.12.0","core":"","indent":"off","fl":"id,g,a,v,p,ec,timestamp,tags","start":"","sort":"score desc,timestamp desc,g asc,a asc,v desc","rows":"20","wt":"json","version":"2.2"}},"response":{"numFound":1,"start":0,"docs":[{"id":"org.apache.commons:commons-lang3:3.12.0","g":"org.apache.commons","a":"commons-lang3","v":"3.12.0","p":"jar","timestamp":1626091980000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar",".jar","-tests.jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]}]}}            
-            """;
-        final String expectedRestURL = "/select/?q=g%3Aorg.apache.commons+AND+a%3Acommons-lang3+AND+v%3A+3.12.0&rows=10000&wt=json";
+        final String jsonResponse = loadJSONResponse();
+        final String expectedRestURL = "/select/?q=g%3Aorg.apache.commons+AND+a%3Acommons-lang3&core=gav&rows=300&wt=json";
         stubFor( get( expectedRestURL ).willReturn( ok( jsonResponse ) ) );
 
         if ( scrapeAdditionalReleaseDates ) {
-            final String jsonResponse2 = """
-{"responseHeader":{"status":0,"QTime":3,"params":{"q":"g:org.apache.commons AND a:commons-lang3 AND v: 3.11","core":"","indent":"off","fl":"id,g,a,v,p,ec,timestamp,tags","start":"","sort":"score desc,timestamp desc,g asc,a asc,v desc","rows":"20","wt":"json","version":"2.2"}},"response":{"numFound":1,"start":0,"docs":[{"id":"org.apache.commons:commons-lang3:3.11","g":"org.apache.commons","a":"commons-lang3","v":"3.11","p":"jar","timestamp":1626001920000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar",".jar","-tests.jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]}]}}            
-            """;
-            final String expectedRestURL2 = "/select/?q=g%3Aorg.apache.commons+AND+a%3Acommons-lang3+AND+v%3A+3.11&rows=10000&wt=json";
+            final String jsonResponse2 = loadJSONResponse();
+            final String expectedRestURL2 = "/select/?q=g%3Aorg.apache.commons+AND+a%3Acommons-lang3&core=gav&rows=300&wt=json";
             stubFor( get( expectedRestURL2 ).willReturn( ok( jsonResponse2 ) ) );
         }
 
-        final IVersionProvider.UpdateResult result = provider.update( info, xxxx );
+        final IVersionProvider.UpdateResult result = provider.update( info, false );
         assertThat(result).isEqualTo( IVersionProvider.UpdateResult.UPDATED );
         assertThat(info.versions).isNotEmpty();
-        assertThat(info.versions).containsExactlyInAnyOrder( Version.of("3.11"), Version.of("3.11.1"), Version.of("3.12.0") );
+        Assertions.assertTrue(info.versions.stream().anyMatch( x -> x.versionString.equals("3.11") ) );
+        Assertions.assertTrue(info.versions.stream().anyMatch( x -> x.versionString.equals("3.12.0") ) );
+
         if ( scrapeAdditionalReleaseDates ) {
             assertThat( info.getVersion( "3.11" ) ).map( x -> x.releaseDate ).hasValueSatisfying( v -> v.toInstant().equals( releaseDate1.toInstant() ) );
         } else {
@@ -120,15 +120,13 @@ private static ZonedDateTime date(String s) {
         return ZonedDateTime.parse( s, format.withZone( ZoneId.of("UTC") ) );
     }
 
-    private static String createMarkupForScraping(Artifact artifact, String versionNumber, ZonedDateTime releaseDate) {
-        /*
-         * <a href="junit-4.12-javadoc.jar" title="junit-4.12-javadoc.jar">junit-4.12-javadoc.jar</a>
-         *                             2014-12-04 16:17    937942
-         */
-        final String prefix = artifact.artifactId+"-"+versionNumber+"-javadoc.jar";
-        DateTimeFormatter format = DateTimeFormatter.ofPattern( "yyyy-MM-dd HH:mm" );
-        return """
-<a href="%s" title="%s">%s</a>
-         %s    937942""".formatted(prefix,prefix,prefix, releaseDate.format( format ));
+    private static String loadJSONResponse() throws IOException
+    {
+        try ( InputStream in = MavenCentralVersionProviderTest.class.getResourceAsStream( "/response.json" ) ) {
+            if ( in == null ) {
+                throw new FileNotFoundException( "Failed to find response.json" );
+            }
+            return new String( in.readAllBytes(), StandardCharsets.UTF_8 );
+        }
     }
 }

common/src/test/java/de/codesourcery/versiontracker/common/server/SonatypeRestAPIUrlBuilderTest.java

@@ -28,7 +28,7 @@ class SonatypeRestAPIUrlBuilderTest
     @Test
     void testQuerySingleVersion() throws IOException
     {
-        assertEquals( url( "https://search.maven.org/solrsearch/select?q=g%3Agroup+AND+a%3Aartifact+AND+v%3A+version&rows=10000&wt=json" ),
+        assertEquals( url( "https://search.maven.org/solrsearch/select?q=g%3Agroup+AND+a%3Aartifact+AND+v%3A+version&rows=300&wt=json" ),
             new SonatypeRestAPIUrlBuilder()
             .artifactId( "artifact" )
             .groupId( "group" )
@@ -39,7 +39,7 @@ void testQuerySingleVersion() throws IOException
     @Test
     void testQueryAllVersions() throws IOException
     {
-        assertEquals( url( "https://search.maven.org/solrsearch/select?q=g%3Agroup+AND+a%3Aartifact&core=gav&rows=10000&wt=json" ),
+        assertEquals( url( "https://search.maven.org/solrsearch/select?q=g%3Agroup+AND+a%3Aartifact&core=gav&rows=300&wt=json" ),
             new SonatypeRestAPIUrlBuilder()
             .artifactId( "artifact" )
             .groupId( "group" )

common/src/test/resources/response.json

@@ -0,0 +1 @@
+{"responseHeader":{"status":0,"QTime":2,"params":{"q":"g:org.apache.commons AND a:commons-lang3","core":"gav","indent":"off","fl":"id,g,a,v,p,ec,timestamp,tags","start":"","sort":"score desc,timestamp desc,g asc,a asc,v desc","rows":"20","wt":"json","version":"2.2"}},"response":{"numFound":20,"start":0,"docs":[{"id":"org.apache.commons:commons-lang3:3.14.0","g":"org.apache.commons","a":"commons-lang3","v":"3.14.0","p":"jar","timestamp":1700319805000,"ec":["-cyclonedx.json","-sources.jar","-cyclonedx.xml",".pom","-test-sources.jar","-javadoc.jar","-tests.jar",".jar",".spdx.rdf.xml"],"tags":["standard","package","considered","classes","hierarchy","existence","utility","commons","that","apache","java","justify","lang"]},{"id":"org.apache.commons:commons-lang3:3.13.0","g":"org.apache.commons","a":"commons-lang3","v":"3.13.0","p":"jar","timestamp":1690141499000,"ec":["-cyclonedx.json","-sources.jar","-cyclonedx.xml",".pom","-javadoc.jar","-test-sources.jar","-tests.jar",".jar",".spdx.rdf.xml"],"tags":["standard","package","considered","classes","hierarchy","existence","utility","commons","that","apache","java","justify","lang"]},{"id":"org.apache.commons:commons-lang3:3.12.0","g":"org.apache.commons","a":"commons-lang3","v":"3.12.0","p":"jar","timestamp":1614372052000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar",".jar","-tests.jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.11","g":"org.apache.commons","a":"commons-lang3","v":"3.11","p":"jar","timestamp":1594560722000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.10","g":"org.apache.commons","a":"commons-lang3","v":"3.10","p":"jar","timestamp":1584970776000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.9","g":"org.apache.commons","a":"commons-lang3","v":"3.9","p":"jar","timestamp":1554946239000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar",".jar","-tests.jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.8.1","g":"org.apache.commons","a":"commons-lang3","v":"3.8.1","p":"jar","timestamp":1537370669000,"ec":["-javadoc.jar","-sources.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.8","g":"org.apache.commons","a":"commons-lang3","v":"3.8","p":"jar","timestamp":1534383430000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar",".jar","-tests.jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.7","g":"org.apache.commons","a":"commons-lang3","v":"3.7","p":"jar","timestamp":1509819379000,"ec":["-javadoc.jar","-sources.jar","-test-sources.jar","-tests.jar",".jar","-src.zip","-src.tar.gz",".pom","-bin.zip","-bin.tar.gz"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.6","g":"org.apache.commons","a":"commons-lang3","v":"3.6","p":"jar","timestamp":1497001265000,"ec":["-javadoc.jar","-sources.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.5","g":"org.apache.commons","a":"commons-lang3","v":"3.5","p":"jar","timestamp":1476388421000,"ec":["-javadoc.jar","-sources.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.4","g":"org.apache.commons","a":"commons-lang3","v":"3.4","p":"jar","timestamp":1428064272000,"ec":["-javadoc.jar","-sources.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.3.2","g":"org.apache.commons","a":"commons-lang3","v":"3.3.2","p":"jar","timestamp":1396786889000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar",".jar","-tests.jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.3.1","g":"org.apache.commons","a":"commons-lang3","v":"3.3.1","p":"jar","timestamp":1394888674000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar",".jar","-tests.jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.3","g":"org.apache.commons","a":"commons-lang3","v":"3.3","p":"jar","timestamp":1393578489000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.2.1","g":"org.apache.commons","a":"commons-lang3","v":"3.2.1","p":"jar","timestamp":1388941333000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.2","g":"org.apache.commons","a":"commons-lang3","v":"3.2","p":"jar","timestamp":1388250411000,"ec":["-sources.jar","-javadoc.jar","-test-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","standard","justify","package","apache","lang","considered","existence","that","utility","commons","java","hierarchy"]},{"id":"org.apache.commons:commons-lang3:3.1","g":"org.apache.commons","a":"commons-lang3","v":"3.1","p":"jar","timestamp":1321342025000,"ec":["-sources.jar","-javadoc.jar","-tests.jar",".jar",".pom"],"tags":["classes","that","utility","standard","commons","justify","package","java","hierarchy","lang","considered","existence"]},{"id":"org.apache.commons:commons-lang3:3.0.1","g":"org.apache.commons","a":"commons-lang3","v":"3.0.1","p":"jar","timestamp":1312947886000,"ec":["-javadoc.jar","-sources.jar","-tests.jar",".jar",".pom"],"tags":["classes","that","utility","standard","commons","justify","package","java","hierarchy","lang","considered","existence"]},{"id":"org.apache.commons:commons-lang3:3.0","g":"org.apache.commons","a":"commons-lang3","v":"3.0","p":"jar","timestamp":1311046565000,"ec":["-sources.jar","-javadoc.jar",".jar","-tests.jar",".pom"],"tags":["classes","that","utility","standard","commons","justify","package","java","hierarchy","lang","considered","existence"]}]}}