Merge pull request #873 from amvanbaren/optimize-sitemap

Optimize Sitemap
eclipse · Mar 20, 2024 · b7ad4a8 · b7ad4a8
2 parents 36f62b9 + 29dc74d
commit b7ad4a8
Show file tree

Hide file tree

Showing 9 changed files with 147 additions and 68 deletions.
diff --git a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java
@@ -30,6 +30,7 @@ public class CacheService {
     public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version";
     public static final String CACHE_NAMESPACE_DETAILS_JSON = "namespace.details.json";
     public static final String CACHE_AVERAGE_REVIEW_RATING = "average.review.rating";
+    public static final String CACHE_SITEMAP = "sitemap";
 
     public static final String GENERATOR_EXTENSION_JSON = "extensionJsonCacheKeyGenerator";
     public static final String GENERATOR_LATEST_EXTENSION_VERSION = "latestExtensionVersionCacheKeyGenerator";

diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java
@@ -18,6 +18,7 @@
 import org.eclipse.openvsx.util.ErrorResultException;
 import org.eclipse.openvsx.util.TempFile;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
 import org.springframework.retry.annotation.Retryable;
 import org.springframework.stereotype.Component;
 
@@ -26,6 +27,8 @@
 import java.io.IOException;
 import java.nio.file.Files;
 
+import static org.eclipse.openvsx.cache.CacheService.CACHE_SITEMAP;
+
 @Component
 public class PublishExtensionVersionService {
 
@@ -84,6 +87,7 @@ public void persistResource(FileResource resource) {
     }
 
     @Transactional
+    @CacheEvict(value = CACHE_SITEMAP, allEntries = true)
     public void activateExtension(ExtensionVersion extVersion, ExtensionService extensions) {
         extVersion.setActive(true);
         extVersion = entityManager.merge(extVersion);

diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java
@@ -11,6 +11,7 @@
 
 import org.eclipse.openvsx.entities.Extension;
 import org.eclipse.openvsx.entities.Namespace;
+import org.eclipse.openvsx.web.SitemapRow;
 import org.jooq.*;
 import org.jooq.Record;
 import org.jooq.impl.DSL;
@@ -200,4 +201,24 @@ public boolean publicIdExists(String publicId) {
                 .fetch()
                 .isNotEmpty();
     }
+
+    public List<SitemapRow> fetchSitemapRows() {
+        var LAST_UPDATED = DSL.toChar(EXTENSION.LAST_UPDATED_DATE, "YYYY-MM-DD");
+        return dsl.select(
+                    NAMESPACE.NAME,
+                    EXTENSION.NAME,
+                    LAST_UPDATED
+                )
+                .from(NAMESPACE)
+                .join(EXTENSION).on(EXTENSION.NAMESPACE_ID.eq(NAMESPACE.ID))
+                .where(EXTENSION.ACTIVE.eq(true))
+                .fetch()
+                .map((record) -> {
+                    return new SitemapRow(
+                            record.get(NAMESPACE.NAME),
+                            record.get(EXTENSION.NAME),
+                            record.get(LAST_UPDATED)
+                    );
+                });
+    }
 }
diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java
@@ -13,6 +13,7 @@
 import org.eclipse.openvsx.entities.*;
 import org.eclipse.openvsx.json.QueryRequest;
 import org.eclipse.openvsx.util.NamingUtil;
+import org.eclipse.openvsx.web.SitemapRow;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
@@ -519,4 +520,8 @@ public boolean extensionPublicIdExists(String publicId) {
     public boolean namespacePublicIdExists(String publicId) {
         return namespaceJooqRepo.publicIdExists(publicId);
     }
+
+    public List<SitemapRow> fetchSitemapRows() {
+        return extensionJooqRepo.fetchSitemapRows();
+    }
 }
diff --git a/server/src/main/java/org/eclipse/openvsx/web/SitemapController.java b/server/src/main/java/org/eclipse/openvsx/web/SitemapController.java
@@ -9,92 +9,28 @@
  ********************************************************************************/
 package org.eclipse.openvsx.web;
 
-import org.apache.commons.lang3.StringUtils;
-import org.eclipse.openvsx.repositories.RepositoryService;
-import org.eclipse.openvsx.util.UrlUtil;
-import org.eclipse.openvsx.util.VersionService;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.CacheControl;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
 
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import java.net.URI;
-import java.time.format.DateTimeFormatter;
 import java.util.concurrent.TimeUnit;
 
 @RestController
 public class SitemapController {
 
-    private static final String NAMESPACE_URI = "http://www.sitemaps.org/schemas/sitemap/0.9";
-
-    @Autowired
-    RepositoryService repositories;
-
     @Autowired
-    VersionService versions;
-
-    @Value("${ovsx.webui.url:}")
-    String webuiUrl;
+    SitemapService service;
 
     @GetMapping(path = "/sitemap.xml", produces = MediaType.APPLICATION_XML_VALUE)
-    public ResponseEntity<StreamingResponseBody> getSitemap() throws ParserConfigurationException {
-        var document = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
-        document.setXmlStandalone(true);
-        var urlset = document.createElementNS(NAMESPACE_URI, "urlset");
-        document.appendChild(urlset);
-
-        var baseUrl = getBaseUrl();
-        var timestampFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
-        repositories.findAllActiveExtensions().forEach(extension -> {
-            var entry = document.createElement("url");
-            var loc = document.createElement("loc");
-            var namespaceName = extension.getNamespace().getName();
-            loc.setTextContent(UrlUtil.createApiUrl(baseUrl, "extension", namespaceName, extension.getName()));
-            entry.appendChild(loc);
-
-            var lastmod = document.createElement("lastmod");
-            var latest = versions.getLatestTrxn(extension, null, false, true);
-            lastmod.setTextContent(latest.getTimestamp().format(timestampFormatter));
-            entry.appendChild(lastmod);
-            urlset.appendChild(entry);
-        });
-
-        StreamingResponseBody stream = out -> {
-            try {
-                var transformer = TransformerFactory.newInstance().newTransformer();
-                transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
-                transformer.setOutputProperty(OutputKeys.INDENT, "yes");
-                transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
-                transformer.transform(new DOMSource(document), new StreamResult(out));
-            } catch (TransformerException exc) {
-                throw new RuntimeException(exc);
-            }
-        };
+    public ResponseEntity<String> getSitemap() throws ParserConfigurationException {
         return ResponseEntity.ok()
                 .cacheControl(CacheControl.maxAge(1, TimeUnit.DAYS).cachePublic())
                 .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_XML_VALUE)
-                .body(stream);
-    }
-
-    private String getBaseUrl() {
-        if (StringUtils.isEmpty(webuiUrl))
-            return UrlUtil.getBaseUrl();
-        else if (URI.create(webuiUrl).isAbsolute())
-            return webuiUrl;
-        else
-            return UrlUtil.createApiUrl(UrlUtil.getBaseUrl(), webuiUrl.split("/"));
+                .body(service.generateSitemap());
     }
-
 }
diff --git a/server/src/main/java/org/eclipse/openvsx/web/SitemapRow.java b/server/src/main/java/org/eclipse/openvsx/web/SitemapRow.java
@@ -0,0 +1,12 @@
+/** ******************************************************************************
+ * Copyright (c) 2024 Precies. Software Ltd and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.web;
+
+public record SitemapRow(String namespace, String extension, String lastUpdated){}
diff --git a/server/src/main/java/org/eclipse/openvsx/web/SitemapService.java b/server/src/main/java/org/eclipse/openvsx/web/SitemapService.java
@@ -0,0 +1,89 @@
+/** ******************************************************************************
+ * Copyright (c) 2024 Precies. Software Ltd and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.web;
+
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.util.UrlUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.stereotype.Component;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URI;
+
+import static org.eclipse.openvsx.cache.CacheService.CACHE_SITEMAP;
+
+@Component
+public class SitemapService {
+
+    @Autowired
+    RepositoryService repositories;
+
+    @Value("${ovsx.webui.url:}")
+    String webuiUrl;
+
+    @Cacheable(CACHE_SITEMAP)
+    public String generateSitemap() throws ParserConfigurationException {
+        var document = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
+        document.setXmlStandalone(true);
+        var namespace = "http://www.sitemaps.org/schemas/sitemap/0.9";
+        var urlset = document.createElementNS(namespace, "urlset");
+        document.appendChild(urlset);
+
+        var baseUrl = getBaseUrl();
+        var rows = repositories.fetchSitemapRows();
+        for(var row : rows) {
+            var entry = document.createElement("url");
+            var loc = document.createElement("loc");
+            loc.setTextContent(baseUrl + row.namespace() + "/" + row.extension());
+            entry.appendChild(loc);
+
+            var lastmod = document.createElement("lastmod");
+            lastmod.setTextContent(row.lastUpdated());
+            entry.appendChild(lastmod);
+            urlset.appendChild(entry);
+        }
+
+        try(var writer = new StringWriter()) {
+            var transformer = TransformerFactory.newInstance().newTransformer();
+            transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
+            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+            transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
+            transformer.transform(new DOMSource(document), new StreamResult(writer));
+            return writer.toString();
+        } catch (TransformerException | IOException exc) {
+            throw new RuntimeException(exc);
+        }
+    }
+
+    private String getBaseUrl() {
+        String url;
+        if (StringUtils.isEmpty(webuiUrl))
+            url = UrlUtil.getBaseUrl();
+        else if (URI.create(webuiUrl).isAbsolute())
+            url = webuiUrl;
+        else
+            url = UrlUtil.createApiUrl(UrlUtil.getBaseUrl(), webuiUrl.split("/"));
+        if(!url.endsWith("/"))
+            url += "/";
+
+        return url + "extension/";
+    }
+}
diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml
@@ -60,4 +60,14 @@
             <disk unit="MB">128</disk>
         </resources>
     </cache>
+    <cache alias="sitemap">
+        <expiry>
+            <ttl unit="seconds">3600</ttl>
+        </expiry>
+        <resources>
+            <heap unit="entries">1</heap>
+            <offheap unit="MB">2</offheap>
+            <disk unit="MB">8</disk>
+        </resources>
+    </cache>
 </config>
diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java
@@ -192,7 +192,8 @@ void testExecuteQueries() {
                 () -> repositories.updateExtensionPublicId(1L, "namespaceName.extensionName"),
                 () -> repositories.updateNamespacePublicIds(Collections.emptyMap()),
                 () -> repositories.extensionPublicIdExists("namespaceName.extensionName"),
-                () -> repositories.namespacePublicIdExists("namespaceName.extensionName")
+                () -> repositories.namespacePublicIdExists("namespaceName.extensionName"),
+                () -> repositories.fetchSitemapRows()
         );
 
         // check that we did not miss anything