Merge pull request #61 from MarginaliaSearch/new-look

Design Revamp For search.marginalia.nu
MarginaliaSearch · Dec 5, 2023 · 21abfc6 · 21abfc6
2 parents c793434 + d1e88df
commit 21abfc6
Show file tree

Hide file tree

Showing 84 changed files with 2,526 additions and 1,040 deletions.
diff --git a/code/common/renderer/build.gradle b/code/common/renderer/build.gradle
@@ -15,6 +15,8 @@ dependencies {
     implementation libs.bundles.slf4j
 
     implementation libs.bundles.handlebars
+    implementation libs.guice
+    implementation libs.spark
 
     testImplementation libs.bundles.slf4j.test
     testImplementation libs.bundles.junit

diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/MustacheRenderer.java b/code/common/renderer/src/main/java/nu/marginalia/renderer/MustacheRenderer.java
@@ -5,50 +5,32 @@
 import com.github.jknack.handlebars.io.ClassPathTemplateLoader;
 import com.github.jknack.handlebars.io.TemplateLoader;
 import lombok.SneakyThrows;
+import nu.marginalia.renderer.config.HandlebarsConfigurator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import spark.Response;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
 
 public class MustacheRenderer<T> {
     private final Template template;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    MustacheRenderer(String templateFile) throws IOException {
+    MustacheRenderer(HandlebarsConfigurator configurator, String templateFile) throws IOException {
 
         TemplateLoader loader = new ClassPathTemplateLoader();
         loader.setPrefix("/templates");
         loader.setSuffix(".hdb");
 
         var handlebars = new Handlebars(loader);
+
         handlebars.registerHelpers(ConditionalHelpers.class);
         handlebars.registerHelper("md", new MarkdownHelper());
-        handlebars.registerHelper("readableUUID", (context, options) -> {
-            if (context == null) return "";
-            String instance = context.toString();
-            if (instance.length() < 31) return "";
-
-            instance = instance.replace("-", "");
-            String color1 = "#"+instance.substring(0, 6);
-            String color2 = "#"+instance.substring(6, 12);
-            String color3 = "#"+instance.substring(12, 18);
-            String color4 = "#"+instance.substring(18, 24);
-
-            String shortName1 = instance.substring(0, 2);
-            String shortName2 = instance.substring(2, 4);
-            String shortName3 = instance.substring(4, 6);
-            String shortName4 = instance.substring(6, 8);
-
-            String ret = "<span title=\"%s\">".formatted(context.toString()) +
-                    "<span style=\"text-shadow: 0 0 0.2ch %s; font-family: monospace;\">%s</span>".formatted(color1, shortName1) +
-                    "<span style=\"text-shadow: 0 0 0.2ch %s; font-family: monospace;\">%s</span>".formatted(color2, shortName2) +
-                    "<span style=\"text-shadow: 0 0 0.2ch %s; font-family: monospace;\">%s</span>".formatted(color3, shortName3) +
-                    "<span style=\"text-shadow :0 0 0.2ch %s; font-family: monospace;\">%s</span>".formatted(color4, shortName4);
-            return ret;
-        });
+
+        configurator.configure(handlebars);
 
         try {
             template = handlebars.compile(templateFile);
@@ -67,6 +49,14 @@ public String render(T model) {
         return template.apply(model);
     }
 
+    @SneakyThrows
+    public Object renderInto(Response response, T model) {
+
+        response.raw().getOutputStream().write(template.apply(model).getBytes(StandardCharsets.UTF_8));
+
+        return "";
+    }
+
     @SneakyThrows
     public <T2> String render(T model, String name, List<T2> children) {
         Context ctx = Context.newBuilder(model).combine(name, children).build();
@@ -75,9 +65,14 @@ public <T2> String render(T model, String name, List<T2> children) {
     }
 
     @SneakyThrows
-    public <T2> String render(T model, Map<String, ?> children) {
+    public String render(T model, Map<String, ?> children) {
         Context ctx = Context.newBuilder(model).combine(children).build();
         return template.apply(ctx);
     }
 
+    @SneakyThrows
+    public void renderInto(Response response, T model, Map<String, ?> children) {
+        Context ctx = Context.newBuilder(model).combine(children).build();
+        response.raw().getOutputStream().write(template.apply(ctx).getBytes(StandardCharsets.UTF_8));
+    }
 }
diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/RendererFactory.java b/code/common/renderer/src/main/java/nu/marginalia/renderer/RendererFactory.java
@@ -1,13 +1,21 @@
 package nu.marginalia.renderer;
 
+import com.google.inject.Inject;
+import nu.marginalia.renderer.config.HandlebarsConfigurator;
+
 import java.io.IOException;
 
 public class RendererFactory {
 
-    public RendererFactory() {
+    private final HandlebarsConfigurator configurator;
+
+    @Inject
+    public RendererFactory(HandlebarsConfigurator configurator) {
+        this.configurator = configurator;
     }
 
+    /** Create a renderer for the given template */
     public <T> MustacheRenderer<T> renderer(String template) throws IOException {
-        return new MustacheRenderer<>(template);
+        return new MustacheRenderer<>(configurator, template);
     }
 }
diff --git a/...n/renderer/src/main/java/nu/marginalia/renderer/config/DefaultHandlebarsConfigurator.java b/...n/renderer/src/main/java/nu/marginalia/renderer/config/DefaultHandlebarsConfigurator.java
@@ -0,0 +1,8 @@
+package nu.marginalia.renderer.config;
+
+import com.github.jknack.handlebars.Handlebars;
+
+public class DefaultHandlebarsConfigurator implements HandlebarsConfigurator {
+    @Override
+    public void configure(Handlebars handlebars) {}
+}
diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/config/HandlebarsConfigurator.java b/code/common/renderer/src/main/java/nu/marginalia/renderer/config/HandlebarsConfigurator.java
@@ -0,0 +1,11 @@
+package nu.marginalia.renderer.config;
+
+import com.github.jknack.handlebars.Handlebars;
+
+/** Configure handlebars rendering by injecting helper methods
+ * into the setup process */
+public interface HandlebarsConfigurator {
+
+    /** Set up helpers for this handlebars instance */
+    void configure(Handlebars handlebars);
+}
diff --git a/code/common/service-client/src/test/java/nu/marginalia/client/AbstractClientTest.java b/code/common/service-client/src/test/java/nu/marginalia/client/AbstractClientTest.java
@@ -109,12 +109,14 @@ public void testPost404() {
 
         assertError(client.post(Context.internal(), 0,"/post", "test"));
     }
+
     @Test
     public void testGet404() {
         testServer.get(this::error404);
 
         assertError(client.get(Context.internal(), 0,"/get"));
     }
+
     @Test
     public void testDelete404() {
         testServer.delete(this::error404);

diff --git a/...ures-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java b/...ures-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java
@@ -26,7 +26,7 @@ public DbBrowseDomainsRandom(HikariDataSource dataSource) {
     public List<BrowseResult> getRandomDomains(int count, DomainBlacklist blacklist, int set) {
 
         final String q = """
-                SELECT DOMAIN_ID, DOMAIN_NAME
+                SELECT DOMAIN_ID, DOMAIN_NAME, INDEXED
                 FROM EC_RANDOM_DOMAINS
                 INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID
                 WHERE STATE<2
@@ -44,9 +44,10 @@ ORDER BY RAND()
                 while (rsp.next()) {
                     int id = rsp.getInt(1);
                     String domain = rsp.getString(2);
+                    boolean indexed = rsp.getBoolean("INDEXED");
 
                     if (!blacklist.isBlacklisted(id)) {
-                        domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0));
+                        domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0, indexed));
                     }
                 }
             }

diff --git a/...arch/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java b/...arch/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java
@@ -3,6 +3,7 @@
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.zaxxer.hikari.HikariDataSource;
+import gnu.trove.set.hash.TIntHashSet;
 import nu.marginalia.browse.model.BrowseResult;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.db.DomainBlacklist;
@@ -23,14 +24,15 @@ public DbBrowseDomainsSimilarCosine(HikariDataSource dataSource) {
         this.dataSource = dataSource;
     }
 
-    public List<BrowseResult> getDomainNeighborsAdjacentCosine(int domainId, DomainBlacklist blacklist, int count) {
+    public List<BrowseResult> getDomainNeighborsAdjacentCosineRequireScreenshot(int domainId, DomainBlacklist blacklist, int count) {
         List<BrowseResult> domains = new ArrayList<>(count);
 
         String q = """
                         SELECT
                             EC_DOMAIN.ID,
                             NV.NEIGHBOR_NAME,
-                            NV.RELATEDNESS
+                            NV.RELATEDNESS,
+                            EC_DOMAIN.INDEXED
                         FROM EC_NEIGHBORS_VIEW NV
                         INNER JOIN DATA_DOMAIN_SCREENSHOT ON DATA_DOMAIN_SCREENSHOT.DOMAIN_NAME=NV.NEIGHBOR_NAME
                         INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=NV.NEIGHBOR_ID
@@ -49,9 +51,10 @@ public List<BrowseResult> getDomainNeighborsAdjacentCosine(int domainId, DomainB
                     int id = rsp.getInt(1);
                     String domain = rsp.getString(2);
                     double relatedness = rsp.getDouble(3);
+                    boolean indexed = rsp.getBoolean("INDEXED");
 
                     if (!blacklist.isBlacklisted(id)) {
-                        domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, relatedness));
+                        domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, relatedness, indexed));
                     }
                 }
             }

diff --git a/...rch/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarOldAlgo.java b/...rch/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarOldAlgo.java
@@ -27,7 +27,7 @@ public List<BrowseResult> getDomainNeighborsAdjacent(int domainId, DomainBlackli
         final Set<BrowseResult> domains = new HashSet<>(count*3);
 
         final String q = """
-                            SELECT EC_DOMAIN.ID AS NEIGHBOR_ID, DOMAIN_NAME, COUNT(*) AS CNT 
+                            SELECT EC_DOMAIN.ID AS NEIGHBOR_ID, DOMAIN_NAME, COUNT(*) AS CNT, INDEXED
                             FROM EC_DOMAIN_NEIGHBORS 
                             INNER JOIN EC_DOMAIN ON NEIGHBOR_ID=EC_DOMAIN.ID 
                             INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID 
@@ -54,14 +54,14 @@ SELECT EC_DOMAIN.ID AS NEIGHBOR_ID, DOMAIN_NAME, COUNT(*) AS CNT
                     String domain = rsp.getString(2);
 
                     if (!blacklist.isBlacklisted(id)) {
-                        domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0));
+                        domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0, rsp.getBoolean("INDEXED")));
                     }
                 }
             }
 
             if (domains.size() < count/2) {
                 final String q2 = """
-                        SELECT EC_DOMAIN.ID, DOMAIN_NAME
+                        SELECT EC_DOMAIN.ID, DOMAIN_NAME, INDEXED
                         FROM EC_DOMAIN
                         INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID 
                         INNER JOIN EC_DOMAIN_LINK B ON DEST_DOMAIN_ID=EC_DOMAIN.ID 
@@ -83,15 +83,15 @@ SELECT EC_DOMAIN.ID AS NEIGHBOR_ID, DOMAIN_NAME, COUNT(*) AS CNT
                         String domain = rsp.getString(2);
 
                         if (!blacklist.isBlacklisted(id)) {
-                            domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0));
+                            domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0, rsp.getBoolean("INDEXED")));
                         }
                     }
                 }
             }
 
             if (domains.size() < count/2) {
                 final String q3 = """
-                    SELECT EC_DOMAIN.ID, DOMAIN_NAME
+                    SELECT EC_DOMAIN.ID, DOMAIN_NAME, INDEXED
                     FROM EC_DOMAIN
                     INNER JOIN DOMAIN_METADATA ON EC_DOMAIN.ID=DOMAIN_METADATA.ID
                     INNER JOIN EC_DOMAIN_LINK B ON B.SOURCE_DOMAIN_ID=EC_DOMAIN.ID
@@ -115,7 +115,7 @@ HAVING COUNT(*) < 100
                         String domain = rsp.getString(2);
 
                         if (!blacklist.isBlacklisted(id)) {
-                            domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0));
+                            domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0, rsp.getBoolean("INDEXED")));
                         }
                     }
                 }
@@ -128,38 +128,5 @@ HAVING COUNT(*) < 100
         return new ArrayList<>(domains);
     }
 
-    public List<BrowseResult> getRandomDomains(int count, DomainBlacklist blacklist, int set) {
-
-        final String q = """
-                SELECT DOMAIN_ID, DOMAIN_NAME
-                FROM EC_RANDOM_DOMAINS
-                INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID
-                WHERE STATE<2
-                AND DOMAIN_SET=?
-                AND DOMAIN_ALIAS IS NULL
-                ORDER BY RAND()
-                LIMIT ?
-                """;
-        List<BrowseResult> domains = new ArrayList<>(count);
-        try (var conn = dataSource.getConnection()) {
-            try (var stmt = conn.prepareStatement(q)) {
-                stmt.setInt(1, set);;
-                stmt.setInt(2, count);
-                var rsp = stmt.executeQuery();
-                while (rsp.next()) {
-                    int id = rsp.getInt(1);
-                    String domain = rsp.getString(2);
-
-                    if (!blacklist.isBlacklisted(id)) {
-                        domains.add(new BrowseResult(new EdgeDomain(domain).toRootUrl(), id, 0));
-                    }
-                }
-            }
-        }
-        catch (SQLException ex) {
-            logger.error("SQL error", ex);
-        }
-        return domains;
-    }
 
 }
diff --git a/...eatures-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java b/...eatures-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java
@@ -2,7 +2,10 @@
 
 import nu.marginalia.model.EdgeUrl;
 
-public record BrowseResult (EdgeUrl url, int domainId, double relatedness) {
+public record BrowseResult (EdgeUrl url,
+                            int domainId,
+                            double relatedness,
+                            boolean indexed) {
 
     public String domainHash() {
         var domain = url.domain;
@@ -11,4 +14,20 @@ public String domainHash() {
         }
         return domain.toString();
     }
+
+    public String displayDomain() {
+        String ret;
+        var domain = url.domain;
+        if ("www".equals(domain.subDomain)) {
+            ret = domain.domain;
+        }
+        else {
+            ret = domain.toString();
+        }
+        if (ret.length() > 25) {
+            ret = ret.substring(0, 22) + "...";
+        }
+        return ret;
+
+    }
 }
diff --git a/.../services-application/dating-service/src/main/java/nu/marginalia/dating/DatingModule.java b/.../services-application/dating-service/src/main/java/nu/marginalia/dating/DatingModule.java
@@ -1,6 +1,11 @@
 package nu.marginalia.dating;
 
 import com.google.inject.AbstractModule;
+import nu.marginalia.renderer.config.DefaultHandlebarsConfigurator;
+import nu.marginalia.renderer.config.HandlebarsConfigurator;
 
 public class DatingModule extends AbstractModule  {
+    public void configure() {
+        bind(HandlebarsConfigurator.class).to(DefaultHandlebarsConfigurator.class);
+    }
 }
diff --git a/...services-application/dating-service/src/main/java/nu/marginalia/dating/DatingService.java b/...services-application/dating-service/src/main/java/nu/marginalia/dating/DatingService.java
@@ -14,9 +14,7 @@
 import spark.Request;
 import spark.Response;
 import spark.Spark;
-import spark.resource.ClassPathResource;
 
-import java.io.FileNotFoundException;
 import java.util.Map;
 import java.util.Optional;
 

diff --git a/...es-application/dating-service/src/main/java/nu/marginalia/dating/DatingSessionObject.java b/...es-application/dating-service/src/main/java/nu/marginalia/dating/DatingSessionObject.java
@@ -28,7 +28,7 @@ public BrowseResult next(DbBrowseDomainsRandom random, DomainBlacklist blacklist
     }
 
     public BrowseResult nextSimilar(int domainId, DbBrowseDomainsSimilarCosine adjacent, DomainBlacklist blacklist) {
-        adjacent.getDomainNeighborsAdjacentCosine(domainId, blacklist, 25).forEach(queue::addFirst);
+        adjacent.getDomainNeighborsAdjacentCosineRequireScreenshot(domainId, blacklist, 25).forEach(queue::addFirst);
 
         while (queue.size() > MAX_QUEUE_SIZE) {
             queue.removeLast();

diff --git a/...vices-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java b/...vices-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java
@@ -26,6 +26,7 @@ public static void main(String... args) {
 
         Injector injector = Guice.createInjector(
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Explorer),
+                new ExplorerModule(),
                 new DatabaseModule()
         );
 

diff --git a/...ces-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerModule.java b/...ces-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerModule.java
@@ -0,0 +1,11 @@
+package nu.marginalia.explorer;
+
+import com.google.inject.AbstractModule;
+import nu.marginalia.renderer.config.DefaultHandlebarsConfigurator;
+import nu.marginalia.renderer.config.HandlebarsConfigurator;
+
+public class ExplorerModule extends AbstractModule {
+    public void configure() {
+        bind(HandlebarsConfigurator.class).to(DefaultHandlebarsConfigurator.class);
+    }
+}