(control) UX-improvements for control service

This commit overhauls a lot of the UX for the control service, adding a new actions menu to the nodes views.  It has many small tweaks to make the work flow better.

It also adds a new /uploads directory in each index node, from which sideloaded data can be selected.  This is a bit of a breaking change, as this directory needs to exist in each index node.

code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java

@@ -8,6 +8,7 @@
 import nu.marginalia.executor.model.transfer.TransferItem;
 import nu.marginalia.executor.model.transfer.TransferSpec;
 import nu.marginalia.executor.storage.FileStorageContent;
+import nu.marginalia.executor.upload.UploadDirContents;
 import nu.marginalia.model.gson.GsonFactory;
 import nu.marginalia.service.descriptor.ServiceDescriptors;
 import nu.marginalia.service.id.ServiceId;
@@ -38,7 +39,7 @@ public void stopProcess(Context ctx, int node, String id) {
     }
 
 
-    public void triggerCrawl(Context ctx, int node, String fid) {
+    public void triggerCrawl(Context ctx, int node, FileStorageId fid) {
         post(ctx, node, "/process/crawl/" + fid, "").blockingSubscribe();
     }
 
@@ -112,6 +113,10 @@ public ActorRunStates getActorStates(Context context, int node) {
         return get(context, node, "/actor", ActorRunStates.class).blockingFirst();
     }
 
+    public UploadDirContents listSideloadDir(Context context, int node) {
+        return get(context, node, "/sideload/", UploadDirContents.class).blockingFirst();
+    }
+
     public FileStorageContent listFileStorage(Context context, int node, FileStorageId fileId) {
         return get(context, node, "/storage/"+fileId.id(), FileStorageContent.class).blockingFirst();
     }

code/api/executor-api/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java

@@ -0,0 +1,6 @@
+package nu.marginalia.executor.upload;
+
+import java.util.List;
+
+public record UploadDirContents(String path, List<UploadDirItem> items) {
+}

code/api/executor-api/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java

@@ -0,0 +1,29 @@
+package nu.marginalia.executor.upload;
+
+import lombok.SneakyThrows;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+public record UploadDirItem (
+        String name,
+        String lastModifiedTime,
+        boolean isDirectory,
+        long size
+) {
+
+    @SneakyThrows
+    public static UploadDirItem fromPath(Path path) {
+        boolean isDir = Files.isDirectory(path);
+        long size = isDir ? 0 : Files.size(path);
+        var mtime = Files.getLastModifiedTime(path);
+
+
+        return new UploadDirItem(path.toString(),
+                LocalDateTime.ofInstant(mtime.toInstant(), ZoneId.systemDefault()).format(DateTimeFormatter.ISO_DATE_TIME), isDir, size);
+    }
+
+}

code/common/config/src/main/java/nu/marginalia/WmsaHome.java

@@ -23,6 +23,10 @@ public static UserAgent getUserAgent() throws IOException {
     }
 
 
+    public static Path getUploadDir() {
+        return Path.of("/uploads");
+    }
+
     public static Path getHomePath() {
         var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElseGet(WmsaHome::findDefaultHomePath);
 

code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java

@@ -455,6 +455,57 @@ public List<FileStorage> getEachFileStorage() {
         return ret;
     }
 
+    public List<FileStorage> getEachFileStorage(FileStorageType type) {
+        return getEachFileStorage(node, type);
+    }
+
+    public List<FileStorage> getEachFileStorage(int node, FileStorageType type) {
+        List<FileStorage> ret = new ArrayList<>();
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT PATH, STATE, TYPE, DESCRIPTION, CREATE_DATE, ID, BASE_ID
+                     FROM FILE_STORAGE_VIEW
+                     WHERE NODE=? AND TYPE=?
+                     """)) {
+
+            stmt.setInt(1, node);
+            stmt.setString(2, type.name());
+
+            long storageId;
+            long baseId;
+            String path;
+            String state;
+            String description;
+            LocalDateTime createDateTime;
+
+            try (var rs = stmt.executeQuery()) {
+                while (rs.next()) {
+                    baseId = rs.getLong("BASE_ID");
+                    storageId = rs.getLong("ID");
+                    path = rs.getString("PATH");
+                    state = rs.getString("STATE");
+
+                    description = rs.getString("DESCRIPTION");
+                    createDateTime = rs.getTimestamp("CREATE_DATE").toLocalDateTime();
+                    var base = getStorageBase(new FileStorageBaseId(baseId));
+
+                    ret.add(new FileStorage(
+                            new FileStorageId(storageId),
+                            base,
+                            type,
+                            createDateTime,
+                            path,
+                            FileStorageState.parse(state),
+                            description
+                    ));
+                }
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+
+        return ret;
+    }
     public void flagFileForDeletion(FileStorageId id) throws SQLException {
         setFileStorageState(id, FileStorageState.DELETE);
     }

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java

@@ -1,10 +1,10 @@
 package nu.marginalia.control;
 
-import com.github.jknack.handlebars.Handlebars;
-import com.github.jknack.handlebars.Helper;
-import com.github.jknack.handlebars.Options;
+import com.github.jknack.handlebars.*;
 import nu.marginalia.renderer.config.HandlebarsConfigurator;
 
+import java.io.IOException;
+
 public class ControlHandlebarsConfigurator implements HandlebarsConfigurator {
     @Override
     public void configure(Handlebars handlebars) {

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlRendererFactory.java

@@ -0,0 +1,38 @@
+package nu.marginalia.control;
+
+import com.google.inject.Inject;
+import lombok.SneakyThrows;
+import nu.marginalia.nodecfg.NodeConfigurationService;
+import nu.marginalia.renderer.RendererFactory;
+
+import java.util.Map;
+
+/** Wrapper for the renderer factory that adds global context
+ * with the nodes listing
+ */
+public class ControlRendererFactory {
+    private final RendererFactory rendererFactory;
+    private final NodeConfigurationService nodeConfigurationService;
+
+    @Inject
+    public ControlRendererFactory(RendererFactory rendererFactory,
+                                  NodeConfigurationService nodeConfigurationService)
+    {
+        this.rendererFactory = rendererFactory;
+        this.nodeConfigurationService = nodeConfigurationService;
+    }
+
+    @SneakyThrows
+    public Renderer renderer(String template) {
+        Map<String, Object> globalContext = Map.of(
+                "nodes", nodeConfigurationService.getAll()
+        );
+        var baseRenderer = rendererFactory.renderer(template);
+
+        return (context) -> baseRenderer.render(context, Map.of("global-context", globalContext));
+    }
+
+    public interface Renderer {
+        String render(Object context);
+    }
+}

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java

@@ -10,7 +10,6 @@
 import nu.marginalia.control.node.svc.ControlNodeService;
 import nu.marginalia.control.sys.svc.*;
 import nu.marginalia.model.gson.GsonFactory;
-import nu.marginalia.renderer.RendererFactory;
 import nu.marginalia.screenshot.ScreenshotService;
 import nu.marginalia.service.server.*;
 import org.slf4j.Logger;
@@ -40,7 +39,7 @@ public ControlService(BaseServiceParams params,
                           ServiceMonitors monitors,
                           HeartbeatService heartbeatService,
                           EventLogService eventLogService,
-                          RendererFactory rendererFactory,
+                          ControlRendererFactory rendererFactory,
                           StaticResources staticResources,
                           MessageQueueService messageQueueService,
                           ControlFileStorageService controlFileStorageService,

code/services-core/control-service/src/main/java/nu/marginalia/control/RedirectControl.java

@@ -1,18 +1,16 @@
 package nu.marginalia.control;
 
 import jakarta.inject.Inject;
-import nu.marginalia.renderer.MustacheRenderer;
-import nu.marginalia.renderer.RendererFactory;
 import spark.ResponseTransformer;
 
 import java.io.IOException;
 import java.util.Map;
 
 public class RedirectControl {
-    private final MustacheRenderer<Object> renderer;
+    private final ControlRendererFactory.Renderer renderer;
 
     @Inject
-    public RedirectControl(RendererFactory rendererFactory) throws IOException {
+    public RedirectControl(ControlRendererFactory rendererFactory) throws IOException {
         renderer = rendererFactory.renderer("control/redirect-ok");
     }
 

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ApiKeyService.java

@@ -2,9 +2,9 @@
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.app.model.ApiKeyModel;
-import nu.marginalia.renderer.RendererFactory;
 import org.eclipse.jetty.util.StringUtil;
 import spark.Request;
 import spark.Response;
@@ -20,11 +20,11 @@
 public class ApiKeyService {
 
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
 
     @Inject
     public ApiKeyService(HikariDataSource dataSource,
-                         RendererFactory rendererFactory
+                         ControlRendererFactory rendererFactory
                          ) {
         this.dataSource = dataSource;
         this.rendererFactory = rendererFactory;

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ControlBlacklistService.java

@@ -2,10 +2,10 @@
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.app.model.BlacklistedDomainModel;
 import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.renderer.RendererFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -22,12 +22,12 @@
 public class ControlBlacklistService {
 
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Inject
     public ControlBlacklistService(HikariDataSource dataSource,
-                                   RendererFactory rendererFactory) {
+                                   ControlRendererFactory rendererFactory) {
         this.dataSource = dataSource;
         this.rendererFactory = rendererFactory;
     }

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/DomainComplaintService.java

@@ -3,11 +3,11 @@
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.app.model.DomainComplaintCategory;
 import nu.marginalia.control.app.model.DomainComplaintModel;
 import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.renderer.RendererFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -26,14 +26,14 @@
  */
 public class DomainComplaintService {
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final ControlBlacklistService blacklistService;
     private final RandomExplorationService randomExplorationService;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Inject
     public DomainComplaintService(HikariDataSource dataSource,
-                                  RendererFactory rendererFactory,
+                                  ControlRendererFactory rendererFactory,
                                   ControlBlacklistService blacklistService,
                                   RandomExplorationService randomExplorationService
     ) {

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/RandomExplorationService.java

@@ -3,8 +3,8 @@
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
 import gnu.trove.list.array.TIntArrayList;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.renderer.RendererFactory;
 import spark.Request;
 import spark.Response;
 import spark.Spark;
@@ -19,12 +19,12 @@
 public class RandomExplorationService {
 
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
 
     @Inject
     public RandomExplorationService(HikariDataSource dataSource,
-                                    RendererFactory rendererFactory
-                                    ) {
+                                    ControlRendererFactory rendererFactory
+    ) {
         this.dataSource = dataSource;
         this.rendererFactory = rendererFactory;
     }
@@ -33,6 +33,7 @@ public void register() throws IOException {
         var reviewRandomDomainsRenderer = rendererFactory.renderer("control/app/review-random-domains");
 
         Spark.get("/public/review-random-domains", this::reviewRandomDomainsModel, reviewRandomDomainsRenderer::render);
+
         Spark.post("/public/review-random-domains", this::reviewRandomDomainsAction);
     }
 

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java

@@ -2,12 +2,13 @@
 
 import com.google.inject.Inject;
 import nu.marginalia.client.Context;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.index.client.model.query.SearchSetIdentifier;
 import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.model.EdgeUrl;
+import nu.marginalia.nodecfg.NodeConfigurationService;
 import nu.marginalia.query.client.QueryClient;
 import nu.marginalia.query.model.QueryParams;
-import nu.marginalia.renderer.RendererFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -20,19 +21,21 @@
 
 public class SearchToBanService {
     private final ControlBlacklistService blacklistService;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final QueryClient queryClient;
     private final Logger logger = LoggerFactory.getLogger(getClass());
+    private final NodeConfigurationService nodeConfigurationService;
 
     @Inject
     public SearchToBanService(ControlBlacklistService blacklistService,
-                              RendererFactory rendererFactory,
-                              QueryClient queryClient)
+                              ControlRendererFactory rendererFactory,
+                              QueryClient queryClient, NodeConfigurationService nodeConfigurationService)
     {
 
         this.blacklistService = blacklistService;
         this.rendererFactory = rendererFactory;
         this.queryClient = queryClient;
+        this.nodeConfigurationService = nodeConfigurationService;
     }
 
     public void register() throws IOException {