Standalone index

code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java

@@ -5,4 +5,6 @@ public class IndexMqEndpoints {
     public static final String INDEX_REPARTITION = "INDEX-REPARTITION";
     public static final String SWITCH_INDEX = "SWITCH-INDEX";
 
+    public static final String SWITCH_LINKDB = "SWITCH_LINKDB";
+
 }

code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/DecoratedSearchResultItem.java

@@ -0,0 +1,82 @@
+package nu.marginalia.index.client.model.results;
+
+import lombok.Getter;
+import lombok.ToString;
+import nu.marginalia.model.EdgeUrl;
+import org.jetbrains.annotations.NotNull;
+
+import javax.annotation.Nullable;
+import java.util.List;
+
+@Getter
+@ToString
+public class DecoratedSearchResultItem {
+    public final SearchResultItem rawIndexResult;
+
+    @NotNull
+    public final EdgeUrl url;
+    @NotNull
+    public final String title;
+    @NotNull
+    public final String description;
+    public final double urlQuality;
+    @NotNull
+    public final String format;
+
+    /** Document features bitmask, see HtmlFeature */
+    public final int features;
+
+    @Nullable
+    public final Integer pubYear;
+    public final long dataHash;
+    public final int wordsTotal;
+    public final double rankingScore;
+
+    public long documentId() {
+        return rawIndexResult.getDocumentId();
+    }
+    public int domainId() {
+        return rawIndexResult.getDomainId();
+    }
+    public int resultsFromDomain() {
+        return rawIndexResult.getResultsFromDomain();
+    }
+
+    public List<SearchResultKeywordScore> keywordScores() {
+        return rawIndexResult.getKeywordScores();
+    }
+
+    public long rankingId() {
+        return rawIndexResult.getRanking();
+    }
+
+    public DecoratedSearchResultItem(SearchResultItem rawIndexResult,
+                                     @NotNull
+                                     EdgeUrl url,
+                                     @NotNull
+                                     String title,
+                                     @NotNull
+                                     String description,
+                                     double urlQuality,
+                                     @NotNull
+                                     String format,
+                                     int features,
+                                     @Nullable
+                                     Integer pubYear,
+                                     long dataHash,
+                                     int wordsTotal,
+                                     double rankingScore)
+    {
+        this.rawIndexResult = rawIndexResult;
+        this.url = url;
+        this.title = title;
+        this.description = description;
+        this.urlQuality = urlQuality;
+        this.format = format;
+        this.features = features;
+        this.pubYear = pubYear;
+        this.dataHash = dataHash;
+        this.wordsTotal = wordsTotal;
+        this.rankingScore = rankingScore;
+    }
+}

code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultSet.java

@@ -8,8 +8,7 @@
 
 @AllArgsConstructor @Getter @ToString
 public class SearchResultSet {
-    public List<SearchResultItem> results;
-    public ResultRankingContext rankingContext;
+    public List<DecoratedSearchResultItem> results;
     public int size() {
         return results.size();
     }

code/api/search-api/build.gradle → code/api/query-api/build.gradle

@@ -1,7 +1,5 @@
 plugins {
     id 'java'
-
-
     id 'jvm-test-suite'
 }
 
@@ -13,8 +11,10 @@ java {
 
 dependencies {
     implementation project(':code:common:model')
+    implementation project(':code:api:index-api')
     implementation project(':code:common:config')
     implementation project(':code:libraries:message-queue')
+    implementation project(':code:features-index:index-query')
     implementation project(':code:common:service-discovery')
     implementation project(':code:common:service-client')
 

code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java

@@ -0,0 +1,65 @@
+package nu.marginalia.query.client;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import io.prometheus.client.Summary;
+import io.reactivex.rxjava3.core.Observable;
+import nu.marginalia.WmsaHome;
+import nu.marginalia.client.AbstractDynamicClient;
+import nu.marginalia.client.Context;
+import nu.marginalia.index.client.model.query.SearchSpecification;
+import nu.marginalia.index.client.model.results.SearchResultSet;
+import nu.marginalia.model.gson.GsonFactory;
+import nu.marginalia.mq.MessageQueueFactory;
+import nu.marginalia.mq.outbox.MqOutbox;
+import nu.marginalia.query.model.QueryParams;
+import nu.marginalia.query.model.QueryResponse;
+import nu.marginalia.service.descriptor.ServiceDescriptors;
+import nu.marginalia.service.id.ServiceId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.CheckReturnValue;
+import java.util.UUID;
+
+@Singleton
+public class QueryClient extends AbstractDynamicClient {
+
+    private static final Summary wmsa_search_index_api_delegate_time = Summary.build().name("wmsa_search_index_api_delegate_time").help("-").register();
+    private static final Summary wmsa_search_index_api_search_time = Summary.build().name("wmsa_search_index_api_search_time").help("-").register();
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    private final MqOutbox outbox;
+
+    @Inject
+    public QueryClient(ServiceDescriptors descriptors,
+                       MessageQueueFactory messageQueueFactory) {
+
+        super(descriptors.forId(ServiceId.Query), WmsaHome.getHostsFile(), GsonFactory::get);
+
+        String inboxName = ServiceId.Query.name + ":" + "0";
+        String outboxName = System.getProperty("service-name", UUID.randomUUID().toString());
+
+        outbox = messageQueueFactory.createOutbox(inboxName, outboxName, UUID.randomUUID());
+
+    }
+
+    /** Delegate an Index API style query directly to the index service */
+    @CheckReturnValue
+    public SearchResultSet delegate(Context ctx, SearchSpecification specs) {
+        return wmsa_search_index_api_delegate_time.time(
+                () -> this.postGet(ctx, "/delegate/", specs, SearchResultSet.class).blockingFirst()
+        );
+    }
+    @CheckReturnValue
+    public QueryResponse search(Context ctx, QueryParams params) {
+        return wmsa_search_index_api_search_time.time(
+                () -> this.postGet(ctx, "/search/", params, QueryResponse.class).blockingFirst()
+        );
+    }
+    public MqOutbox outbox() {
+        return outbox;
+    }
+
+}

code/api/query-api/src/main/java/nu/marginalia/query/model/ProcessedQuery.java

@@ -0,0 +1,21 @@
+package nu.marginalia.query.model;
+
+import nu.marginalia.index.client.model.query.SearchSpecification;
+
+import java.util.*;
+
+public class ProcessedQuery {
+    public final SearchSpecification specs;
+    public final List<String> searchTermsHuman;
+    public final String domain;
+
+    public ProcessedQuery(SearchSpecification specs, List<String> searchTermsHuman, String domain) {
+        this.specs = specs;
+        this.searchTermsHuman = searchTermsHuman;
+        this.domain = domain;
+    }
+
+    public ProcessedQuery(SearchSpecification justSpecs) {
+        this(justSpecs, List.of(), null);
+    }
+}

code/api/query-api/src/main/java/nu/marginalia/query/model/QueryParams.java

@@ -0,0 +1,41 @@
+package nu.marginalia.query.model;
+
+import nu.marginalia.index.client.model.query.SearchSetIdentifier;
+import nu.marginalia.index.client.model.query.SearchSpecification;
+import nu.marginalia.index.query.limit.QueryLimits;
+import nu.marginalia.index.query.limit.SpecificationLimit;
+
+import java.util.List;
+
+public record QueryParams(
+        String humanQuery,
+        String nearDomain,
+        List<String> tacitIncludes,
+        List<String> tacitExcludes,
+        List<String> tacitPriority,
+        List<String> tacitAdvice,
+        SpecificationLimit quality,
+        SpecificationLimit year,
+        SpecificationLimit size,
+        SpecificationLimit rank,
+        List<Integer> domainIds,
+        QueryLimits limits,
+        SearchSetIdentifier identifier
+)
+{
+    public QueryParams(String query, QueryLimits limits, SearchSetIdentifier identifier) {
+        this(query, null,
+                List.of(),
+                List.of(),
+                List.of(),
+                List.of(),
+                SpecificationLimit.none(),
+                SpecificationLimit.none(),
+                SpecificationLimit.none(),
+                SpecificationLimit.none(),
+                List.of(),
+                limits,
+                identifier
+                );
+    }
+}

code/api/query-api/src/main/java/nu/marginalia/query/model/QueryResponse.java

@@ -0,0 +1,23 @@
+package nu.marginalia.query.model;
+
+import nu.marginalia.index.client.model.query.SearchSpecification;
+import nu.marginalia.index.client.model.results.DecoratedSearchResultItem;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public record QueryResponse(SearchSpecification specs,
+                            List<DecoratedSearchResultItem> results,
+                            List<String> searchTermsHuman,
+                            List<String> problems,
+                            String domain)
+{
+    public Set<String> getAllKeywords() {
+        Set<String> keywords = new HashSet<>(100);
+        for (var sq : specs.subqueries) {
+            keywords.addAll(sq.searchTermsInclude);
+        }
+        return keywords;
+    }
+}

code/api/readme.md

@@ -3,7 +3,7 @@
 ## Core Services
 
 * [assistant-api](assistant-api/)
-* [search-api](search-api/)
+* [query-api](query-api/)
 * [index-api](index-api/)
 
 These are clients for the [core services](../services-core/), along with what models
@@ -13,11 +13,11 @@ are necessary for speaking to them. They each implement the abstract client clas
 All that is necessary is to `@Inject` them into the constructor and then 
 requests can be sent. 
 
-**Note:** If you are looking for the public API, it's handled by the api service in [services-satellite/api-service](../services-satellite/api-service).
+**Note:** If you are looking for the public API, it's handled by the api service in [services-application/api-service](../services-application/api-service).
 
 ## MQ-API Process API
 
 [process-mqapi](process-mqapi/) defines requests and inboxes for the message queue based API used 
 for interacting with processes.   
 
-See [libraries/message-queue](../libraries/message-queue) and [services-satellite/control-service](../services-satellite/control-service). 
+See [libraries/message-queue](../libraries/message-queue) and [services-application/control-service](../services-core/control-service). 

code/common/linkdb/readme.md

@@ -8,4 +8,4 @@ this information take effect in production immediately, even before
 the information was searchable.
 
 It is constructed by the [loading-process](../../processes/loading-process), and consumed 
-by the [search-service](../../services-core/search-service).
+by the [index-service](../../services-core/index-service).

code/common/linkdb/src/main/java/nu/marginalia/linkdb/LinkdbReader.java

@@ -62,6 +62,15 @@ public void switchInput(Path newDbFile) throws IOException, SQLException {
         connection = createConnection();
     }
 
+    /** Re-establishes the connection, useful in tests and not
+     * much else */
+    public void reconnect() throws SQLException {
+        if (connection != null)
+            connection.close();
+
+        connection = createConnection();
+    }
+
     public List<String> getUrlsFromDomain(int domainId) throws SQLException {
         if (connection == null ||
                 connection.isClosed())