New Cottontail API, PolyphenyDB (#224)

* Upgraded to new Cottontail API
* New Polypheny-DB Connector
* New DB Abstractions
* LSC Import Code which needed the new Cottontail API

Co-authored-by: Ralph Gasser <ralph.gasser@unibas.ch>
Co-authored-by: Loris Sauter <loris.sauter@unibas.ch>
Former-commit-id: c450c76

.github/workflows/gradle.yml

@@ -17,16 +17,24 @@ jobs:
     # Setup Cottontail DB service container
     services:
       cottontail:
-        image: vitrivr/cottontaildb:0.12.11
+        image: vitrivr/cottontaildb:0.13.3
         ports:
           - 1865:1865
         options: -it
+      polypheny-ci:
+        image: silvanheller/polypheny-vitrivr-ci:0.4.1
+        ports:
+          - 8070:8070
+          - 13137:13137
+          - 20591:20591
 
     # Start actual job.
     steps:
     - uses: actions/checkout@v2
     - name: nc connection test
       run: nc -zv 127.0.0.1 1865
+    - name: polypheny connection test
+      run: nc -zv 127.0.0.1 13137
     - name: Set up JDK 11
       uses: actions/setup-java@v2
       with:

build.gradle

@@ -11,7 +11,7 @@ allprojects {
     group = 'org.vitrivr'
 
     /* Our current version, on dev branch this should always be release+1-SNAPSHOT */
-    version = '3.6.4'
+    version = '3.7.0'
 
     apply plugin: 'java-library'
     apply plugin: 'maven-publish'

cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java

@@ -93,8 +93,7 @@ public class APIEndpoint {
   /**
    * The retrieval logic used to retrieve
    */
-  public static ContinuousRetrievalLogic retrievalLogic = new ContinuousRetrievalLogic(
-      Config.sharedConfig().getDatabase());
+  public static ContinuousRetrievalLogic retrievalLogic = new ContinuousRetrievalLogic(Config.sharedConfig().getDatabase());
 
   /**
    * The single instance of this class

cineast-api/src/main/java/org/vitrivr/cineast/api/GRPCEndpoint.java

@@ -34,7 +34,6 @@ public static void start() {
     try {
       server.start();
     } catch (IOException e) {
-      e.printStackTrace();
     }
 
   }

cineast-core/build.gradle

@@ -98,9 +98,12 @@ signing {
 dependencies {
 
     /** THE Cottontail DB proto dependency */
-    api group: 'org.vitrivr', name: 'cottontaildb-proto', version: version_cottontail_proto
+    api group: 'org.vitrivr', name: 'cottontaildb-proto', version: version_cottontaildb_proto
     api group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: version_kotlin_stdlib
 
+    /** THE Polypheny dependency */
+    api group: "org.polypheny", name: "polypheny-jdbc-driver", version: version_polypheny
+
     /** THE ADAMpro proto dependency (legacy) */
     api group: 'org.vitrivr', name: 'adampro-proto', version: version_adampro
 

cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java

@@ -2,91 +2,34 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import java.io.File;
 import java.util.function.Supplier;
+import org.vitrivr.cineast.core.db.DBSelector;
 import org.vitrivr.cineast.core.db.DBSelectorSupplier;
-import org.vitrivr.cineast.core.db.NoDBSelector;
-import org.vitrivr.cineast.core.db.NoDBWriter;
+import org.vitrivr.cineast.core.db.DataSource;
+import org.vitrivr.cineast.core.db.PersistencyWriter;
 import org.vitrivr.cineast.core.db.PersistencyWriterSupplier;
-import org.vitrivr.cineast.core.db.adampro.ADAMproEntityCreator;
-import org.vitrivr.cineast.core.db.adampro.ADAMproSelector;
-import org.vitrivr.cineast.core.db.adampro.ADAMproStreamingSelector;
-import org.vitrivr.cineast.core.db.adampro.ADAMproWrapper;
-import org.vitrivr.cineast.core.db.adampro.ADAMproWriter;
-import org.vitrivr.cineast.core.db.cottontaildb.CottontailEntityCreator;
-import org.vitrivr.cineast.core.db.cottontaildb.CottontailSelector;
-import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper;
-import org.vitrivr.cineast.core.db.cottontaildb.CottontailWriter;
-import org.vitrivr.cineast.core.db.json.JsonFileWriter;
-import org.vitrivr.cineast.core.db.json.JsonSelector;
-import org.vitrivr.cineast.core.db.memory.InMemoryEntityCreator;
-import org.vitrivr.cineast.core.db.memory.InMemoryWriter;
 import org.vitrivr.cineast.core.db.setup.EntityCreator;
-import org.vitrivr.cineast.core.db.setup.NoEntityCreator;
 
 public final class DatabaseConfig {
 
-
   /**
    * Default value for batchsize.
    */
   public static final int DEFAULT_BATCH_SIZE = 1000;
   public static final String DEFAULT_HOST = "localhost";
   public static final int DEFAULT_PORT = 5890;
   public static final boolean DEFAULT_PLAINTEXT = true;
-  public static final boolean SINGLE_CONNECTION = true;
 
   private String host = DEFAULT_HOST;
   private int port = DEFAULT_PORT;
   private boolean plaintext = DEFAULT_PLAINTEXT;
-  private Writer writer = Writer.COTTONTAIL;
-  private Selector selector = Selector.COTTONTAIL;
+  private DataSource writer = DataSource.COTTONTAIL;
+  private DataSource selector = DataSource.COTTONTAIL;
 
   private Integer batchsize = DEFAULT_BATCH_SIZE;
 
-  private static final PersistencyWriterSupplier NO_WRITER_SUPPLY = NoDBWriter::new;
-
-  private static final DBSelectorSupplier NO_SELECTOR_SUPPLY = NoDBSelector::new;
-
-  private static final Supplier<EntityCreator> NO_CREATOR_SUPPLY = NoEntityCreator::new;
-
-
-  public enum Writer {
-    NONE,
-    JSON,
-    ADAMPRO,
-    COTTONTAIL,
-    INMEMORY
-  }
-
-  public enum Selector {
-    NONE,
-    JSON,
-    ADAMPRO,
-    ADAMPROSTREAM,
-    COTTONTAIL,
-    INMEMORY
-  }
-
-  private ADAMproWrapper adaMproWrapper = null;
-
-  private synchronized void ensureAdamProWrapper() {
-    if (this.adaMproWrapper == null) {
-      this.adaMproWrapper = new ADAMproWrapper(this);
-    }
-  }
-
-  private CottontailWrapper cottontailWrapper = null;
-
-  private synchronized void ensureCottontailWrapper() {
-    if (this.cottontailWrapper == null) {
-      this.cottontailWrapper = new CottontailWrapper(this, true);
-    }
-  }
-
   @JsonCreator
   public DatabaseConfig() {
-
   }
 
   @JsonProperty
@@ -105,7 +48,6 @@ public void setHost(String host) {
   public int getPort() {
     return this.port;
   }
-
   public void setPort(int port) {
     if (port < 1 || port > 65535) {
       throw new IllegalArgumentException(port + " is outside of valid port range");
@@ -124,118 +66,40 @@ public void setPlaintext(boolean plaintext) {
 
   @JsonProperty
   public Integer getBatchsize() {
-    return batchsize;
+    return this.batchsize;
   }
 
   public void setBatchsize(Integer batchsize) {
     this.batchsize = batchsize;
   }
 
   @JsonProperty
-  public Writer getWriter() {
+  public DataSource getWriter() {
     return this.writer;
   }
 
-  public void setWriter(Writer writer) {
+  public void setWriter(DataSource writer) {
     this.writer = writer;
   }
 
   @JsonProperty
-  public Selector getSelector() {
+  public DataSource getSelector() {
     return this.selector;
   }
 
-  public void setSelector(Selector selector) {
+  public void setSelector(DataSource selector) {
     this.selector = selector;
   }
 
-  public synchronized PersistencyWriterSupplier getWriterSupplier() {
-    switch (this.writer) {
-      case NONE:
-        return NO_WRITER_SUPPLY;
-      case ADAMPRO: {
-        if (SINGLE_CONNECTION) {
-          ensureAdamProWrapper();
-          return () -> new ADAMproWriter(this.adaMproWrapper);
-        }
-        return () -> new ADAMproWriter(new ADAMproWrapper(this));
-      }
-      case JSON: {
-        return () -> new JsonFileWriter(new File(this.host));
-      }
-      case COTTONTAIL: {
-        if (SINGLE_CONNECTION) {
-          ensureCottontailWrapper();
-          return () -> new CottontailWriter(this.cottontailWrapper);
-        }
-        return () -> new CottontailWriter(new CottontailWrapper(this, false));
-      }
-      case INMEMORY: {
-        return InMemoryWriter::new;
-      }
-      default:
-        throw new IllegalStateException("No supplier for writer " + this.writer);
-
-    }
+  public PersistencyWriterSupplier getWriterSupplier() {
+    return this.writer.getWriterSupplier(this);
   }
 
-  public synchronized DBSelectorSupplier getSelectorSupplier() {
-    switch (this.selector) {
-      case ADAMPRO: {
-        if (SINGLE_CONNECTION) {
-          ensureAdamProWrapper();
-          return () -> new ADAMproSelector(this.adaMproWrapper);
-        }
-        return () -> new ADAMproSelector(new ADAMproWrapper(this));
-      }
-      case ADAMPROSTREAM: {
-        if (SINGLE_CONNECTION) {
-          ensureAdamProWrapper();
-          return () -> new ADAMproStreamingSelector(this.adaMproWrapper);
-        }
-        return () -> new ADAMproStreamingSelector(new ADAMproWrapper(this));
-      }
-      case JSON: {
-        return () -> new JsonSelector(new File(this.host));
-      }
-      case NONE:
-        return NO_SELECTOR_SUPPLY;
-      case COTTONTAIL: {
-        if (SINGLE_CONNECTION) {
-          ensureCottontailWrapper();
-          return () -> new CottontailSelector(this.cottontailWrapper);
-        }
-        return () -> new CottontailSelector(new CottontailWrapper(this, false));
-      }
-      default:
-        throw new IllegalStateException("No supplier for selector " + this.selector);
-
-    }
+  public Supplier<EntityCreator> getEntityCreatorSupplier() {
+    return this.writer.getEntityCreatorSupplier(this);
   }
 
-  public synchronized Supplier<EntityCreator> getEntityCreatorSupplier() {
-    switch (this.selector) {
-      case ADAMPRO:
-      case ADAMPROSTREAM: {
-        if (SINGLE_CONNECTION) {
-          ensureAdamProWrapper();
-          return () -> new ADAMproEntityCreator(this.adaMproWrapper);
-        }
-        return () -> new ADAMproEntityCreator(new ADAMproWrapper(this));
-      }
-      case NONE:
-        return NO_CREATOR_SUPPLY;
-      case COTTONTAIL: {
-        if (SINGLE_CONNECTION) {
-          ensureCottontailWrapper();
-          return () -> new CottontailEntityCreator(this.cottontailWrapper);
-        }
-        return () -> new CottontailEntityCreator(new CottontailWrapper(this, false));
-      }
-      case INMEMORY:
-        return InMemoryEntityCreator::new;
-      default:
-        throw new IllegalStateException("No supplier for EntityCreator " + this.selector);
-    }
+  public DBSelectorSupplier getSelectorSupplier() {
+    return this.writer.getSelectorSupplier(this);
   }
 }

cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java

@@ -16,6 +16,7 @@
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.tuple.Triple;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.vitrivr.cineast.core.config.ReadableQueryConfig;
 import org.vitrivr.cineast.core.data.distance.DistanceElement;
 import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider;
@@ -26,6 +27,8 @@
 
 public interface DBSelector extends Closeable {
 
+  Logger LOGGER = LogManager.getLogger();
+
   boolean open(String name);
 
   void close();

cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java

@@ -2,6 +2,12 @@
 
 import java.util.function.Supplier;
 
-public interface DBSelectorSupplier extends Supplier<DBSelector> {
+/**
+ * This interface describes a {@link Supplier} for {@link DBSelector}s.
+ *
+ * <strong>Important:</strong> This class is required because of signature clashes in {@link org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule} due
+ * to type erasure!
+ */
+public interface DBSelectorSupplier extends Supplier<DBSelector>{
 
 }