Skip to content

Commit

Permalink
Merge pull request #226 from vitrivr/data-model-refactoring
Browse files Browse the repository at this point in the history 
Data Model Clarification & Clear Deprecation of QueryComponent

Former-commit-id: 947a071
  • Loading branch information
@silvanheller
silvanheller committed Oct 24, 2021
2 parents 17bb5bb + 17db5c6 commit d72993c
Show file tree
Hide file tree
Showing 37 changed files with 219 additions and 374 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ allprojects {
group = 'org.vitrivr'

/* Our current version */
version = '3.2.0-SNAPSHOT'
version = '3.3.0'

apply plugin: 'java-library'
apply plugin: 'maven-publish'
Expand Down
7 changes: 2 additions & 5 deletions cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/CineastQueryService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import io.grpc.stub.StreamObserver;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.time.StopWatch;
Expand All @@ -13,7 +12,6 @@
import org.vitrivr.cineast.api.grpc.util.MediaObjectUtil;
import org.vitrivr.cineast.api.grpc.util.MediaSegmentUtil;
import org.vitrivr.cineast.api.grpc.util.QueryContainerUtil;
import org.vitrivr.cineast.api.messages.query.StagedSimilarityQuery;
import org.vitrivr.cineast.api.util.QueryUtil;
import org.vitrivr.cineast.core.config.QueryConfig;
import org.vitrivr.cineast.core.config.ReadableQueryConfig;
Expand All @@ -22,7 +20,7 @@
import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor;
import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor;
import org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor;
import org.vitrivr.cineast.core.data.query.containers.QueryContainer;
import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer;
import org.vitrivr.cineast.core.data.score.SegmentScoreElement;
import org.vitrivr.cineast.core.db.dao.reader.MediaObjectMetadataReader;
import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader;
Expand Down Expand Up @@ -190,12 +188,11 @@ public void getSimilar(CineastGrpc.TemporalQuery query, StreamObserver<CineastGr
LOGGER.warn("QueryTerm was null for stage {}", stage);
return;
}
QueryContainer qc = qt.getContainer();
AbstractQueryTermContainer qc = qt.getContainer();
if (qc == null) {
LOGGER.warn("Likely an empty query, as it could not be converted to a query container. Ignoring it");
return;
}
qc.setContainerId(finalContainerIdx);

List<Thread> categoryThreads = new ArrayList<>();

Expand Down
8 changes: 4 additions & 4 deletions cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/data/QueryTerm.java
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,27 @@
package org.vitrivr.cineast.api.grpc.data;

import org.vitrivr.cineast.core.config.ReadableQueryConfig;
import org.vitrivr.cineast.core.data.query.containers.QueryContainer;
import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

public class QueryTerm {

private final QueryContainer container;
private final AbstractQueryTermContainer container;
private final ReadableQueryConfig queryConfig;
private final float weight;
private final List<String> categories = new ArrayList<>();

public QueryTerm(QueryContainer container, ReadableQueryConfig queryConfig, float weight, Collection<String> categories){
public QueryTerm(AbstractQueryTermContainer container, ReadableQueryConfig queryConfig, float weight, Collection<String> categories){
this.container = container;
this.queryConfig = queryConfig;
this.weight = weight;
this.categories.addAll(categories);
}

public QueryContainer getContainer() {
public AbstractQueryTermContainer getContainer() {
return container;
}

Expand Down
42 changes: 21 additions & 21 deletions cineast-api/src/main/java/org/vitrivr/cineast/api/grpc/util/QueryContainerUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,18 @@ public class QueryContainerUtil {

private static final Logger LOGGER = LogManager.getLogger();

public static AudioQueryContainer audioQueryContainer(CineastGrpc.AudioQueryContainer container) {
return new AudioQueryContainer(container.getAudioFramesList().stream().map(QueryContainerUtil::audioFrame).collect(Collectors.toList()));
public static AudioQueryTermContainer audioQueryContainer(CineastGrpc.AudioQueryContainer container) {
return new AudioQueryTermContainer(container.getAudioFramesList().stream().map(QueryContainerUtil::audioFrame).collect(Collectors.toList()));
}

public static AudioFrame audioFrame(CineastGrpc.AudioFrame frame) {
AudioDescriptor descriptor = new AudioDescriptor(frame.getSamplingRate(), frame.getChannels(), frame.getDuration());
return new AudioFrame(frame.getIdx(), frame.getTimestamp(), frame.getData().toByteArray(), descriptor);
}

public static BooleanQueryContainer booleanQueryContainer(CineastGrpc.BooleanQueryContainer container) {
public static BooleanQueryTermContainer booleanQueryContainer(CineastGrpc.BooleanQueryContainer container) {

return new BooleanQueryContainer(
return new BooleanQueryTermContainer(
container.getExpressionsList().stream().map(QueryContainerUtil::booleanExpression).collect(Collectors.toList())
);

Expand Down Expand Up @@ -94,13 +94,13 @@ public static PrimitiveTypeProvider primitiveTypeProvider(CineastGrpc.PrimitiveT
return new NothingProvider();
}

public static IdQueryContainer idQueryContainer(CineastGrpc.IdQueryContainer container) {
return new IdQueryContainer(container.getSegmentId().getId());
public static IdQueryTermContainer idQueryContainer(CineastGrpc.IdQueryContainer container) {
return new IdQueryTermContainer(container.getSegmentId().getId());
}

public static ImageQueryContainer imageQueryContainer(CineastGrpc.ImageQueryContainer container) {
public static ImageQueryTermContainer imageQueryContainer(CineastGrpc.ImageQueryContainer container) {
try {
return new ImageQueryContainer(
return new ImageQueryTermContainer(
ImageIO.read(new ByteArrayInputStream(container.getImage().toByteArray()))
);
} catch (IOException e) {
Expand All @@ -109,21 +109,21 @@ public static ImageQueryContainer imageQueryContainer(CineastGrpc.ImageQueryCont
return null;
}

public static InstantQueryContainer instantQueryContainer(CineastGrpc.InstantQueryContainer container) {
return new InstantQueryContainer(container.getInstant());
public static InstantQueryTermContainer instantQueryContainer(CineastGrpc.InstantQueryContainer container) {
return new InstantQueryTermContainer(container.getInstant());
}

public static LocationQueryContainer locationQueryContainer(CineastGrpc.LocationQueryContainer container) {
return new LocationQueryContainer(Location.of(container.getLongitude(), container.getLatitude()));
public static LocationQueryTermContainer locationQueryContainer(CineastGrpc.LocationQueryContainer container) {
return new LocationQueryTermContainer(Location.of(container.getLongitude(), container.getLatitude()));
}

public static ModelQueryContainer modelQueryContainer(CineastGrpc.ModelQueryContainer container) {
public static ModelQueryTermContainer modelQueryContainer(CineastGrpc.ModelQueryContainer container) {
//TODO figure out better mesh representation
return null;
}

public static MotionQueryContainer motionQueryContainer(CineastGrpc.MotionQueryContainer container) {
MotionQueryContainer motionQueryContainer = new MotionQueryContainer();
public static MotionQueryTermContainer motionQueryContainer(CineastGrpc.MotionQueryContainer container) {
MotionQueryTermContainer motionQueryContainer = new MotionQueryTermContainer();
container.getBackgroundPathList().stream().forEach(path -> motionQueryContainer.addBgPath(motionPath(path)));
container.getForegroundPathList().stream().forEach(path -> motionQueryContainer.addPath(motionPath(path)));
return motionQueryContainer;
Expand All @@ -139,9 +139,9 @@ public static Point2D_F32 point(CineastGrpc.MotionQueryContainer.MotionPath.Poin
return new Point2D_F32(point.getX(), point.getY());
}

public static SemanticMapQueryContainer semanticMapQueryContainer(CineastGrpc.SemanticMapQueryContainer container) {
public static SemanticMapQueryTermContainer semanticMapQueryContainer(CineastGrpc.SemanticMapQueryContainer container) {
try {
return new SemanticMapQueryContainer(new SemanticMap(
return new SemanticMapQueryTermContainer(new SemanticMap(
ImageIO.read(new ByteArrayInputStream(container.getImage().toByteArray())),
container.getConceptsMap())
);
Expand All @@ -151,16 +151,16 @@ public static SemanticMapQueryContainer semanticMapQueryContainer(CineastGrpc.Se
return null;
}

public static TagQueryContainer tagQueryContainer(CineastGrpc.TagQueryContainer container) {
public static TagQueryTermContainer tagQueryContainer(CineastGrpc.TagQueryContainer container) {
return null; //TODO do we even still need that one?
}

public static TextQueryContainer textQueryContainer(CineastGrpc.TextQueryContainer container) {
return new TextQueryContainer(container.getText());
public static TextQueryTermContainer textQueryContainer(CineastGrpc.TextQueryContainer container) {
return new TextQueryTermContainer(container.getText());
}


public static QueryContainer queryTermContainer(CineastGrpc.QueryTerm term) {
public static AbstractQueryTermContainer queryTermContainer(CineastGrpc.QueryTerm term) {
switch(term.getContainerCase()){

case AUDIOQUERYCONTAINER: return audioQueryContainer(term.getAudioQueryContainer());
Expand Down
69 changes: 3 additions & 66 deletions cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@
import org.apache.commons.lang3.builder.ToStringStyle;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.data.query.containers.QueryContainer;
import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer;

/**
* The wording is suboptimal. A {@link QueryComponent} has only one containerID, but multiple {@link QueryContainer}s are created out of it.
* The wording is suboptimal. A {@link QueryComponent} has only one containerID, but multiple {@link AbstractQueryTermContainer}s are created out of it.
* <p>
* These all have the {@link QueryComponent#containerId} of their parent.
*
* @deprecated use {@link TemporalQuery} instead
* @deprecated use {@link TemporalQuery} instead, which uses {@link StagedSimilarityQuery} which contains {@link QueryStage} which in turn contains {@link QueryTerm}
*/
@Deprecated
public class QueryComponent {
Expand Down Expand Up @@ -47,69 +47,6 @@ public List<QueryTerm> getTerms() {
return this.terms;
}

/**
* Converts the provided collection of QueryComponent objects to a map that maps feature categories defined in the query-terms to @{@link QueryContainer} derived from the {@link QueryTerm}.
*
* @return Category map.
*/
public static HashMap<String, ArrayList<QueryContainer>> toCategoryMap(Collection<QueryComponent> components) {
final HashMap<String, ArrayList<QueryContainer>> categoryMap = new HashMap<>();
if (components.isEmpty()) {
LOGGER.warn("Empty components collection, returning empty map");
return categoryMap;
}
for (QueryComponent component : components) {
if (component.getTerms().isEmpty()) {
LOGGER.warn("No terms for component {}", component);
continue;
}
for (QueryTerm term : component.getTerms()) {
if (term.getCategories().isEmpty()) {
LOGGER.warn("No categories for term {}", term);
}
for (String category : term.getCategories()) {
if (!categoryMap.containsKey(category)) {
categoryMap.put(category, new ArrayList<>());
}
final QueryContainer container = term.toContainer();
if (container != null) {
container.setContainerId(component.containerId);
categoryMap.get(category).add(container);
} else {
LOGGER.warn("Null container generated for term {}", term);
}
}
}
}
return categoryMap;
}

/**
* Converts the provided collection of {@link QueryComponent} object to a map of {@link QueryContainer} and their categories.
*
* @return A map of querycontainers with their associated categories
*/
public static HashMap<QueryContainer, List<String>> toContainerMap(Collection<QueryComponent> components) {
final HashMap<QueryContainer, List<String>> map = new HashMap<>();
if (components.isEmpty()) {
LOGGER.warn("Empty components collection, returning empty list of containers");
return map;
}
for (QueryComponent component : components) {
for (QueryTerm qt : component.getTerms()) {
if (qt == null) {
/* FIXME in rare instances, it is possible to have null as query component*/
LOGGER.warn("QueryTerm was null for component {}", component);
continue;
}
QueryContainer qc = qt.toContainer();
qc.setContainerId(component.containerId);
map.put(qc, qt.getCategories());
}
}
return map;
}

@Override
public String toString() {
return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE);
Expand Down
20 changes: 10 additions & 10 deletions cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/QueryTerm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import java.util.List;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.vitrivr.cineast.core.data.query.containers.QueryContainer;
import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer;

/**
* Contains the data of a particular {@link QueryTerm}.
Expand All @@ -30,9 +30,9 @@ public class QueryTerm {
private final String data;

/**
* Cached version of the {@link QueryContainer} representation of this {@link QueryTerm}.
* Cached version of the {@link AbstractQueryTermContainer} representation of this {@link QueryTerm}.
*/
private QueryContainer cachedQueryContainer;
private AbstractQueryTermContainer cachedQueryTermContainer;

@Override
public String toString() {
Expand Down Expand Up @@ -75,18 +75,18 @@ public QueryTermType getType() {


/**
* Converts the {@link QueryTerm} to a {@link QueryContainer} that can be processed by the retrieval pipeline. This includes conversion of query-objects from the Base64 encoded representation.
* Converts the {@link QueryTerm} to a {@link AbstractQueryTermContainer} that can be processed by the retrieval pipeline. This includes conversion of query-objects from the Base64 encoded representation.
*
* <strong>IMPORTANT:</strong> Subsequent calls to this method return a cached version of the original {@link QueryContainer}.
* <strong>IMPORTANT:</strong> Subsequent calls to this method return a cached version of the original {@link AbstractQueryTermContainer}.
*
* @return {@link QueryContainer} representation of the {@link QueryTerm}.
* @return {@link AbstractQueryTermContainer} representation of the {@link QueryTerm}.
*/
public QueryContainer toContainer() {
if (this.cachedQueryContainer == null) {
public AbstractQueryTermContainer toContainer() {
if (this.cachedQueryTermContainer == null) {
if (this.data != null) {
this.cachedQueryContainer = this.type.getQueryContainer(this.data).orElse(null);
this.cachedQueryTermContainer = this.type.getQueryContainer(this.data).orElse(null);
}
}
return this.cachedQueryContainer;
return this.cachedQueryTermContainer;
}
}
Loading

0 comments on commit d72993c

Please sign in to comment.