diff --git a/core/model/src/main/java/it/unibz/inf/ontop/substitution/impl/ImmutableUnificationTools.java b/core/model/src/main/java/it/unibz/inf/ontop/substitution/impl/ImmutableUnificationTools.java index 73062e2938e..631b8b35b54 100644 --- a/core/model/src/main/java/it/unibz/inf/ontop/substitution/impl/ImmutableUnificationTools.java +++ b/core/model/src/main/java/it/unibz/inf/ontop/substitution/impl/ImmutableUnificationTools.java @@ -189,7 +189,7 @@ public Optional> computeMGUS2(Immutable * Computes one Most General Unifier (MGU) of (two) substitutions. */ public Optional> computeMGUS(ImmutableSubstitution substitution1, - ImmutableSubstitution substitution2) { + ImmutableSubstitution substitution2) { ImmutableList.Builder firstArgListBuilder = ImmutableList.builder(); ImmutableList.Builder secondArgListBuilder = ImmutableList.builder(); @@ -214,13 +214,8 @@ public Optional> computeAtomMGUS( ImmutableSubstitution substitution1, ImmutableSubstitution substitution2) { Optional> optionalMGUS = computeMGUS(substitution1, substitution2); - if (optionalMGUS.isPresent()) { - return Optional.of(substitutionTools.convertIntoVariableOrGroundTermSubstitution( - optionalMGUS.get())); - } - else { - return Optional.empty(); - } + return optionalMGUS + .map(substitutionTools::convertIntoVariableOrGroundTermSubstitution); } diff --git a/core/model/src/test/java/it/unibz/inf/ontop/substitution/SubstitutionTest.java b/core/model/src/test/java/it/unibz/inf/ontop/substitution/SubstitutionTest.java index 3f4d26f9ebd..e04bfccf42a 100644 --- a/core/model/src/test/java/it/unibz/inf/ontop/substitution/SubstitutionTest.java +++ b/core/model/src/test/java/it/unibz/inf/ontop/substitution/SubstitutionTest.java @@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableMap; import it.unibz.inf.ontop.model.term.Constant; import it.unibz.inf.ontop.model.term.ImmutableTerm; -import it.unibz.inf.ontop.model.term.NonConstantTerm; import it.unibz.inf.ontop.model.term.Variable; import org.junit.Test; diff --git a/core/optimization/src/main/java/it/unibz/inf/ontop/iq/executor/join/RedundantSelfJoinExecutor.java b/core/optimization/src/main/java/it/unibz/inf/ontop/iq/executor/join/RedundantSelfJoinExecutor.java index ba907f66369..1c25278bfa0 100644 --- a/core/optimization/src/main/java/it/unibz/inf/ontop/iq/executor/join/RedundantSelfJoinExecutor.java +++ b/core/optimization/src/main/java/it/unibz/inf/ontop/iq/executor/join/RedundantSelfJoinExecutor.java @@ -3,11 +3,13 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; import it.unibz.inf.ontop.injection.IntermediateQueryFactory; import it.unibz.inf.ontop.iq.IntermediateQuery; import it.unibz.inf.ontop.iq.exception.EmptyQueryException; import it.unibz.inf.ontop.iq.exception.InvalidQueryOptimizationProposalException; import it.unibz.inf.ontop.iq.impl.QueryTreeComponent; +import it.unibz.inf.ontop.iq.node.DataNode; import it.unibz.inf.ontop.iq.node.EmptyNode; import it.unibz.inf.ontop.iq.node.ExtensionalDataNode; import it.unibz.inf.ontop.iq.node.InnerJoinNode; @@ -15,10 +17,14 @@ import it.unibz.inf.ontop.iq.proposal.NodeCentricOptimizationResults; import it.unibz.inf.ontop.iq.proposal.impl.NodeCentricOptimizationResultsImpl; import it.unibz.inf.ontop.model.atom.RelationPredicate; +import it.unibz.inf.ontop.model.term.ImmutableExpression; import it.unibz.inf.ontop.model.term.TermFactory; import it.unibz.inf.ontop.model.term.Variable; +import it.unibz.inf.ontop.model.term.VariableOrGroundTerm; +import it.unibz.inf.ontop.substitution.ImmutableSubstitution; import it.unibz.inf.ontop.substitution.SubstitutionFactory; import it.unibz.inf.ontop.substitution.impl.ImmutableUnificationTools; +import it.unibz.inf.ontop.utils.ImmutableCollectors; import java.util.Optional; @@ -37,12 +43,14 @@ public abstract class RedundantSelfJoinExecutor extends SelfJoinLikeExecutor imp */ private static final int MAX_ITERATIONS = 100; private final IntermediateQueryFactory iqFactory; + private final TermFactory termFactory; protected RedundantSelfJoinExecutor(IntermediateQueryFactory iqFactory, SubstitutionFactory substitutionFactory, ImmutableUnificationTools unificationTools, TermFactory termFactory) { super(substitutionFactory, unificationTools, termFactory); this.iqFactory = iqFactory; + this.termFactory = termFactory; } @@ -137,7 +145,38 @@ private Optional propose(InnerJoinNode joinNode, ImmutableMult predicateProposal.ifPresent(proposalListBuilder::add); } - return createConcreteProposal(proposalListBuilder.build(), priorityVariables); + return createConcreteProposal(proposalListBuilder.build(), initialDataNodeMap, priorityVariables); + } + + protected Optional createConcreteProposal( + ImmutableList predicateProposals, + ImmutableMultimap initialDataNodeMap, ImmutableList priorityVariables) { + + + + Optional> optionalMergedSubstitution; + try { + optionalMergedSubstitution = mergeSubstitutions(extractSubstitutions(predicateProposals), initialDataNodeMap, priorityVariables); + } catch (AtomUnificationException e) { + return Optional.empty(); + } + + ImmutableSet removedDataNodes =predicateProposals.stream() + .flatMap(p -> p.getRemovedDataNodes().stream()) + .collect(ImmutableCollectors.toSet()); + + if (removedDataNodes.isEmpty() + && (! optionalMergedSubstitution.isPresent())) + return Optional.empty(); + + Optional isNotConjunction = termFactory.getConjunction(predicateProposals.stream() + .map(PredicateLevelProposal::getIsNotNullConjunction) + .filter(Optional::isPresent) + .map(Optional::get) + .flatMap(ImmutableExpression::flattenAND) + .distinct()); + + return Optional.of(new ConcreteProposal(optionalMergedSubstitution, removedDataNodes, isNotConjunction)); } protected abstract Optional proposePerPredicate(InnerJoinNode joinNode, ImmutableCollection initialNodes, diff --git a/core/optimization/src/main/java/it/unibz/inf/ontop/iq/executor/join/SelfJoinLikeExecutor.java b/core/optimization/src/main/java/it/unibz/inf/ontop/iq/executor/join/SelfJoinLikeExecutor.java index 7120f4f2d77..96636f3a7c9 100644 --- a/core/optimization/src/main/java/it/unibz/inf/ontop/iq/executor/join/SelfJoinLikeExecutor.java +++ b/core/optimization/src/main/java/it/unibz/inf/ontop/iq/executor/join/SelfJoinLikeExecutor.java @@ -1,6 +1,7 @@ package it.unibz.inf.ontop.iq.executor.join; import com.google.common.collect.*; +import it.unibz.inf.ontop.exception.MinorOntopInternalBugException; import it.unibz.inf.ontop.iq.exception.EmptyQueryException; import it.unibz.inf.ontop.iq.node.*; import it.unibz.inf.ontop.model.atom.DataAtom; @@ -20,6 +21,8 @@ import it.unibz.inf.ontop.model.term.VariableOrGroundTerm; import it.unibz.inf.ontop.substitution.ImmutableSubstitution; import it.unibz.inf.ontop.utils.ImmutableCollectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.*; import java.util.stream.Stream; @@ -28,6 +31,8 @@ public class SelfJoinLikeExecutor { + private static final Logger LOGGER = LoggerFactory.getLogger(SelfJoinLikeExecutor.class); + /** * TODO: explain * @@ -219,6 +224,7 @@ protected PredicateLevelProposal proposeForGroupingMap( }) .filter(s -> !s.isEmpty()) .collect(ImmutableCollectors.toSet()); + /* * All the nodes that have been at least once dominated (--> could thus be removed). * @@ -249,34 +255,6 @@ protected PredicateLevelProposal proposeForGroupingMap( } } - protected Optional createConcreteProposal( - ImmutableList predicateProposals, - ImmutableList priorityVariables) { - Optional> optionalMergedSubstitution; - try { - optionalMergedSubstitution = mergeSubstitutions(extractSubstitutions(predicateProposals), priorityVariables); - } catch (AtomUnificationException e) { - return Optional.empty(); - } - - ImmutableSet removedDataNodes =predicateProposals.stream() - .flatMap(p -> p.getRemovedDataNodes().stream()) - .collect(ImmutableCollectors.toSet()); - - if (removedDataNodes.isEmpty() - && (! optionalMergedSubstitution.isPresent())) - return Optional.empty(); - - Optional isNotConjunction = termFactory.getConjunction(predicateProposals.stream() - .map(PredicateLevelProposal::getIsNotNullConjunction) - .filter(Optional::isPresent) - .map(Optional::get) - .flatMap(ImmutableExpression::flattenAND) - .distinct()); - - return Optional.of(new ConcreteProposal(optionalMergedSubstitution, removedDataNodes, isNotConjunction)); - } - protected ImmutableSubstitution unifyRedundantNodes( Collection redundantNodes) throws AtomUnificationException { @@ -291,31 +269,109 @@ protected ImmutableSubstitution unifyRedundantNodes( return accumulatedSubstitution; } + ImmutableMap> occurrenceVariableMap = redundantNodes.stream() + .flatMap(n -> n.getVariables().stream() + .map(v -> Maps.immutableEntry(v, n))) + .collect(ImmutableCollectors.toMultimap()).asMap(); + + // Variables used in more than one data node + ImmutableSet sharedVariables = occurrenceVariableMap.entrySet().stream() + .filter(e -> ImmutableSet.copyOf(e.getValue()).size() > 1) + .map(Map.Entry::getKey) + .collect(ImmutableCollectors.toSet()); + ImmutableSet nonSharedVariables = Sets.difference(occurrenceVariableMap.keySet(), sharedVariables) + .immutableCopy(); + Iterator nodeIterator = redundantNodes.iterator(); + /* + * For performance purposes, we can detach some fragments from the substitution to be "unified" with the following atom. + */ + ImmutableList.Builder> nonSharedSubstitutionListBuilder = ImmutableList.builder(); + // Non-final DataAtom accumulatedAtom = nodeIterator.next().getProjectionAtom(); + while (nodeIterator.hasNext()) { DataAtom newAtom = nodeIterator.next().getProjectionAtom(); + + /* + * Before the following unification, we detach a fragment about non-shared variables from the accumulated substitution + * + * Particularly useful when dealing with tables with a large number of columns (e.g. views after collapsing some JSON objects) + * + */ + ImmutableSubstitution nonSharedSubstitution = accumulatedSubstitution.reduceDomainToIntersectionWith(nonSharedVariables); + if (!nonSharedSubstitution.isEmpty()) + nonSharedSubstitutionListBuilder.add(nonSharedSubstitution); + + ImmutableSubstitution substitutionToUnify = nonSharedSubstitution.isEmpty() + ? accumulatedSubstitution + : accumulatedSubstitution.reduceDomainToIntersectionWith(sharedVariables); + // May throw an exception - accumulatedSubstitution = updateSubstitution(accumulatedSubstitution, accumulatedAtom, newAtom); + accumulatedSubstitution = updateSubstitution(substitutionToUnify, accumulatedAtom, newAtom); + accumulatedAtom = accumulatedSubstitution.applyToDataAtom(accumulatedAtom); } - return accumulatedSubstitution; + + return Stream.concat( + nonSharedSubstitutionListBuilder.build().stream(), + Stream.of(accumulatedSubstitution)) + .reduce((v1, v2) -> v2.composeWith2(v1)) + .orElseThrow(() -> new MinorOntopInternalBugException("At least one substitution was expected")); } protected Optional> mergeSubstitutions( - ImmutableList> substitutions, ImmutableList priorityVariables) + ImmutableList> substitutions, + ImmutableMultimap initialDataNodeMap, + ImmutableList priorityVariables) throws AtomUnificationException { + ImmutableMap> occurrenceVariableMap = initialDataNodeMap.asMap().entrySet().stream() + .flatMap(e -> e.getValue().stream() + .flatMap(n -> n.getVariables().stream()) + .map(v -> Maps.immutableEntry(v, e.getKey()))) + .collect(ImmutableCollectors.toMultimap()).asMap(); + + // Variables appearing for for more one relation predicate + ImmutableSet sharedVariables = occurrenceVariableMap.entrySet().stream() + .filter(e -> ImmutableSet.copyOf(e.getValue()).size() > 1) + .map(Map.Entry::getKey) + .collect(ImmutableCollectors.toSet()); + ImmutableSet nonSharedVariables = Sets.difference(occurrenceVariableMap.keySet(), sharedVariables) + .immutableCopy(); + + /* + * For performance purposes, we can detach some fragments from the substitution to be "unified" with the following atom. + */ + ImmutableList.Builder> nonSharedSubstitutionListBuilder = ImmutableList.builder(); + // Non-final Optional> optionalAccumulatedSubstitution = Optional.empty(); for (ImmutableSubstitution substitution : substitutions) { if (!substitution.isEmpty()) { if (optionalAccumulatedSubstitution.isPresent()) { + + ImmutableSubstitution accumulatedSubstitution = optionalAccumulatedSubstitution.get(); + + /* + * Before the following unification, we detach a fragment about non-shared variables from the accumulated substitution + * + * Particularly useful when dealing with tables with a large number of columns (e.g. views after collapsing some JSON objects) + * + */ + ImmutableSubstitution nonSharedSubstitution = accumulatedSubstitution.reduceDomainToIntersectionWith(nonSharedVariables); + if (!nonSharedSubstitution.isEmpty()) + nonSharedSubstitutionListBuilder.add(nonSharedSubstitution); + + ImmutableSubstitution substitutionToUnify = nonSharedSubstitution.isEmpty() + ? accumulatedSubstitution + : accumulatedSubstitution.reduceDomainToIntersectionWith(sharedVariables); + Optional> optionalMGUS = unificationTools.computeAtomMGUS( - optionalAccumulatedSubstitution.get(), substitution); + substitutionToUnify, substitution); if (optionalMGUS.isPresent()) { optionalAccumulatedSubstitution = optionalMGUS; } @@ -331,6 +387,11 @@ protected Optional> mergeSubstitutio } return optionalAccumulatedSubstitution + .map(s -> Stream.concat( + nonSharedSubstitutionListBuilder.build().stream(), + Stream.of(s)) + .reduce((v1, v2) -> v2.composeWith2(v1)) + .orElseThrow(() -> new MinorOntopInternalBugException("At least one substitution was expected"))) .map(s -> s.orientate(priorityVariables)); } diff --git a/core/optimization/src/main/java/it/unibz/inf/ontop/iq/optimizer/impl/GeneralStructuralAndSemanticIQOptimizerImpl.java b/core/optimization/src/main/java/it/unibz/inf/ontop/iq/optimizer/impl/GeneralStructuralAndSemanticIQOptimizerImpl.java index d836cc07c54..5bc03150a94 100644 --- a/core/optimization/src/main/java/it/unibz/inf/ontop/iq/optimizer/impl/GeneralStructuralAndSemanticIQOptimizerImpl.java +++ b/core/optimization/src/main/java/it/unibz/inf/ontop/iq/optimizer/impl/GeneralStructuralAndSemanticIQOptimizerImpl.java @@ -58,9 +58,12 @@ public IQ optimize(IQ query, ExecutorRegistry executorRegistry) { LOGGER.debug("After projection shrinking: \n" + intermediateQuery.toString()); - + long beginningJoinLike = System.currentTimeMillis(); intermediateQuery = joinLikeOptimizer.optimize(intermediateQuery); - LOGGER.debug("New query after fixed point join optimization: \n" + intermediateQuery.toString()); + LOGGER.debug(String.format( + "New query after fixed point join optimization (%d ms): \n%s", + System.currentTimeMillis() - beginningJoinLike, + intermediateQuery.toString())); intermediateQuery = flattenUnionOptimizer.optimize(intermediateQuery); LOGGER.debug("New query after flattening Unions: \n" + intermediateQuery.toString()); diff --git a/engine/reformulation/core/src/main/java/it/unibz/inf/ontop/answering/reformulation/impl/QuestQueryProcessor.java b/engine/reformulation/core/src/main/java/it/unibz/inf/ontop/answering/reformulation/impl/QuestQueryProcessor.java index b9227665067..de7cd904664 100644 --- a/engine/reformulation/core/src/main/java/it/unibz/inf/ontop/answering/reformulation/impl/QuestQueryProcessor.java +++ b/engine/reformulation/core/src/main/java/it/unibz/inf/ontop/answering/reformulation/impl/QuestQueryProcessor.java @@ -85,6 +85,8 @@ private QuestQueryProcessor(@Assisted OBDASpecification obdaSpecification, public IQ reformulateIntoNativeQuery(InputQuery inputQuery) throws OntopReformulationException { + long beginning = System.currentTimeMillis(); + IQ cachedQuery = queryCache.get(inputQuery); if (cachedQuery != null) return cachedQuery; @@ -103,8 +105,10 @@ public IQ reformulateIntoNativeQuery(InputQuery inputQuery) log.debug("Start the unfolding..."); IQ unfoldedIQ = queryUnfolder.optimize(rewrittenIQ); - if (unfoldedIQ.getTree().isDeclaredAsEmpty()) - return unfoldedIQ; + if (unfoldedIQ.getTree().isDeclaredAsEmpty()) { + log.debug(String.format("Reformulation time: %d ms", System.currentTimeMillis() - beginning)); + return unfoldedIQ; + } log.debug("Unfolded query: \n" + unfoldedIQ.toString()); IQ optimizedQuery = generalOptimizer.optimize(unfoldedIQ, executorRegistry); @@ -113,6 +117,7 @@ public IQ reformulateIntoNativeQuery(InputQuery inputQuery) IQ executableQuery = generateExecutableQuery(plannedQuery); queryCache.put(inputQuery, executableQuery); + log.debug(String.format("Reformulation time: %d ms", System.currentTimeMillis() - beginning)); return executableQuery; }