Skip to content

Commit

Permalink
cardinality estimation for exclusive group uses the same formula now
Browse files Browse the repository at this point in the history 
changed costs for hash and bind joins
new results
  • Loading branch information
apotocki committed Oct 10, 2016
1 parent 96fe13f commit 1aa31ac
Show file tree
Hide file tree
Showing 10 changed files with 139 additions and 109 deletions.
44 changes: 22 additions & 22 deletions costfed/results/cfed-report.csv
View file
@@ -1,23 +1,23 @@
Query,#Results,Sample #0,Sample #1,Sample #2,Sample #3,Sample #4,Sample #5,Sample #6,Sample #7,Sample #8,Sample #9,Sample #10,Sample #11,Sample #12,Sample #13,Sample #14,Sample #15
S1,90,142,19,17,20,21,21,20,16,15,16,20,21,19,20,20,15
S2,1,9,5,4,5,5,4,6,5,4,4,4,5,3,5,4,4
S3,2,25,16,17,17,17,15,18,17,17,16,15,88,18,15,17,16
S4,1,10,9,7,6,6,7,6,5,7,6,6,6,7,7,6,6
S5,2,220,173,231,168,170,228,166,171,230,299,168,229,171,168,232,168
S6,11,278,212,177,172,232,169,162,160,164,163,229,163,160,234,162,227
S7,1,253,218,218,285,216,281,212,215,216,218,219,218,216,214,208,214
S8,1159,18,16,14,15,14,15,16,16,16,16,15,16,17,15,15,17
S9,333,27,26,20,21,22,22,22,18,18,23,21,18,18,22,22,21
S10,9054,2574,2577,2532,2532,2568,2578,2511,2498,2621,2515,2565,2464,2615,2499,2567,2493
S11,3,8,8,7,7,7,7,7,7,7,6,6,7,7,7,7,6
S12,393,515,495,565,497,570,501,502,494,499,492,496,564,492,497,494,565
S13,28,35,34,35,31,33,34,33,104,97,34,34,32,32,31,32,33
S14,1620,2220,1804,1933,1748,1746,1731,1747,1702,1682,1681,1697,1730,1701,1662,1765,1736
C1,1000,1967,1393,2429,1416,2493,1436,1374,1752,1414,1389,2478,1382,1491,1718,1364,2187
C2,4,909,858,985,858,858,862,859,859,858,850,940,854,857,851,855,845
C3,9,238,382,220,223,216,211,216,220,211,388,213,214,215,215,216,215
C6,148,1633,769,1628,1346,1880,1545,1370,932,928,1353,947,941,1358,938,939,1349
C7,112,121,94,98,94,92,95,89,93,93,90,92,90,90,93,94,92
C8,3073,971,866,905,817,835,948,857,847,877,852,862,844,844,821,821,833
C9,100,16304,15930,16822,16127,16586,16341,16042,15744,16194,15982,16023,16054,15659,16023,15854,16185
C10,0,2,0,0,0,1,0,1,1,0,0,0,0,1,0,0,0
S1,90,76,23,16,20,20,20,20,19,15,20,19,19,19,16,19,20
S2,1,10,5,5,5,4,5,5,6,5,4,4,5,4,5,5,4
S3,2,24,19,19,17,17,15,14,16,16,17,16,15,16,15,18,14
S4,1,11,8,8,7,6,7,7,6,6,7,6,5,6,5,6,5
S5,2,13,7,8,7,9,7,7,7,8,8,8,7,9,10,8,7
S6,11,122,117,92,93,86,80,92,149,89,84,74,173,76,76,75,83
S7,1,295,220,208,212,212,226,209,215,212,273,206,213,277,218,277,210
S8,1159,17,14,15,15,14,15,14,15,15,15,19,14,16,15,14,15
S9,333,31,26,22,22,23,22,24,21,22,21,21,22,23,22,22,22
S10,9054,2808,2647,2623,2808,2632,2806,2744,2738,2637,2715,2661,2651,2666,2630,2657,2639
S11,3,10,8,8,8,7,7,7,7,7,7,6,7,7,8,7,7
S12,393,729,554,597,552,596,571,604,604,657,607,651,603,600,600,605,604
S13,28,35,33,91,35,34,34,99,34,33,34,31,34,35,35,33,95
S14,1620,1920,1745,1731,1749,1738,2541,1824,1776,1788,1705,1758,1773,1705,1780,1753,1709
C1,1000,1673,1465,1588,1750,2514,1862,1623,1767,1876,1769,1748,1764,1489,1782,1413,1875
C2,4,972,960,1010,1392,1132,1038,950,945,950,956,929,1011,931,939,929,921
C3,9,253,216,219,214,224,232,219,215,227,220,218,221,220,211,214,217
C6,148,980,997,971,1764,1013,1008,962,965,1008,1007,1021,1807,1228,1040,939,1012
C7,112,122,87,132,87,88,93,87,88,89,93,90,92,89,87,86,88
C8,3073,962,851,912,837,851,917,827,829,846,834,830,849,841,845,826,824
C9,100,16615,16831,17770,16043,16936,17867,16107,16003,16023,16164,16155,15981,16173,16175,15956,16057
C10,0,1,0,1,0,1,0,0,1,0,0,0,0,1,0,0,0
38 changes: 19 additions & 19 deletions costfed/results/cfed-summary.csv
View file
@@ -1,23 +1,23 @@
Query,Average,SD,SE
S1,26,30,7
S1,23,14,3
S2,5,1,0
S3,22,17,4
S3,17,2,1
S4,7,1,0
S5,200,38,10
S6,192,36,9
S7,226,23,6
S8,16,1,0
S9,21,3,1
S10,2544,44,11
S5,8,2,0
S6,98,28,7
S7,230,30,7
S8,15,1,0
S9,23,2,1
S10,2691,66,17
S11,7,1,0
S12,515,30,7
S13,42,22,6
S14,1768,132,33
C1,1730,422,106
C2,872,37,9
C3,238,56,14
C6,1241,321,80
C7,94,7,2
C8,862,43,11
C9,16117,286,72
C10,0,1,0
S12,608,41,10
S13,45,24,6
S14,1812,195,49
C1,1747,242,61
C2,998,115,29
C3,221,10,2
C6,1108,264,66
C7,94,13,3
C8,855,38,10
C9,16428,600,150
C10,0,0,0
36 changes: 18 additions & 18 deletions costfed/results/cmp-summary.csv
View file
@@ -1,24 +1,24 @@
Query,FedX,Semagrow,CostFed+,CostFed++
S1,10,1686,16,26
S1,10,1686,16,23
S2,1,50,4,5
S3,27,155,22,22
S3,27,155,22,17
S4,33,176,8,7
S5,21,104,8,200
S6,479,857,196,192
S7,545,1004,275,226
S8,17,80,17,16
S9,33,787,17,21
S10,4818,6544,2698,2544
S5,21,104,8,8
S6,479,857,196,98
S7,545,1004,275,230
S8,17,80,17,15
S9,33,787,17,23
S10,4818,6544,2698,2691
S11,8,89,7,7
S12,1336,1614,701,515
S13,94519,162,34,42
S14,2702,3542,2169,1768
C1,3710,-,1849,1730
C2,1205,1347,1057,872
C3,13155,11580,654,238
C4,-,5695,837,840
C6,24151,47185,2057,1241
S12,1336,1614,701,608
S13,94519,162,34,45
S14,2702,3542,2169,1812
C1,3710,-,1849,1747
C2,1205,1347,1057,998
C3,13155,11580,654,221
C4,-,5695,837,1200
C6,24151,47185,2057,1108
C7,340,6114,116,94
C8,2084,2437,996,862
C9,17668,-,18040,16117
C8,2084,2437,996,855
C9,17668,-,18040,16428
C10,645,611,136,-
4 changes: 2 additions & 2 deletions costfed/src/main/java/org/aksw/simba/quetsal/configuration/CostFedSummary.java
View file
Expand Up @@ -693,11 +693,11 @@ public double getTriplePatternObjectMVKoef(StatementPattern stmt, List<Statement
String s = boundSubject(stmt);
String p = boundPredicate(stmt);
String o = boundObject(stmt);
if (s != null || o != null) return 1.0;
if (p == null) {
// not implemented
return 1.0;
}
if (s != null || o != null) return 0.71;
long totalObjects = 0, totalTriples = 0;
for (StatementSource ss : stmtSrces)
{
Expand All @@ -715,11 +715,11 @@ public double getTriplePatternSubjectMVKoef(StatementPattern stmt, List<Statemen
String s = boundSubject(stmt);
String p = boundPredicate(stmt);
String o = boundObject(stmt);
if (s != null || o != null) return 1.0;
if (p == null) {
// not implemented
return 1.0;
}
if (s != null || o != null) return 0.71;
long totalSubjects = 0, totalTriples = 0;
for (StatementSource ss : stmtSrces)
{
Expand Down
72 changes: 48 additions & 24 deletions costfed/src/main/java/org/aksw/simba/quetsal/core/CardinalityVisitor.java
View file
Expand Up @@ -15,10 +15,16 @@
import com.fluidops.fedx.algebra.ExclusiveStatement;
import com.fluidops.fedx.algebra.StatementSource;
import com.fluidops.fedx.optimizer.OptimizerUtil;
import com.fluidops.fedx.structures.QueryInfo;

public class CardinalityVisitor extends AbstractQueryModelVisitor<RuntimeException>
{
protected static int ESTIMATION_TYPE = 0;
final QueryInfo queryInfo;

public CardinalityVisitor(QueryInfo queryInfo) {
this.queryInfo = queryInfo;
}

public static class NodeDescriptor {
public long card = Long.MAX_VALUE;
Expand All @@ -44,10 +50,10 @@ public String toString() {

public static NodeDescriptor getJoinCardinality(Collection<String> commonvars, CardPair left, CardPair right)
{
CardinalityVisitor.NodeDescriptor result = new CardinalityVisitor.NodeDescriptor();
NodeDescriptor result = new NodeDescriptor();
if (CardinalityVisitor.ESTIMATION_TYPE == 0) {
if (commonvars != null && !commonvars.isEmpty()) {
result.card = (long)Math.ceil((Math.min(left.nd.card, right.nd.card) / 2));
result.card = (long)Math.ceil((Math.min(left.nd.card, right.nd.card)));

// multivalue fixes
if (left.expr instanceof StatementPattern) {
Expand All @@ -68,7 +74,6 @@ public static NodeDescriptor getJoinCardinality(Collection<String> commonvars, C
result.card *= right.nd.mvobjkoef;
}
}

} else {
result.card = (long)Math.ceil(left.nd.card * right.nd.card);
}
Expand Down Expand Up @@ -114,6 +119,16 @@ public static Collection<String> getCommonVars(Collection<String> vars, TupleExp
return commonvars;
}

@Override
public void meet(StatementPattern stmt) {
List<StatementSource> stmtSrces = queryInfo.getSourceSelection().getStmtToSources().get(stmt);
current.card = Cardinality.getTriplePatternCardinality(stmt, stmtSrces);
assert(current.card != 0);
current.sel = current.card/(double)Cardinality.getTotalTripleCount(stmtSrces);
current.mvsbjkoef = Cardinality.getTriplePatternSubjectMVKoef(stmt, stmtSrces);
current.mvobjkoef = Cardinality.getTriplePatternObjectMVKoef(stmt, stmtSrces);
}

@Override
public void meet(Filter filter) {
filter.getArg().visit(this);
Expand All @@ -123,16 +138,20 @@ public void meet(ExclusiveGroup eg) {
List<ExclusiveStatement> slst = new LinkedList<ExclusiveStatement>(eg.getStatements()); // make copy
List<StatementSource> stmtSrces = slst.get(0).getStatementSources();
assert (stmtSrces.size() == 1);
long total = Cardinality.getTotalTripleCount(stmtSrces);

//long total = Cardinality.getTotalTripleCount(stmtSrces);

ExclusiveStatement leftArg = slst.get(0);
slst.remove(0);

Set<String> joinVars = new HashSet<String>();
joinVars.addAll(OptimizerUtil.getFreeVars(leftArg));

long leftCard = Cardinality.getTriplePatternCardinality(leftArg, stmtSrces);
double leftSelectivity = leftCard/(double)total;
//long leftCard = Cardinality.getTriplePatternCardinality(leftArg, stmtSrces);
leftArg.visit(this);
CardPair left = new CardPair(leftArg, getDescriptor());
reset();
//double leftSelectivity = leftCard/(double)total;

// find possible join order
while (!slst.isEmpty()) {
Expand All @@ -150,28 +169,33 @@ public void meet(ExclusiveGroup eg) {
rightArg = slst.get(0);
slst.remove(0);
}
long rightCard = Cardinality.getTriplePatternCardinality(rightArg, stmtSrces);
rightArg.visit(this);
CardPair right = new CardPair(rightArg, getDescriptor());
reset();
//long rightCard = Cardinality.getTriplePatternCardinality(rightArg, stmtSrces);

joinVars.addAll(OptimizerUtil.getFreeVars(rightArg));

double sel = 1;
//double sel = 1;

if (ESTIMATION_TYPE == 0) {
if (commonvars != null && !commonvars.isEmpty()) {
leftCard = (long)Math.ceil((Math.min(leftCard, rightCard))/2);
} else {
leftCard = (long)Math.ceil(leftCard * rightCard);
}
} else {
double rightSelectivity = rightCard/(double)total;
if (commonvars != null && !commonvars.isEmpty()) {
sel *= Math.min(leftSelectivity, rightSelectivity);
}
leftCard = (long)Math.ceil(leftCard * rightCard * sel);
leftSelectivity = sel;
}
left.nd = getJoinCardinality(commonvars, left, right);

//if (ESTIMATION_TYPE == 0) {
//if (commonvars != null && !commonvars.isEmpty()) {
// leftCard = (long)Math.ceil((Math.min(leftCard, rightCard))/2);
//} else {
// leftCard = (long)Math.ceil(leftCard * rightCard);
//}
//} else {
// double rightSelectivity = rightCard/(double)total;
// if (commonvars != null && !commonvars.isEmpty()) {
// sel *= Math.min(leftSelectivity, rightSelectivity);
// }
// leftCard = (long)Math.ceil(leftCard * rightCard * sel);
// leftSelectivity = sel;
//}
}
current.card = leftCard;
current.sel = leftSelectivity;
current = left.nd;
//current.sel = leftSelectivity;
}
}
28 changes: 14 additions & 14 deletions costfed/src/main/java/org/aksw/simba/quetsal/core/JoinOrderOptimizer.java
View file
Expand Up @@ -29,22 +29,16 @@
public class JoinOrderOptimizer extends StatementGroupOptimizer {
public static Logger log = Logger.getLogger(JoinOrderOptimizer.class);

protected static double C_TRANSFER_TUPLE = 0.02;
protected static double C_TRANSFER_QUERY = 50;

protected static double C_HANDLE_TUPLE = 0.0025;
protected static double C_TRANSFER_TUPLE = 0.01;
protected static double C_TRANSFER_QUERY = 100;
Map<QueryModelNode, CardinalityVisitor.NodeDescriptor> ds = new HashMap<QueryModelNode, CardinalityVisitor.NodeDescriptor>();

public class EstimatorVisitor extends CardinalityVisitor
{
@Override
public void meet(StatementPattern stmt) {
List<StatementSource> stmtSrces = queryInfo.getSourceSelection().getStmtToSources().get(stmt);
current.card = Cardinality.getTriplePatternCardinality(stmt, stmtSrces);
assert(current.card != 0);
current.sel = current.card/(double)Cardinality.getTotalTripleCount(stmtSrces);
current.mvsbjkoef = Cardinality.getTriplePatternSubjectMVKoef(stmt, stmtSrces);
current.mvobjkoef = Cardinality.getTriplePatternObjectMVKoef(stmt, stmtSrces);
EstimatorVisitor() {
super(JoinOrderOptimizer.this.queryInfo);
}

@Override
Expand Down Expand Up @@ -185,9 +179,15 @@ public int compare(CardinalityVisitor.CardPair cpl, CardinalityVisitor.CardPair

CardinalityVisitor.NodeDescriptor rd = CardinalityVisitor.getJoinCardinality(commonvars, leftArg, rightArg);

double hashCost = rightArg.nd.card * C_TRANSFER_TUPLE + 2 * C_TRANSFER_QUERY;
double bindCost = leftArg.nd.card / Config.getConfig().getBoundJoinBlockSize() * C_TRANSFER_QUERY + rd.card * C_TRANSFER_TUPLE;
long threads = Config.getConfig().getWorkerThreads();

double hashCost = rightArg.nd.card * C_TRANSFER_TUPLE + (2 + threads - 1)/threads * C_TRANSFER_QUERY + (rightArg.nd.card + rightArg.nd.card) * C_HANDLE_TUPLE;

long bsize = Config.getConfig().getBoundJoinBlockSize();

long numOfBindRequest = (leftArg.nd.card + bsize - 1) / bsize;
double bindCost = C_TRANSFER_QUERY + (numOfBindRequest + threads - 1) / threads * C_TRANSFER_QUERY + (leftArg.nd.card /*+ rd.card */) * C_TRANSFER_TUPLE;
//bindCost = numOfBindRequest * C_TRANSFER_QUERY + rd.card * C_TRANSFER_TUPLE;
if (log.isTraceEnabled()) {
log.debug(String.format("join card: %s, hash cost: %s, bind cost: %s", rd.card, hashCost, bindCost));
}
Expand Down
2 changes: 2 additions & 0 deletions costfed/src/main/java/org/aksw/simba/quetsal/core/TBSSFederationEvalStrategy.java
View file
Expand Up @@ -27,6 +27,7 @@
import com.fluidops.fedx.evaluation.iterator.RestartableCloseableIteration;
import com.fluidops.fedx.evaluation.join.ParallelBoundJoinTask;
import com.fluidops.fedx.evaluation.join.ParallelCheckJoinTask;
import com.fluidops.fedx.structures.QueryInfo;

import info.aduna.iteration.CloseableIteration;

Expand Down Expand Up @@ -71,6 +72,7 @@ public void evaluate(QueueIteration<BindingSet> qit, TupleExpr expr, List<Bindin
CloseableIteration<BindingSet, QueryEvaluationException> evaluateJoinRestarter(JoinRestarter jri, BindingSet bindings)
{
return new RestarterIteration<BindingSet>(
jri.getQueryInfo(),
(RestartableCloseableIteration<BindingSet>)evaluate(jri.getArg(0), bindings),
jri
);
Expand Down
10 changes: 9 additions & 1 deletion costfed/src/main/java/org/aksw/simba/quetsal/core/evaluation/RestarterIteration.java
View file
Expand Up @@ -21,6 +21,7 @@
import com.fluidops.fedx.algebra.ExclusiveGroup;
import com.fluidops.fedx.algebra.StatementSource;
import com.fluidops.fedx.evaluation.iterator.RestartableCloseableIteration;
import com.fluidops.fedx.structures.QueryInfo;

import info.aduna.iteration.LookAheadIteration;

Expand All @@ -35,13 +36,20 @@ public class RestarterIteration<E> extends LookAheadIteration<E, QueryEvaluation
static double rCost = 50; // transfer query cost
static double tCost = 0.02; // transfer tuple cost

public RestarterIteration(RestartableCloseableIteration<E> arg, JoinRestarter jr) {
final QueryInfo queryInfo;

public RestarterIteration(QueryInfo queryInfo, RestartableCloseableIteration<E> arg, JoinRestarter jr) {
this.queryInfo = queryInfo;
this.arg = arg;
this.jr = jr;
}

class CVisitor extends CardinalityVisitor
{
CVisitor() {
super(RestarterIteration.this.queryInfo);
}

@Override
public void meet(StatementPattern stmt) {
List<StatementSource> stmtSrces = jr.getQueryInfo().getSourceSelection().getStmtToSources().get(stmt);
Expand Down

0 comments on commit 1aa31ac

Please sign in to comment.