Skip to content

Commit

Permalink
Tidy up development debug printing in the optimizer
Browse files Browse the repository at this point in the history
  • Loading branch information
@afs
afs committed Feb 25, 2022
1 parent 2d6aa4e commit c94054e
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 82 deletions.
87 changes: 44 additions & 43 deletions jena-arq/src/main/java/org/apache/jena/sparql/algebra/optimize/OptimizerStd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,35 +40,35 @@ public OptimizerStd(Context context) {

/** Alternative name for compatibility only */
public static final Symbol filterPlacementOldName = SystemARQ.allocSymbol("filterPlacement") ;

@Override
public Op rewrite(Op op) {
// Record optimizer
if ( context.get(ARQConstants.sysOptimizer) == null )
context.set(ARQConstants.sysOptimizer, this) ;

// Old name, new name fixup.
if ( context.isDefined(filterPlacementOldName) ) {
if ( context.isUndef(ARQ.optFilterPlacement) )
context.set(ARQ.optFilterPlacement, context.get(filterPlacementOldName)) ;
}

if ( false ) {
// Removal of "group of one" join (AKA SPARQL "simplification")
// Removal of "group of one" join (AKA SPARQL "simplification")
// is done during algebra generation in AlgebraGenerator
op = apply("Simplify", new TransformSimplify(), op) ;
op = apply("Delabel", new TransformRemoveLabels(), op) ;
}

// ** TransformScopeRename
// This is a requirement for the linearization execution that the default
// ARQ query engine uses where possible.
// This transformation must be done (e.g. by QueryEngineBase) if no other optimization is done.
// ARQ query engine uses where possible.
// This transformation must be done (e.g. by QueryEngineBase) if no other optimization is done.
op = TransformScopeRename.transform(op) ;

// Prepare expressions.
OpWalker.walk(op, new OpVisitorExprPrepare(context)) ;

// Convert paths to triple patterns if possible.
if ( context.isTrueOrUndef(ARQ.optPathFlatten) ) {
op = apply("Path flattening", new TransformPathFlattern(), op) ;
Expand All @@ -80,11 +80,11 @@ public Op rewrite(Op op) {
// Having done the required transforms, specifically TransformScopeRename,
// do each optimization via an overrideable method. Subclassing can modify the
// transform performed.

// Expression constant folding
if ( context.isTrueOrUndef(ARQ.optExprConstantFolding) )
op = transformExprConstantFolding(op) ;

if ( context.isTrueOrUndef(ARQ.propertyFunctions) )
op = transformPropertyFunctions(op) ;

Expand All @@ -95,65 +95,65 @@ public Op rewrite(Op op) {
// Expand IN and NOT IN which then allows other optimizations to be applied.
if ( context.isTrueOrUndef(ARQ.optFilterExpandOneOf) )
op = transformFilterExpandOneOf(op) ;

// Eliminate/Inline assignments where possible
// Do this before we do some of the filter transformation work as inlining assignments
// Do this before we do some of the filter transformation work as inlining assignments
// may give us more flexibility in optimizing the resulting filters
if ( context.isTrue(ARQ.optInlineAssignments) )
op = transformInlineAssignments(op) ;

// Apply some general purpose filter transformations
if ( context.isTrueOrUndef(ARQ.optFilterImplicitJoin) )
op = transformFilterImplicitJoin(op) ;

if ( context.isTrueOrUndef(ARQ.optImplicitLeftJoin) )
op = transformFilterImplicitLeftJoin(op) ;

if ( context.isTrueOrUndef(ARQ.optFilterDisjunction) )
op = transformFilterDisjunction(op) ;

// Some ORDER BY-LIMIT N queries can be done more efficiently by only recording
// the top N items, so a full sort is not needed.
if ( context.isTrueOrUndef(ARQ.optTopNSorting) )
op = transformTopNSorting(op) ;

// ORDER BY+DISTINCT optimizations
// We apply the one that changes evaluation order first since when it does apply it will give much
// better performance than just transforming DISTINCT to REDUCED

if ( context.isTrueOrUndef(ARQ.optOrderByDistinctApplication) )
op = transformOrderByDistinctApplication(op) ;

// Transform some DISTINCT to REDUCED, slightly more liberal transform that ORDER BY+DISTINCT application
// Reduces memory consumption.
if ( context.isTrueOrUndef(ARQ.optDistinctToReduced) )
op = transformDistinctToReduced(op) ;

// Find joins/leftJoin that can be done by index joins (generally preferred as fixed memory overhead).
if ( context.isTrueOrUndef(ARQ.optIndexJoinStrategy) )
op = transformJoinStrategy(op) ;

// Place filters close to where their dependency variables are defined.
// This prunes the output of that step as early as possible.
// If done before TransformJoinStrategy, you can get two applications
// of a filter in a (sequence) from each half of a (join). This is harmless,
// because filters are generally cheap, but it looks a bit bad.
if ( context.isTrueOrUndef(ARQ.optFilterPlacement) )
op = transformFilterPlacement(op) ;
// Replace suitable FILTER(?x = TERM) with (assign) and write the TERM for ?x in the pattern.

// Replace suitable FILTER(?x = TERM) with (assign) and write the TERM for ?x in the pattern.
// Apply (possible a second time) after FILTER placement as it can create new possibilities.
// See JENA-616.
if ( context.isTrueOrUndef(ARQ.optFilterEquality) )
op = transformFilterEquality(op) ;

// Replace suitable FILTER(?x != TERM) with (minus (original) (table)) where the table contains
// the candidate rows to be eliminated
// Off by default due to minimal performance difference
if ( context.isTrue(ARQ.optFilterInequality) )
op = transformFilterInequality(op) ;
// Promote table empty as late as possible since this will only be produced by other

// Promote table empty as late as possible since this will only be produced by other
// optimizations and never directly from algebra generation
if ( context.isTrueOrUndef(ARQ.optPromoteTableEmpty) )
op = transformPromoteTableEmpty(op) ;
Expand All @@ -165,17 +165,17 @@ public Op rewrite(Op op) {
// Normally, leave to the specific engines.
if ( context.isTrue(ARQ.optReorderBGP) )
op = transformReorder(op) ;

// Merge (extend) and (assign) stacks
if ( context.isTrueOrUndef(ARQ.optMergeExtends) )
op = transformExtendCombine(op) ;

// Mark
if ( false )
op = OpLabel.create("Transformed", op) ;
return op ;
}

protected Op transformExprConstantFolding(Op op) {
return Transformer.transform(new TransformCopy(), new ExprTransformConstantFold(), op);
}
Expand Down Expand Up @@ -226,8 +226,8 @@ protected Op transformJoinStrategy(Op op) {
protected Op transformFilterPlacement(Op op) {
if ( context.isTrue(ARQ.optFilterPlacementConservative))
op = apply("Filter Placement (conservative)", new TransformFilterPlacementConservative(), op) ;
else {
// Whether to push into BGPs
else {
// Whether to push into BGPs
boolean b = context.isTrueOrUndef(ARQ.optFilterPlacementBGP) ;
op = apply("Filter Placement", new TransformFilterPlacement(b), op) ;
}
Expand Down Expand Up @@ -264,32 +264,33 @@ public static Op apply(Transform transform, Op op) {
return op2 ;
return op ;
}


private static final boolean debug = false ;
private static final boolean printNoAction = false;

public static Op apply(String label, Transform transform, Op op) {
Op op2 = Transformer.transformSkipService(transform, op) ;
final boolean debug = false ;


if ( debug ) {
if ( label != null && log.isInfoEnabled() )
if ( printNoAction && label != null )
log.info("Transform: " + label) ;
if ( op == op2 ) {
if ( log.isInfoEnabled() )
if ( printNoAction )
log.info("No change (==)") ;
return op2 ;
}

if ( op.equals(op2) ) {
if ( log.isInfoEnabled() )
if ( printNoAction )
log.info("No change (equals)") ;
return op2 ;
}
if ( log.isInfoEnabled() ) {
log.info("\n" + op.toString()) ;
log.info("\n" + op2.toString()) ;
}

if ( ! printNoAction && label != null );
log.info("Transform: " + label) ;
log.info("\n" + op.toString()) ;
log.info("\n" + op2.toString()) ;
}
if ( op2 != op )
return op2 ;
return op ;
return op2 ;
}
}
38 changes: 20 additions & 18 deletions jena-arq/src/main/java/org/apache/jena/sparql/engine/main/JoinClassifier.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,20 +42,20 @@ static public boolean isLinear(Op _left, Op _right) {
// Modifiers that we don't touch
// OpSlice, OpTopN, OpOrder (which gets lost - could remove it!)
// (These could be first and top - i.e. in call once position, and be safe)

Op left = effectiveOp(_left) ;
Op right = effectiveOp(_right) ;

if ( ! isSafeForLinear(left) || ! isSafeForLinear(right) )
return false ;

// Modifiers.
if ( right instanceof OpExtend ) return false ;
if ( right instanceof OpAssign ) return false ;
if ( right instanceof OpGroup ) return false ;
// if ( right instanceof OpDiff ) return false ;
// if ( right instanceof OpMinus ) return false ;

if ( right instanceof OpSlice ) return false ;
if ( right instanceof OpTopN ) return false ;
if ( right instanceof OpOrder ) return false ;
Expand All @@ -66,7 +66,7 @@ static public boolean isLinear(Op _left, Op _right) {

// -- pre check for ops we can't handle in a linear fashion.
// These are the negation patterns (minus and diff)
// FILTER NOT EXISTS is safe - it's defined by iteration like the linear execution algorithm.
// FILTER NOT EXISTS is safe - it's defined by iteration like the linear execution algorithm.
private static class UnsafeLineraOpException extends RuntimeException {}
private static OpVisitor checkForUnsafeVisitor = new OpVisitorBase() {
@Override public void visit(OpMinus opMinus) { throw new UnsafeLineraOpException(); }
Expand All @@ -77,10 +77,11 @@ private static boolean isSafeForLinear(Op op) {
catch (UnsafeLineraOpException e) { return false; }
}
// --

// Check left can stream into right
static private boolean check(Op leftOp, Op rightOp) {
if ( print ) {
System.err.println("== JoinClassifier");
System.err.println("Left::");
System.err.println(leftOp) ;
System.err.println("Right::");
Expand All @@ -101,7 +102,7 @@ static private boolean check(Op leftOp, Op rightOp) {
System.err.println("Right") ;
vfRight.print(System.err) ;
}

Set<Var> vRightFixed = vfRight.getFixed() ;
Set<Var> vRightOpt = vfRight.getOpt() ;
Set<Var> vRightFilter = vfRight.getFilter() ;
Expand All @@ -126,7 +127,7 @@ static private boolean check(Op leftOp, Op rightOp) {
// System.err.println("vRightFilterOnly.not isEmpty");
// return false;
}

// Step 2 : remove any variable definitely fixed from the floating sets
// because the nature of the "join" will deal with that.
vLeftOpt = SetUtils.difference(vLeftOpt, vLeftFixed) ;
Expand Down Expand Up @@ -168,7 +169,9 @@ static private boolean check(Op leftOp, Op rightOp) {
boolean bad1 = r11 || r12 ;

if ( print )
System.err.println("Case 1 = " + bad1) ;
System.err.println("J: Case 1 (false=ok) = " + bad1) ;
if ( bad1 )
return false;

// Case 2 : a filter in the RHS is uses a variable from the LHS (whether
// fixed or optional)
Expand All @@ -179,7 +182,9 @@ static private boolean check(Op leftOp, Op rightOp) {

boolean bad2 = SetUtils.intersectionP(vRightFilter, vLeftFixed) ;
if ( print )
System.err.println("Case 2 = " + bad2) ;
System.err.println("J: Case 2 (false=ok) = " + bad2) ;
if ( bad2 )
return false;

// Case 3 : an assign in the RHS uses a variable not introduced
// Scoping means we must hide the LHS value from the RHS
Expand All @@ -189,18 +194,15 @@ static private boolean check(Op leftOp, Op rightOp) {
// the RHS
// vRightAssign.removeAll(vRightFixed);
// boolean bad3 = vRightAssign.size() > 0;

boolean bad3 = SetUtils.intersectionP(vRightAssign, vLeftFixed) ;
if ( print )
System.err.println("Case 3 = " + bad3) ;
System.err.println("J: Case 3 (false=ok) = " + bad3) ;
if ( bad3 )
return false;

// Linear if all conditions are false
boolean result = !bad1 && !bad2 && !bad3 ;

if ( print ) {
System.err.println("Result: "+result) ;
}
return result ;
System.err.println("J: Result: OK");
return true;
}

/** Find the "effective op" - i.e. the one that may be sensitive to linearization */
Expand Down
Loading

0 comments on commit c94054e

Please sign in to comment.