Skip to content

Commit

Permalink
Merge mutualy exclusive bool properties to single string property.
Browse files Browse the repository at this point in the history 
WARNING: This change may invalidate old configurations!
  • Loading branch information
@fiedukow @cberner
fiedukow authored and cberner committed Jun 21, 2016
1 parent 0340248 commit 45516fa
Show file tree
Hide file tree
Showing 7 changed files with 91 additions and 89 deletions.
7 changes: 6 additions & 1 deletion presto-docs/src/main/sphinx/release/release-0.149.rst
View file
Expand Up @@ -5,4 +5,9 @@ Release 0.149
General Changes
---------------
* ``query.max-age`` property was renamed to ``query.min-expire-age``.

* ``optimizer.columnar-processing`` and ``optimizer.columnar-processing-dictionary``
properties were merged to ``optimizer.processing-optimization`` with possible
values ``disabled``, ``columnar`` and ``columnar_dictionary``
* ``columnar_processing`` and ``columnar_processing_dictionary`` session
properties were merged to ``processing_optimization`` with possible values
``disabled``, ``columnar`` and ``columnar_dictionary``
25 changes: 7 additions & 18 deletions presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java
View file
Expand Up @@ -53,8 +53,7 @@ public final class SystemSessionProperties
public static final String REDISTRIBUTE_WRITES = "redistribute_writes";
public static final String PUSH_TABLE_WRITE_THROUGH_UNION = "push_table_write_through_union";
public static final String EXECUTION_POLICY = "execution_policy";
public static final String COLUMNAR_PROCESSING = "columnar_processing";
public static final String COLUMNAR_PROCESSING_DICTIONARY = "columnar_processing_dictionary";
public static final String PROCESSING_OPTIMIZATION = "processing_optimization";
public static final String DICTIONARY_AGGREGATION = "dictionary_aggregation";
public static final String PLAN_WITH_TABLE_NODE_PARTITIONING = "plan_with_table_node_partitioning";
public static final String COLOCATED_JOIN = "colocated_join";
Expand Down Expand Up @@ -177,15 +176,10 @@ public SystemSessionProperties(
"Use resources which are not guaranteed to be available to the query",
false,
false),
booleanSessionProperty(
COLUMNAR_PROCESSING,
"Use columnar processing",
featuresConfig.isColumnarProcessing(),
false),
booleanSessionProperty(
COLUMNAR_PROCESSING_DICTIONARY,
"Use columnar processing with optimizations for dictionaries",
featuresConfig.isColumnarProcessingDictionary(),
stringSessionProperty(
PROCESSING_OPTIMIZATION,
"Type of optimization for query processing",
featuresConfig.getProcessingOptimization(),
false),
booleanSessionProperty(
DICTIONARY_AGGREGATION,
Expand Down Expand Up @@ -288,14 +282,9 @@ public static boolean isShareIndexLoading(Session session)
return session.getProperty(TASK_SHARE_INDEX_LOADING, Boolean.class);
}

public static boolean isColumnarProcessingEnabled(Session session)
{
return session.getProperty(COLUMNAR_PROCESSING, Boolean.class);
}

public static boolean isColumnarProcessingDictionaryEnabled(Session session)
public static String getProcessingOptimization(Session session)
{
return session.getProperty(COLUMNAR_PROCESSING_DICTIONARY, Boolean.class);
return session.getProperty(PROCESSING_OPTIMIZATION, String.class);
}

public static boolean isDictionaryAggregationEnabled(Session session)
Expand Down
45 changes: 24 additions & 21 deletions presto-main/src/main/java/com/facebook/presto/operator/FilterAndProjectOperator.java
View file
Expand Up @@ -16,14 +16,14 @@
import com.facebook.presto.spi.Page;
import com.facebook.presto.spi.PageBuilder;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.sql.analyzer.FeaturesConfig;
import com.facebook.presto.sql.planner.plan.PlanNodeId;
import com.google.common.collect.ImmutableList;

import java.util.List;
import java.util.function.Supplier;

import static com.facebook.presto.SystemSessionProperties.isColumnarProcessingDictionaryEnabled;
import static com.facebook.presto.SystemSessionProperties.isColumnarProcessingEnabled;
import static com.facebook.presto.SystemSessionProperties.getProcessingOptimization;
import static com.google.common.base.Preconditions.checkState;
import static java.util.Objects.requireNonNull;

Expand All @@ -35,8 +35,7 @@ public class FilterAndProjectOperator

private final PageBuilder pageBuilder;
private final PageProcessor processor;
private final boolean columnarProcessingEnabled;
private final boolean columnarProcessingDictionaryEnabled;
private final String processingOptimization;
private Page currentPage;
private int currentPosition;
private boolean finishing;
Expand All @@ -46,8 +45,7 @@ public FilterAndProjectOperator(OperatorContext operatorContext, Iterable<? exte
this.processor = requireNonNull(processor, "processor is null");
this.operatorContext = requireNonNull(operatorContext, "operatorContext is null");
this.types = ImmutableList.copyOf(requireNonNull(types, "types is null"));
this.columnarProcessingEnabled = isColumnarProcessingEnabled(operatorContext.getSession());
this.columnarProcessingDictionaryEnabled = isColumnarProcessingDictionaryEnabled(operatorContext.getSession());
this.processingOptimization = getProcessingOptimization(operatorContext.getSession());
this.pageBuilder = new PageBuilder(getTypes());
}

Expand Down Expand Up @@ -96,24 +94,29 @@ public final void addInput(Page page)
public final Page getOutput()
{
if (!pageBuilder.isFull() && currentPage != null) {
if (columnarProcessingDictionaryEnabled) {
Page page = processor.processColumnarDictionary(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
currentPage = null;
currentPosition = 0;
return page;
}
else if (columnarProcessingEnabled) {
Page page = processor.processColumnar(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
currentPage = null;
currentPosition = 0;
return page;
}
else {
currentPosition = processor.process(operatorContext.getSession().toConnectorSession(), currentPage, currentPosition, currentPage.getPositionCount(), pageBuilder);
if (currentPosition == currentPage.getPositionCount()) {
switch (processingOptimization) {
case FeaturesConfig.ProcessingOptimization.COLUMNAR: {
Page page = processor.processColumnar(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
currentPage = null;
currentPosition = 0;
return page;
}
case FeaturesConfig.ProcessingOptimization.COLUMNAR_DICTIONARY: {
Page page = processor.processColumnarDictionary(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
currentPage = null;
currentPosition = 0;
return page;
}
case FeaturesConfig.ProcessingOptimization.DISABLED: {
currentPosition = processor.process(operatorContext.getSession().toConnectorSession(), currentPage, currentPosition, currentPage.getPositionCount(), pageBuilder);
if (currentPosition == currentPage.getPositionCount()) {
currentPage = null;
currentPosition = 0;
}
break;
}
default:
throw new IllegalStateException();
}
}

Expand Down
47 changes: 26 additions & 21 deletions presto-main/src/main/java/com/facebook/presto/operator/ScanFilterAndProjectOperator.java
View file
Expand Up @@ -36,8 +36,10 @@
import java.util.Optional;
import java.util.function.Supplier;

import static com.facebook.presto.SystemSessionProperties.isColumnarProcessingDictionaryEnabled;
import static com.facebook.presto.SystemSessionProperties.isColumnarProcessingEnabled;
import static com.facebook.presto.SystemSessionProperties.getProcessingOptimization;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.ProcessingOptimization.COLUMNAR;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.ProcessingOptimization.COLUMNAR_DICTIONARY;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.ProcessingOptimization.DISABLED;
import static com.google.common.base.Preconditions.checkState;
import static java.util.Objects.requireNonNull;

Expand All @@ -57,8 +59,7 @@ public class ScanFilterAndProjectOperator
private final LocalMemoryContext pageSourceMemoryContext;
private final LocalMemoryContext pageBuilderMemoryContext;
private final SettableFuture<?> blocked = SettableFuture.create();
private final boolean columnarProcessingEnabled;
private final boolean columnarProcessingDictionaryEnabled;
private final String processingOptimization;

private RecordCursor cursor;
private ConnectorPageSource pageSource;
Expand Down Expand Up @@ -90,8 +91,7 @@ protected ScanFilterAndProjectOperator(
this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null"));
this.pageSourceMemoryContext = operatorContext.getSystemMemoryContext().newLocalMemoryContext();
this.pageBuilderMemoryContext = operatorContext.getSystemMemoryContext().newLocalMemoryContext();
this.columnarProcessingEnabled = isColumnarProcessingEnabled(operatorContext.getSession());
this.columnarProcessingDictionaryEnabled = isColumnarProcessingDictionaryEnabled(operatorContext.getSession());
this.processingOptimization = getProcessingOptimization(operatorContext.getSession());

this.pageBuilder = new PageBuilder(getTypes());
}
Expand Down Expand Up @@ -243,24 +243,29 @@ public Page getOutput()
}

if (currentPage != null) {
if (columnarProcessingDictionaryEnabled) {
Page page = pageProcessor.processColumnarDictionary(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
currentPage = null;
currentPosition = 0;
return page;
}
else if (columnarProcessingEnabled) {
Page page = pageProcessor.processColumnar(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
currentPage = null;
currentPosition = 0;
return page;
}
else {
currentPosition = pageProcessor.process(operatorContext.getSession().toConnectorSession(), currentPage, currentPosition, currentPage.getPositionCount(), pageBuilder);
if (currentPosition == currentPage.getPositionCount()) {
switch (processingOptimization) {
case COLUMNAR: {
Page page = pageProcessor.processColumnar(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
currentPage = null;
currentPosition = 0;
return page;
}
case COLUMNAR_DICTIONARY: {
Page page = pageProcessor.processColumnarDictionary(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
currentPage = null;
currentPosition = 0;
return page;
}
case DISABLED: {
currentPosition = pageProcessor.process(operatorContext.getSession().toConnectorSession(), currentPage, currentPosition, currentPage.getPositionCount(), pageBuilder);
if (currentPosition == currentPage.getPositionCount()) {
currentPage = null;
currentPosition = 0;
}
break;
}
default:
throw new IllegalStateException(String.format("Found unexpected value %s for processingOptimization", processingOptimization));
}
}

Expand Down
40 changes: 21 additions & 19 deletions presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java
View file
Expand Up @@ -13,17 +13,29 @@
*/
package com.facebook.presto.sql.analyzer;

import com.google.common.collect.ImmutableList;
import io.airlift.configuration.Config;
import io.airlift.configuration.ConfigDescription;
import io.airlift.configuration.LegacyConfig;

import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;

import java.util.List;

import static com.facebook.presto.sql.analyzer.RegexLibrary.JONI;

public class FeaturesConfig
{
public static class ProcessingOptimization
{
public static final String DISABLED = "disabled";
public static final String COLUMNAR = "columnar";
public static final String COLUMNAR_DICTIONARY = "columnar_dictionary";

public static final List<String> AVAILABLE_OPTIONS = ImmutableList.of(DISABLED, COLUMNAR, COLUMNAR_DICTIONARY);
}

public static final String FILE_BASED_RESOURCE_GROUP_MANAGER = "file";
private boolean experimentalSyntaxEnabled;
private boolean distributedIndexJoinsEnabled;
Expand All @@ -35,8 +47,7 @@ public class FeaturesConfig
private boolean optimizeSingleDistinct = true;
private boolean pushTableWriteThroughUnion = true;

private boolean columnarProcessing;
private boolean columnarProcessingDictionary;
private String processingOptimization = ProcessingOptimization.DISABLED;
private boolean dictionaryAggregation;
private boolean resourceGroups;

Expand Down Expand Up @@ -181,27 +192,18 @@ public FeaturesConfig setPushTableWriteThroughUnion(boolean pushTableWriteThroug
return this;
}

public boolean isColumnarProcessing()
{
return columnarProcessing;
}

@Config("optimizer.columnar-processing")
public FeaturesConfig setColumnarProcessing(boolean columnarProcessing)
{
this.columnarProcessing = columnarProcessing;
return this;
}

public boolean isColumnarProcessingDictionary()
public String getProcessingOptimization()
{
return columnarProcessingDictionary;
return processingOptimization;
}

@Config("optimizer.columnar-processing-dictionary")
public FeaturesConfig setColumnarProcessingDictionary(boolean columnarProcessingDictionary)
@Config("optimizer.processing-optimization")
public FeaturesConfig setProcessingOptimization(String processingOptimization)
{
this.columnarProcessingDictionary = columnarProcessingDictionary;
if (!ProcessingOptimization.AVAILABLE_OPTIONS.contains(processingOptimization)) {
throw new IllegalStateException(String.format("Value %s is not valid for processingOptimization.", processingOptimization));
}
this.processingOptimization = processingOptimization;
return this;
}

Expand Down
14 changes: 6 additions & 8 deletions presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java
View file
Expand Up @@ -20,6 +20,8 @@
import java.util.Map;

import static com.facebook.presto.sql.analyzer.FeaturesConfig.FILE_BASED_RESOURCE_GROUP_MANAGER;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.ProcessingOptimization.COLUMNAR_DICTIONARY;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.ProcessingOptimization.DISABLED;
import static com.facebook.presto.sql.analyzer.RegexLibrary.JONI;
import static com.facebook.presto.sql.analyzer.RegexLibrary.RE2J;
import static io.airlift.configuration.testing.ConfigAssertions.assertDeprecatedEquivalence;
Expand All @@ -42,8 +44,7 @@ public void testDefaults()
.setOptimizeHashGeneration(true)
.setOptimizeSingleDistinct(true)
.setPushTableWriteThroughUnion(true)
.setColumnarProcessing(false)
.setColumnarProcessingDictionary(false)
.setProcessingOptimization(DISABLED)
.setDictionaryAggregation(false)
.setRegexLibrary(JONI)
.setRe2JDfaStatesLimit(Integer.MAX_VALUE)
Expand All @@ -65,8 +66,7 @@ public void testExplicitPropertyMappings()
.put("optimizer.optimize-hash-generation", "false")
.put("optimizer.optimize-single-distinct", "false")
.put("optimizer.push-table-write-through-union", "false")
.put("optimizer.columnar-processing", "true")
.put("optimizer.columnar-processing-dictionary", "true")
.put("optimizer.processing-optimization", "columnar_dictionary")
.put("optimizer.dictionary-aggregation", "true")
.put("regex-library", "RE2J")
.put("re2j.dfa-states-limit", "42")
Expand All @@ -84,8 +84,7 @@ public void testExplicitPropertyMappings()
.put("optimizer.optimize-hash-generation", "false")
.put("optimizer.optimize-single-distinct", "false")
.put("optimizer.push-table-write-through-union", "false")
.put("optimizer.columnar-processing", "true")
.put("optimizer.columnar-processing-dictionary", "true")
.put("optimizer.processing-optimization", "columnar_dictionary")
.put("optimizer.dictionary-aggregation", "true")
.put("regex-library", "RE2J")
.put("re2j.dfa-states-limit", "42")
Expand All @@ -104,8 +103,7 @@ public void testExplicitPropertyMappings()
.setOptimizeHashGeneration(false)
.setOptimizeSingleDistinct(false)
.setPushTableWriteThroughUnion(false)
.setColumnarProcessing(true)
.setColumnarProcessingDictionary(true)
.setProcessingOptimization(COLUMNAR_DICTIONARY)
.setDictionaryAggregation(true)
.setRegexLibrary(RE2J)
.setRe2JDfaStatesLimit(42)
Expand Down
2 changes: 1 addition & 1 deletion presto-raptor/src/test/java/com/facebook/presto/raptor/RaptorQueryRunner.java
View file
Expand Up @@ -135,7 +135,7 @@ private static Session createSession(String schema)
return testSessionBuilder(sessionPropertyManager)
.setCatalog("raptor")
.setSchema(schema)
.setSystemProperties(ImmutableMap.of("columnar_processing_dictionary", "true", "dictionary_aggregation", "true"))
.setSystemProperties(ImmutableMap.of("processing_optimization", "columnar_dictionary", "dictionary_aggregation", "true"))
.build();
}

Expand Down

0 comments on commit 45516fa

Please sign in to comment.