From cb6cc78b9991e9d6e384a2cc28b70332a9d92a22 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Tue, 9 Sep 2014 13:35:54 -0700 Subject: [PATCH 01/58] Initial commit --- contrib/format-maprdb/README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 contrib/format-maprdb/README.md diff --git a/contrib/format-maprdb/README.md b/contrib/format-maprdb/README.md new file mode 100644 index 00000000000..ff192851a67 --- /dev/null +++ b/contrib/format-maprdb/README.md @@ -0,0 +1,2 @@ +drill-mapr-plugin +================= From 95face3575817719a389fe049d3d8c1d65bb12dd Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Tue, 23 Sep 2014 11:45:27 -0700 Subject: [PATCH 02/58] First version of maprdb plugin + Moved files under contrib/format-maprdb --- contrib/format-maprdb/pom.xml | 170 +++++++ .../store/maprdb/MapRDBFormatMatcher.java | 60 +++ .../exec/store/maprdb/MapRDBFormatPlugin.java | 134 ++++++ .../maprdb/MapRDBFormatPluginConfig.java | 36 ++ .../exec/store/maprdb/MapRDBGroupScan.java | 455 ++++++++++++++++++ .../store/maprdb/MapRDBScanBatchCreator.java | 53 ++ .../exec/store/maprdb/MapRDBSubScan.java | 225 +++++++++ .../exec/store/maprdb/MapRDBTableStats.java | 50 ++ .../resources/bootstrap-storage-plugins.json | 12 + .../src/main/resources/checkstyle-config.xml | 31 ++ .../resources/checkstyle-suppressions.xml | 19 + .../src/main/resources/drill-module.conf | 34 ++ .../src/test/resources/hbase-site.xml | 143 ++++++ .../hbase/hbase_scan_screen_physical.json | 29 ++ ...se_scan_screen_physical_column_select.json | 32 ++ ...se_scan_screen_physical_family_select.json | 32 ++ .../src/test/resources/logback.xml | 64 +++ 17 files changed, 1579 insertions(+) create mode 100644 contrib/format-maprdb/pom.xml create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java create mode 100644 contrib/format-maprdb/src/main/resources/bootstrap-storage-plugins.json create mode 100644 contrib/format-maprdb/src/main/resources/checkstyle-config.xml create mode 100644 contrib/format-maprdb/src/main/resources/checkstyle-suppressions.xml create mode 100644 contrib/format-maprdb/src/main/resources/drill-module.conf create mode 100644 contrib/format-maprdb/src/test/resources/hbase-site.xml create mode 100644 contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical.json create mode 100644 contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json create mode 100644 contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json create mode 100644 contrib/format-maprdb/src/test/resources/logback.xml diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml new file mode 100644 index 00000000000..a09d64f3ba2 --- /dev/null +++ b/contrib/format-maprdb/pom.xml @@ -0,0 +1,170 @@ + + + + 4.0.0 + + drill-contrib-parent + org.apache.drill.contrib + 0.6.0-incubating-SNAPSHOT + + + drill-storage-maprdb + + maprdb-storage-plugin + + + **/HBaseTestsSuite.class + + + + + org.apache.drill.contrib + drill-storage-hbase + ${project.version} + + + + + org.apache.drill.exec + drill-java-exec + ${project.version} + + + + org.apache.drill.contrib + drill-storage-hbase + ${project.version} + + + + + org.apache.drill.exec + drill-java-exec + tests + ${project.version} + test + + + org.apache.drill + drill-common + tests + ${project.version} + test + + + com.yammer.metrics + metrics-core + 2.1.1 + test + + + org.apache.hbase + hbase + 0.94.13-mapr-1401-m7-3.0.2 + + + commons-logging + commons-logging + + + slf4j-log4j12 + org.slf4j + + + slf4j-log4j12 + org.slf4j + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-client + + + asm + asm + + + libthrift + org.apache.thrift + + + jruby-complete + org.jruby + + + + + com.mapr.hadoop + maprfs + + + com.mapr.fs + mapr-hbase + 4.0.2-mapr-SNAPSHOT + + + + + mapr-releases + http://maven.corp.maprtech.com/nexus/content/repositories/public + + true + + + true + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${hbase.TestSuite} + + + + hbase.test.root + ${project.build.directory}/data + + + logback.log.dir + ${project.build.directory}/surefire-reports + + + + + + + diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java new file mode 100644 index 00000000000..9f1d7b7ddbe --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import com.mapr.fs.MapRFileStatus; +import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.dfs.FormatMatcher; +import org.apache.drill.exec.store.dfs.FormatPlugin; +import org.apache.drill.exec.store.dfs.FormatSelection; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.hadoop.fs.FileStatus; + +import java.io.IOException; + +public class MapRDBFormatMatcher extends FormatMatcher { + + private FormatPlugin plugin; + private DrillFileSystem fs; + + public MapRDBFormatMatcher(FormatPlugin plugin, DrillFileSystem fs) { + this.plugin = plugin; + this.fs = fs; + } + + @Override + public boolean supportDirectoryReads() { + return false; + } + + @Override + public FormatSelection isReadable(FileSelection selection) throws IOException { + FileStatus status = selection.getFirstPath(fs); + if (status instanceof MapRFileStatus) { + if (((MapRFileStatus) status).isTable()) { + return new FormatSelection(getFormatPlugin().getConfig(), selection); + } + } + return null; + } + + @Override + public FormatPlugin getFormatPlugin() { + return plugin; + } +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java new file mode 100644 index 00000000000..d8c6187ba2f --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import com.google.common.collect.ImmutableSet; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.physical.base.AbstractWriter; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.dfs.FormatMatcher; +import org.apache.drill.exec.store.dfs.FormatPlugin; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.drill.exec.store.hbase.HBaseScanSpec; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +public class MapRDBFormatPlugin implements FormatPlugin{ + + private final StoragePluginConfig storageConfig; + private final MapRDBFormatPluginConfig config; + private final MapRDBFormatMatcher matcher; + private final DrillFileSystem fs; + private final DrillbitContext context; + private final String name; + + public MapRDBFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig){ + this(name, context, fs, storageConfig, new MapRDBFormatPluginConfig()); + } + + public MapRDBFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig){ + this.context = context; + this.config = formatConfig; + this.matcher = new MapRDBFormatMatcher(this, fs); + this.storageConfig = storageConfig; + this.fs = fs; + this.name = name == null ? "maprdb" : name; + } + @Override + public boolean supportsRead() { + return true; + } + + @Override + public boolean supportsWrite() { + return false; + } + + @Override + public FormatMatcher getMatcher() { + return matcher; + } + + @Override + public AbstractWriter getWriter(PhysicalOperator child, String location) + throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public AbstractGroupScan getGroupScan(FileSelection selection) + throws IOException { + return getGroupScan(selection, null); + } + + @Override + public Set getOptimizerRules() { + return ImmutableSet.of(); + } + + @Override + public AbstractGroupScan getGroupScan(FileSelection selection, + List columns) throws IOException { + List files = selection.getAsFiles(); + assert(files.size() == 1); + String tableName = files.get(0); + HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); + try { + return new MapRDBGroupScan((FileSystemPlugin)(context.getStorage().getPlugin(storageConfig)), this, scanSpec, columns); + } catch (ExecutionSetupException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public FormatPluginConfig getConfig() { + return config; + } + + @Override + public StoragePluginConfig getStorageConfig() { + return storageConfig; + } + + @Override + public DrillFileSystem getFileSystem() { + return fs; + } + + @Override + public DrillbitContext getContext() { + return context; + } + + @Override + public String getName() { + return name; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java new file mode 100644 index 00000000000..7eff4e4536b --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.drill.common.logical.FormatPluginConfig; + +@JsonTypeName("maprdb") +public class MapRDBFormatPluginConfig implements FormatPluginConfig { + + @Override + public int hashCode() { + return 53; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof MapRDBFormatPluginConfig; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java new file mode 100644 index 00000000000..8da40de20c5 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -0,0 +1,455 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.EndpointAffinity; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.exec.store.dfs.FileSystemFormatConfig; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.hbase.DrillHBaseConstants; +import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.drill.exec.store.hbase.HBaseSubScan.HBaseSubScanSpec; +import org.apache.drill.exec.store.hbase.HBaseUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HTable; +import org.codehaus.jackson.annotate.JsonCreator; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Comparator; +import java.util.TreeMap; +import java.util.HashMap; +import java.util.Queue; +import java.util.ArrayList; +import java.util.Set; +import java.util.Iterator; +import java.util.PriorityQueue; + +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +@JsonTypeName("maprdb-scan") +public class MapRDBGroupScan extends AbstractGroupScan implements DrillHBaseConstants { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBGroupScan.class); + + private static final Comparator> LIST_SIZE_COMPARATOR = new Comparator>() { + @Override + public int compare(List list1, List list2) { + return list1.size() - list2.size(); + } + }; + + private static final Comparator> LIST_SIZE_COMPARATOR_REV = Collections.reverseOrder(LIST_SIZE_COMPARATOR); + + private List columns; + + private HBaseScanSpec hbaseScanSpec; + + private FileSystemPlugin storagePlugin; + + private MapRDBFormatPlugin formatPlugin; + + private Stopwatch watch = new Stopwatch(); + + private Map> endpointFragmentMapping; + + private NavigableMap regionsToScan; + + private HTableDescriptor hTableDesc; + + private boolean filterPushedDown = false; + + private MapRDBTableStats tableStats; + + @JsonCreator + public MapRDBGroupScan(@JsonProperty("hbaseScanSpec") HBaseScanSpec hbaseScanSpec, + @JsonProperty("storage") FileSystemConfig storagePluginConfig, + @JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig, + @JsonProperty("columns") List columns, + @JacksonInject StoragePluginRegistry pluginRegistry) throws IOException, ExecutionSetupException { + this ((FileSystemPlugin) pluginRegistry.getPlugin(storagePluginConfig), + (MapRDBFormatPlugin) pluginRegistry.getFormatPlugin(storagePluginConfig, formatPluginConfig), + hbaseScanSpec, + columns); + } + + public MapRDBGroupScan(FileSystemPlugin storagePlugin, MapRDBFormatPlugin formatPlugin, HBaseScanSpec scanSpec, List columns) { + this.storagePlugin = storagePlugin; + this.formatPlugin = formatPlugin; + this.hbaseScanSpec = scanSpec; + this.columns = columns; + init(); + } + + /** + * Private constructor, used for cloning. + * @param that The HBaseGroupScan to clone + */ + private MapRDBGroupScan(MapRDBGroupScan that) { + this.columns = that.columns; + this.hbaseScanSpec = that.hbaseScanSpec; + this.endpointFragmentMapping = that.endpointFragmentMapping; + this.regionsToScan = that.regionsToScan; + this.storagePlugin = that.storagePlugin; + this.formatPlugin = that.formatPlugin; + this.hTableDesc = that.hTableDesc; + this.filterPushedDown = that.filterPushedDown; + this.tableStats = that.tableStats; + } + + @Override + public GroupScan clone(List columns) { + MapRDBGroupScan newScan = new MapRDBGroupScan(this); + newScan.columns = columns; + newScan.verifyColumns(); + return newScan; + } + + private void init() { + logger.debug("Getting region locations"); + try { + tableStats = new MapRDBTableStats(hbaseScanSpec.getTableName()); + HTable table = new HTable(HBaseConfiguration.create(), hbaseScanSpec.getTableName()); + this.hTableDesc = table.getTableDescriptor(); + NavigableMap regionsMap = table.getRegionLocations(); + table.close(); + + boolean foundStartRegion = false; + regionsToScan = new TreeMap(); + for (Entry mapEntry : regionsMap.entrySet()) { + HRegionInfo regionInfo = mapEntry.getKey(); + if (!foundStartRegion && hbaseScanSpec.getStartRow() != null && hbaseScanSpec.getStartRow().length != 0 && !regionInfo.containsRow(hbaseScanSpec.getStartRow())) { + continue; + } + foundStartRegion = true; + regionsToScan.put(regionInfo, mapEntry.getValue()); + if (hbaseScanSpec.getStopRow() != null && hbaseScanSpec.getStopRow().length != 0 && regionInfo.containsRow(hbaseScanSpec.getStopRow())) { + break; + } + } + } catch (Exception e) { + throw new DrillRuntimeException("Error getting region info for table: " + hbaseScanSpec.getTableName(), e); + } + verifyColumns(); + } + + private void verifyColumns() { + /* + if (columns != null) { + for (SchemaPath column : columns) { + if (!(column.equals(ROW_KEY_PATH) || hTableDesc.hasFamily(HBaseUtils.getBytes(column.getRootSegment().getPath())))) { + DrillRuntimeException.format("The column family '%s' does not exist in HBase table: %s .", + column.getRootSegment().getPath(), hTableDesc.getNameAsString()); + } + } + } + */ + } + + @Override + public List getOperatorAffinity() { + watch.reset(); + watch.start(); + Map endpointMap = new HashMap(); + for (DrillbitEndpoint ep : formatPlugin.getContext().getBits()) { + endpointMap.put(ep.getAddress(), ep); + } + + Map affinityMap = new HashMap(); + for (ServerName sn : regionsToScan.values()) { + DrillbitEndpoint ep = endpointMap.get(sn.getHostname()); + if (ep != null) { + EndpointAffinity affinity = affinityMap.get(ep); + if (affinity == null) { + affinityMap.put(ep, new EndpointAffinity(ep, 1)); + } else { + affinity.addAffinity(1); + } + } + } + logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS)/1000); + return Lists.newArrayList(affinityMap.values()); + } + + /** + * + * @param incomingEndpoints + */ + @Override + public void applyAssignments(List incomingEndpoints) { + watch.reset(); + watch.start(); + + final int numSlots = incomingEndpoints.size(); + Preconditions.checkArgument(numSlots <= regionsToScan.size(), + String.format("Incoming endpoints %d is greater than number of scan regions %d", numSlots, regionsToScan.size())); + + /* + * Minimum/Maximum number of assignment per slot + */ + final int minPerEndpointSlot = (int) Math.floor((double)regionsToScan.size() / numSlots); + final int maxPerEndpointSlot = (int) Math.ceil((double)regionsToScan.size() / numSlots); + + /* + * initialize (endpoint index => HBaseSubScanSpec list) map + */ + endpointFragmentMapping = Maps.newHashMapWithExpectedSize(numSlots); + + /* + * another map with endpoint (hostname => corresponding index list) in 'incomingEndpoints' list + */ + Map> endpointHostIndexListMap = Maps.newHashMap(); + + /* + * Initialize these two maps + */ + for (int i = 0; i < numSlots; ++i) { + endpointFragmentMapping.put(i, new ArrayList(maxPerEndpointSlot)); + String hostname = incomingEndpoints.get(i).getAddress(); + Queue hostIndexQueue = endpointHostIndexListMap.get(hostname); + if (hostIndexQueue == null) { + hostIndexQueue = Lists.newLinkedList(); + endpointHostIndexListMap.put(hostname, hostIndexQueue); + } + hostIndexQueue.add(i); + } + + Set> regionsToAssignSet = Sets.newHashSet(regionsToScan.entrySet()); + + /* + * First, we assign regions which are hosted on region servers running on drillbit endpoints + */ + for (Iterator> regionsIterator = regionsToAssignSet.iterator(); regionsIterator.hasNext(); /*nothing*/) { + Entry regionEntry = regionsIterator.next(); + /* + * Test if there is a drillbit endpoint which is also an HBase RegionServer that hosts the current HBase region + */ + Queue endpointIndexlist = endpointHostIndexListMap.get(regionEntry.getValue().getHostname()); + if (endpointIndexlist != null) { + Integer slotIndex = endpointIndexlist.poll(); + List endpointSlotScanList = endpointFragmentMapping.get(slotIndex); + endpointSlotScanList.add(regionInfoToSubScanSpec(regionEntry.getKey())); + // add to the tail of the slot list, to add more later in round robin fashion + endpointIndexlist.offer(slotIndex); + // this region has been assigned + regionsIterator.remove(); + } + } + + /* + * Build priority queues of slots, with ones which has tasks lesser than 'minPerEndpointSlot' and another which have more. + */ + PriorityQueue> minHeap = new PriorityQueue>(numSlots, LIST_SIZE_COMPARATOR); + PriorityQueue> maxHeap = new PriorityQueue>(numSlots, LIST_SIZE_COMPARATOR_REV); + for(List listOfScan : endpointFragmentMapping.values()) { + if (listOfScan.size() < minPerEndpointSlot) { + minHeap.offer(listOfScan); + } else if (listOfScan.size() > minPerEndpointSlot){ + maxHeap.offer(listOfScan); + } + } + + /* + * Now, let's process any regions which remain unassigned and assign them to slots with minimum number of assignments. + */ + if (regionsToAssignSet.size() > 0) { + for (Entry regionEntry : regionsToAssignSet) { + List smallestList = minHeap.poll(); + smallestList.add(regionInfoToSubScanSpec(regionEntry.getKey())); + if (smallestList.size() < maxPerEndpointSlot) { + minHeap.offer(smallestList); + } + } + } + + /* + * While there are slots with lesser than 'minPerEndpointSlot' unit work, balance from those with more. + */ + while(minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) { + List smallestList = minHeap.poll(); + List largestList = maxHeap.poll(); + smallestList.add(largestList.remove(largestList.size()-1)); + if (largestList.size() > minPerEndpointSlot) { + maxHeap.offer(largestList); + } + if (smallestList.size() < minPerEndpointSlot) { + minHeap.offer(smallestList); + } + } + + /* no slot should be empty at this point */ + assert (minHeap.peek() == null || minHeap.peek().size() > 0) : String.format( + "Unable to assign tasks to some endpoints.\nEndpoints: {}.\nAssignment Map: {}.", + incomingEndpoints, endpointFragmentMapping.toString()); + + logger.debug("Built assignment map in {} µs.\nEndpoints: {}.\nAssignment Map: {}", + watch.elapsed(TimeUnit.NANOSECONDS)/1000, incomingEndpoints, endpointFragmentMapping.toString()); + } + + private HBaseSubScanSpec regionInfoToSubScanSpec(HRegionInfo ri) { + HBaseScanSpec spec = hbaseScanSpec; + HBaseSubScanSpec subScanSpec = new HBaseSubScanSpec(spec.getTableName(), + regionsToScan.get(ri).getHostname(), + (!isNullOrEmpty(spec.getStartRow()) && ri.containsRow(spec.getStartRow())) ? spec.getStartRow() : ri.getStartKey(), + (!isNullOrEmpty(spec.getStopRow()) && ri.containsRow(spec.getStopRow())) ? spec.getStopRow() : ri.getEndKey(), + spec.getSerializedFilter(), + null); + return subScanSpec; + } + + private boolean isNullOrEmpty(byte[] key) { + return key == null || key.length == 0; + } + + @Override + public MapRDBSubScan getSpecificScan(int minorFragmentId) { + assert minorFragmentId < endpointFragmentMapping.size() : String.format( + "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), + minorFragmentId); + return new MapRDBSubScan(storagePlugin, storagePlugin.getConfig(), endpointFragmentMapping.get(minorFragmentId), columns); + } + + @Override + public int getMaxParallelizationWidth() { + return regionsToScan.size(); + } + + @Override + public ScanStats getScanStats() { + //TODO: look at stats for this. + int rowCount = (int) (hbaseScanSpec.getFilter() != null ? .5 : 1) * tableStats.getNumRows(); + int avgColumnSize = 10; + int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size(); + return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount); + } + + @Override + @JsonIgnore + public PhysicalOperator getNewWithChildren(List children) { + Preconditions.checkArgument(children.isEmpty()); + return new MapRDBGroupScan(this); + } + + @JsonIgnore + public MapRDBFormatPlugin getFormatPlugin() { + return formatPlugin; + } + + @JsonIgnore + public Configuration getHBaseConf() { + return HBaseConfiguration.create(); + } + + @JsonIgnore + public String getTableName() { + return getHBaseScanSpec().getTableName(); + } + + @Override + public String getDigest() { + return toString(); + } + + @Override + public String toString() { + return "MapRDBGroupScan [HBaseScanSpec=" + + hbaseScanSpec + ", columns=" + + columns + "]"; + } + + @JsonProperty("storage") + public FileSystemConfig getStorageConfig() { + return (FileSystemConfig) storagePlugin.getConfig(); + } + + @JsonProperty + public List getColumns() { + return columns; + } + + @JsonProperty + public HBaseScanSpec getHBaseScanSpec() { + return hbaseScanSpec; + } + + @JsonIgnore + public boolean canPushdownProjects(List columns) { + return true; + } + + @JsonIgnore + public void setFilterPushedDown(boolean b) { + this.filterPushedDown = true; + } + + @JsonIgnore + public boolean isFilterPushedDown() { + return filterPushedDown; + } + + /** + * Empty constructor, do not use, only for testing. + */ + @VisibleForTesting + public MapRDBGroupScan() { } + + /** + * Do not use, only for testing. + */ + @VisibleForTesting + public void setHBaseScanSpec(HBaseScanSpec hbaseScanSpec) { + this.hbaseScanSpec = hbaseScanSpec; + } + + /** + * Do not use, only for testing. + */ + @JsonIgnore + @VisibleForTesting + public void setRegionsToScan(NavigableMap regionsToScan) { + this.regionsToScan = regionsToScan; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java new file mode 100644 index 00000000000..74ea7a8b304 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.physical.impl.BatchCreator; +import org.apache.drill.exec.physical.impl.ScanBatch; +import org.apache.drill.exec.record.RecordBatch; +import org.apache.drill.exec.store.RecordReader; +import org.apache.drill.exec.store.hbase.HBaseRecordReader; +import org.apache.drill.exec.store.hbase.HBaseSubScan; +import org.apache.hadoop.hbase.HBaseConfiguration; + +import java.util.List; + +public class MapRDBScanBatchCreator implements BatchCreator{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBScanBatchCreator.class); + + @Override + public RecordBatch getBatch(FragmentContext context, MapRDBSubScan subScan, List children) throws ExecutionSetupException { + Preconditions.checkArgument(children.isEmpty()); + List readers = Lists.newArrayList(); + for(HBaseSubScan.HBaseSubScanSpec scanSpec : subScan.getRegionScanSpecList()){ + try { + readers.add( + new HBaseRecordReader(HBaseConfiguration.create(), scanSpec, subScan.getColumns(), context) + ); + } catch (Exception e1) { + throw new ExecutionSetupException(e1); + } + } + return new ScanBatch(subScan, context, readers.iterator()); + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java new file mode 100644 index 00000000000..40901e7d623 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java @@ -0,0 +1,225 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractBase; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.PhysicalVisitor; +import org.apache.drill.exec.physical.base.SubScan; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.hbase.HBaseSubScan; +import org.apache.drill.exec.store.hbase.HBaseUtils; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.ParseFilter; +import org.apache.hadoop.hbase.util.Bytes; + +import java.nio.charset.CharacterCodingException; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +// Class containing information for reading a single HBase region +@JsonTypeName("maprdb-sub-scan") +public class MapRDBSubScan extends AbstractBase implements SubScan { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBSubScan.class); + + @JsonProperty + public final StoragePluginConfig storage; + @JsonIgnore + private final FileSystemPlugin fsStoragePlugin; + private final List regionScanSpecList; + private final List columns; + + @JsonCreator + public MapRDBSubScan(@JacksonInject StoragePluginRegistry registry, + @JsonProperty("storage") StoragePluginConfig storage, + @JsonProperty("regionScanSpecList") LinkedList regionScanSpecList, + @JsonProperty("columns") List columns) throws ExecutionSetupException { + this.fsStoragePlugin = (FileSystemPlugin) registry.getPlugin(storage); + this.regionScanSpecList = regionScanSpecList; + this.storage = storage; + this.columns = columns; + } + + public MapRDBSubScan(FileSystemPlugin storagePlugin, StoragePluginConfig config, + List hBaseSubScanSpecs, List columns) { + fsStoragePlugin = storagePlugin; + storage = config; + this.regionScanSpecList = hBaseSubScanSpecs; + this.columns = columns; + } + + public List getRegionScanSpecList() { + return regionScanSpecList; + } + + public List getColumns() { + return columns; + } + + @Override + public boolean isExecutable() { + return false; + } + + @Override + public T accept(PhysicalVisitor physicalVisitor, X value) throws E { + return physicalVisitor.visitSubScan(this, value); + } + + @Override + public PhysicalOperator getNewWithChildren(List children) { + Preconditions.checkArgument(children.isEmpty()); + return new MapRDBSubScan(fsStoragePlugin, storage, regionScanSpecList, columns); + } + + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } + + public static class HBaseSubScanSpec { + + protected String tableName; + protected String regionServer; + protected byte[] startRow; + protected byte[] stopRow; + protected byte[] serializedFilter; + + @parquet.org.codehaus.jackson.annotate.JsonCreator + public HBaseSubScanSpec(@JsonProperty("tableName") String tableName, + @JsonProperty("regionServer") String regionServer, + @JsonProperty("startRow") byte[] startRow, + @JsonProperty("stopRow") byte[] stopRow, + @JsonProperty("serializedFilter") byte[] serializedFilter, + @JsonProperty("filterString") String filterString) { + if (serializedFilter != null && filterString != null) { + throw new IllegalArgumentException("The parameters 'serializedFilter' or 'filterString' cannot be specified at the same time."); + } + this.tableName = tableName; + this.regionServer = regionServer; + this.startRow = startRow; + this.stopRow = stopRow; + if (serializedFilter != null) { + this.serializedFilter = serializedFilter; + } else { + this.serializedFilter = HBaseUtils.serializeFilter(parseFilterString(filterString)); + } + } + + static final ParseFilter PARSE_FILTER = new ParseFilter(); + + static Filter parseFilterString(String filterString) { + if (filterString == null) { + return null; + } + try { + return PARSE_FILTER.parseFilterString(filterString); + } catch (CharacterCodingException e) { + throw new DrillRuntimeException("Error parsing filter string: " + filterString, e); + } + } + + /* package */ HBaseSubScanSpec() { + // empty constructor, to be used with builder pattern; + } + + @JsonIgnore + private Filter scanFilter; + public Filter getScanFilter() { + if (scanFilter == null && serializedFilter != null) { + scanFilter = HBaseUtils.deserializeFilter(serializedFilter); + } + return scanFilter; + } + + public String getTableName() { + return tableName; + } + + public HBaseSubScanSpec setTableName(String tableName) { + this.tableName = tableName; + return this; + } + + public String getRegionServer() { + return regionServer; + } + + public HBaseSubScanSpec setRegionServer(String regionServer) { + this.regionServer = regionServer; + return this; + } + + public byte[] getStartRow() { + return startRow; + } + + public HBaseSubScanSpec setStartRow(byte[] startRow) { + this.startRow = startRow; + return this; + } + + public byte[] getStopRow() { + return stopRow; + } + + public HBaseSubScanSpec setStopRow(byte[] stopRow) { + this.stopRow = stopRow; + return this; + } + + public byte[] getSerializedFilter() { + return serializedFilter; + } + + public HBaseSubScanSpec setSerializedFilter(byte[] serializedFilter) { + this.serializedFilter = serializedFilter; + this.scanFilter = null; + return this; + } + + @Override + public String toString() { + return "HBaseScanSpec [tableName=" + tableName + + ", startRow=" + (startRow == null ? null : Bytes.toStringBinary(startRow)) + + ", stopRow=" + (stopRow == null ? null : Bytes.toStringBinary(stopRow)) + + ", filter=" + (getScanFilter() == null ? null : getScanFilter().toString()) + + ", regionServer=" + regionServer + "]"; + } + + } + + @Override + public int getOperatorType() { + return 1001; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java new file mode 100644 index 00000000000..4d539f4b786 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.HBaseConfiguration; + +import com.mapr.fs.hbase.HBaseAdminImpl; +import org.apache.hadoop.hbase.client.mapr.BaseTableMappingRules; +import org.apache.hadoop.hbase.client.mapr.TableMappingRulesFactory; + +public class MapRDBTableStats { + + private int numRows; + private Configuration config = HBaseConfiguration.create(); + + public MapRDBTableStats(HTable table) throws Exception { + HBaseAdminImpl admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(config)); + numRows = admin.getNumRows(new String(table.getTableName())); + } + + public MapRDBTableStats(String tableName) throws Exception { + HBaseAdminImpl admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(config)); + numRows = admin.getNumRows(tableName); + } + + public int getNumRows() { + return numRows; + } + +} diff --git a/contrib/format-maprdb/src/main/resources/bootstrap-storage-plugins.json b/contrib/format-maprdb/src/main/resources/bootstrap-storage-plugins.json new file mode 100644 index 00000000000..3e0e8c04174 --- /dev/null +++ b/contrib/format-maprdb/src/main/resources/bootstrap-storage-plugins.json @@ -0,0 +1,12 @@ +{ + "storage":{ + hbase : { + type:"hbase", + enabled: false, + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } + } + } +} diff --git a/contrib/format-maprdb/src/main/resources/checkstyle-config.xml b/contrib/format-maprdb/src/main/resources/checkstyle-config.xml new file mode 100644 index 00000000000..6743466b568 --- /dev/null +++ b/contrib/format-maprdb/src/main/resources/checkstyle-config.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/format-maprdb/src/main/resources/checkstyle-suppressions.xml b/contrib/format-maprdb/src/main/resources/checkstyle-suppressions.xml new file mode 100644 index 00000000000..c30ff09cc21 --- /dev/null +++ b/contrib/format-maprdb/src/main/resources/checkstyle-suppressions.xml @@ -0,0 +1,19 @@ + + + + + + + + diff --git a/contrib/format-maprdb/src/main/resources/drill-module.conf b/contrib/format-maprdb/src/main/resources/drill-module.conf new file mode 100644 index 00000000000..0edceafadbc --- /dev/null +++ b/contrib/format-maprdb/src/main/resources/drill-module.conf @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file tells Drill to consider this module when class path scanning. +// This file can also include any supplementary configuration information. +// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. + +drill.exec: { + + sys.store.provider: { + # HBasePStoreProvider is disabled by default + # class: "org.apache.drill.exec.store.hbase.config.HBasePStoreProvider", + hbase: { + table : "drill_store", + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } + } + } + +} diff --git a/contrib/format-maprdb/src/test/resources/hbase-site.xml b/contrib/format-maprdb/src/test/resources/hbase-site.xml new file mode 100644 index 00000000000..54425d440c6 --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/hbase-site.xml @@ -0,0 +1,143 @@ + + + + + + hbase.regionserver.msginterval + 1000 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.client.pause + 1000 + General client pause value. Used mostly as value to wait + before running a retry of a failed get, region lookup, etc. + + + hbase.client.retries.number + 10 + Maximum retries. Used as maximum for all retryable + operations such as fetching of the root region from root region + server, getting a cell's value, starting a row update, etc. + Default: 10. + + + + hbase.server.thread.wakefrequency + 1000 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as META scanner and log roller. + + + + hbase.master.event.waiting.time + 50 + Time to sleep between checks to see if a table event took place. + + + + hbase.regionserver.handler.count + 5 + Count of RPC Server instances spun up on RegionServers + Same property is used by the HMaster for count of master handlers. + Default is 10. + + + + + hbase.regionserver.info.port + -1 + The port for the hbase regionserver web UI + Set to -1 if you do not want the info server to run. + + + + hbase.regionserver.info.port.auto + true + Info server auto port bind. Enables automatic port + search if hbase.regionserver.info.port is already in use. + Enabled for testing to run multiple tests on one machine. + + + + hbase.master.lease.thread.wakefrequency + 3000 + The interval between checks for expired region server leases. + This value has been reduced due to the other reduced values above so that + the master will notice a dead region server sooner. The default is 15 seconds. + + + + hbase.regionserver.safemode + false + + Turn on/off safe mode in region server. Always on for production, always off + for tests. + + + + hbase.hregion.max.filesize + 67108864 + + Maximum desired file size for an HRegion. If filesize exceeds + value + (value / 2), the HRegion is split in two. Default: 256M. + + Keep the maximum filesize small so we split more often in tests. + + + + hbase.zookeeper.property.clientPort + 2181 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + hbase.defaults.for.version.skip + true + + Set to true to skip the 'hbase.defaults.for.version'. + Setting this to true can be useful in contexts other than + the other side of a maven generation; i.e. running in an + ide. You'll want to set this boolean to true to avoid + seeing the RuntimException complaint: "hbase-default.xml file + seems to be for and old version of HBase (@@@VERSION@@@), this + version is X.X.X-SNAPSHOT" + + + + hbase.ipc.client.fallback-to-simple-auth-allowed + true + + \ No newline at end of file diff --git a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical.json b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical.json new file mode 100644 index 00000000000..7f9015bdfa8 --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical.json @@ -0,0 +1,29 @@ +{ + head : { + type : "APACHE_DRILL_PHYSICAL", + version : 1, + generator : { + type : "manual" + } + }, + graph : [ { + pop : "hbase-scan", + @id : 1, + hbaseScanSpec : { + tableName : "[TABLE_NAME]" + }, + storage: + { + "type":"hbase", + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } + } + }, + { + pop : "screen", + @id : 2, + child : 1 + } ] +} \ No newline at end of file diff --git a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json new file mode 100644 index 00000000000..f399f6f1c37 --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json @@ -0,0 +1,32 @@ +{ + head : { + type : "APACHE_DRILL_PHYSICAL", + version : 1, + generator : { + type : "manual" + } + }, + graph : [ { + pop : "hbase-scan", + @id : 1, + hbaseScanSpec : { + tableName : "[TABLE_NAME]" + }, + storage: + { + "type":"hbase", + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } + }, + columns: [ + "`f2`.c1", "`f2`.c2", "row_key" + ] + }, + { + pop : "screen", + @id : 2, + child : 1 + } ] +} \ No newline at end of file diff --git a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json new file mode 100644 index 00000000000..0002164fe70 --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json @@ -0,0 +1,32 @@ +{ + head : { + type : "APACHE_DRILL_PHYSICAL", + version : 1, + generator : { + type : "manual" + } + }, + graph : [ { + pop : "hbase-scan", + @id : 1, + hbaseScanSpec : { + tableName : "[TABLE_NAME]" + }, + storage: + { + "type":"hbase", + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } + }, + columns: [ + "f2" + ] + }, + { + pop : "screen", + @id : 2, + child : 1 + } ] +} \ No newline at end of file diff --git a/contrib/format-maprdb/src/test/resources/logback.xml b/contrib/format-maprdb/src/test/resources/logback.xml new file mode 100644 index 00000000000..00cc85b303f --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/logback.xml @@ -0,0 +1,64 @@ + + + + + + + true + 10000 + true + ${LILITH_HOSTNAME:-localhost} + + + + + ${logback.log.dir:-./target/surefire-reports}/hbase-tests-${bySecond}.log + false + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + + + + + + + + + + + From bfd27ec82851446a918b65f06805c63f4f0690ae Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Wed, 8 Oct 2014 08:20:11 -0700 Subject: [PATCH 03/58] Updated plugin version to 0.7.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index a09d64f3ba2..63295812042 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 0.6.0-incubating-SNAPSHOT + 0.7.0-incubating-SNAPSHOT drill-storage-maprdb From 846451926a9f9c43b874ed0cdfb565186419bef9 Mon Sep 17 00:00:00 2001 From: Anbang Xu Date: Tue, 21 Oct 2014 15:49:30 -0700 Subject: [PATCH 04/58] MD-12: Filters are not being pushed down through maprdb plugin + Reverted mapr-hbase back to MapR 4.0.1 release + Apply DRILL-1508 and DRILL-1567 to MapRDB plugin 1. https://github.com/apache/drill/commit/67df8cfa27041c9696255ee96e2d066e034f6f58 2. https://github.com/apache/drill/commit/71f5ad447823cba96cf73fb6709f8ed6cc0b63e9 --- contrib/format-maprdb/pom.xml | 2 +- .../maprdb/CompareFunctionsProcessor.java | 234 ++++++++++++++ .../store/maprdb/MapRDBFilterBuilder.java | 302 ++++++++++++++++++ .../exec/store/maprdb/MapRDBFormatPlugin.java | 12 +- .../exec/store/maprdb/MapRDBGroupScan.java | 5 + .../maprdb/MapRDBPushFilterIntoScan.java | 93 ++++++ 6 files changed, 643 insertions(+), 5 deletions(-) create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index 63295812042..b4e001e93ff 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -127,7 +127,7 @@ com.mapr.fs mapr-hbase - 4.0.2-mapr-SNAPSHOT + 4.0.1.28318-mapr diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java new file mode 100644 index 00000000000..fde8de0bea9 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +import java.nio.ByteOrder; + +import org.apache.drill.common.expression.CastExpression; +import org.apache.drill.common.expression.ConvertExpression; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; +import org.apache.drill.common.expression.ValueExpressions.DateExpression; +import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; +import org.apache.drill.common.expression.ValueExpressions.FloatExpression; +import org.apache.drill.common.expression.ValueExpressions.IntExpression; +import org.apache.drill.common.expression.ValueExpressions.LongExpression; +import org.apache.drill.common.expression.ValueExpressions.QuotedString; +import org.apache.drill.common.expression.ValueExpressions.TimeExpression; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +class CompareFunctionsProcessor extends AbstractExprVisitor { + private byte[] value; + private boolean success; + private boolean isEqualityFn; + private SchemaPath path; + private String functionName; + + public static boolean isCompareFunction(String functionName) { + return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName); + } + + public static CompareFunctionsProcessor process(FunctionCall call, boolean nullComparatorSupported) { + String functionName = call.getName(); + LogicalExpression nameArg = call.args.get(0); + LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null; + CompareFunctionsProcessor evaluator = new CompareFunctionsProcessor(functionName); + + if (valueArg != null) { // binary function + if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) { + LogicalExpression swapArg = valueArg; + valueArg = nameArg; + nameArg = swapArg; + evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName); + } + evaluator.success = nameArg.accept(evaluator, valueArg); + } else if (nullComparatorSupported && call.args.get(0) instanceof SchemaPath) { + evaluator.success = true; + evaluator.path = (SchemaPath) nameArg; + } + + return evaluator; + } + + public CompareFunctionsProcessor(String functionName) { + this.success = false; + this.functionName = functionName; + this.isEqualityFn = COMPARE_FUNCTIONS_TRANSPOSE_MAP.containsKey(functionName) + && COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName).equals(functionName); + } + + public byte[] getValue() { + return value; + } + + public boolean isSuccess() { + return success; + } + + public SchemaPath getPath() { + return path; + } + + public String getFunctionName() { + return functionName; + } + + @Override + public Boolean visitCastExpression(CastExpression e, LogicalExpression valueArg) throws RuntimeException { + if (e.getInput() instanceof CastExpression || e.getInput() instanceof SchemaPath) { + return e.getInput().accept(this, valueArg); + } + return false; + } + + @Override + public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression valueArg) throws RuntimeException { + if (e.getConvertFunction() == ConvertExpression.CONVERT_FROM && e.getInput() instanceof SchemaPath) { + ByteBuf bb = null; + String encodingType = e.getEncodingType(); + switch (encodingType) { + case "INT_BE": + case "INT": + case "UINT_BE": + case "UINT": + case "UINT4_BE": + case "UINT4": + if (valueArg instanceof IntExpression + && (isEqualityFn || encodingType.startsWith("U"))) { + bb = Unpooled.wrappedBuffer(new byte[4]).order(encodingType.endsWith("_BE") ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + bb.writeInt(((IntExpression)valueArg).getInt()); + } + break; + case "BIGINT_BE": + case "BIGINT": + case "UINT8_BE": + case "UINT8": + if (valueArg instanceof LongExpression + && (isEqualityFn || encodingType.startsWith("U"))) { + bb = Unpooled.wrappedBuffer(new byte[8]).order(encodingType.endsWith("_BE") ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + bb.writeLong(((LongExpression)valueArg).getLong()); + } + break; + case "FLOAT": + if (valueArg instanceof FloatExpression && isEqualityFn) { + bb = Unpooled.wrappedBuffer(new byte[4]).order(ByteOrder.BIG_ENDIAN); + bb.writeFloat(((FloatExpression)valueArg).getFloat()); + } + break; + case "DOUBLE": + if (valueArg instanceof DoubleExpression && isEqualityFn) { + bb = Unpooled.wrappedBuffer(new byte[8]).order(ByteOrder.BIG_ENDIAN);; + bb.writeDouble(((DoubleExpression)valueArg).getDouble()); + } + break; + case "TIME_EPOCH": + case "TIME_EPOCH_BE": + if (valueArg instanceof TimeExpression) { + bb = Unpooled.wrappedBuffer(new byte[8]).order(encodingType.endsWith("_BE") ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + bb.writeLong(((TimeExpression)valueArg).getTime()); + } + break; + case "DATE_EPOCH": + case "DATE_EPOCH_BE": + if (valueArg instanceof DateExpression) { + bb = Unpooled.wrappedBuffer(new byte[8]).order(encodingType.endsWith("_BE") ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + bb.writeLong(((DateExpression)valueArg).getDate()); + } + break; + case "BOOLEAN_BYTE": + if (valueArg instanceof BooleanExpression) { + bb = Unpooled.wrappedBuffer(new byte[1]); + bb.writeByte(((BooleanExpression)valueArg).getBoolean() ? 1 : 0); + } + break; + case "UTF8": + // let visitSchemaPath() handle this. + return e.getInput().accept(this, valueArg); + } + + if (bb != null) { + this.value = bb.array(); + this.path = (SchemaPath)e.getInput(); + return true; + } + } + return false; + } + + @Override + public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) throws RuntimeException { + return false; + } + + @Override + public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) throws RuntimeException { + if (valueArg instanceof QuotedString) { + this.value = ((QuotedString) valueArg).value.getBytes(Charsets.UTF_8); + this.path = path; + return true; + } + return false; + } + + private static final ImmutableSet> VALUE_EXPRESSION_CLASSES; + static { + ImmutableSet.Builder> builder = ImmutableSet.builder(); + VALUE_EXPRESSION_CLASSES = builder + .add(BooleanExpression.class) + .add(DateExpression.class) + .add(DoubleExpression.class) + .add(FloatExpression.class) + .add(IntExpression.class) + .add(LongExpression.class) + .add(QuotedString.class) + .add(TimeExpression.class) + .build(); + } + + private static final ImmutableMap COMPARE_FUNCTIONS_TRANSPOSE_MAP; + static { + ImmutableMap.Builder builder = ImmutableMap.builder(); + COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder + // unary functions + .put("isnotnull", "isnotnull") + .put("isNotNull", "isNotNull") + .put("is not null", "is not null") + .put("isnull", "isnull") + .put("isNull", "isNull") + .put("is null", "is null") + // binary functions + .put("like", "like") + .put("equal", "equal") + .put("not_equal", "not_equal") + .put("greater_than_or_equal_to", "less_than_or_equal_to") + .put("greater_than", "less_than") + .put("less_than_or_equal_to", "greater_than_or_equal_to") + .put("less_than", "greater_than") + .build(); + } + +} \ No newline at end of file diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java new file mode 100644 index 00000000000..4683c9dabc0 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java @@ -0,0 +1,302 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import java.util.Arrays; + +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.drill.exec.store.hbase.DrillHBaseConstants; +import org.apache.drill.exec.store.hbase.HBaseRegexParser; +import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.drill.exec.store.hbase.HBaseUtils; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.NullComparator; +import org.apache.hadoop.hbase.filter.RegexStringComparator; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; + +public class MapRDBFilterBuilder extends AbstractExprVisitor implements DrillHBaseConstants { + + final private MapRDBGroupScan groupScan; + + final private LogicalExpression le; + + private boolean allExpressionsConverted = true; + + private static Boolean nullComparatorSupported = null; + + MapRDBFilterBuilder(MapRDBGroupScan groupScan, LogicalExpression le) { + this.groupScan = groupScan; + this.le = le; + } + + public HBaseScanSpec parseTree() { + HBaseScanSpec parsedSpec = le.accept(this, null); + if (parsedSpec != null) { + parsedSpec = mergeScanSpecs("booleanAnd", this.groupScan.getHBaseScanSpec(), parsedSpec); + /* + * If RowFilter is THE filter attached to the scan specification, + * remove it since its effect is also achieved through startRow and stopRow. + */ + Filter filter = parsedSpec.getFilter(); + if (filter instanceof RowFilter && + ((RowFilter)filter).getComparator() instanceof BinaryComparator) { + filter = null; + } + } + return parsedSpec; + } + + public boolean isAllExpressionsConverted() { + return allExpressionsConverted; + } + + @Override + public HBaseScanSpec visitUnknown(LogicalExpression e, Void value) throws RuntimeException { + allExpressionsConverted = false; + return null; + } + + @Override + public HBaseScanSpec visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException { + return visitFunctionCall(op, value); + } + + @Override + public HBaseScanSpec visitFunctionCall(FunctionCall call, Void value) throws RuntimeException { + HBaseScanSpec nodeScanSpec = null; + String functionName = call.getName(); + ImmutableList args = call.args; + + if (CompareFunctionsProcessor.isCompareFunction(functionName)) { + /* + * HBASE-10848: Bug in HBase versions (0.94.[0-18], 0.96.[0-2], 0.98.[0-1]) + * causes a filter with NullComparator to fail. Enable only if specified in + * the configuration (after ensuring that the HBase cluster has the fix). + */ + if (nullComparatorSupported == null) { + nullComparatorSupported = groupScan.getHBaseConf().getBoolean("drill.hbase.supports.null.comparator", false); + } + + CompareFunctionsProcessor processor = CompareFunctionsProcessor.process(call, nullComparatorSupported); + if (processor.isSuccess()) { + nodeScanSpec = createHBaseScanSpec(call, processor); + } + } else { + switch (functionName) { + case "booleanAnd": + case "booleanOr": + HBaseScanSpec firstScanSpec = args.get(0).accept(this, null); + for (int i = 1; i < args.size(); ++i) { + HBaseScanSpec nextScanSpec = args.get(i).accept(this, null); + if (firstScanSpec != null && nextScanSpec != null) { + nodeScanSpec = mergeScanSpecs(functionName, firstScanSpec, nextScanSpec); + } else { + allExpressionsConverted = false; + if ("booleanAnd".equals(functionName)) { + nodeScanSpec = firstScanSpec == null ? nextScanSpec : firstScanSpec; + } + } + firstScanSpec = nodeScanSpec; + } + break; + } + } + + if (nodeScanSpec == null) { + allExpressionsConverted = false; + } + + return nodeScanSpec; + } + + private HBaseScanSpec mergeScanSpecs(String functionName, HBaseScanSpec leftScanSpec, HBaseScanSpec rightScanSpec) { + Filter newFilter = null; + byte[] startRow = HConstants.EMPTY_START_ROW; + byte[] stopRow = HConstants.EMPTY_END_ROW; + + switch (functionName) { + case "booleanAnd": + newFilter = HBaseUtils.andFilterAtIndex(leftScanSpec.getFilter(), -1, rightScanSpec.getFilter()); //HBaseUtils.LAST_FILTER + startRow = HBaseUtils.maxOfStartRows(leftScanSpec.getStartRow(), rightScanSpec.getStartRow()); + stopRow = HBaseUtils.minOfStopRows(leftScanSpec.getStopRow(), rightScanSpec.getStopRow()); + break; + case "booleanOr": + newFilter = HBaseUtils.orFilterAtIndex(leftScanSpec.getFilter(), -1, rightScanSpec.getFilter()); //HBaseUtils.LAST_FILTER + startRow = HBaseUtils.minOfStartRows(leftScanSpec.getStartRow(), rightScanSpec.getStartRow()); + stopRow = HBaseUtils.maxOfStopRows(leftScanSpec.getStopRow(), rightScanSpec.getStopRow()); + } + return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, newFilter); + } + + private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsProcessor processor) { + String functionName = processor.getFunctionName(); + SchemaPath field = processor.getPath(); + byte[] fieldValue = processor.getValue(); + boolean isRowKey = field.getAsUnescapedPath().equals(ROW_KEY); + if (!(isRowKey + || (!field.getRootSegment().isLastPath() + && field.getRootSegment().getChild().isLastPath() + && field.getRootSegment().getChild().isNamed()) + ) + ) { + /* + * if the field in this function is neither the row_key nor a qualified HBase column, return. + */ + return null; + } + + CompareOp compareOp = null; + boolean isNullTest = false; + WritableByteArrayComparable comparator = new BinaryComparator(fieldValue); + byte[] startRow = HConstants.EMPTY_START_ROW; + byte[] stopRow = HConstants.EMPTY_END_ROW; + switch (functionName) { + case "equal": + compareOp = CompareOp.EQUAL; + if (isRowKey) { + startRow = stopRow = fieldValue; + compareOp = null; + } + break; + case "not_equal": + compareOp = CompareOp.NOT_EQUAL; + break; + case "greater_than_or_equal_to": + compareOp = CompareOp.GREATER_OR_EQUAL; + if (isRowKey) { + startRow = fieldValue; + } + break; + case "greater_than": + compareOp = CompareOp.GREATER; + if (isRowKey) { + // startRow should be just greater than 'value' + startRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + } + break; + case "less_than_or_equal_to": + compareOp = CompareOp.LESS_OR_EQUAL; + if (isRowKey) { + // stopRow should be just greater than 'value' + stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + } + break; + case "less_than": + compareOp = CompareOp.LESS; + if (isRowKey) { + stopRow = fieldValue; + } + break; + case "isnull": + case "isNull": + case "is null": + if (isRowKey) { + return null; + } + isNullTest = true; + compareOp = CompareOp.EQUAL; + comparator = new NullComparator(); + break; + case "isnotnull": + case "isNotNull": + case "is not null": + if (isRowKey) { + return null; + } + compareOp = CompareOp.NOT_EQUAL; + comparator = new NullComparator(); + break; + case "like": + /* + * Convert the LIKE operand to Regular Expression pattern so that we can + * apply RegexStringComparator() + */ + HBaseRegexParser parser = new HBaseRegexParser(call).parse(); + compareOp = CompareOp.EQUAL; + comparator = new RegexStringComparator(parser.getRegexString()); + + /* + * We can possibly do better if the LIKE operator is on the row_key + */ + if (isRowKey) { + String prefix = parser.getPrefixString(); + if (prefix != null) { // group 3 is literal + /* + * If there is a literal prefix, it can help us prune the scan to a sub range + */ + if (prefix.equals(parser.getLikeString())) { + /* The operand value is literal. This turns the LIKE operator to EQUAL operator */ + startRow = stopRow = fieldValue; + compareOp = null; + } else { + startRow = prefix.getBytes(Charsets.UTF_8); + stopRow = startRow.clone(); + boolean isMaxVal = true; + for (int i = stopRow.length - 1; i >= 0 ; --i) { + int nextByteValue = (0xff & stopRow[i]) + 1; + if (nextByteValue < 0xff) { + stopRow[i] = (byte) nextByteValue; + isMaxVal = false; + break; + } else { + stopRow[i] = 0; + } + } + if (isMaxVal) { + stopRow = HConstants.EMPTY_END_ROW; + } + } + } + } + break; + } + + if (compareOp != null || startRow != HConstants.EMPTY_START_ROW || stopRow != HConstants.EMPTY_END_ROW) { + Filter filter = null; + if (isRowKey) { + if (compareOp != null) { + filter = new RowFilter(compareOp, comparator); + } + } else { + byte[] family = HBaseUtils.getBytes(field.getRootSegment().getPath()); + byte[] qualifier = HBaseUtils.getBytes(field.getRootSegment().getChild().getNameSegment().getPath()); + filter = new SingleColumnValueFilter(family, qualifier, compareOp, comparator); + ((SingleColumnValueFilter)filter).setLatestVersionOnly(true); + if (!isNullTest) { + ((SingleColumnValueFilter)filter).setFilterIfMissing(true); + } + } + return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter); + } + // else + return null; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java index d8c6187ba2f..0e0d57cef9c 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -17,7 +17,9 @@ */ package org.apache.drill.exec.store.maprdb; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.FormatPluginConfig; @@ -39,7 +41,8 @@ import java.util.Set; public class MapRDBFormatPlugin implements FormatPlugin{ - + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBFormatPlugin.class); + private final StoragePluginConfig storageConfig; private final MapRDBFormatPluginConfig config; private final MapRDBFormatMatcher matcher; @@ -82,13 +85,14 @@ public AbstractWriter getWriter(PhysicalOperator child, String location) @Override public AbstractGroupScan getGroupScan(FileSelection selection) - throws IOException { - return getGroupScan(selection, null); + throws IOException { + return getGroupScan(selection, null); } @Override + @JsonIgnore public Set getOptimizerRules() { - return ImmutableSet.of(); + return ImmutableSet.of(MapRDBPushFilterIntoScan.INSTANCE); } @Override diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java index 8da40de20c5..a5237204f51 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -404,6 +404,11 @@ public FileSystemConfig getStorageConfig() { return (FileSystemConfig) storagePlugin.getConfig(); } + @JsonIgnore + public FileSystemPlugin getStoragePlugin(){ + return storagePlugin; + } + @JsonProperty public List getColumns() { return columns; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java new file mode 100644 index 00000000000..d86e1eebba1 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.store.maprdb; + +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.eigenbase.rel.RelNode; +import org.eigenbase.relopt.RelOptRuleCall; +import org.eigenbase.rex.RexNode; + +import com.google.common.collect.ImmutableList; + +public class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRule { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBPushFilterIntoScan.class); + + public static final StoragePluginOptimizerRule INSTANCE = new MapRDBPushFilterIntoScan(); + + private MapRDBPushFilterIntoScan() { + super(RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), "MapRDBPushFilterIntoScan"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + final FilterPrel filter = (FilterPrel) call.rel(0); + final RexNode condition = filter.getCondition(); + + MapRDBGroupScan groupScan = (MapRDBGroupScan)scan.getGroupScan(); + if (groupScan.isFilterPushedDown()) { + /* + * The rule can get triggered again due to the transformed "scan => filter" sequence + * created by the earlier execution of this rule when we could not do a complete + * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon + * this flag to not do a re-processing of the rule on the already transformed call. + */ + return; + } + + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(), scan, condition); + MapRDBFilterBuilder hbaseFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp); + HBaseScanSpec newScanSpec = hbaseFilterBuilder.parseTree(); + if (newScanSpec == null) { + return; //no filter pushdown ==> No transformation. + } + + final MapRDBGroupScan newGroupsScan = new MapRDBGroupScan(groupScan.getStoragePlugin(), groupScan.getFormatPlugin(), + newScanSpec, groupScan.getColumns()); + newGroupsScan.setFilterPushedDown(true); + + final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType()); + if (hbaseFilterBuilder.isAllExpressionsConverted()) { + /* + * Since we could convert the entire filter condition expression into an HBase filter, + * we can eliminate the filter operator altogether. + */ + call.transformTo(newScanPrel); + } else { + call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of((RelNode)newScanPrel))); + } + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof MapRDBGroupScan) { + return super.matches(call); + } + return false; + } + +} From 0e5b67e964946380fc33f278f0a20340ec338109 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Wed, 3 Dec 2014 12:03:29 -0800 Subject: [PATCH 05/58] Updated plugin version to 0.8.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index b4e001e93ff..0d6bd0f2ddf 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 0.7.0-incubating-SNAPSHOT + 0.8.0-SNAPSHOT drill-storage-maprdb From 7e59a1ee16bbff98f636dc290eda6bb8785c0ab2 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Wed, 31 Dec 2014 11:28:59 -0800 Subject: [PATCH 06/58] DRILL-1900: Fix numeric overflow problem in HBase stats calculation. --- .../exec/store/maprdb/MapRDBGroupScan.java | 58 +++++++++---------- .../exec/store/maprdb/MapRDBTableStats.java | 29 ++++------ 2 files changed, 38 insertions(+), 49 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java index a5237204f51..256d1f88cad 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -17,16 +17,22 @@ */ package org.apache.drill.exec.store.maprdb; -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.PriorityQueue; +import java.util.Queue; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; @@ -39,12 +45,10 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.dfs.FileSystemConfig; -import org.apache.drill.exec.store.dfs.FileSystemFormatConfig; import org.apache.drill.exec.store.dfs.FileSystemPlugin; import org.apache.drill.exec.store.hbase.DrillHBaseConstants; import org.apache.drill.exec.store.hbase.HBaseScanSpec; import org.apache.drill.exec.store.hbase.HBaseSubScan.HBaseSubScanSpec; -import org.apache.drill.exec.store.hbase.HBaseUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; @@ -53,22 +57,16 @@ import org.apache.hadoop.hbase.client.HTable; import org.codehaus.jackson.annotate.JsonCreator; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Comparator; -import java.util.TreeMap; -import java.util.HashMap; -import java.util.Queue; -import java.util.ArrayList; -import java.util.Set; -import java.util.Iterator; -import java.util.PriorityQueue; - -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; @JsonTypeName("maprdb-scan") public class MapRDBGroupScan extends AbstractGroupScan implements DrillHBaseConstants { @@ -150,8 +148,8 @@ public GroupScan clone(List columns) { private void init() { logger.debug("Getting region locations"); try { - tableStats = new MapRDBTableStats(hbaseScanSpec.getTableName()); HTable table = new HTable(HBaseConfiguration.create(), hbaseScanSpec.getTableName()); + tableStats = new MapRDBTableStats(table); this.hTableDesc = table.getTableDescriptor(); NavigableMap regionsMap = table.getRegionLocations(); table.close(); @@ -359,7 +357,7 @@ public int getMaxParallelizationWidth() { @Override public ScanStats getScanStats() { //TODO: look at stats for this. - int rowCount = (int) (hbaseScanSpec.getFilter() != null ? .5 : 1) * tableStats.getNumRows(); + long rowCount = (long) ((hbaseScanSpec.getFilter() != null ? .5 : 1) * tableStats.getNumRows()); int avgColumnSize = 10; int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size(); return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount); diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java index 4d539f4b786..6eef196433e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java @@ -17,34 +17,25 @@ */ package org.apache.drill.exec.store.maprdb; -import java.io.IOException; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.mapr.TableMappingRulesFactory; import com.mapr.fs.hbase.HBaseAdminImpl; -import org.apache.hadoop.hbase.client.mapr.BaseTableMappingRules; -import org.apache.hadoop.hbase.client.mapr.TableMappingRulesFactory; public class MapRDBTableStats { - - private int numRows; - private Configuration config = HBaseConfiguration.create(); - public MapRDBTableStats(HTable table) throws Exception { - HBaseAdminImpl admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(config)); - numRows = admin.getNumRows(new String(table.getTableName())); - } + private long numRows; - public MapRDBTableStats(String tableName) throws Exception { + public MapRDBTableStats(HTable table) throws Exception { + Configuration config = table.getConfiguration(); HBaseAdminImpl admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(config)); - numRows = admin.getNumRows(tableName); + numRows = admin.getNumRows(new String(table.getTableName())); + admin.close(); + } + + public long getNumRows() { + return numRows; } - public int getNumRows() { - return numRows; - } - } From cb0a676dd542eaadba49776d22bdaf7b5b335720 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Wed, 28 Jan 2015 02:21:26 -0800 Subject: [PATCH 07/58] MD-125: Update HBase storage plugin to support HBase 0.98 --- contrib/format-maprdb/pom.xml | 108 +++++------------- .../store/maprdb/MapRDBFilterBuilder.java | 4 +- 2 files changed, 29 insertions(+), 83 deletions(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index 0d6bd0f2ddf..1aee921c224 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -33,24 +33,25 @@ - + + org.apache.drill.exec + drill-java-exec + ${project.version} + + org.apache.drill.contrib drill-storage-hbase ${project.version} - - - org.apache.drill.exec - drill-java-exec - ${project.version} + com.mapr.hadoop + maprfs + + + com.mapr.fs + mapr-hbase + 4.0.1.28318-mapr - - - org.apache.drill.contrib - drill-storage-hbase - ${project.version} - @@ -73,78 +74,22 @@ 2.1.1 test - - org.apache.hbase - hbase - 0.94.13-mapr-1401-m7-3.0.2 - - - commons-logging - commons-logging - - - slf4j-log4j12 - org.slf4j - - - slf4j-log4j12 - org.slf4j - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-client - - - asm - asm - - - libthrift - org.apache.thrift - - - jruby-complete - org.jruby - - - - - com.mapr.hadoop - maprfs - - - com.mapr.fs - mapr-hbase - 4.0.1.28318-mapr - - - - mapr-releases - http://maven.corp.maprtech.com/nexus/content/repositories/public - - true - - - true - - - + + + mapr-releases + http://maven.corp.maprtech.com/nexus/content/repositories/public + + true + + + true + + + - + org.apache.maven.plugins @@ -167,4 +112,5 @@ + diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java index 4683c9dabc0..45fe22f5f13 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java @@ -30,13 +30,13 @@ import org.apache.drill.exec.store.hbase.HBaseUtils; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.NullComparator; import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; -import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; @@ -174,7 +174,7 @@ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsPro CompareOp compareOp = null; boolean isNullTest = false; - WritableByteArrayComparable comparator = new BinaryComparator(fieldValue); + ByteArrayComparable comparator = new BinaryComparator(fieldValue); byte[] startRow = HConstants.EMPTY_START_ROW; byte[] stopRow = HConstants.EMPTY_END_ROW; switch (functionName) { From bed705556be5bd0ed55366620f116d93aaf2237e Mon Sep 17 00:00:00 2001 From: vkorukanti Date: Fri, 6 Feb 2015 07:18:06 -0800 Subject: [PATCH 08/58] MD-143: Update DrillFileSystem package location after DRILL-2080. --- .../org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java | 2 +- .../org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java index 9f1d7b7ddbe..63cd737da35 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java @@ -22,7 +22,7 @@ import org.apache.drill.exec.store.dfs.FormatMatcher; import org.apache.drill.exec.store.dfs.FormatPlugin; import org.apache.drill.exec.store.dfs.FormatSelection; -import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.hadoop.fs.FileStatus; import java.io.IOException; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java index 0e0d57cef9c..da3541f9f03 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -33,7 +33,7 @@ import org.apache.drill.exec.store.dfs.FileSystemPlugin; import org.apache.drill.exec.store.dfs.FormatMatcher; import org.apache.drill.exec.store.dfs.FormatPlugin; -import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.drill.exec.store.hbase.HBaseScanSpec; import java.io.IOException; From d5dcccce0b5566370e84abbc9c8bbadb64b7e153 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Tue, 24 Feb 2015 17:12:08 -0800 Subject: [PATCH 09/58] MD-131: Queries fail when using maprdb plugin --- .../drill/exec/store/maprdb/MapRDBTableStats.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java index 6eef196433e..3d5320e2bd7 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java @@ -26,12 +26,18 @@ public class MapRDBTableStats { private long numRows; + private volatile HBaseAdminImpl admin = null; public MapRDBTableStats(HTable table) throws Exception { - Configuration config = table.getConfiguration(); - HBaseAdminImpl admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(config)); + if (admin == null) { + synchronized (MapRDBTableStats.class) { + if (admin == null) { + Configuration config = table.getConfiguration(); + admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(config)); + } + } + } numRows = admin.getNumRows(new String(table.getTableName())); - admin.close(); } public long getNumRows() { From b64ee82837d4ccb5d33d509787507bf9704e0820 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Wed, 25 Mar 2015 14:57:28 -0700 Subject: [PATCH 10/58] MD-192: CONVERT_FROM in where clause * Set the writeIndex of ByteBuf returned by Unpooled.wrappedBuffer() to 0. --- .../maprdb/CompareFunctionsProcessor.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java index fde8de0bea9..de8e080a181 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java @@ -119,7 +119,7 @@ public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression val case "UINT4": if (valueArg instanceof IntExpression && (isEqualityFn || encodingType.startsWith("U"))) { - bb = Unpooled.wrappedBuffer(new byte[4]).order(encodingType.endsWith("_BE") ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + bb = newByteBuf(4, encodingType.endsWith("_BE")); bb.writeInt(((IntExpression)valueArg).getInt()); } break; @@ -129,39 +129,39 @@ public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression val case "UINT8": if (valueArg instanceof LongExpression && (isEqualityFn || encodingType.startsWith("U"))) { - bb = Unpooled.wrappedBuffer(new byte[8]).order(encodingType.endsWith("_BE") ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + bb = newByteBuf(8, encodingType.endsWith("_BE")); bb.writeLong(((LongExpression)valueArg).getLong()); } break; case "FLOAT": if (valueArg instanceof FloatExpression && isEqualityFn) { - bb = Unpooled.wrappedBuffer(new byte[4]).order(ByteOrder.BIG_ENDIAN); + bb = newByteBuf(4, true); bb.writeFloat(((FloatExpression)valueArg).getFloat()); } break; case "DOUBLE": if (valueArg instanceof DoubleExpression && isEqualityFn) { - bb = Unpooled.wrappedBuffer(new byte[8]).order(ByteOrder.BIG_ENDIAN);; + bb = newByteBuf(8, true); bb.writeDouble(((DoubleExpression)valueArg).getDouble()); } break; case "TIME_EPOCH": case "TIME_EPOCH_BE": if (valueArg instanceof TimeExpression) { - bb = Unpooled.wrappedBuffer(new byte[8]).order(encodingType.endsWith("_BE") ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + bb = newByteBuf(8, encodingType.endsWith("_BE")); bb.writeLong(((TimeExpression)valueArg).getTime()); } break; case "DATE_EPOCH": case "DATE_EPOCH_BE": if (valueArg instanceof DateExpression) { - bb = Unpooled.wrappedBuffer(new byte[8]).order(encodingType.endsWith("_BE") ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + bb = newByteBuf(8, encodingType.endsWith("_BE")); bb.writeLong(((DateExpression)valueArg).getDate()); } break; case "BOOLEAN_BYTE": if (valueArg instanceof BooleanExpression) { - bb = Unpooled.wrappedBuffer(new byte[1]); + bb = newByteBuf(1, false /* does not matter */); bb.writeByte(((BooleanExpression)valueArg).getBoolean() ? 1 : 0); } break; @@ -194,6 +194,12 @@ public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) thro return false; } + private static ByteBuf newByteBuf(int size, boolean bigEndian) { + return Unpooled.wrappedBuffer(new byte[size]) + .order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN) + .writerIndex(0); + } + private static final ImmutableSet> VALUE_EXPRESSION_CLASSES; static { ImmutableSet.Builder> builder = ImmutableSet.builder(); From 1c3013af5053136b167dd948f87f88092d2aa9d9 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Wed, 15 Apr 2015 13:01:32 -0700 Subject: [PATCH 11/58] Updated plugin version to 0.9.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index 1aee921c224..a70591a3075 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT drill-storage-maprdb From 419e7f38f78428605000fd12873f4aa6eb7211c7 Mon Sep 17 00:00:00 2001 From: vkorukanti Date: Tue, 21 Apr 2015 17:51:37 -0700 Subject: [PATCH 12/58] MD-244: Update MapRDB plugin after DRILL-2514 and DRILL-2413 --- .../store/maprdb/MapRDBFormatMatcher.java | 8 ++-- .../exec/store/maprdb/MapRDBFormatPlugin.java | 37 ++++++++----------- .../exec/store/maprdb/MapRDBGroupScan.java | 15 +++++--- .../maprdb/MapRDBPushFilterIntoScan.java | 4 +- .../exec/store/maprdb/MapRDBSubScan.java | 7 +++- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java index 63cd737da35..f2e6ceb2700 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java @@ -29,12 +29,10 @@ public class MapRDBFormatMatcher extends FormatMatcher { - private FormatPlugin plugin; - private DrillFileSystem fs; + private final FormatPlugin plugin; - public MapRDBFormatMatcher(FormatPlugin plugin, DrillFileSystem fs) { + public MapRDBFormatMatcher(FormatPlugin plugin) { this.plugin = plugin; - this.fs = fs; } @Override @@ -43,7 +41,7 @@ public boolean supportDirectoryReads() { } @Override - public FormatSelection isReadable(FileSelection selection) throws IOException { + public FormatSelection isReadable(DrillFileSystem fs, FileSelection selection) throws IOException { FileStatus status = selection.getFirstPath(fs); if (status instanceof MapRFileStatus) { if (((MapRFileStatus) status).isTable()) { diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java index da3541f9f03..305d58177ec 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -36,6 +36,8 @@ import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.hadoop.conf.Configuration; + import java.io.IOException; import java.util.List; import java.util.Set; @@ -46,20 +48,20 @@ public class MapRDBFormatPlugin implements FormatPlugin{ private final StoragePluginConfig storageConfig; private final MapRDBFormatPluginConfig config; private final MapRDBFormatMatcher matcher; - private final DrillFileSystem fs; + private final Configuration fsConf; private final DrillbitContext context; private final String name; - public MapRDBFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig){ - this(name, context, fs, storageConfig, new MapRDBFormatPluginConfig()); + public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig storageConfig){ + this(name, context, fsConf, storageConfig, new MapRDBFormatPluginConfig()); } - public MapRDBFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig){ + public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig){ this.context = context; this.config = formatConfig; - this.matcher = new MapRDBFormatMatcher(this, fs); + this.matcher = new MapRDBFormatMatcher(this); this.storageConfig = storageConfig; - this.fs = fs; + this.fsConf = fsConf; this.name = name == null ? "maprdb" : name; } @Override @@ -77,33 +79,31 @@ public FormatMatcher getMatcher() { return matcher; } + public Configuration getFsConf() { + return fsConf; + } + @Override public AbstractWriter getWriter(PhysicalOperator child, String location) throws IOException { throw new UnsupportedOperationException(); } - @Override - public AbstractGroupScan getGroupScan(FileSelection selection) - throws IOException { - return getGroupScan(selection, null); - } - @Override @JsonIgnore public Set getOptimizerRules() { - return ImmutableSet.of(MapRDBPushFilterIntoScan.INSTANCE); + return ImmutableSet.of(MapRDBPushFilterIntoScan.INSTANCE); } @Override - public AbstractGroupScan getGroupScan(FileSelection selection, - List columns) throws IOException { + public AbstractGroupScan getGroupScan(String userName, FileSelection selection, + List columns) throws IOException { List files = selection.getAsFiles(); assert(files.size() == 1); String tableName = files.get(0); HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); try { - return new MapRDBGroupScan((FileSystemPlugin)(context.getStorage().getPlugin(storageConfig)), this, scanSpec, columns); + return new MapRDBGroupScan(userName, (FileSystemPlugin)(context.getStorage().getPlugin(storageConfig)), this, scanSpec, columns); } catch (ExecutionSetupException e) { e.printStackTrace(); return null; @@ -120,11 +120,6 @@ public StoragePluginConfig getStorageConfig() { return storageConfig; } - @Override - public DrillFileSystem getFileSystem() { - return fs; - } - @Override public DrillbitContext getContext() { return context; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java index 256d1f88cad..f1649aec3ab 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -102,18 +102,20 @@ public int compare(List list1, List list2) { private MapRDBTableStats tableStats; @JsonCreator - public MapRDBGroupScan(@JsonProperty("hbaseScanSpec") HBaseScanSpec hbaseScanSpec, + public MapRDBGroupScan(@JsonProperty("userName") final String userName, + @JsonProperty("hbaseScanSpec") HBaseScanSpec hbaseScanSpec, @JsonProperty("storage") FileSystemConfig storagePluginConfig, @JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig, @JsonProperty("columns") List columns, @JacksonInject StoragePluginRegistry pluginRegistry) throws IOException, ExecutionSetupException { - this ((FileSystemPlugin) pluginRegistry.getPlugin(storagePluginConfig), + this (userName, (FileSystemPlugin) pluginRegistry.getPlugin(storagePluginConfig), (MapRDBFormatPlugin) pluginRegistry.getFormatPlugin(storagePluginConfig, formatPluginConfig), hbaseScanSpec, columns); } - public MapRDBGroupScan(FileSystemPlugin storagePlugin, MapRDBFormatPlugin formatPlugin, HBaseScanSpec scanSpec, List columns) { + public MapRDBGroupScan(String userName, FileSystemPlugin storagePlugin, MapRDBFormatPlugin formatPlugin, HBaseScanSpec scanSpec, List columns) { + super(userName); this.storagePlugin = storagePlugin; this.formatPlugin = formatPlugin; this.hbaseScanSpec = scanSpec; @@ -126,6 +128,7 @@ public MapRDBGroupScan(FileSystemPlugin storagePlugin, MapRDBFormatPlugin format * @param that The HBaseGroupScan to clone */ private MapRDBGroupScan(MapRDBGroupScan that) { + super(that); this.columns = that.columns; this.hbaseScanSpec = that.hbaseScanSpec; this.endpointFragmentMapping = that.endpointFragmentMapping; @@ -346,7 +349,7 @@ public MapRDBSubScan getSpecificScan(int minorFragmentId) { assert minorFragmentId < endpointFragmentMapping.size() : String.format( "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), minorFragmentId); - return new MapRDBSubScan(storagePlugin, storagePlugin.getConfig(), endpointFragmentMapping.get(minorFragmentId), columns); + return new MapRDBSubScan(getUserName(), storagePlugin, storagePlugin.getConfig(), endpointFragmentMapping.get(minorFragmentId), columns); } @Override @@ -436,7 +439,9 @@ public boolean isFilterPushedDown() { * Empty constructor, do not use, only for testing. */ @VisibleForTesting - public MapRDBGroupScan() { } + public MapRDBGroupScan() { + super((String)null); + } /** * Do not use, only for testing. diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index d86e1eebba1..0d222595672 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -65,8 +65,8 @@ public void onMatch(RelOptRuleCall call) { return; //no filter pushdown ==> No transformation. } - final MapRDBGroupScan newGroupsScan = new MapRDBGroupScan(groupScan.getStoragePlugin(), groupScan.getFormatPlugin(), - newScanSpec, groupScan.getColumns()); + final MapRDBGroupScan newGroupsScan = new MapRDBGroupScan(groupScan.getUserName(), + groupScan.getStoragePlugin(), groupScan.getFormatPlugin(), newScanSpec, groupScan.getColumns()); newGroupsScan.setFilterPushedDown(true); final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType()); diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java index 40901e7d623..c80ae3024b4 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java @@ -59,17 +59,20 @@ public class MapRDBSubScan extends AbstractBase implements SubScan { @JsonCreator public MapRDBSubScan(@JacksonInject StoragePluginRegistry registry, + @JsonProperty("userName") String userName, @JsonProperty("storage") StoragePluginConfig storage, @JsonProperty("regionScanSpecList") LinkedList regionScanSpecList, @JsonProperty("columns") List columns) throws ExecutionSetupException { + super(userName); this.fsStoragePlugin = (FileSystemPlugin) registry.getPlugin(storage); this.regionScanSpecList = regionScanSpecList; this.storage = storage; this.columns = columns; } - public MapRDBSubScan(FileSystemPlugin storagePlugin, StoragePluginConfig config, + public MapRDBSubScan(String userName, FileSystemPlugin storagePlugin, StoragePluginConfig config, List hBaseSubScanSpecs, List columns) { + super(userName); fsStoragePlugin = storagePlugin; storage = config; this.regionScanSpecList = hBaseSubScanSpecs; @@ -97,7 +100,7 @@ public T accept(PhysicalVisitor physicalVis @Override public PhysicalOperator getNewWithChildren(List children) { Preconditions.checkArgument(children.isEmpty()); - return new MapRDBSubScan(fsStoragePlugin, storage, regionScanSpecList, columns); + return new MapRDBSubScan(getUserName(), fsStoragePlugin, storage, regionScanSpecList, columns); } @Override From c71ec6760ace959146174489e9551666880deb69 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Wed, 22 Apr 2015 11:02:15 -0700 Subject: [PATCH 13/58] MD-245: Optiq is now Calcite. --- .../drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index 0d222595672..3afee782c84 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -26,9 +26,9 @@ import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.eigenbase.rel.RelNode; -import org.eigenbase.relopt.RelOptRuleCall; -import org.eigenbase.rex.RexNode; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rex.RexNode; import com.google.common.collect.ImmutableList; From ec901a5f1412e9e119f243745b816532365d54fb Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Mon, 4 May 2015 12:08:12 -0700 Subject: [PATCH 14/58] Updated version to 1.0.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index a70591a3075..7fa7691e8a5 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 0.9.0-SNAPSHOT + 1.0.0-SNAPSHOT drill-storage-maprdb From 1c43e3d2f23cf96027d0b2940c7a6362880a5781 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Tue, 5 May 2015 17:28:37 -0700 Subject: [PATCH 15/58] DRILL-2826: Simplify and centralize Operator cleanup --- .../drill/exec/store/maprdb/MapRDBScanBatchCreator.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java index 74ea7a8b304..1beabc929a1 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java @@ -17,8 +17,8 @@ */ package org.apache.drill.exec.store.maprdb; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import java.util.List; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.BatchCreator; @@ -29,13 +29,14 @@ import org.apache.drill.exec.store.hbase.HBaseSubScan; import org.apache.hadoop.hbase.HBaseConfiguration; -import java.util.List; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; public class MapRDBScanBatchCreator implements BatchCreator{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBScanBatchCreator.class); @Override - public RecordBatch getBatch(FragmentContext context, MapRDBSubScan subScan, List children) throws ExecutionSetupException { + public ScanBatch getBatch(FragmentContext context, MapRDBSubScan subScan, List children) throws ExecutionSetupException { Preconditions.checkArgument(children.isEmpty()); List readers = Lists.newArrayList(); for(HBaseSubScan.HBaseSubScanSpec scanSpec : subScan.getRegionScanSpecList()){ From d7fe7825ce833c37a320ab7b94b07f5440a5c69f Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Mon, 11 May 2015 15:40:33 -0700 Subject: [PATCH 16/58] MD-272: Incorporate DRILL-2848 (option to disable decimal data type) --- .../drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index 3afee782c84..7d79a8e5edc 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.planner.logical.DrillParseContext; import org.apache.drill.exec.planner.logical.RelOptHelper; import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.apache.drill.exec.store.hbase.HBaseScanSpec; @@ -58,7 +59,7 @@ public void onMatch(RelOptRuleCall call) { return; } - LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(), scan, condition); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); MapRDBFilterBuilder hbaseFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp); HBaseScanSpec newScanSpec = hbaseFilterBuilder.parseTree(); if (newScanSpec == null) { From a19d0164a17952d44ece1e98b0f6192ecfb5b2e8 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Wed, 20 May 2015 14:52:48 -0700 Subject: [PATCH 17/58] Updated plugin version to 1.1.0-SNAPSHOT +Using MapR v4.1.0 dependencies --- contrib/format-maprdb/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index 7fa7691e8a5..2e4ef35f838 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 1.0.0-SNAPSHOT + 1.1.0-SNAPSHOT drill-storage-maprdb @@ -50,7 +50,7 @@ com.mapr.fs mapr-hbase - 4.0.1.28318-mapr + 4.1.0-mapr @@ -79,7 +79,7 @@ mapr-releases - http://maven.corp.maprtech.com/nexus/content/repositories/public + http://repository.mapr.com/nexus/content/repositories/releases true From c5c24792b83f71826aea61ec9870146de48cfe98 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Fri, 26 Aug 2016 11:37:00 -0700 Subject: [PATCH 18/58] MD-290: Fix build failure due to upstream changes. + Includes MD-292 --- .../exec/store/maprdb/MapRDBFormatPlugin.java | 177 +++++++++--------- 1 file changed, 93 insertions(+), 84 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java index 305d58177ec..eb72db0681e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -17,8 +17,9 @@ */ package org.apache.drill.exec.store.maprdb; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.List; +import java.util.Set; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; @@ -33,101 +34,109 @@ import org.apache.drill.exec.store.dfs.FileSystemPlugin; import org.apache.drill.exec.store.dfs.FormatMatcher; import org.apache.drill.exec.store.dfs.FormatPlugin; -import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.drill.exec.store.hbase.HBaseScanSpec; - import org.apache.hadoop.conf.Configuration; -import java.io.IOException; -import java.util.List; -import java.util.Set; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.ImmutableSet; -public class MapRDBFormatPlugin implements FormatPlugin{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBFormatPlugin.class); - - private final StoragePluginConfig storageConfig; - private final MapRDBFormatPluginConfig config; - private final MapRDBFormatMatcher matcher; - private final Configuration fsConf; - private final DrillbitContext context; - private final String name; - - public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig storageConfig){ - this(name, context, fsConf, storageConfig, new MapRDBFormatPluginConfig()); - } +public class MapRDBFormatPlugin implements FormatPlugin { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(MapRDBFormatPlugin.class); - public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig){ - this.context = context; - this.config = formatConfig; - this.matcher = new MapRDBFormatMatcher(this); - this.storageConfig = storageConfig; - this.fsConf = fsConf; - this.name = name == null ? "maprdb" : name; - } - @Override - public boolean supportsRead() { - return true; - } - - @Override - public boolean supportsWrite() { - return false; - } - - @Override - public FormatMatcher getMatcher() { + private final StoragePluginConfig storageConfig; + private final MapRDBFormatPluginConfig config; + private final MapRDBFormatMatcher matcher; + private final Configuration fsConf; + private final DrillbitContext context; + private final String name; + + public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, + StoragePluginConfig storageConfig) { + this(name, context, fsConf, storageConfig, new MapRDBFormatPluginConfig()); + } + + public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, + StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig) { + this.context = context; + this.config = formatConfig; + this.matcher = new MapRDBFormatMatcher(this); + this.storageConfig = storageConfig; + this.fsConf = fsConf; + this.name = name == null ? "maprdb" : name; + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public boolean supportsWrite() { + return false; + } + + @Override + public boolean supportsAutoPartitioning() { + return false; + } + + @Override + public FormatMatcher getMatcher() { return matcher; - } + } public Configuration getFsConf() { return fsConf; } - @Override - public AbstractWriter getWriter(PhysicalOperator child, String location) - throws IOException { - throw new UnsupportedOperationException(); - } + @Override + public AbstractWriter getWriter(PhysicalOperator child, String location, + List partitionColumns) throws IOException { + throw new UnsupportedOperationException(); + } - @Override + @Override @JsonIgnore - public Set getOptimizerRules() { - return ImmutableSet.of(MapRDBPushFilterIntoScan.INSTANCE); - } - - @Override - public AbstractGroupScan getGroupScan(String userName, FileSelection selection, - List columns) throws IOException { - List files = selection.getAsFiles(); - assert(files.size() == 1); - String tableName = files.get(0); - HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); - try { - return new MapRDBGroupScan(userName, (FileSystemPlugin)(context.getStorage().getPlugin(storageConfig)), this, scanSpec, columns); - } catch (ExecutionSetupException e) { - e.printStackTrace(); - return null; - } + public Set getOptimizerRules() { + return ImmutableSet.of(MapRDBPushFilterIntoScan.INSTANCE); + } + + @Override + public AbstractGroupScan getGroupScan(String userName, FileSelection selection, + List columns) throws IOException { + List files = selection.getAsFiles(); + assert (files.size() == 1); + String tableName = files.get(0); + HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); + try { + return new MapRDBGroupScan(userName, + (FileSystemPlugin) (context.getStorage().getPlugin(storageConfig)), this, scanSpec, + columns); + } catch (ExecutionSetupException e) { + e.printStackTrace(); + return null; } + } + + @Override + public FormatPluginConfig getConfig() { + return config; + } + + @Override + public StoragePluginConfig getStorageConfig() { + return storageConfig; + } + + @Override + public DrillbitContext getContext() { + return context; + } + + @Override + public String getName() { + return name; + } - @Override - public FormatPluginConfig getConfig() { - return config; - } - - @Override - public StoragePluginConfig getStorageConfig() { - return storageConfig; - } - - @Override - public DrillbitContext getContext() { - return context; - } - - @Override - public String getName() { - return name; - } - } From ba8a785c9c932f7e9e90a28f1b67042e50126597 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Sun, 5 Jul 2015 20:30:03 -0700 Subject: [PATCH 19/58] Updated version to 1.2.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index 2e4ef35f838..b862cfa3166 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 1.1.0-SNAPSHOT + 1.2.0-SNAPSHOT drill-storage-maprdb From df19a019117db3448a01f34daf804df747ca94b8 Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Thu, 6 Aug 2015 15:50:44 -0700 Subject: [PATCH 20/58] Port DRILL-3492 and DRILL-3364 to maprdb plugin + Fix build issues due to API changes in DRILL-3535 --- .../maprdb/CompareFunctionsProcessor.java | 421 +++++++++++++++--- .../store/maprdb/MapRDBFilterBuilder.java | 79 +++- .../store/maprdb/MapRDBFormatMatcher.java | 14 +- .../exec/store/maprdb/MapRDBFormatPlugin.java | 2 +- .../maprdb/MapRDBPushFilterIntoScan.java | 130 ++++-- 5 files changed, 529 insertions(+), 117 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java index de8e080a181..c6c25047c33 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java @@ -20,6 +20,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.apache.drill.common.expression.CastExpression; @@ -35,7 +36,16 @@ import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.expression.ValueExpressions.TimeExpression; +import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; + +import org.apache.drill.exec.store.hbase.DrillHBaseConstants; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.PrefixFilter; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; @@ -47,6 +57,15 @@ class CompareFunctionsProcessor extends AbstractExprVisitor= 2 ? call.args.get(1) : null; + LogicalExpression valueArg2 = call.args.size() >= 3 ? call.args.get(2) : null; + + if (((nameArg instanceof SchemaPath) == false) || + (valueArg1 == null) || ((valueArg1 instanceof IntExpression) == false) || + (valueArg2 == null) || ((valueArg2 instanceof IntExpression) == false)) { + return false; } - break; - case "DOUBLE": - if (valueArg instanceof DoubleExpression && isEqualityFn) { - bb = newByteBuf(8, true); - bb.writeDouble(((DoubleExpression)valueArg).getDouble()); + + boolean isRowKey = ((SchemaPath)nameArg).getAsUnescapedPath().equals(DrillHBaseConstants.ROW_KEY); + int offset = ((IntExpression)valueArg1).getInt(); + + if (!isRowKey || (offset != 1)) { + return false; + } + + this.path = (SchemaPath)nameArg; + prefixLength = ((IntExpression)valueArg2).getInt(); + this.isRowKeyPrefixComparison = true; + return visitRowKeyPrefixConvertExpression(e, prefixLength, valueArg); + } + + if (e.getInput() instanceof SchemaPath) { + ByteBuf bb = null; + + switch (encodingType) { + case "INT_BE": + case "INT": + case "UINT_BE": + case "UINT": + case "UINT4_BE": + case "UINT4": + if (valueArg instanceof IntExpression + && (isEqualityFn || encodingType.startsWith("U"))) { + bb = newByteBuf(4, encodingType.endsWith("_BE")); + bb.writeInt(((IntExpression)valueArg).getInt()); + } + break; + case "BIGINT_BE": + case "BIGINT": + case "UINT8_BE": + case "UINT8": + if (valueArg instanceof LongExpression + && (isEqualityFn || encodingType.startsWith("U"))) { + bb = newByteBuf(8, encodingType.endsWith("_BE")); + bb.writeLong(((LongExpression)valueArg).getLong()); + } + break; + case "FLOAT": + if (valueArg instanceof FloatExpression && isEqualityFn) { + bb = newByteBuf(4, true); + bb.writeFloat(((FloatExpression)valueArg).getFloat()); + } + break; + case "DOUBLE": + if (valueArg instanceof DoubleExpression && isEqualityFn) { + bb = newByteBuf(8, true); + bb.writeDouble(((DoubleExpression)valueArg).getDouble()); + } + break; + case "TIME_EPOCH": + case "TIME_EPOCH_BE": + if (valueArg instanceof TimeExpression) { + bb = newByteBuf(8, encodingType.endsWith("_BE")); + bb.writeLong(((TimeExpression)valueArg).getTime()); + } + break; + case "DATE_EPOCH": + case "DATE_EPOCH_BE": + if (valueArg instanceof DateExpression) { + bb = newByteBuf(8, encodingType.endsWith("_BE")); + bb.writeLong(((DateExpression)valueArg).getDate()); + } + break; + case "BOOLEAN_BYTE": + if (valueArg instanceof BooleanExpression) { + bb = newByteBuf(1, false /* does not matter */); + bb.writeByte(((BooleanExpression)valueArg).getBoolean() ? 1 : 0); + } + break; + case "DOUBLE_OB": + case "DOUBLE_OBD": + if (valueArg instanceof DoubleExpression) { + bb = newByteBuf(9, true); + PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, 9); + if (encodingType.endsWith("_OBD")) { + org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br, + ((DoubleExpression)valueArg).getDouble(), Order.DESCENDING); + this.sortOrderAscending = false; + } else { + org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br, + ((DoubleExpression)valueArg).getDouble(), Order.ASCENDING); + } + } + break; + case "FLOAT_OB": + case "FLOAT_OBD": + if (valueArg instanceof FloatExpression) { + bb = newByteBuf(5, true); + PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, 5); + if (encodingType.endsWith("_OBD")) { + org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br, + ((FloatExpression)valueArg).getFloat(), Order.DESCENDING); + this.sortOrderAscending = false; + } else { + org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br, + ((FloatExpression)valueArg).getFloat(), Order.ASCENDING); + } + } + break; + case "BIGINT_OB": + case "BIGINT_OBD": + if (valueArg instanceof LongExpression) { + bb = newByteBuf(9, true); + PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, 9); + if (encodingType.endsWith("_OBD")) { + org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br, + ((LongExpression)valueArg).getLong(), Order.DESCENDING); + this.sortOrderAscending = false; + } else { + org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br, + ((LongExpression)valueArg).getLong(), Order.ASCENDING); + } + } + break; + case "INT_OB": + case "INT_OBD": + if (valueArg instanceof IntExpression) { + bb = newByteBuf(5, true); + PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, 5); + if (encodingType.endsWith("_OBD")) { + org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br, + ((IntExpression)valueArg).getInt(), Order.DESCENDING); + this.sortOrderAscending = false; + } else { + org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br, + ((IntExpression)valueArg).getInt(), Order.ASCENDING); + } + } + break; + case "UTF8_OB": + case "UTF8_OBD": + if (valueArg instanceof QuotedString) { + int stringLen = ((QuotedString) valueArg).value.getBytes(Charsets.UTF_8).length; + bb = newByteBuf(stringLen + 2, true); + PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, stringLen + 2); + if (encodingType.endsWith("_OBD")) { + org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br, + ((QuotedString)valueArg).value, Order.DESCENDING); + this.sortOrderAscending = false; + } else { + org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br, + ((QuotedString)valueArg).value, Order.ASCENDING); + } + } + break; + case "UTF8": + // let visitSchemaPath() handle this. + return e.getInput().accept(this, valueArg); + } + + if (bb != null) { + this.value = bb.array(); + this.path = (SchemaPath)e.getInput(); + return true; } - break; - case "TIME_EPOCH": - case "TIME_EPOCH_BE": - if (valueArg instanceof TimeExpression) { - bb = newByteBuf(8, encodingType.endsWith("_BE")); - bb.writeLong(((TimeExpression)valueArg).getTime()); + } + } + return false; + } + + private Boolean visitRowKeyPrefixConvertExpression(ConvertExpression e, + int prefixLength, LogicalExpression valueArg) { + String encodingType = e.getEncodingType(); + rowKeyPrefixStartRow = HConstants.EMPTY_START_ROW; + rowKeyPrefixStopRow = HConstants.EMPTY_START_ROW; + rowKeyPrefixFilter = null; + + if ((encodingType.compareTo("UINT4_BE") == 0) || + (encodingType.compareTo("UINT_BE") == 0)) { + if (prefixLength != 4) { + throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); + } + + int val; + if ((valueArg instanceof IntExpression) == false) { + return false; + } + + val = ((IntExpression)valueArg).getInt(); + + // For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >= + switch (functionName) { + case "equal": + rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(4).putInt(val).array()); + rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array(); + rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array(); + return true; + case "greater_than_or_equal_to": + rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array(); + return true; + case "greater_than": + rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val + 1).array(); + return true; + case "less_than_or_equal_to": + rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array(); + return true; + case "less_than": + rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val).array(); + return true; + } + + return false; + } + + if ((encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) || + (encodingType.compareTo("TIME_EPOCH_BE") == 0) || + (encodingType.compareTo("UINT8_BE") == 0)) { + + if (prefixLength != 8) { + throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); + } + + long val; + if (encodingType.compareTo("TIME_EPOCH_BE") == 0) { + if ((valueArg instanceof TimeExpression) == false) { + return false; } - break; - case "DATE_EPOCH": - case "DATE_EPOCH_BE": - if (valueArg instanceof DateExpression) { - bb = newByteBuf(8, encodingType.endsWith("_BE")); - bb.writeLong(((DateExpression)valueArg).getDate()); + + val = ((TimeExpression)valueArg).getTime(); + } else if (encodingType.compareTo("UINT8_BE") == 0){ + if ((valueArg instanceof LongExpression) == false) { + return false; } - break; - case "BOOLEAN_BYTE": - if (valueArg instanceof BooleanExpression) { - bb = newByteBuf(1, false /* does not matter */); - bb.writeByte(((BooleanExpression)valueArg).getBoolean() ? 1 : 0); + + val = ((LongExpression)valueArg).getLong(); + } else if (encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) { + if ((valueArg instanceof TimeStampExpression) == false) { + return false; } - break; - case "UTF8": - // let visitSchemaPath() handle this. - return e.getInput().accept(this, valueArg); + + val = ((TimeStampExpression)valueArg).getTimeStamp(); + } else { + // Should not reach here. + return false; } - if (bb != null) { - this.value = bb.array(); - this.path = (SchemaPath)e.getInput(); + // For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >= + switch (functionName) { + case "equal": + rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(8).putLong(val).array()); + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array(); + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array(); + return true; + case "greater_than_or_equal_to": + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array(); return true; + case "greater_than": + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val + 1).array(); + return true; + case "less_than_or_equal_to": + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array(); + return true; + case "less_than": + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val).array(); + return true; + } + + return false; + } + + if (encodingType.compareTo("DATE_EPOCH_BE") == 0) { + if ((valueArg instanceof DateExpression) == false) { + return false; + } + + if (prefixLength != 8) { + throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); } + + final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; + long dateToSet; + // For DATE encoding, the operators that we push-down are =, <>, <, <=, >, >= + switch (functionName) { + case "equal": + long startDate = ((DateExpression)valueArg).getDate(); + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(startDate).array(); + long stopDate = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY; + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(stopDate).array(); + return true; + case "greater_than_or_equal_to": + dateToSet = ((DateExpression)valueArg).getDate(); + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); + return true; + case "greater_than": + dateToSet = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY; + rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); + return true; + case "less_than_or_equal_to": + dateToSet = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY; + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); + return true; + case "less_than": + dateToSet = ((DateExpression)valueArg).getDate(); + rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); + return true; + } + + return false; } + return false; } @@ -237,4 +544,4 @@ private static ByteBuf newByteBuf(int size, boolean bigEndian) { .build(); } -} \ No newline at end of file +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java index 45fe22f5f13..a4a39383c30 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.NullComparator; +import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; @@ -159,6 +160,7 @@ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsPro String functionName = processor.getFunctionName(); SchemaPath field = processor.getPath(); byte[] fieldValue = processor.getValue(); + boolean sortOrderAscending = processor.isSortOrderAscending(); boolean isRowKey = field.getAsUnescapedPath().equals(ROW_KEY); if (!(isRowKey || (!field.getRootSegment().isLastPath() @@ -172,6 +174,10 @@ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsPro return null; } + if (processor.isRowKeyPrefixComparison()) { + return createRowKeyPrefixScanSpec(call, processor); + } + CompareOp compareOp = null; boolean isNullTest = false; ByteArrayComparable comparator = new BinaryComparator(fieldValue); @@ -189,29 +195,59 @@ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsPro compareOp = CompareOp.NOT_EQUAL; break; case "greater_than_or_equal_to": - compareOp = CompareOp.GREATER_OR_EQUAL; - if (isRowKey) { - startRow = fieldValue; + if (sortOrderAscending) { + compareOp = CompareOp.GREATER_OR_EQUAL; + if (isRowKey) { + startRow = fieldValue; + } + } else { + compareOp = CompareOp.LESS_OR_EQUAL; + if (isRowKey) { + // stopRow should be just greater than 'value' + stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + } } break; case "greater_than": - compareOp = CompareOp.GREATER; - if (isRowKey) { - // startRow should be just greater than 'value' - startRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + if (sortOrderAscending) { + compareOp = CompareOp.GREATER; + if (isRowKey) { + // startRow should be just greater than 'value' + startRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + } + } else { + compareOp = CompareOp.LESS; + if (isRowKey) { + stopRow = fieldValue; + } } break; case "less_than_or_equal_to": - compareOp = CompareOp.LESS_OR_EQUAL; - if (isRowKey) { - // stopRow should be just greater than 'value' - stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + if (sortOrderAscending) { + compareOp = CompareOp.LESS_OR_EQUAL; + if (isRowKey) { + // stopRow should be just greater than 'value' + stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + } + } else { + compareOp = CompareOp.GREATER_OR_EQUAL; + if (isRowKey) { + startRow = fieldValue; + } } break; case "less_than": - compareOp = CompareOp.LESS; - if (isRowKey) { - stopRow = fieldValue; + if (sortOrderAscending) { + compareOp = CompareOp.LESS; + if (isRowKey) { + stopRow = fieldValue; + } + } else { + compareOp = CompareOp.GREATER; + if (isRowKey) { + // startRow should be just greater than 'value' + startRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + } } break; case "isnull": @@ -299,4 +335,19 @@ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsPro return null; } + private HBaseScanSpec createRowKeyPrefixScanSpec(FunctionCall call, + CompareFunctionsProcessor processor) { + byte[] startRow = processor.getRowKeyPrefixStartRow(); + byte[] stopRow = processor.getRowKeyPrefixStopRow(); + Filter filter = processor.getRowKeyPrefixFilter(); + + if (startRow != HConstants.EMPTY_START_ROW || + stopRow != HConstants.EMPTY_END_ROW || + filter != null) { + return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter); + } + + // else + return null; + } } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java index f2e6ceb2700..b048e3759c0 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java @@ -43,12 +43,16 @@ public boolean supportDirectoryReads() { @Override public FormatSelection isReadable(DrillFileSystem fs, FileSelection selection) throws IOException { FileStatus status = selection.getFirstPath(fs); - if (status instanceof MapRFileStatus) { - if (((MapRFileStatus) status).isTable()) { - return new FormatSelection(getFormatPlugin().getConfig(), selection); - } + if (!isFileReadable(fs, status)) { + return null; } - return null; + + return new FormatSelection(getFormatPlugin().getConfig(), selection); + } + + @Override + public boolean isFileReadable(DrillFileSystem fs, FileStatus status) throws IOException { + return (status instanceof MapRFileStatus) && ((MapRFileStatus) status).isTable(); } @Override diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java index eb72db0681e..6a10d224d24 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -99,7 +99,7 @@ public AbstractWriter getWriter(PhysicalOperator child, String location, @Override @JsonIgnore public Set getOptimizerRules() { - return ImmutableSet.of(MapRDBPushFilterIntoScan.INSTANCE); + return ImmutableSet.of(MapRDBPushFilterIntoScan.FILTER_ON_SCAN, MapRDBPushFilterIntoScan.FILTER_ON_PROJECT); } @Override diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index 7d79a8e5edc..50f3d95cfd0 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -24,71 +24,121 @@ import org.apache.drill.exec.planner.logical.RelOptHelper; import org.apache.drill.exec.planner.physical.FilterPrel; import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ProjectPrel; import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.apache.drill.exec.store.hbase.HBaseScanSpec; import org.apache.calcite.rel.RelNode; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rex.RexNode; import com.google.common.collect.ImmutableList; -public class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRule { +public abstract class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRule { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBPushFilterIntoScan.class); - public static final StoragePluginOptimizerRule INSTANCE = new MapRDBPushFilterIntoScan(); +// public static final StoragePluginOptimizerRule INSTANCE = new MapRDBPushFilterIntoScan(); - private MapRDBPushFilterIntoScan() { - super(RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), "MapRDBPushFilterIntoScan"); + private MapRDBPushFilterIntoScan(RelOptRuleOperand operand, String description) { + super(operand, description); } - @Override - public void onMatch(RelOptRuleCall call) { - final ScanPrel scan = (ScanPrel) call.rel(1); - final FilterPrel filter = (FilterPrel) call.rel(0); - final RexNode condition = filter.getCondition(); - - MapRDBGroupScan groupScan = (MapRDBGroupScan)scan.getGroupScan(); - if (groupScan.isFilterPushedDown()) { - /* - * The rule can get triggered again due to the transformed "scan => filter" sequence - * created by the earlier execution of this rule when we could not do a complete - * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon - * this flag to not do a re-processing of the rule on the already transformed call. - */ - return; + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new MapRDBPushFilterIntoScan(RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), "MapRDBPushFilterIntoScan:Filter_On_Scan") { + + @Override + public void onMatch(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + final FilterPrel filter = (FilterPrel) call.rel(0); + final RexNode condition = filter.getCondition(); + + MapRDBGroupScan groupScan = (MapRDBGroupScan)scan.getGroupScan(); + if (groupScan.isFilterPushedDown()) { + /* + * The rule can get triggered again due to the transformed "scan => filter" sequence + * created by the earlier execution of this rule when we could not do a complete + * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon + * this flag to not do a re-processing of the rule on the already transformed call. + */ + return; + } + + doPushFilterToScan(call, filter, null, scan, groupScan, condition); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof MapRDBGroupScan) { + return super.matches(call); + } + return false; } + }; - LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); - MapRDBFilterBuilder hbaseFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp); - HBaseScanSpec newScanSpec = hbaseFilterBuilder.parseTree(); + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new MapRDBPushFilterIntoScan(RelOptHelper.some(FilterPrel.class, RelOptHelper.some(ProjectPrel.class, RelOptHelper.any(ScanPrel.class))), "MapRDBPushFilterIntoScan:Filter_On_Project") { + + @Override + public void onMatch(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(2); + final ProjectPrel project = (ProjectPrel) call.rel(1); + final FilterPrel filter = (FilterPrel) call.rel(0); + + MapRDBGroupScan groupScan = (MapRDBGroupScan)scan.getGroupScan(); + if (groupScan.isFilterPushedDown()) { + /* + * The rule can get triggered again due to the transformed "scan => filter" sequence + * created by the earlier execution of this rule when we could not do a complete + * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon + * this flag to not do a re-processing of the rule on the already transformed call. + */ + return; + } + + // convert the filter to one that references the child of the project + final RexNode condition = RelOptUtil.pushFilterPastProject(filter.getCondition(), project); + + doPushFilterToScan(call, filter, project, scan, groupScan, condition); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(2); + if (scan.getGroupScan() instanceof MapRDBGroupScan) { + return super.matches(call); + } + return false; + } + }; + + protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel filter, final ProjectPrel project, final ScanPrel scan, final MapRDBGroupScan groupScan, final RexNode condition) { + + final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + final MapRDBFilterBuilder maprdbFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp); + final HBaseScanSpec newScanSpec = maprdbFilterBuilder.parseTree(); if (newScanSpec == null) { return; //no filter pushdown ==> No transformation. } - final MapRDBGroupScan newGroupsScan = new MapRDBGroupScan(groupScan.getUserName(), - groupScan.getStoragePlugin(), groupScan.getFormatPlugin(), newScanSpec, groupScan.getColumns()); + final MapRDBGroupScan newGroupsScan = new MapRDBGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), + groupScan.getFormatPlugin(), newScanSpec, groupScan.getColumns()); newGroupsScan.setFilterPushedDown(true); final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType()); - if (hbaseFilterBuilder.isAllExpressionsConverted()) { - /* - * Since we could convert the entire filter condition expression into an HBase filter, - * we can eliminate the filter operator altogether. - */ - call.transformTo(newScanPrel); - } else { - call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of((RelNode)newScanPrel))); - } - } - @Override - public boolean matches(RelOptRuleCall call) { - final ScanPrel scan = (ScanPrel) call.rel(1); - if (scan.getGroupScan() instanceof MapRDBGroupScan) { - return super.matches(call); + // Depending on whether is a project in the middle, assign either scan or copy of project to childRel. + final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode)newScanPrel));; + + if (maprdbFilterBuilder.isAllExpressionsConverted()) { + /* + * Since we could convert the entire filter condition expression into an HBase filter, + * we can eliminate the filter operator altogether. + */ + call.transformTo(childRel); + } else { + call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel))); } - return false; } } From a0f9658e446d22ddaf12f8f4512832e997ad9253 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Mon, 19 Oct 2015 13:03:14 -0700 Subject: [PATCH 21/58] Updated plugin version to 1.3.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index b862cfa3166..e8a7b75600b 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT drill-storage-maprdb From 2a45749ad59a807474e006ba9d03b31c290c28aa Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 5 Nov 2015 17:09:51 -0800 Subject: [PATCH 22/58] MD-524: Drill API change broke the MapRDB Plugin --- .../exec/store/maprdb/MapRDBFormatMatcher.java | 18 ++++++++++++------ .../drill/exec/store/maprdb/MapRDBSubScan.java | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java index b048e3759c0..fa084f0c3d1 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java @@ -17,15 +17,19 @@ */ package org.apache.drill.exec.store.maprdb; -import com.mapr.fs.MapRFileStatus; +import java.io.IOException; + +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DynamicDrillTable; +import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; import org.apache.drill.exec.store.dfs.FormatMatcher; import org.apache.drill.exec.store.dfs.FormatPlugin; import org.apache.drill.exec.store.dfs.FormatSelection; -import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.hadoop.fs.FileStatus; -import java.io.IOException; +import com.mapr.fs.MapRFileStatus; public class MapRDBFormatMatcher extends FormatMatcher { @@ -40,14 +44,16 @@ public boolean supportDirectoryReads() { return false; } - @Override - public FormatSelection isReadable(DrillFileSystem fs, FileSelection selection) throws IOException { + public DrillTable isReadable(DrillFileSystem fs, + FileSelection selection, FileSystemPlugin fsPlugin, + String storageEngineName, String userName) throws IOException { FileStatus status = selection.getFirstPath(fs); if (!isFileReadable(fs, status)) { return null; } - return new FormatSelection(getFormatPlugin().getConfig(), selection); + return new DynamicDrillTable(fsPlugin, storageEngineName, userName, + new FormatSelection(getFormatPlugin().getConfig(), selection)); } @Override diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java index c80ae3024b4..57ef0b65497 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java @@ -116,7 +116,7 @@ public static class HBaseSubScanSpec { protected byte[] stopRow; protected byte[] serializedFilter; - @parquet.org.codehaus.jackson.annotate.JsonCreator + @JsonCreator public HBaseSubScanSpec(@JsonProperty("tableName") String tableName, @JsonProperty("regionServer") String regionServer, @JsonProperty("startRow") byte[] startRow, From 75aa0511f174288c612f105fc8aebdc6bb40d7e1 Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 6 Nov 2015 15:45:21 -0800 Subject: [PATCH 23/58] Applying DRILL-3621 (Wrong results when Drill on HBase query contains rowkey "or" or "IN") to MapRDB plugin Already in json_support branch, exclude while merging. --- .../apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java index a4a39383c30..a2407199a2f 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java @@ -187,8 +187,10 @@ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsPro case "equal": compareOp = CompareOp.EQUAL; if (isRowKey) { - startRow = stopRow = fieldValue; - compareOp = null; + startRow = fieldValue; + /* stopRow should be just greater than 'value'*/ + stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1); + compareOp = CompareOp.EQUAL; } break; case "not_equal": From 9bd65246e58271ad787ba6deb94de7c071ec4839 Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 6 Nov 2015 15:56:36 -0800 Subject: [PATCH 24/58] Applying DRILL-4020 (The not-equal operator returns incorrect results when used on the HBase row key) --- .../org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java index a2407199a2f..857d7990a38 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java @@ -67,6 +67,7 @@ public HBaseScanSpec parseTree() { */ Filter filter = parsedSpec.getFilter(); if (filter instanceof RowFilter && + ((RowFilter)filter).getOperator() != CompareOp.NOT_EQUAL && ((RowFilter)filter).getComparator() instanceof BinaryComparator) { filter = null; } From 3a3c956d95e0636d07dc08f4cfb46e0324f378d6 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 3 Dec 2015 11:54:17 -0800 Subject: [PATCH 25/58] Updating the plugin version to 1.4.0-SNAPSHOT and fixing for interface change. --- contrib/format-maprdb/pom.xml | 2 +- .../org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index e8a7b75600b..4c6770b94f2 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 1.3.0-SNAPSHOT + 1.4.0-SNAPSHOT drill-storage-maprdb diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java index 6a10d224d24..aa9190137c2 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -105,7 +105,7 @@ public Set getOptimizerRules() { @Override public AbstractGroupScan getGroupScan(String userName, FileSelection selection, List columns) throws IOException { - List files = selection.getAsFiles(); + List files = selection.getFiles(); assert (files.size() == 1); String tableName = files.get(0); HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); From fb8872bd49fe615ce3d9ee4310e91621d0474568 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Tue, 8 Dec 2015 11:25:16 -0800 Subject: [PATCH 26/58] Updated plugin version to 1.5.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index 4c6770b94f2..9e299b54cb2 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,7 +21,7 @@ drill-contrib-parent org.apache.drill.contrib - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT drill-storage-maprdb From 115a85e7611f39394b046a713578c8da6a52932e Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Wed, 10 Feb 2016 17:57:37 +0000 Subject: [PATCH 27/58] Fix the private-drill-mapr-plugin build. --- .../org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java index f1649aec3ab..9358753dadb 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -89,7 +89,7 @@ public int compare(List list1, List list2) { private MapRDBFormatPlugin formatPlugin; - private Stopwatch watch = new Stopwatch(); + private Stopwatch watch = Stopwatch.createUnstarted(); private Map> endpointFragmentMapping; From e19c048c4b3a78df6582531eaa80c188631657f7 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 16 Jul 2015 17:59:52 -0700 Subject: [PATCH 28/58] Cleaning up code. Removed some unnecessary files from the module. --- .../resources/bootstrap-storage-plugins.json | 12 -- .../src/main/resources/drill-module.conf | 16 +- .../src/test/resources/hbase-site.xml | 143 ------------------ .../hbase/hbase_scan_screen_physical.json | 29 ---- ...se_scan_screen_physical_column_select.json | 32 ---- ...se_scan_screen_physical_family_select.json | 32 ---- 6 files changed, 1 insertion(+), 263 deletions(-) delete mode 100644 contrib/format-maprdb/src/main/resources/bootstrap-storage-plugins.json delete mode 100644 contrib/format-maprdb/src/test/resources/hbase-site.xml delete mode 100644 contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical.json delete mode 100644 contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json delete mode 100644 contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json diff --git a/contrib/format-maprdb/src/main/resources/bootstrap-storage-plugins.json b/contrib/format-maprdb/src/main/resources/bootstrap-storage-plugins.json deleted file mode 100644 index 3e0e8c04174..00000000000 --- a/contrib/format-maprdb/src/main/resources/bootstrap-storage-plugins.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "storage":{ - hbase : { - type:"hbase", - enabled: false, - config : { - "hbase.zookeeper.quorum" : "localhost", - "hbase.zookeeper.property.clientPort" : 2181 - } - } - } -} diff --git a/contrib/format-maprdb/src/main/resources/drill-module.conf b/contrib/format-maprdb/src/main/resources/drill-module.conf index 0edceafadbc..8d42355f368 100644 --- a/contrib/format-maprdb/src/main/resources/drill-module.conf +++ b/contrib/format-maprdb/src/main/resources/drill-module.conf @@ -17,18 +17,4 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.exec: { - - sys.store.provider: { - # HBasePStoreProvider is disabled by default - # class: "org.apache.drill.exec.store.hbase.config.HBasePStoreProvider", - hbase: { - table : "drill_store", - config : { - "hbase.zookeeper.quorum" : "localhost", - "hbase.zookeeper.property.clientPort" : 2181 - } - } - } - -} +# This file currently does not contain any configuration \ No newline at end of file diff --git a/contrib/format-maprdb/src/test/resources/hbase-site.xml b/contrib/format-maprdb/src/test/resources/hbase-site.xml deleted file mode 100644 index 54425d440c6..00000000000 --- a/contrib/format-maprdb/src/test/resources/hbase-site.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - - - - hbase.regionserver.msginterval - 1000 - Interval between messages from the RegionServer to HMaster - in milliseconds. Default is 15. Set this value low if you want unit - tests to be responsive. - - - - hbase.client.pause - 1000 - General client pause value. Used mostly as value to wait - before running a retry of a failed get, region lookup, etc. - - - hbase.client.retries.number - 10 - Maximum retries. Used as maximum for all retryable - operations such as fetching of the root region from root region - server, getting a cell's value, starting a row update, etc. - Default: 10. - - - - hbase.server.thread.wakefrequency - 1000 - Time to sleep in between searches for work (in milliseconds). - Used as sleep interval by service threads such as META scanner and log roller. - - - - hbase.master.event.waiting.time - 50 - Time to sleep between checks to see if a table event took place. - - - - hbase.regionserver.handler.count - 5 - Count of RPC Server instances spun up on RegionServers - Same property is used by the HMaster for count of master handlers. - Default is 10. - - - - - hbase.regionserver.info.port - -1 - The port for the hbase regionserver web UI - Set to -1 if you do not want the info server to run. - - - - hbase.regionserver.info.port.auto - true - Info server auto port bind. Enables automatic port - search if hbase.regionserver.info.port is already in use. - Enabled for testing to run multiple tests on one machine. - - - - hbase.master.lease.thread.wakefrequency - 3000 - The interval between checks for expired region server leases. - This value has been reduced due to the other reduced values above so that - the master will notice a dead region server sooner. The default is 15 seconds. - - - - hbase.regionserver.safemode - false - - Turn on/off safe mode in region server. Always on for production, always off - for tests. - - - - hbase.hregion.max.filesize - 67108864 - - Maximum desired file size for an HRegion. If filesize exceeds - value + (value / 2), the HRegion is split in two. Default: 256M. - - Keep the maximum filesize small so we split more often in tests. - - - - hbase.zookeeper.property.clientPort - 2181 - Property from ZooKeeper's config zoo.cfg. - The port at which the clients will connect. - - - - hbase.defaults.for.version.skip - true - - Set to true to skip the 'hbase.defaults.for.version'. - Setting this to true can be useful in contexts other than - the other side of a maven generation; i.e. running in an - ide. You'll want to set this boolean to true to avoid - seeing the RuntimException complaint: "hbase-default.xml file - seems to be for and old version of HBase (@@@VERSION@@@), this - version is X.X.X-SNAPSHOT" - - - - hbase.ipc.client.fallback-to-simple-auth-allowed - true - - \ No newline at end of file diff --git a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical.json b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical.json deleted file mode 100644 index 7f9015bdfa8..00000000000 --- a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - head : { - type : "APACHE_DRILL_PHYSICAL", - version : 1, - generator : { - type : "manual" - } - }, - graph : [ { - pop : "hbase-scan", - @id : 1, - hbaseScanSpec : { - tableName : "[TABLE_NAME]" - }, - storage: - { - "type":"hbase", - config : { - "hbase.zookeeper.quorum" : "localhost", - "hbase.zookeeper.property.clientPort" : 2181 - } - } - }, - { - pop : "screen", - @id : 2, - child : 1 - } ] -} \ No newline at end of file diff --git a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json deleted file mode 100644 index f399f6f1c37..00000000000 --- a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - head : { - type : "APACHE_DRILL_PHYSICAL", - version : 1, - generator : { - type : "manual" - } - }, - graph : [ { - pop : "hbase-scan", - @id : 1, - hbaseScanSpec : { - tableName : "[TABLE_NAME]" - }, - storage: - { - "type":"hbase", - config : { - "hbase.zookeeper.quorum" : "localhost", - "hbase.zookeeper.property.clientPort" : 2181 - } - }, - columns: [ - "`f2`.c1", "`f2`.c2", "row_key" - ] - }, - { - pop : "screen", - @id : 2, - child : 1 - } ] -} \ No newline at end of file diff --git a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json b/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json deleted file mode 100644 index 0002164fe70..00000000000 --- a/contrib/format-maprdb/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - head : { - type : "APACHE_DRILL_PHYSICAL", - version : 1, - generator : { - type : "manual" - } - }, - graph : [ { - pop : "hbase-scan", - @id : 1, - hbaseScanSpec : { - tableName : "[TABLE_NAME]" - }, - storage: - { - "type":"hbase", - config : { - "hbase.zookeeper.quorum" : "localhost", - "hbase.zookeeper.property.clientPort" : 2181 - } - }, - columns: [ - "f2" - ] - }, - { - pop : "screen", - @id : 2, - child : 1 - } ] -} \ No newline at end of file From f97a33321887458996cf7e311252fc9e8719feb3 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 14 Oct 2015 23:49:09 -0700 Subject: [PATCH 29/58] Adding support for Json tables. + Re-factored code to separate binary table specific code from the common code. + Added test cases --- contrib/format-maprdb/pom.xml | 221 +++++++--- .../exec/store/maprdb/MapRDBFormatPlugin.java | 51 ++- .../exec/store/maprdb/MapRDBGroupScan.java | 270 ++---------- .../store/maprdb/MapRDBScanBatchCreator.java | 21 +- .../exec/store/maprdb/MapRDBSubScan.java | 177 ++------ .../exec/store/maprdb/MapRDBSubScanSpec.java | 113 +++++ .../exec/store/maprdb/MapRDBTableStats.java | 11 +- .../exec/store/maprdb/TabletFragmentInfo.java | 108 +++++ .../maprdb/binary/BinaryTableGroupScan.java | 216 ++++++++++ .../CompareFunctionsProcessor.java | 2 +- .../{ => binary}/MapRDBFilterBuilder.java | 8 +- .../MapRDBPushFilterIntoScan.java | 17 +- .../store/maprdb/json/JsonTableGroupScan.java | 186 +++++++++ .../maprdb/json/MaprDBJsonRecordReader.java | 386 ++++++++++++++++++ .../exec/store/maprdb/util/CommonFns.java | 26 ++ .../drill/maprdb/tests/MaprDBTestsSuite.java | 162 ++++++++ .../binary/TestMapRDBFilterPushDown.java | 47 +++ .../maprdb/tests/binary/TestMapRDBSimple.java | 53 +++ .../maprdb/tests/json/TestSimpleJson.java | 75 ++++ .../src/test/resources/hbase-site.xml | 25 ++ .../src/test/resources/json/business.json | 10 + .../src/test/resources/logback.xml | 12 +- 22 files changed, 1739 insertions(+), 458 deletions(-) create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/TabletFragmentInfo.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/{ => binary}/CompareFunctionsProcessor.java (99%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/{ => binary}/MapRDBFilterBuilder.java (97%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/{ => binary}/MapRDBPushFilterIntoScan.java (91%) create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java create mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java create mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java create mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java create mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java create mode 100644 contrib/format-maprdb/src/test/resources/hbase-site.xml create mode 100644 contrib/format-maprdb/src/test/resources/json/business.json diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index 9e299b54cb2..bcb6c295972 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -15,102 +15,223 @@ See the License for the specific language governing permissions and limitations under the License. --> - 4.0.0 + - drill-contrib-parent - org.apache.drill.contrib - 1.5.0-SNAPSHOT + com.mapr + mapr-release + 5.1.0-mapr-SNAPSHOT + drill-storage-maprdb - maprdb-storage-plugin + 1.5.0-SNAPSHOT - **/HBaseTestsSuite.class + ${project.parent.version} + ${project.version} + 0.98.12-mapr-1506 + **/MaprDBTestsSuite.class + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${maprdb.TestSuite} + + + + logback.log.dir + ${project.build.directory}/surefire-reports + + + + + + + + + com.mapr.hadoop + maprfs + + + commons-logging + commons-logging + + + slf4j-log4j12 + org.slf4j + + + + + com.mapr.fs + mapr-hbase + + + slf4j-log4j12 + org.slf4j + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + commons-logging + commons-logging + + + asm + asm + + + org.mortbay.jetty + servlet-api-2.5 + + + + + com.mapr.db + maprdb + + + slf4j-log4j12 + org.slf4j + + + org.apache.drill.exec drill-java-exec - ${project.version} + ${drill.version} + + + log4j-over-slf4j + org.slf4j + + org.apache.drill.contrib drill-storage-hbase - ${project.version} + ${drill.version} + + + log4j-over-slf4j + org.slf4j + + + hbase-client + org.apache.hbase + + + + - com.mapr.hadoop - maprfs + ch.qos.logback + logback-core - com.mapr.fs - mapr-hbase - 4.1.0-mapr + ch.qos.logback + logback-classic + + + de.huxhorn.lilith + de.huxhorn.lilith.logback.appender.multiplex-classic + 0.9.44 + test - + + com.mapr + mapr-java-utils + tests + + + slf4j-log4j12 + org.slf4j + + + org.apache.drill.exec drill-java-exec + ${drill.version} tests - ${project.version} test org.apache.drill drill-common + ${drill.version} + tests + test + + + log4j-over-slf4j + org.slf4j + + + + + org.apache.drill.contrib + drill-storage-hbase + ${drill.version} tests - ${project.version} test - com.yammer.metrics - metrics-core - 2.1.1 + org.apache.hbase + hbase-server test + tests + + + commons-logging + commons-logging + + + servlet-api-2.5 + org.mortbay.jetty + + + servlet-api + javax.servlet + + + slf4j-log4j12 + org.slf4j + + + mapr-releases - http://repository.mapr.com/nexus/content/repositories/releases - - true - - - true - + http://repository.mapr.com/nexus/content/repositories/releases + true + true - - - - org.apache.maven.plugins - maven-surefire-plugin - - - ${hbase.TestSuite} - - - - hbase.test.root - ${project.build.directory}/data - - - logback.log.dir - ${project.build.directory}/surefire-reports - - - - - - - diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java index aa9190137c2..d22434d09c6 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -17,7 +17,11 @@ */ package org.apache.drill.exec.store.maprdb; +import static com.mapr.fs.jni.MapRConstants.MAPRFS_PREFIX; + import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.List; import java.util.Set; @@ -31,26 +35,36 @@ import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.dfs.FileSystemConfig; import org.apache.drill.exec.store.dfs.FileSystemPlugin; import org.apache.drill.exec.store.dfs.FormatMatcher; import org.apache.drill.exec.store.dfs.FormatPlugin; import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan; +import org.apache.drill.exec.store.maprdb.binary.MapRDBPushFilterIntoScan; +import org.apache.drill.exec.store.maprdb.json.JsonTableGroupScan; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; +import com.mapr.fs.MapRFileSystem; +import com.mapr.fs.tables.TableProperties; public class MapRDBFormatPlugin implements FormatPlugin { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory .getLogger(MapRDBFormatPlugin.class); - private final StoragePluginConfig storageConfig; + private final FileSystemConfig storageConfig; private final MapRDBFormatPluginConfig config; private final MapRDBFormatMatcher matcher; private final Configuration fsConf; private final DrillbitContext context; private final String name; + private volatile FileSystemPlugin storagePlugin; + private volatile MapRFileSystem maprfs; + public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig storageConfig) { this(name, context, fsConf, storageConfig, new MapRDBFormatPluginConfig()); @@ -61,9 +75,15 @@ public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fs this.context = context; this.config = formatConfig; this.matcher = new MapRDBFormatMatcher(this); - this.storageConfig = storageConfig; + this.storageConfig = (FileSystemConfig) storageConfig; this.fsConf = fsConf; this.name = name == null ? "maprdb" : name; + try { + this.maprfs = new MapRFileSystem(); + maprfs.initialize(new URI(MAPRFS_PREFIX), fsConf); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } } @Override @@ -108,14 +128,14 @@ public AbstractGroupScan getGroupScan(String userName, FileSelection selection, List files = selection.getFiles(); assert (files.size() == 1); String tableName = files.get(0); - HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); - try { - return new MapRDBGroupScan(userName, - (FileSystemPlugin) (context.getStorage().getPlugin(storageConfig)), this, scanSpec, - columns); - } catch (ExecutionSetupException e) { - e.printStackTrace(); - return null; + TableProperties props = maprfs.getTableProperties(new Path(tableName)); + + if (props.getAttr().getJson()) { + MapRDBSubScanSpec scanSpec = new MapRDBSubScanSpec().setTableName(tableName); + return new JsonTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns); + } else { + HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); + return new BinaryTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns); } } @@ -139,4 +159,15 @@ public String getName() { return name; } + public synchronized FileSystemPlugin getStoragePlugin() { + if (this.storagePlugin == null) { + try { + this.storagePlugin = (FileSystemPlugin) (context.getStorage().getPlugin(storageConfig)); + } catch (ExecutionSetupException e) { + throw new RuntimeException(e); + } + } + return storagePlugin; + } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java index 9358753dadb..cbfb18c2767 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.maprdb; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -30,163 +29,64 @@ import java.util.PriorityQueue; import java.util.Queue; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.TimeUnit; -import org.apache.drill.common.exceptions.DrillRuntimeException; -import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.EndpointAffinity; import org.apache.drill.exec.physical.base.AbstractGroupScan; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.base.ScanStats; -import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.dfs.FileSystemConfig; import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.hbase.DrillHBaseConstants; -import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.hbase.HBaseSubScan.HBaseSubScanSpec; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.client.HTable; -import org.codehaus.jackson.annotate.JsonCreator; - -import com.fasterxml.jackson.annotation.JacksonInject; + import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -@JsonTypeName("maprdb-scan") -public class MapRDBGroupScan extends AbstractGroupScan implements DrillHBaseConstants { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBGroupScan.class); - - private static final Comparator> LIST_SIZE_COMPARATOR = new Comparator>() { - @Override - public int compare(List list1, List list2) { - return list1.size() - list2.size(); - } - }; - - private static final Comparator> LIST_SIZE_COMPARATOR_REV = Collections.reverseOrder(LIST_SIZE_COMPARATOR); - - private List columns; - - private HBaseScanSpec hbaseScanSpec; +public abstract class MapRDBGroupScan extends AbstractGroupScan { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBGroupScan.class); private FileSystemPlugin storagePlugin; private MapRDBFormatPlugin formatPlugin; - private Stopwatch watch = Stopwatch.createUnstarted(); + protected List columns; - private Map> endpointFragmentMapping; + protected Map> endpointFragmentMapping; - private NavigableMap regionsToScan; - - private HTableDescriptor hTableDesc; + protected NavigableMap regionsToScan; private boolean filterPushedDown = false; - private MapRDBTableStats tableStats; - - @JsonCreator - public MapRDBGroupScan(@JsonProperty("userName") final String userName, - @JsonProperty("hbaseScanSpec") HBaseScanSpec hbaseScanSpec, - @JsonProperty("storage") FileSystemConfig storagePluginConfig, - @JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig, - @JsonProperty("columns") List columns, - @JacksonInject StoragePluginRegistry pluginRegistry) throws IOException, ExecutionSetupException { - this (userName, (FileSystemPlugin) pluginRegistry.getPlugin(storagePluginConfig), - (MapRDBFormatPlugin) pluginRegistry.getFormatPlugin(storagePluginConfig, formatPluginConfig), - hbaseScanSpec, - columns); - } + private Stopwatch watch = new Stopwatch(); - public MapRDBGroupScan(String userName, FileSystemPlugin storagePlugin, MapRDBFormatPlugin formatPlugin, HBaseScanSpec scanSpec, List columns) { - super(userName); - this.storagePlugin = storagePlugin; - this.formatPlugin = formatPlugin; - this.hbaseScanSpec = scanSpec; - this.columns = columns; - init(); - } + private static final Comparator> LIST_SIZE_COMPARATOR = new Comparator>() { + @Override + public int compare(List list1, List list2) { + return list1.size() - list2.size(); + } + }; - /** - * Private constructor, used for cloning. - * @param that The HBaseGroupScan to clone - */ - private MapRDBGroupScan(MapRDBGroupScan that) { + private static final Comparator> LIST_SIZE_COMPARATOR_REV = Collections.reverseOrder(LIST_SIZE_COMPARATOR); + + public MapRDBGroupScan(MapRDBGroupScan that) { super(that); this.columns = that.columns; - this.hbaseScanSpec = that.hbaseScanSpec; - this.endpointFragmentMapping = that.endpointFragmentMapping; - this.regionsToScan = that.regionsToScan; - this.storagePlugin = that.storagePlugin; this.formatPlugin = that.formatPlugin; - this.hTableDesc = that.hTableDesc; + this.storagePlugin = that.storagePlugin; + this.regionsToScan = that.regionsToScan; this.filterPushedDown = that.filterPushedDown; - this.tableStats = that.tableStats; - } - - @Override - public GroupScan clone(List columns) { - MapRDBGroupScan newScan = new MapRDBGroupScan(this); - newScan.columns = columns; - newScan.verifyColumns(); - return newScan; - } - - private void init() { - logger.debug("Getting region locations"); - try { - HTable table = new HTable(HBaseConfiguration.create(), hbaseScanSpec.getTableName()); - tableStats = new MapRDBTableStats(table); - this.hTableDesc = table.getTableDescriptor(); - NavigableMap regionsMap = table.getRegionLocations(); - table.close(); - - boolean foundStartRegion = false; - regionsToScan = new TreeMap(); - for (Entry mapEntry : regionsMap.entrySet()) { - HRegionInfo regionInfo = mapEntry.getKey(); - if (!foundStartRegion && hbaseScanSpec.getStartRow() != null && hbaseScanSpec.getStartRow().length != 0 && !regionInfo.containsRow(hbaseScanSpec.getStartRow())) { - continue; - } - foundStartRegion = true; - regionsToScan.put(regionInfo, mapEntry.getValue()); - if (hbaseScanSpec.getStopRow() != null && hbaseScanSpec.getStopRow().length != 0 && regionInfo.containsRow(hbaseScanSpec.getStopRow())) { - break; - } - } - } catch (Exception e) { - throw new DrillRuntimeException("Error getting region info for table: " + hbaseScanSpec.getTableName(), e); - } - verifyColumns(); } - private void verifyColumns() { - /* - if (columns != null) { - for (SchemaPath column : columns) { - if (!(column.equals(ROW_KEY_PATH) || hTableDesc.hasFamily(HBaseUtils.getBytes(column.getRootSegment().getPath())))) { - DrillRuntimeException.format("The column family '%s' does not exist in HBase table: %s .", - column.getRootSegment().getPath(), hTableDesc.getNameAsString()); - } - } - } - */ + public MapRDBGroupScan(FileSystemPlugin storagePlugin, + MapRDBFormatPlugin formatPlugin, List columns, String userName) { + super(userName); + this.storagePlugin = storagePlugin; + this.formatPlugin = formatPlugin; + this.columns = columns; } @Override @@ -199,8 +99,8 @@ public List getOperatorAffinity() { } Map affinityMap = new HashMap(); - for (ServerName sn : regionsToScan.values()) { - DrillbitEndpoint ep = endpointMap.get(sn.getHostname()); + for (String serverName : regionsToScan.values()) { + DrillbitEndpoint ep = endpointMap.get(serverName); if (ep != null) { EndpointAffinity affinity = affinityMap.get(ep); if (affinity == null) { @@ -247,7 +147,7 @@ public void applyAssignments(List incomingEndpoints) { * Initialize these two maps */ for (int i = 0; i < numSlots; ++i) { - endpointFragmentMapping.put(i, new ArrayList(maxPerEndpointSlot)); + endpointFragmentMapping.put(i, new ArrayList(maxPerEndpointSlot)); String hostname = incomingEndpoints.get(i).getAddress(); Queue hostIndexQueue = endpointHostIndexListMap.get(hostname); if (hostIndexQueue == null) { @@ -257,21 +157,21 @@ public void applyAssignments(List incomingEndpoints) { hostIndexQueue.add(i); } - Set> regionsToAssignSet = Sets.newHashSet(regionsToScan.entrySet()); + Set> regionsToAssignSet = Sets.newHashSet(regionsToScan.entrySet()); /* * First, we assign regions which are hosted on region servers running on drillbit endpoints */ - for (Iterator> regionsIterator = regionsToAssignSet.iterator(); regionsIterator.hasNext(); /*nothing*/) { - Entry regionEntry = regionsIterator.next(); + for (Iterator> regionsIterator = regionsToAssignSet.iterator(); regionsIterator.hasNext(); /*nothing*/) { + Entry regionEntry = regionsIterator.next(); /* * Test if there is a drillbit endpoint which is also an HBase RegionServer that hosts the current HBase region */ - Queue endpointIndexlist = endpointHostIndexListMap.get(regionEntry.getValue().getHostname()); + Queue endpointIndexlist = endpointHostIndexListMap.get(regionEntry.getValue()); if (endpointIndexlist != null) { Integer slotIndex = endpointIndexlist.poll(); - List endpointSlotScanList = endpointFragmentMapping.get(slotIndex); - endpointSlotScanList.add(regionInfoToSubScanSpec(regionEntry.getKey())); + List endpointSlotScanList = endpointFragmentMapping.get(slotIndex); + endpointSlotScanList.add(getSubScanSpec(regionEntry.getKey())); // add to the tail of the slot list, to add more later in round robin fashion endpointIndexlist.offer(slotIndex); // this region has been assigned @@ -282,9 +182,9 @@ public void applyAssignments(List incomingEndpoints) { /* * Build priority queues of slots, with ones which has tasks lesser than 'minPerEndpointSlot' and another which have more. */ - PriorityQueue> minHeap = new PriorityQueue>(numSlots, LIST_SIZE_COMPARATOR); - PriorityQueue> maxHeap = new PriorityQueue>(numSlots, LIST_SIZE_COMPARATOR_REV); - for(List listOfScan : endpointFragmentMapping.values()) { + PriorityQueue> minHeap = new PriorityQueue>(numSlots, LIST_SIZE_COMPARATOR); + PriorityQueue> maxHeap = new PriorityQueue>(numSlots, LIST_SIZE_COMPARATOR_REV); + for(List listOfScan : endpointFragmentMapping.values()) { if (listOfScan.size() < minPerEndpointSlot) { minHeap.offer(listOfScan); } else if (listOfScan.size() > minPerEndpointSlot){ @@ -296,9 +196,9 @@ public void applyAssignments(List incomingEndpoints) { * Now, let's process any regions which remain unassigned and assign them to slots with minimum number of assignments. */ if (regionsToAssignSet.size() > 0) { - for (Entry regionEntry : regionsToAssignSet) { - List smallestList = minHeap.poll(); - smallestList.add(regionInfoToSubScanSpec(regionEntry.getKey())); + for (Entry regionEntry : regionsToAssignSet) { + List smallestList = minHeap.poll(); + smallestList.add(getSubScanSpec(regionEntry.getKey())); if (smallestList.size() < maxPerEndpointSlot) { minHeap.offer(smallestList); } @@ -309,8 +209,8 @@ public void applyAssignments(List incomingEndpoints) { * While there are slots with lesser than 'minPerEndpointSlot' unit work, balance from those with more. */ while(minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) { - List smallestList = minHeap.poll(); - List largestList = maxHeap.poll(); + List smallestList = (List) minHeap.poll(); + List largestList = (List) maxHeap.poll(); smallestList.add(largestList.remove(largestList.size()-1)); if (largestList.size() > minPerEndpointSlot) { maxHeap.offer(largestList); @@ -329,77 +229,21 @@ public void applyAssignments(List incomingEndpoints) { watch.elapsed(TimeUnit.NANOSECONDS)/1000, incomingEndpoints, endpointFragmentMapping.toString()); } - private HBaseSubScanSpec regionInfoToSubScanSpec(HRegionInfo ri) { - HBaseScanSpec spec = hbaseScanSpec; - HBaseSubScanSpec subScanSpec = new HBaseSubScanSpec(spec.getTableName(), - regionsToScan.get(ri).getHostname(), - (!isNullOrEmpty(spec.getStartRow()) && ri.containsRow(spec.getStartRow())) ? spec.getStartRow() : ri.getStartKey(), - (!isNullOrEmpty(spec.getStopRow()) && ri.containsRow(spec.getStopRow())) ? spec.getStopRow() : ri.getEndKey(), - spec.getSerializedFilter(), - null); - return subScanSpec; - } - - private boolean isNullOrEmpty(byte[] key) { - return key == null || key.length == 0; - } - - @Override - public MapRDBSubScan getSpecificScan(int minorFragmentId) { - assert minorFragmentId < endpointFragmentMapping.size() : String.format( - "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), - minorFragmentId); - return new MapRDBSubScan(getUserName(), storagePlugin, storagePlugin.getConfig(), endpointFragmentMapping.get(minorFragmentId), columns); - } - @Override public int getMaxParallelizationWidth() { return regionsToScan.size(); } - @Override - public ScanStats getScanStats() { - //TODO: look at stats for this. - long rowCount = (long) ((hbaseScanSpec.getFilter() != null ? .5 : 1) * tableStats.getNumRows()); - int avgColumnSize = 10; - int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size(); - return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount); - } - - @Override - @JsonIgnore - public PhysicalOperator getNewWithChildren(List children) { - Preconditions.checkArgument(children.isEmpty()); - return new MapRDBGroupScan(this); - } - @JsonIgnore public MapRDBFormatPlugin getFormatPlugin() { return formatPlugin; } - @JsonIgnore - public Configuration getHBaseConf() { - return HBaseConfiguration.create(); - } - - @JsonIgnore - public String getTableName() { - return getHBaseScanSpec().getTableName(); - } - @Override public String getDigest() { return toString(); } - @Override - public String toString() { - return "MapRDBGroupScan [HBaseScanSpec=" - + hbaseScanSpec + ", columns=" - + columns + "]"; - } - @JsonProperty("storage") public FileSystemConfig getStorageConfig() { return (FileSystemConfig) storagePlugin.getConfig(); @@ -407,7 +251,7 @@ public FileSystemConfig getStorageConfig() { @JsonIgnore public FileSystemPlugin getStoragePlugin(){ - return storagePlugin; + return storagePlugin; } @JsonProperty @@ -415,11 +259,6 @@ public List getColumns() { return columns; } - @JsonProperty - public HBaseScanSpec getHBaseScanSpec() { - return hbaseScanSpec; - } - @JsonIgnore public boolean canPushdownProjects(List columns) { return true; @@ -435,29 +274,6 @@ public boolean isFilterPushedDown() { return filterPushedDown; } - /** - * Empty constructor, do not use, only for testing. - */ - @VisibleForTesting - public MapRDBGroupScan() { - super((String)null); - } - - /** - * Do not use, only for testing. - */ - @VisibleForTesting - public void setHBaseScanSpec(HBaseScanSpec hbaseScanSpec) { - this.hbaseScanSpec = hbaseScanSpec; - } - - /** - * Do not use, only for testing. - */ - @JsonIgnore - @VisibleForTesting - public void setRegionsToScan(NavigableMap regionsToScan) { - this.regionsToScan = regionsToScan; - } + protected abstract MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo key); } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java index 1beabc929a1..058de615980 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java @@ -26,7 +26,10 @@ import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.hbase.HBaseRecordReader; -import org.apache.drill.exec.store.hbase.HBaseSubScan; +import org.apache.drill.exec.store.hbase.HBaseSubScan.HBaseSubScanSpec; +import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan; +import org.apache.drill.exec.store.maprdb.json.MaprDBJsonRecordReader; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import com.google.common.base.Preconditions; @@ -39,11 +42,14 @@ public class MapRDBScanBatchCreator implements BatchCreator{ public ScanBatch getBatch(FragmentContext context, MapRDBSubScan subScan, List children) throws ExecutionSetupException { Preconditions.checkArgument(children.isEmpty()); List readers = Lists.newArrayList(); - for(HBaseSubScan.HBaseSubScanSpec scanSpec : subScan.getRegionScanSpecList()){ + Configuration conf = HBaseConfiguration.create(); + for(MapRDBSubScanSpec scanSpec : subScan.getRegionScanSpecList()){ try { - readers.add( - new HBaseRecordReader(HBaseConfiguration.create(), scanSpec, subScan.getColumns(), context) - ); + if (BinaryTableGroupScan.TABLE_BINARY.equals(subScan.getTableType())) { + readers.add(new HBaseRecordReader(conf, getHBaseSubScanSpec(scanSpec), subScan.getColumns(), context)); + } else { + readers.add(new MaprDBJsonRecordReader(scanSpec, subScan.getColumns(), context)); + } } catch (Exception e1) { throw new ExecutionSetupException(e1); } @@ -51,4 +57,9 @@ public ScanBatch getBatch(FragmentContext context, MapRDBSubScan subScan, List regionScanSpecList; + private final List regionScanSpecList; private final List columns; + private final String tableType; @JsonCreator public MapRDBSubScan(@JacksonInject StoragePluginRegistry registry, - @JsonProperty("userName") String userName, - @JsonProperty("storage") StoragePluginConfig storage, - @JsonProperty("regionScanSpecList") LinkedList regionScanSpecList, - @JsonProperty("columns") List columns) throws ExecutionSetupException { + @JsonProperty("userName") String userName, + @JsonProperty("storage") StoragePluginConfig storage, + @JsonProperty("regionScanSpecList") List regionScanSpecList, + @JsonProperty("columns") List columns, + @JsonProperty("tableType") String tableType) throws ExecutionSetupException { super(userName); this.fsStoragePlugin = (FileSystemPlugin) registry.getPlugin(storage); this.regionScanSpecList = regionScanSpecList; this.storage = storage; this.columns = columns; + this.tableType = tableType; } - public MapRDBSubScan(String userName, FileSystemPlugin storagePlugin, StoragePluginConfig config, - List hBaseSubScanSpecs, List columns) { - super(userName); - fsStoragePlugin = storagePlugin; - storage = config; - this.regionScanSpecList = hBaseSubScanSpecs; - this.columns = columns; - } + public MapRDBSubScan(String userName, FileSystemPlugin storagePlugin, StoragePluginConfig config, + List maprSubScanSpecs, List columns, String tableType) { + super(userName); + fsStoragePlugin = storagePlugin; + storage = config; + this.regionScanSpecList = maprSubScanSpecs; + this.columns = columns; + this.tableType = tableType; + } - public List getRegionScanSpecList() { + public List getRegionScanSpecList() { return regionScanSpecList; } @@ -100,7 +97,7 @@ public T accept(PhysicalVisitor physicalVis @Override public PhysicalOperator getNewWithChildren(List children) { Preconditions.checkArgument(children.isEmpty()); - return new MapRDBSubScan(getUserName(), fsStoragePlugin, storage, regionScanSpecList, columns); + return new MapRDBSubScan(getUserName(), fsStoragePlugin, storage, regionScanSpecList, columns, tableType); } @Override @@ -108,121 +105,13 @@ public Iterator iterator() { return Iterators.emptyIterator(); } - public static class HBaseSubScanSpec { - - protected String tableName; - protected String regionServer; - protected byte[] startRow; - protected byte[] stopRow; - protected byte[] serializedFilter; - - @JsonCreator - public HBaseSubScanSpec(@JsonProperty("tableName") String tableName, - @JsonProperty("regionServer") String regionServer, - @JsonProperty("startRow") byte[] startRow, - @JsonProperty("stopRow") byte[] stopRow, - @JsonProperty("serializedFilter") byte[] serializedFilter, - @JsonProperty("filterString") String filterString) { - if (serializedFilter != null && filterString != null) { - throw new IllegalArgumentException("The parameters 'serializedFilter' or 'filterString' cannot be specified at the same time."); - } - this.tableName = tableName; - this.regionServer = regionServer; - this.startRow = startRow; - this.stopRow = stopRow; - if (serializedFilter != null) { - this.serializedFilter = serializedFilter; - } else { - this.serializedFilter = HBaseUtils.serializeFilter(parseFilterString(filterString)); - } - } - - static final ParseFilter PARSE_FILTER = new ParseFilter(); - - static Filter parseFilterString(String filterString) { - if (filterString == null) { - return null; - } - try { - return PARSE_FILTER.parseFilterString(filterString); - } catch (CharacterCodingException e) { - throw new DrillRuntimeException("Error parsing filter string: " + filterString, e); - } - } - - /* package */ HBaseSubScanSpec() { - // empty constructor, to be used with builder pattern; - } - - @JsonIgnore - private Filter scanFilter; - public Filter getScanFilter() { - if (scanFilter == null && serializedFilter != null) { - scanFilter = HBaseUtils.deserializeFilter(serializedFilter); - } - return scanFilter; - } - - public String getTableName() { - return tableName; - } - - public HBaseSubScanSpec setTableName(String tableName) { - this.tableName = tableName; - return this; - } - - public String getRegionServer() { - return regionServer; - } - - public HBaseSubScanSpec setRegionServer(String regionServer) { - this.regionServer = regionServer; - return this; - } - - public byte[] getStartRow() { - return startRow; - } - - public HBaseSubScanSpec setStartRow(byte[] startRow) { - this.startRow = startRow; - return this; - } - - public byte[] getStopRow() { - return stopRow; - } - - public HBaseSubScanSpec setStopRow(byte[] stopRow) { - this.stopRow = stopRow; - return this; - } - - public byte[] getSerializedFilter() { - return serializedFilter; - } - - public HBaseSubScanSpec setSerializedFilter(byte[] serializedFilter) { - this.serializedFilter = serializedFilter; - this.scanFilter = null; - return this; - } - - @Override - public String toString() { - return "HBaseScanSpec [tableName=" + tableName - + ", startRow=" + (startRow == null ? null : Bytes.toStringBinary(startRow)) - + ", stopRow=" + (stopRow == null ? null : Bytes.toStringBinary(stopRow)) - + ", filter=" + (getScanFilter() == null ? null : getScanFilter().toString()) - + ", regionServer=" + regionServer + "]"; - } - - } - @Override public int getOperatorType() { return 1001; } + public String getTableType() { + return tableType; + } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java new file mode 100644 index 00000000000..5e8d84cfb54 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.mapr.fs.jni.MapRConstants; +import com.mapr.org.apache.hadoop.hbase.util.Bytes; + +public class MapRDBSubScanSpec { + + protected String tableName; + protected String regionServer; + protected byte[] startRow; + protected byte[] stopRow; + protected byte[] serializedFilter; + + @parquet.org.codehaus.jackson.annotate.JsonCreator + public MapRDBSubScanSpec(@JsonProperty("tableName") String tableName, + @JsonProperty("regionServer") String regionServer, + @JsonProperty("startRow") byte[] startRow, + @JsonProperty("stopRow") byte[] stopRow, + @JsonProperty("serializedFilter") byte[] serializedFilter, + @JsonProperty("filterString") String filterString) { + if (serializedFilter != null && filterString != null) { + throw new IllegalArgumentException("The parameters 'serializedFilter' or 'filterString' cannot be specified at the same time."); + } + this.tableName = tableName; + this.regionServer = regionServer; + this.startRow = startRow; + this.stopRow = stopRow; + this.serializedFilter = serializedFilter; + } + + /* package */ MapRDBSubScanSpec() { + // empty constructor, to be used with builder pattern; + } + + public String getTableName() { + return tableName; + } + + public MapRDBSubScanSpec setTableName(String tableName) { + this.tableName = tableName; + return this; + } + + public String getRegionServer() { + return regionServer; + } + + public MapRDBSubScanSpec setRegionServer(String regionServer) { + this.regionServer = regionServer; + return this; + } + + /** + * @return the raw (not-encoded) start row key for this sub-scan + */ + public byte[] getStartRow() { + return startRow == null ? MapRConstants.EMPTY_BYTE_ARRAY: startRow; + } + + public MapRDBSubScanSpec setStartRow(byte[] startRow) { + this.startRow = startRow; + return this; + } + + /** + * @return the raw (not-encoded) stop row key for this sub-scan + */ + public byte[] getStopRow() { + return stopRow == null ? MapRConstants.EMPTY_BYTE_ARRAY : stopRow; + } + + public MapRDBSubScanSpec setStopRow(byte[] stopRow) { + this.stopRow = stopRow; + return this; + } + + public byte[] getSerializedFilter() { + return serializedFilter; + } + + public MapRDBSubScanSpec setSerializedFilter(byte[] serializedFilter) { + this.serializedFilter = serializedFilter; + return this; + } + + @Override + public String toString() { + return "MapRDBSubScanSpec [tableName=" + tableName + + ", startRow=" + (startRow == null ? null : Bytes.toStringBinary(startRow)) + + ", stopRow=" + (stopRow == null ? null : Bytes.toStringBinary(stopRow)) + + ", filter=" + (getSerializedFilter() == null ? null : Bytes.toBase64(getSerializedFilter())) + + ", regionServer=" + regionServer + "]"; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java index 3d5320e2bd7..d2b1453c62e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java @@ -18,26 +18,25 @@ package org.apache.drill.exec.store.maprdb; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.mapr.TableMappingRulesFactory; import com.mapr.fs.hbase.HBaseAdminImpl; public class MapRDBTableStats { + private static volatile HBaseAdminImpl admin = null; private long numRows; - private volatile HBaseAdminImpl admin = null; - public MapRDBTableStats(HTable table) throws Exception { + public MapRDBTableStats(Configuration conf, String tablePath) throws Exception { if (admin == null) { synchronized (MapRDBTableStats.class) { if (admin == null) { - Configuration config = table.getConfiguration(); - admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(config)); + Configuration config = conf; + admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(conf)); } } } - numRows = admin.getNumRows(new String(table.getTableName())); + numRows = admin.getNumRows(tablePath); } public long getNumRows() { diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/TabletFragmentInfo.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/TabletFragmentInfo.java new file mode 100644 index 00000000000..389f00dfc70 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/TabletFragmentInfo.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb; + +import org.apache.hadoop.hbase.HRegionInfo; + +import com.mapr.db.impl.TabletInfoImpl; + +public class TabletFragmentInfo implements Comparable { + + final private HRegionInfo regionInfo; + final private TabletInfoImpl tabletInfoImpl; + + public TabletFragmentInfo(HRegionInfo regionInfo) { + this(null, regionInfo); + } + + public TabletFragmentInfo(TabletInfoImpl tabletInfoImpl) { + this(tabletInfoImpl, null); + } + + TabletFragmentInfo(TabletInfoImpl tabletInfoImpl, HRegionInfo regionInfo) { + this.regionInfo = regionInfo; + this.tabletInfoImpl = tabletInfoImpl; + } + + public HRegionInfo getRegionInfo() { + return regionInfo; + } + + public TabletInfoImpl getTabletInfoImpl() { + return tabletInfoImpl; + } + + public boolean containsRow(byte[] row) { + return tabletInfoImpl != null ? tabletInfoImpl.containsRow(row) : + regionInfo.containsRow(row); + } + + public byte[] getStartKey() { + return tabletInfoImpl != null ? tabletInfoImpl.getStartRow() : + regionInfo.getStartKey(); + } + + public byte[] getEndKey() { + return tabletInfoImpl != null ? tabletInfoImpl.getStopRow() : + regionInfo.getEndKey(); + } + + @Override + public int compareTo(TabletFragmentInfo o) { + return tabletInfoImpl != null ? tabletInfoImpl.compareTo(o.tabletInfoImpl) : + regionInfo.compareTo(o.regionInfo); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((regionInfo == null) ? 0 : regionInfo.hashCode()); + result = prime * result + ((tabletInfoImpl == null) ? 0 : tabletInfoImpl.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + TabletFragmentInfo other = (TabletFragmentInfo) obj; + if (regionInfo == null) { + if (other.regionInfo != null) + return false; + } else if (!regionInfo.equals(other.regionInfo)) + return false; + if (tabletInfoImpl == null) { + if (other.tabletInfoImpl != null) + return false; + } else if (!tabletInfoImpl.equals(other.tabletInfoImpl)) + return false; + return true; + } + + @Override + public String toString() { + return "TabletFragmentInfo [regionInfo=" + regionInfo + ", tabletInfoImpl=" + tabletInfoImpl + + "]"; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java new file mode 100644 index 00000000000..69fda9c5efd --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java @@ -0,0 +1,216 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb.binary; + +import java.io.IOException; +import java.util.List; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.TreeMap; + +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.hbase.DrillHBaseConstants; +import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.drill.exec.store.maprdb.MapRDBFormatPlugin; +import org.apache.drill.exec.store.maprdb.MapRDBFormatPluginConfig; +import org.apache.drill.exec.store.maprdb.MapRDBGroupScan; +import org.apache.drill.exec.store.maprdb.MapRDBSubScan; +import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; +import org.apache.drill.exec.store.maprdb.MapRDBTableStats; +import org.apache.drill.exec.store.maprdb.TabletFragmentInfo; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HTable; +import org.codehaus.jackson.annotate.JsonCreator; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Preconditions; + +@JsonTypeName("maprdb-binary-scan") +public class BinaryTableGroupScan extends MapRDBGroupScan implements DrillHBaseConstants { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BinaryTableGroupScan.class); + + public static final String TABLE_BINARY = "binary"; + + private HBaseScanSpec hbaseScanSpec; + + private HTableDescriptor hTableDesc; + + private MapRDBTableStats tableStats; + + @JsonCreator + public BinaryTableGroupScan(@JsonProperty("userName") final String userName, + @JsonProperty("hbaseScanSpec") HBaseScanSpec scanSpec, + @JsonProperty("storage") FileSystemConfig storagePluginConfig, + @JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig, + @JsonProperty("columns") List columns, + @JacksonInject StoragePluginRegistry pluginRegistry) throws IOException, ExecutionSetupException { + this (userName, + (FileSystemPlugin) pluginRegistry.getPlugin(storagePluginConfig), + (MapRDBFormatPlugin) pluginRegistry.getFormatPlugin(storagePluginConfig, formatPluginConfig), + scanSpec, columns); + } + + public BinaryTableGroupScan(String userName, FileSystemPlugin storagePlugin, + MapRDBFormatPlugin formatPlugin, HBaseScanSpec scanSpec, List columns) { + super(storagePlugin, formatPlugin, columns, userName); + this.hbaseScanSpec = scanSpec; + init(); + } + + /** + * Private constructor, used for cloning. + * @param that The HBaseGroupScan to clone + */ + private BinaryTableGroupScan(BinaryTableGroupScan that) { + super(that); + this.hbaseScanSpec = that.hbaseScanSpec; + this.endpointFragmentMapping = that.endpointFragmentMapping; + this.hTableDesc = that.hTableDesc; + this.tableStats = that.tableStats; + } + + @Override + public GroupScan clone(List columns) { + BinaryTableGroupScan newScan = new BinaryTableGroupScan(this); + newScan.columns = columns; + newScan.verifyColumns(); + return newScan; + } + + private void init() { + logger.debug("Getting region locations"); + try { + Configuration conf = HBaseConfiguration.create(); + HTable table = new HTable(conf, hbaseScanSpec.getTableName()); + tableStats = new MapRDBTableStats(conf, hbaseScanSpec.getTableName()); + this.hTableDesc = table.getTableDescriptor(); + NavigableMap regionsMap = table.getRegionLocations(); + table.close(); + + boolean foundStartRegion = false; + regionsToScan = new TreeMap(); + for (Entry mapEntry : regionsMap.entrySet()) { + HRegionInfo regionInfo = mapEntry.getKey(); + if (!foundStartRegion && hbaseScanSpec.getStartRow() != null && hbaseScanSpec.getStartRow().length != 0 && !regionInfo.containsRow(hbaseScanSpec.getStartRow())) { + continue; + } + foundStartRegion = true; + regionsToScan.put(new TabletFragmentInfo(regionInfo), mapEntry.getValue().getHostname()); + if (hbaseScanSpec.getStopRow() != null && hbaseScanSpec.getStopRow().length != 0 && regionInfo.containsRow(hbaseScanSpec.getStopRow())) { + break; + } + } + } catch (Exception e) { + throw new DrillRuntimeException("Error getting region info for table: " + hbaseScanSpec.getTableName(), e); + } + verifyColumns(); + } + + private void verifyColumns() { + /* + if (columns != null) { + for (SchemaPath column : columns) { + if (!(column.equals(ROW_KEY_PATH) || hTableDesc.hasFamily(HBaseUtils.getBytes(column.getRootSegment().getPath())))) { + DrillRuntimeException.format("The column family '%s' does not exist in HBase table: %s .", + column.getRootSegment().getPath(), hTableDesc.getNameAsString()); + } + } + } + */ + } + + protected MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) { + HBaseScanSpec spec = hbaseScanSpec; + MapRDBSubScanSpec subScanSpec = new MapRDBSubScanSpec( + spec.getTableName(), + regionsToScan.get(tfi), + (!isNullOrEmpty(spec.getStartRow()) && tfi.containsRow(spec.getStartRow())) ? spec.getStartRow() : tfi.getStartKey(), + (!isNullOrEmpty(spec.getStopRow()) && tfi.containsRow(spec.getStopRow())) ? spec.getStopRow() : tfi.getEndKey(), + spec.getSerializedFilter(), + null); + return subScanSpec; + } + + private boolean isNullOrEmpty(byte[] key) { + return key == null || key.length == 0; + } + + @Override + public MapRDBSubScan getSpecificScan(int minorFragmentId) { + assert minorFragmentId < endpointFragmentMapping.size() : String.format( + "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), + minorFragmentId); + return new MapRDBSubScan(getUserName(), getStoragePlugin(), getStoragePlugin().getConfig(), + endpointFragmentMapping.get(minorFragmentId), columns, TABLE_BINARY); + } + + @Override + public ScanStats getScanStats() { + //TODO: look at stats for this. + long rowCount = (long) ((hbaseScanSpec.getFilter() != null ? .5 : 1) * tableStats.getNumRows()); + int avgColumnSize = 10; + int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size(); + return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount); + } + + @Override + @JsonIgnore + public PhysicalOperator getNewWithChildren(List children) { + Preconditions.checkArgument(children.isEmpty()); + return new BinaryTableGroupScan(this); + } + + @JsonIgnore + public Configuration getHBaseConf() { + return HBaseConfiguration.create(); + } + + @JsonIgnore + public String getTableName() { + return getHBaseScanSpec().getTableName(); + } + + @Override + public String toString() { + return "BinaryTableGroupScan [ScanSpec=" + + hbaseScanSpec + ", columns=" + + columns + "]"; + } + + @JsonProperty + public HBaseScanSpec getHBaseScanSpec() { + return hbaseScanSpec; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/CompareFunctionsProcessor.java similarity index 99% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/CompareFunctionsProcessor.java index c6c25047c33..f06786dcd00 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/CompareFunctionsProcessor.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.maprdb.binary; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java similarity index 97% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java index 857d7990a38..800d155e223 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFilterBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.maprdb.binary; import java.util.Arrays; @@ -44,7 +44,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor implements DrillHBaseConstants { - final private MapRDBGroupScan groupScan; + final private BinaryTableGroupScan groupScan; final private LogicalExpression le; @@ -52,7 +52,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor filter" sequence @@ -70,7 +67,7 @@ public void onMatch(RelOptRuleCall call) { @Override public boolean matches(RelOptRuleCall call) { final ScanPrel scan = (ScanPrel) call.rel(1); - if (scan.getGroupScan() instanceof MapRDBGroupScan) { + if (scan.getGroupScan() instanceof BinaryTableGroupScan) { return super.matches(call); } return false; @@ -85,7 +82,7 @@ public void onMatch(RelOptRuleCall call) { final ProjectPrel project = (ProjectPrel) call.rel(1); final FilterPrel filter = (FilterPrel) call.rel(0); - MapRDBGroupScan groupScan = (MapRDBGroupScan)scan.getGroupScan(); + BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); if (groupScan.isFilterPushedDown()) { /* * The rule can get triggered again due to the transformed "scan => filter" sequence @@ -105,14 +102,14 @@ public void onMatch(RelOptRuleCall call) { @Override public boolean matches(RelOptRuleCall call) { final ScanPrel scan = (ScanPrel) call.rel(2); - if (scan.getGroupScan() instanceof MapRDBGroupScan) { + if (scan.getGroupScan() instanceof BinaryTableGroupScan) { return super.matches(call); } return false; } }; - protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel filter, final ProjectPrel project, final ScanPrel scan, final MapRDBGroupScan groupScan, final RexNode condition) { + protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel filter, final ProjectPrel project, final ScanPrel scan, final BinaryTableGroupScan groupScan, final RexNode condition) { final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); final MapRDBFilterBuilder maprdbFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp); @@ -121,7 +118,7 @@ protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel fi return; //no filter pushdown ==> No transformation. } - final MapRDBGroupScan newGroupsScan = new MapRDBGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), + final BinaryTableGroupScan newGroupsScan = new BinaryTableGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), groupScan.getFormatPlugin(), newScanSpec, groupScan.getColumns()); newGroupsScan.setFilterPushedDown(true); diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java new file mode 100644 index 00000000000..e798c52e0c4 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb.json; + +import static org.apache.drill.exec.store.maprdb.util.CommonFns.isNullOrEmpty; + +import java.io.IOException; +import java.util.List; +import java.util.TreeMap; + +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.maprdb.MapRDBFormatPlugin; +import org.apache.drill.exec.store.maprdb.MapRDBFormatPluginConfig; +import org.apache.drill.exec.store.maprdb.MapRDBGroupScan; +import org.apache.drill.exec.store.maprdb.MapRDBSubScan; +import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; +import org.apache.drill.exec.store.maprdb.MapRDBTableStats; +import org.apache.drill.exec.store.maprdb.TabletFragmentInfo; +import org.apache.hadoop.conf.Configuration; +import org.codehaus.jackson.annotate.JsonCreator; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Preconditions; +import com.mapr.db.MapRDB; +import com.mapr.db.Table; +import com.mapr.db.TabletInfo; +import com.mapr.db.impl.TabletInfoImpl; + +@JsonTypeName("maprdb-json-scan") +public class JsonTableGroupScan extends MapRDBGroupScan { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonTableGroupScan.class); + + public static final String TABLE_JSON = "json"; + + private MapRDBTableStats tableStats; + + private MapRDBSubScanSpec subscanSpec; + + @JsonCreator + public JsonTableGroupScan(@JsonProperty("userName") final String userName, + @JsonProperty("subscanSpec") MapRDBSubScanSpec subscanSpec, + @JsonProperty("storage") FileSystemConfig storagePluginConfig, + @JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig, + @JsonProperty("columns") List columns, + @JacksonInject StoragePluginRegistry pluginRegistry) throws IOException, ExecutionSetupException { + this (userName, + (FileSystemPlugin) pluginRegistry.getPlugin(storagePluginConfig), + (MapRDBFormatPlugin) pluginRegistry.getFormatPlugin(storagePluginConfig, formatPluginConfig), + subscanSpec, columns); + } + + public JsonTableGroupScan(String userName, FileSystemPlugin storagePlugin, + MapRDBFormatPlugin formatPlugin, MapRDBSubScanSpec subscanSpec, List columns) { + super(storagePlugin, formatPlugin, columns, userName); + this.subscanSpec = subscanSpec; + init(); + } + + /** + * Private constructor, used for cloning. + * @param that The HBaseGroupScan to clone + */ + private JsonTableGroupScan(JsonTableGroupScan that) { + super(that); + this.subscanSpec = that.subscanSpec; + this.endpointFragmentMapping = that.endpointFragmentMapping; + this.tableStats = that.tableStats; + } + + @Override + public GroupScan clone(List columns) { + JsonTableGroupScan newScan = new JsonTableGroupScan(this); + newScan.columns = columns; + return newScan; + } + + private void init() { + logger.debug("Getting tablet locations"); + try { + Configuration conf = new Configuration(); + Table t = MapRDB.getTable(subscanSpec.getTableName()); + TabletInfo[] tabletInfos = t.getTabletInfos(); + tableStats = new MapRDBTableStats(conf, subscanSpec.getTableName()); + + boolean foundStartRegion = false; + regionsToScan = new TreeMap(); + for (TabletInfo tabletInfo : tabletInfos) { + TabletInfoImpl tabletInfoImpl = (TabletInfoImpl) tabletInfo; + if (!foundStartRegion + && !isNullOrEmpty(subscanSpec.getStartRow()) + && !tabletInfoImpl.containsRow(subscanSpec.getStartRow())) { + continue; + } + foundStartRegion = true; + regionsToScan.put(new TabletFragmentInfo(tabletInfoImpl), tabletInfo.getLocations()[0]); + if (!isNullOrEmpty(subscanSpec.getStopRow()) + && tabletInfoImpl.containsRow(subscanSpec.getStopRow())) { + break; + } + } + } catch (Exception e) { + throw new DrillRuntimeException("Error getting region info for table: " + subscanSpec.getTableName(), e); + } + } + + protected MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) { + MapRDBSubScanSpec spec = subscanSpec; + MapRDBSubScanSpec subScanSpec = new MapRDBSubScanSpec( + spec.getTableName(), + regionsToScan.get(tfi), + (!isNullOrEmpty(spec.getStartRow()) && tfi.containsRow(spec.getStartRow())) ? spec.getStartRow() : tfi.getStartKey(), + (!isNullOrEmpty(spec.getStopRow()) && tfi.containsRow(spec.getStopRow())) ? spec.getStopRow() : tfi.getEndKey(), + spec.getSerializedFilter(), + null); + return subScanSpec; + } + + @Override + public MapRDBSubScan getSpecificScan(int minorFragmentId) { + assert minorFragmentId < endpointFragmentMapping.size() : String.format( + "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), + minorFragmentId); + return new MapRDBSubScan(getUserName(), getStoragePlugin(), getStoragePlugin().getConfig(), + endpointFragmentMapping.get(minorFragmentId), columns, TABLE_JSON); + } + + @Override + public ScanStats getScanStats() { + //TODO: look at stats for this. + long rowCount = (long) ((subscanSpec.getSerializedFilter() != null ? .5 : 1) * tableStats.getNumRows()); + int avgColumnSize = 10; + int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size(); + return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount); + } + + @Override + @JsonIgnore + public PhysicalOperator getNewWithChildren(List children) { + Preconditions.checkArgument(children.isEmpty()); + return new JsonTableGroupScan(this); + } + + @JsonIgnore + public String getTableName() { + return subscanSpec.getTableName(); + } + + @Override + public String toString() { + return "JsonTableGroupScan [ScanSpec=" + + subscanSpec + ", columns=" + + columns + "]"; + } + + public MapRDBSubScanSpec getSubscanSpec() { + return subscanSpec; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java new file mode 100644 index 00000000000..2f2e5618d8a --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -0,0 +1,386 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb.json; + +import static org.ojai.DocumentConstants.ID_FIELD; +import static org.ojai.DocumentConstants.ID_KEY; +import io.netty.buffer.DrillBuf; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.ops.OperatorContext; +import org.apache.drill.exec.ops.OperatorStats; +import org.apache.drill.exec.physical.impl.OutputMutator; +import org.apache.drill.exec.store.AbstractRecordReader; +import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; +import org.apache.drill.exec.store.maprdb.util.CommonFns; +import org.apache.drill.exec.vector.BaseValueVector; +import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; +import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter; +import org.apache.drill.exec.vector.complex.writer.VarCharWriter; +import org.ojai.DocumentReader; +import org.ojai.DocumentReader.EventType; +import org.ojai.DocumentStream; +import org.ojai.FieldPath; +import org.ojai.Value; +import org.ojai.store.QueryCondition; +import org.ojai.store.QueryCondition.Op; + +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.mapr.db.DBDocument; +import com.mapr.db.MapRDB; +import com.mapr.db.Table; +import com.mapr.db.Table.TableOption; +import com.mapr.db.exceptions.DBException; +import com.mapr.db.impl.IdCodec; +import com.mapr.db.ojai.DBDocumentReader; +import com.mapr.db.util.ByteBufs; +import com.mapr.org.apache.hadoop.hbase.util.Bytes; + +public class MaprDBJsonRecordReader extends AbstractRecordReader { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MaprDBJsonRecordReader.class); + + public static final SchemaPath ID_PATH = SchemaPath.getSimplePath(ID_KEY); + + private Table table; + private QueryCondition condition; + private FieldPath[] projectedFields; + + private String tableName; + private OperatorContext operatorContext; + private VectorContainerWriter writer; + + @SuppressWarnings("unused") + private boolean idOnly; + + private DrillBuf buffer; + + private DocumentStream documentStream; + + private Iterator documentReaderIterators; + + public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, + List projectedColumns, FragmentContext context) { + buffer = context.getManagedBuffer(); + tableName = Preconditions.checkNotNull(subScanSpec, "MapRDB reader needs a sub-scan spec").getTableName(); + condition = MapRDB.newCondition().and(); + addKeyCondition(condition, Op.GREATER_OR_EQUAL, subScanSpec.getStartRow()); + addKeyCondition(condition, Op.LESS, subScanSpec.getStopRow()); + if (subScanSpec.getSerializedFilter() != null) { + condition.condition(com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter()))); + } + condition.close().build(); + setColumns(projectedColumns); + } + + private void addKeyCondition(QueryCondition condition, Op op, byte[] key) { + if (!CommonFns.isNullOrEmpty(key)) { + Value value = IdCodec.decode(key); + switch (value.getType()) { + case STRING: + condition.is(ID_FIELD, op, value.getString()); + return; + case BINARY: + condition.is(ID_FIELD, op, value.getBinary()); + return; + default: + throw new UnsupportedOperationException(""); + } + } + } + + @Override + protected Collection transformColumns(Collection columns) { + Set transformed = Sets.newLinkedHashSet(); + idOnly = true; // TODO: handle the case when only ID is requested. + if (!isStarQuery()) { + ArrayList projectedFieldsList = Lists.newArrayList(); + for (SchemaPath column : columns) { + if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) { + transformed.add(ID_PATH); + continue; + } + idOnly = false; + projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath())); + } + projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]); + } else { + idOnly = false; + transformed.add(ID_PATH); + } + + return transformed; + } + + @Override + public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { + this.writer = new VectorContainerWriter(output); + this.operatorContext = context; + + try { + table = MapRDB.getTable(tableName); + table.setOption(TableOption.EXCLUDEID, true); + documentStream = table.find(condition, projectedFields); + documentReaderIterators = documentStream.documentReaders().iterator(); + } catch (DBException e) { + throw new ExecutionSetupException(e); + } + } + + @Override + public int next() { + Stopwatch watch = new Stopwatch(); + watch.start(); + + writer.allocate(); + writer.reset(); + + int recordCount = 0; + + while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION) { + DBDocumentReader reader = nextDocumentReader(); + if (reader == null) break; + writer.setPosition(recordCount); + if (reader.next() != EventType.START_MAP) { + throw new IllegalStateException("The document did not start with START_MAP!"); + } + try { + MapWriter map = writer.rootAsMap(); + if (reader.getId() != null) { + switch (reader.getId().getType()) { + case BINARY: + writeBinary(map.varBinary(ID_KEY), reader.getId().getBinary()); + break; + case STRING: + writeString(map.varChar(ID_KEY), reader.getId().getString()); + break; + default: + throw new UnsupportedOperationException(reader.getId().getType() + + " is not a supported type for _id field."); + } + } + writeToMap(reader, map); + recordCount++; + } catch (IllegalStateException e) { + logger.warn(String.format("Possible schema change at _id: %s", + IdCodec.asString(reader.getId())), e); + } + } + + writer.setValueCount(recordCount); + logger.debug("Took {} ms to get {} records", watch.elapsed(TimeUnit.MILLISECONDS), recordCount); + return recordCount; + } + + private void writeToMap(DBDocumentReader reader, MapWriter map) { + String fieldName = null; + map.start(); + outside: while (true) { + EventType event = reader.next(); + if (event == null) break outside; + switch (event) { + case FIELD_NAME: + fieldName = reader.getFieldName(); + break; + case NULL: + map.varChar(fieldName).write(null); // treat as VARCHAR for now + case BINARY: + writeBinary(map.varBinary(fieldName), reader.getBinary()); + break; + case BOOLEAN: + map.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); + break; + case STRING: + writeString(map.varChar(fieldName), reader.getString()); + break; + case BYTE: + map.tinyInt(fieldName).writeTinyInt(reader.getByte()); + break; + case SHORT: + map.smallInt(fieldName).writeSmallInt(reader.getShort()); + break; + case INT: + map.integer(fieldName).writeInt(reader.getInt()); + break; + case LONG: + map.bigInt(fieldName).writeBigInt(reader.getLong()); + break; + case FLOAT: + map.float4(fieldName).writeFloat4(reader.getFloat()); + break; + case DOUBLE: + map.float8(fieldName).writeFloat8(reader.getDouble()); + break; + case DECIMAL: + throw new UnsupportedOperationException("Decimals are currently not supported."); + case DATE: + map.date(fieldName).writeDate(reader.getDate().getTime()); + break; + case TIME: + map.time(fieldName).writeTime(reader.getTimeInt()); + break; + case TIMESTAMP: + map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); + break; + case INTERVAL: + throw new UnsupportedOperationException("Interval is currently not supported."); + case START_MAP: + writeToMap(reader, map.map(fieldName)); + break; + case END_MAP: + break outside; + case START_ARRAY: + writeToList(reader, map.list(fieldName)); + break; + case END_ARRAY: + throw new IllegalStateException("Shouldn't get a END_ARRAY inside a map"); + default: + throw new UnsupportedOperationException("Unsupported type: " + event); + } + } + map.end(); + } + + private void writeToList(DBDocumentReader reader, ListWriter list) { + list.start(); + outside: while (true) { + EventType event = reader.next(); + if (event == null) break outside; + switch (event) { + case FIELD_NAME: + throw new IllegalStateException("Shouldn't get a field name inside a list"); + case NULL: + list.varChar().write(null); // treat as VARCHAR for now + case BINARY: + writeBinary(list.varBinary(), reader.getBinary()); + break; + case BOOLEAN: + list.bit().writeBit(reader.getBoolean() ? 1 : 0); + break; + case STRING: + writeString(list.varChar(), reader.getString()); + break; + case BYTE: + list.tinyInt().writeTinyInt(reader.getByte()); + break; + case SHORT: + list.smallInt().writeSmallInt(reader.getShort()); + break; + case INT: + list.integer().writeInt(reader.getInt()); + break; + case LONG: + list.bigInt().writeBigInt(reader.getLong()); + break; + case FLOAT: + list.float4().writeFloat4(reader.getFloat()); + break; + case DOUBLE: + list.float8().writeFloat8(reader.getDouble()); + break; + case DECIMAL: + throw new UnsupportedOperationException("Decimals are currently not supported."); + case DATE: + list.date().writeDate(reader.getDate().getTime()); + break; + case TIME: + list.time().writeTime(reader.getTimeInt()); + break; + case TIMESTAMP: + list.timeStamp().writeTimeStamp(reader.getTimestampLong()); + break; + case INTERVAL: + throw new UnsupportedOperationException("Interval is currently not supported."); + case START_MAP: + writeToMap(reader, list.map()); + break; + case END_MAP: + throw new IllegalStateException("Shouldn't get a END_MAP inside a list"); + case START_ARRAY: + writeToList(reader, list.list()); + break; + case END_ARRAY: + break outside; + default: + throw new UnsupportedOperationException("Unsupported type: " + event); + } + } + list.end(); + } + + private void writeBinary(VarBinaryWriter binaryWriter, ByteBuffer buf) { + buffer.reallocIfNeeded(buf.remaining()); + buffer.setBytes(0, buf, buf.position(), buf.remaining()); + binaryWriter.writeVarBinary(0, buf.remaining(), buffer); + } + + private void writeString(VarCharWriter varCharWriter, String string) { + final byte[] strBytes = Bytes.toBytes(string); + buffer.reallocIfNeeded(strBytes.length); + buffer.setBytes(0, strBytes); + varCharWriter.writeVarChar(0, strBytes.length, buffer); + } + + private DBDocumentReader nextDocumentReader() { + final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats(); + try { + if (operatorStats != null) { + operatorStats.startWait(); + } + try { + if (!documentReaderIterators.hasNext()) { + return null; + } else { + return (DBDocumentReader) documentReaderIterators.next(); + } + } finally { + if (operatorStats != null) { + operatorStats.stopWait(); + } + } + } catch (DBException e) { + throw new DrillRuntimeException(e); + } + } + + @Override + public void close() { + if (documentStream != null) { + documentStream.close(); + } + if (table != null) { + table.close(); + } + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java new file mode 100644 index 00000000000..894e5bd4ea9 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb.util; + +public class CommonFns { + + public static boolean isNullOrEmpty(final byte[] key) { + return key == null || key.length == 0; + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java new file mode 100644 index 00000000000..cd1333ad7c1 --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.hbase.HBaseTestsSuite; +import org.apache.hadoop.conf.Configuration; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; +import org.ojai.Document; +import org.ojai.DocumentStream; +import org.ojai.json.Json; + +import com.mapr.db.Admin; +import com.mapr.db.MapRDB; +import com.mapr.db.Table; +import com.mapr.drill.maprdb.tests.binary.TestMapRDBFilterPushDown; +import com.mapr.drill.maprdb.tests.binary.TestMapRDBSimple; +import com.mapr.drill.maprdb.tests.json.TestSimpleJson; + +@RunWith(Suite.class) +@SuiteClasses({ + TestMapRDBSimple.class, + TestMapRDBFilterPushDown.class, + TestSimpleJson.class +}) +public class MaprDBTestsSuite { + private static final String TMP_BUSINESS_TABLE = "/tmp/business"; + + private static final boolean IS_DEBUG = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0; + + private static volatile AtomicInteger initCount = new AtomicInteger(0); + private static volatile Configuration conf; + + private static Admin admin; + + @BeforeClass + public static void setupTests() throws Exception { + if (initCount.get() == 0) { + synchronized (MaprDBTestsSuite.class) { + if (initCount.get() == 0) { + HBaseTestsSuite.configure(false, true); + HBaseTestsSuite.initCluster(); + createJsonTables(); + + // Sleep to allow table data to be flushed to tables. + // Without this, the row count stats to return 0, + // causing the planner to reject optimized plans. + System.out.println("Sleeping for 5 seconds to allow table flushes"); + Thread.sleep(5000); + + conf = HBaseTestsSuite.getConf(); + initCount.incrementAndGet(); // must increment while inside the synchronized block + return; + } + } + } + initCount.incrementAndGet(); + return; + } + + @AfterClass + public static void cleanupTests() throws Exception { + synchronized (MaprDBTestsSuite.class) { + if (initCount.decrementAndGet() == 0) { + HBaseTestsSuite.tearDownCluster(); + deleteJsonTables(); + } + } + } + + private static volatile boolean pluginCreated; + + public static Configuration createPluginAndGetConf(DrillbitContext ctx) throws Exception { + if (!pluginCreated) { + synchronized (MaprDBTestsSuite.class) { + if (!pluginCreated) { + String pluginConfStr = "{" + + " \"type\": \"file\"," + + " \"enabled\": true," + + " \"connection\": \"maprfs:///\"," + + " \"workspaces\": {" + + " \"default\": {" + + " \"location\": \"/tmp\"," + + " \"writable\": false," + + " \"defaultInputFormat\": \"maprdb\"" + + " }" + + " }," + + " \"formats\": {" + + " \"maprdb\": {" + + " \"type\": \"maprdb\"" + + " }" + + " }" + + "}"; + + FileSystemConfig pluginConfig = ctx.getConfig().getMapper().readValue(pluginConfStr, FileSystemConfig.class); + // create the plugin with "hbase" name so that we can run HBase unit tests against them + ctx.getStorage().createOrUpdate("hbase", pluginConfig, true); + } + } + } + return conf; + } + + public static boolean isDebug() { + return IS_DEBUG; + } + + public static InputStream getJsonStream(String resourceName) { + return MaprDBTestsSuite.class.getClassLoader().getResourceAsStream(resourceName); + } + + public static void createJsonTables() throws IOException { + admin = MapRDB.newAdmin(); + if (admin.tableExists(TMP_BUSINESS_TABLE)) { + admin.deleteTable(TMP_BUSINESS_TABLE); + } + + try (Table table = admin.createTable(TMP_BUSINESS_TABLE); + InputStream in = getJsonStream("json/business.json"); + DocumentStream stream = Json.newDocumentStream(in)) { + for (Document document : stream) { + table.insert(document, "business_id"); + } + table.flush(); + } + } + + public static void deleteJsonTables() { + if (admin != null) { + if (admin.tableExists(TMP_BUSINESS_TABLE)) { + admin.deleteTable(TMP_BUSINESS_TABLE); + } + admin.close(); + } + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java new file mode 100644 index 00000000000..b049f37424b --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.binary; + +import org.apache.drill.hbase.TestHBaseFilterPushDown; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +/** + * This class does not define any test method but includes all test methods + * defined in the parent class, all of which are tested against MapRDB instead + * of HBase. + */ +@Category(ClusterTest.class) +public class TestMapRDBFilterPushDown extends TestHBaseFilterPushDown { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java new file mode 100644 index 00000000000..894e64d0860 --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.binary; + +import org.apache.drill.hbase.BaseHBaseTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +@Category(ClusterTest.class) +public class TestMapRDBSimple extends BaseHBaseTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + + @Test + public void testMe() throws Exception { + setColumnWidths(new int[] {8, 38, 38}); + final String sql = "SELECT\n" + + " *\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` tableName"; + runHBaseSQLVerifyCount(sql, 7); + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java new file mode 100644 index 00000000000..c92fc448b26 --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.json; + +import java.util.List; + +import org.apache.drill.BaseTestQuery; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +@Category(ClusterTest.class) +public class TestSimpleJson extends BaseTestQuery { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + + @Test + public void testMe() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business"; + runSQLAndVerifyCount(sql, 10); + } + + protected List runHBaseSQLlWithResults(String sql) throws Exception { + System.out.println("Running query:\n" + sql); + return testSqlWithResults(sql); + } + + protected void runSQLAndVerifyCount(String sql, int expectedRowCount) throws Exception{ + List results = runHBaseSQLlWithResults(sql); + printResultAndVerifyRowCount(results, expectedRowCount); + } + + private void printResultAndVerifyRowCount(List results, int expectedRowCount) throws SchemaChangeException { + int rowCount = printResult(results); + if (expectedRowCount != -1) { + Assert.assertEquals(expectedRowCount, rowCount); + } + } + +} diff --git a/contrib/format-maprdb/src/test/resources/hbase-site.xml b/contrib/format-maprdb/src/test/resources/hbase-site.xml new file mode 100644 index 00000000000..92e8a867486 --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/hbase-site.xml @@ -0,0 +1,25 @@ + + + + + + hbase.table.namespace.mappings + *:/tmp/ + + + diff --git a/contrib/format-maprdb/src/test/resources/json/business.json b/contrib/format-maprdb/src/test/resources/json/business.json new file mode 100644 index 00000000000..e1d46ac13c0 --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/json/business.json @@ -0,0 +1,10 @@ +{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business"} diff --git a/contrib/format-maprdb/src/test/resources/logback.xml b/contrib/format-maprdb/src/test/resources/logback.xml index 00cc85b303f..38c2fc870e5 100644 --- a/contrib/format-maprdb/src/test/resources/logback.xml +++ b/contrib/format-maprdb/src/test/resources/logback.xml @@ -28,7 +28,7 @@ - ${logback.log.dir:-./target/surefire-reports}/hbase-tests-${bySecond}.log + ${logback.log.dir:-./target/surefire-reports}/maprdb-tests-${bySecond}.log false %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n @@ -41,6 +41,16 @@ + + + + + + + + + + From aa6312171f8a325d5fccdd7d91ba91d0eb2b366c Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Fri, 26 Aug 2016 13:12:41 -0700 Subject: [PATCH 30/58] Use DBDocumentReaderBase instead of DBDocumentReader. --- .../exec/store/maprdb/MapRDBSubScanSpec.java | 3 ++- .../maprdb/json/MaprDBJsonRecordReader.java | 16 ++++++++-------- .../drill/maprdb/tests/MaprDBTestsSuite.java | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java index 5e8d84cfb54..cc8bc5df436 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.store.maprdb; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.mapr.fs.jni.MapRConstants; import com.mapr.org.apache.hadoop.hbase.util.Bytes; @@ -29,7 +30,7 @@ public class MapRDBSubScanSpec { protected byte[] stopRow; protected byte[] serializedFilter; - @parquet.org.codehaus.jackson.annotate.JsonCreator + @JsonCreator public MapRDBSubScanSpec(@JsonProperty("tableName") String tableName, @JsonProperty("regionServer") String regionServer, @JsonProperty("startRow") byte[] startRow, diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 2f2e5618d8a..590c6e398b0 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -63,7 +63,7 @@ import com.mapr.db.Table.TableOption; import com.mapr.db.exceptions.DBException; import com.mapr.db.impl.IdCodec; -import com.mapr.db.ojai.DBDocumentReader; +import com.mapr.db.ojai.DBDocumentReaderBase; import com.mapr.db.util.ByteBufs; import com.mapr.org.apache.hadoop.hbase.util.Bytes; @@ -168,7 +168,7 @@ public int next() { int recordCount = 0; while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION) { - DBDocumentReader reader = nextDocumentReader(); + DBDocumentReaderBase reader = nextDocumentReader(); if (reader == null) break; writer.setPosition(recordCount); if (reader.next() != EventType.START_MAP) { @@ -202,7 +202,7 @@ public int next() { return recordCount; } - private void writeToMap(DBDocumentReader reader, MapWriter map) { + private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { String fieldName = null; map.start(); outside: while (true) { @@ -271,8 +271,8 @@ private void writeToMap(DBDocumentReader reader, MapWriter map) { map.end(); } - private void writeToList(DBDocumentReader reader, ListWriter list) { - list.start(); + private void writeToList(DBDocumentReaderBase reader, ListWriter list) { + list.startList(); outside: while (true) { EventType event = reader.next(); if (event == null) break outside; @@ -335,7 +335,7 @@ private void writeToList(DBDocumentReader reader, ListWriter list) { throw new UnsupportedOperationException("Unsupported type: " + event); } } - list.end(); + list.endList(); } private void writeBinary(VarBinaryWriter binaryWriter, ByteBuffer buf) { @@ -351,7 +351,7 @@ private void writeString(VarCharWriter varCharWriter, String string) { varCharWriter.writeVarChar(0, strBytes.length, buffer); } - private DBDocumentReader nextDocumentReader() { + private DBDocumentReaderBase nextDocumentReader() { final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats(); try { if (operatorStats != null) { @@ -361,7 +361,7 @@ private DBDocumentReader nextDocumentReader() { if (!documentReaderIterators.hasNext()) { return null; } else { - return (DBDocumentReader) documentReaderIterators.next(); + return (DBDocumentReaderBase) documentReaderIterators.next(); } } finally { if (operatorStats != null) { diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java index cd1333ad7c1..0f54796e142 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java @@ -117,7 +117,7 @@ public static Configuration createPluginAndGetConf(DrillbitContext ctx) throws E " }" + "}"; - FileSystemConfig pluginConfig = ctx.getConfig().getMapper().readValue(pluginConfStr, FileSystemConfig.class); + FileSystemConfig pluginConfig = ctx.getLpPersistence().getMapper().readValue(pluginConfStr, FileSystemConfig.class); // create the plugin with "hbase" name so that we can run HBase unit tests against them ctx.getStorage().createOrUpdate("hbase", pluginConfig, true); } From d1adebd836e9c7c08bfd9abce11e638b582864aa Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Sat, 24 Oct 2015 15:48:35 -0700 Subject: [PATCH 31/58] Filter push-down support for JSON tables. --- .../exec/store/maprdb/MapRDBFormatPlugin.java | 4 +- .../MapRDBPushFilterIntoScan.java | 114 +++++-- .../maprdb/binary/MapRDBFilterBuilder.java | 2 +- .../json/CompareFunctionsProcessor.java | 201 ++++++++++++ .../maprdb/json/JsonConditionBuilder.java | 231 +++++++++++++ .../exec/store/maprdb/json/JsonScanSpec.java | 94 ++++++ .../store/maprdb/json/JsonSubScanSpec.java | 94 ++++++ .../store/maprdb/json/JsonTableGroupScan.java | 57 ++-- .../maprdb/json/MaprDBJsonRecordReader.java | 21 +- .../binary/TestMapRDBCFAsJSONString.java | 47 +++ .../binary/TestMapRDBProjectPushDown.java | 47 +++ .../tests/binary/TestMapRDBQueries.java | 47 +++ .../maprdb/tests/json/TestSimpleJson.java | 309 ++++++++++++++++++ .../src/test/resources/json/business.json | 20 +- 14 files changed, 1206 insertions(+), 82 deletions(-) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/{binary => }/MapRDBPushFilterIntoScan.java (52%) create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java create mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java create mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java create mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java index d22434d09c6..0694f5b2590 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java @@ -41,7 +41,7 @@ import org.apache.drill.exec.store.dfs.FormatPlugin; import org.apache.drill.exec.store.hbase.HBaseScanSpec; import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan; -import org.apache.drill.exec.store.maprdb.binary.MapRDBPushFilterIntoScan; +import org.apache.drill.exec.store.maprdb.json.JsonScanSpec; import org.apache.drill.exec.store.maprdb.json.JsonTableGroupScan; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -131,7 +131,7 @@ public AbstractGroupScan getGroupScan(String userName, FileSelection selection, TableProperties props = maprfs.getTableProperties(new Path(tableName)); if (props.getAttr().getJson()) { - MapRDBSubScanSpec scanSpec = new MapRDBSubScanSpec().setTableName(tableName); + JsonScanSpec scanSpec = new JsonScanSpec(tableName, null/*condition*/); return new JsonTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns); } else { HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java similarity index 52% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBPushFilterIntoScan.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index 5adff38c9db..714221fae50 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.binary; +package org.apache.drill.exec.store.maprdb; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.exec.planner.logical.DrillOptiq; @@ -27,11 +27,17 @@ import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan; +import org.apache.drill.exec.store.maprdb.binary.MapRDBFilterBuilder; +import org.apache.drill.exec.store.maprdb.json.JsonConditionBuilder; +import org.apache.drill.exec.store.maprdb.json.JsonScanSpec; +import org.apache.drill.exec.store.maprdb.json.JsonTableGroupScan; import org.apache.calcite.rel.RelNode; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rex.RexNode; +import org.ojai.store.QueryCondition; import com.google.common.collect.ImmutableList; @@ -50,24 +56,21 @@ public void onMatch(RelOptRuleCall call) { final FilterPrel filter = (FilterPrel) call.rel(0); final RexNode condition = filter.getCondition(); - BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); - if (groupScan.isFilterPushedDown()) { - /* - * The rule can get triggered again due to the transformed "scan => filter" sequence - * created by the earlier execution of this rule when we could not do a complete - * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon - * this flag to not do a re-processing of the rule on the already transformed call. - */ - return; + if (scan.getGroupScan() instanceof BinaryTableGroupScan) { + BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); + doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition); + } else { + assert(scan.getGroupScan() instanceof JsonTableGroupScan); + JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan(); + doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition); } - - doPushFilterToScan(call, filter, null, scan, groupScan, condition); } @Override public boolean matches(RelOptRuleCall call) { final ScanPrel scan = (ScanPrel) call.rel(1); - if (scan.getGroupScan() instanceof BinaryTableGroupScan) { + if (scan.getGroupScan() instanceof BinaryTableGroupScan || + scan.getGroupScan() instanceof JsonTableGroupScan) { return super.matches(call); } return false; @@ -82,34 +85,90 @@ public void onMatch(RelOptRuleCall call) { final ProjectPrel project = (ProjectPrel) call.rel(1); final FilterPrel filter = (FilterPrel) call.rel(0); - BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); - if (groupScan.isFilterPushedDown()) { - /* - * The rule can get triggered again due to the transformed "scan => filter" sequence - * created by the earlier execution of this rule when we could not do a complete - * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon - * this flag to not do a re-processing of the rule on the already transformed call. - */ - return; - } - // convert the filter to one that references the child of the project final RexNode condition = RelOptUtil.pushFilterPastProject(filter.getCondition(), project); - doPushFilterToScan(call, filter, project, scan, groupScan, condition); + if (scan.getGroupScan() instanceof BinaryTableGroupScan) { + BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); + doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition); + } else { + assert(scan.getGroupScan() instanceof JsonTableGroupScan); + JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan(); + doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition); + } } @Override public boolean matches(RelOptRuleCall call) { final ScanPrel scan = (ScanPrel) call.rel(2); - if (scan.getGroupScan() instanceof BinaryTableGroupScan) { + if (scan.getGroupScan() instanceof BinaryTableGroupScan || + scan.getGroupScan() instanceof JsonTableGroupScan) { return super.matches(call); } return false; } }; - protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel filter, final ProjectPrel project, final ScanPrel scan, final BinaryTableGroupScan groupScan, final RexNode condition) { + protected void doPushFilterIntoJsonGroupScan(RelOptRuleCall call, + FilterPrel filter, final ProjectPrel project, ScanPrel scan, + JsonTableGroupScan groupScan, RexNode condition) { + + if (groupScan.isFilterPushedDown()) { + /* + * The rule can get triggered again due to the transformed "scan => filter" sequence + * created by the earlier execution of this rule when we could not do a complete + * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon + * this flag to not do a re-processing of the rule on the already transformed call. + */ + return; + } + + final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + final JsonConditionBuilder jsonConditionBuilder = new JsonConditionBuilder(groupScan, conditionExp); + final JsonScanSpec newScanSpec = jsonConditionBuilder.parseTree(); + if (newScanSpec == null) { + return; //no filter pushdown ==> No transformation. + } + + final JsonTableGroupScan newGroupsScan = new JsonTableGroupScan(groupScan.getUserName(), + groupScan.getStoragePlugin(), + groupScan.getFormatPlugin(), + newScanSpec, + groupScan.getColumns()); + newGroupsScan.setFilterPushedDown(true); + + final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType()); + + // Depending on whether is a project in the middle, assign either scan or copy of project to childRel. + final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode)newScanPrel));; + + if (jsonConditionBuilder.isAllExpressionsConverted()) { + /* + * Since we could convert the entire filter condition expression into an HBase filter, + * we can eliminate the filter operator altogether. + */ + call.transformTo(childRel); + } else { + call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel))); + } + } + + protected void doPushFilterIntoBinaryGroupScan(final RelOptRuleCall call, + final FilterPrel filter, + final ProjectPrel project, + final ScanPrel scan, + final BinaryTableGroupScan groupScan, + final RexNode condition) { + + if (groupScan.isFilterPushedDown()) { + /* + * The rule can get triggered again due to the transformed "scan => filter" sequence + * created by the earlier execution of this rule when we could not do a complete + * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon + * this flag to not do a re-processing of the rule on the already transformed call. + */ + return; + } final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); final MapRDBFilterBuilder maprdbFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp); @@ -137,5 +196,4 @@ protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel fi call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel))); } } - } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java index 800d155e223..07c33648110 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java @@ -52,7 +52,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor { + + private String functionName; + private Boolean success; + private Value value; + private SchemaPath path; + + public CompareFunctionsProcessor(String functionName) { + this.functionName = functionName; + this.success = false; + this.value = null; + } + + public static boolean isCompareFunction(String functionName) { + return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName); + } + + public static CompareFunctionsProcessor process(FunctionCall call) { + String functionName = call.getName(); + LogicalExpression nameArg = call.args.get(0); + LogicalExpression valueArg = call.args.size() >= 2? call.args.get(1) : null; + CompareFunctionsProcessor evaluator = new CompareFunctionsProcessor(functionName); + + //if (valueArg != null) { + if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) { + LogicalExpression swapArg = valueArg; + valueArg = nameArg; + nameArg = swapArg; + evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName); + } + evaluator.success = nameArg.accept(evaluator, valueArg); + //} + + return evaluator; + } + + public boolean isSuccess() { + // TODO Auto-generated method stub + return success; + } + + public SchemaPath getPath() { + return path; + } + + public Value getValue() { + return value; + } + + public String getFunctionName() { + return functionName; + } + + @Override + public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) throws RuntimeException { + // If valueArg is null, this might be a IS NULL/IS NOT NULL type of query + if (valueArg == null) { + this.path = path; + return true; + } + + if (valueArg instanceof QuotedString) { + this.value = KeyValueBuilder.initFrom(((QuotedString) valueArg).value); + this.path = path; + return true; + } + + if (valueArg instanceof IntExpression) { + this.value = KeyValueBuilder.initFrom(((IntExpression)valueArg).getInt()); + this.path = path; + return true; + } + + if (valueArg instanceof FloatExpression) { + this.value = KeyValueBuilder.initFrom(((FloatExpression)valueArg).getFloat()); + this.path = path; + return true; + } + + if (valueArg instanceof BooleanExpression) { + this.value = KeyValueBuilder.initFrom(((BooleanExpression)valueArg).getBoolean()); + this.path = path; + return true; + } + + if (valueArg instanceof Decimal28Expression) { + this.value = KeyValueBuilder.initFrom(((Decimal28Expression)valueArg).getBigDecimal()); + this.path = path; + return true; + } + + if (valueArg instanceof Decimal38Expression) { + this.value = KeyValueBuilder.initFrom(((Decimal38Expression)valueArg).getBigDecimal()); + this.path = path; + return true; + } + + if (valueArg instanceof DoubleExpression) { + this.value = KeyValueBuilder.initFrom(((DoubleExpression)valueArg).getDouble()); + this.path = path; + return true; + } + + if (valueArg instanceof LongExpression) { + this.value = KeyValueBuilder.initFrom(((LongExpression)valueArg).getLong()); + this.path = path; + return true; + } + + if (valueArg instanceof DateExpression) { + this.value = KeyValueBuilder.initFrom(new Date(((DateExpression)valueArg).getDate())); + this.path = path; + return true; + } + + if (valueArg instanceof TimeExpression) { + this.value = KeyValueBuilder.initFrom(new Time(((TimeExpression)valueArg).getTime())); + this.path = path; + return true; + } + + if (valueArg instanceof TimeStampExpression) { + this.value = KeyValueBuilder.initFrom(new Timestamp(((TimeStampExpression)valueArg).getTimeStamp())); + this.path = path; + return true; + } + + return false; + } + + private static final ImmutableSet> VALUE_EXPRESSION_CLASSES; + static { + ImmutableSet.Builder> builder = ImmutableSet.builder(); + VALUE_EXPRESSION_CLASSES = builder + .add(BooleanExpression.class) + .add(DateExpression.class) + .add(DoubleExpression.class) + .add(FloatExpression.class) + .add(IntExpression.class) + .add(LongExpression.class) + .add(QuotedString.class) + .add(TimeExpression.class) + .build(); + } + + private static final ImmutableMap COMPARE_FUNCTIONS_TRANSPOSE_MAP; + static { + ImmutableMap.Builder builder = ImmutableMap.builder(); + COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder + // unary functions + .put("isnotnull", "isnotnull") + .put("isNotNull", "isNotNull") + .put("is not null", "is not null") + .put("isnull", "isnull") + .put("isNull", "isNull") + .put("is null", "is null") + // binary functions + .put("like", "like") + .put("equal", "equal") + .put("not_equal", "not_equal") + .put("greater_than_or_equal_to", "less_than_or_equal_to") + .put("greater_than", "less_than") + .put("less_than_or_equal_to", "greater_than_or_equal_to") + .put("less_than", "greater_than") + .build(); + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java new file mode 100644 index 00000000000..a48d784b0b5 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java @@ -0,0 +1,231 @@ +package org.apache.drill.exec.store.maprdb.json; + +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.drill.exec.store.hbase.DrillHBaseConstants; +import org.apache.hadoop.hbase.HConstants; +import org.bouncycastle.util.Arrays; +import org.ojai.Value; + +import static org.ojai.DocumentConstants.ID_KEY; + +import org.ojai.store.QueryCondition; +import org.ojai.store.QueryCondition.Op; + +import com.google.common.collect.ImmutableList; +import com.mapr.db.MapRDB; +import com.mapr.db.impl.IdCodec; + +public class JsonConditionBuilder extends AbstractExprVisitor implements DrillHBaseConstants { + + final private JsonTableGroupScan groupScan; + + final private LogicalExpression le; + + private boolean allExpressionsConverted = true; + + public JsonConditionBuilder(JsonTableGroupScan groupScan, + LogicalExpression conditionExp) { + this.groupScan = groupScan; + this.le = conditionExp; + } + + public JsonScanSpec parseTree() { + JsonScanSpec parsedSpec = le.accept(this, null); + if (parsedSpec != null) { + parsedSpec.mergeScanSpec("booleanAnd", this.groupScan.getScanSpec()); + } + return parsedSpec; + } + + public boolean isAllExpressionsConverted() { + // TODO Auto-generated method stub + return allExpressionsConverted; + } + + @Override + public JsonScanSpec visitUnknown(LogicalExpression e, Void value) throws RuntimeException { + allExpressionsConverted = false; + return null; + } + + @Override + public JsonScanSpec visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException { + return visitFunctionCall(op, value); + } + + @Override + public JsonScanSpec visitFunctionCall(FunctionCall call, Void value) throws RuntimeException { + JsonScanSpec nodeScanSpec = null; + String functionName = call.getName(); + ImmutableList args = call.args; + + if (CompareFunctionsProcessor.isCompareFunction(functionName)) { + CompareFunctionsProcessor processor = CompareFunctionsProcessor.process(call); + if (processor.isSuccess()) { + nodeScanSpec = createJsonScanSpec(call, processor); + } + } else { + switch(functionName) { + case "booleanAnd": + case "booleanOr": + nodeScanSpec = args.get(0).accept(this, null); + for (int i = 1; i < args.size(); ++i) { + JsonScanSpec nextScanSpec = args.get(i).accept(this, null); + if (nodeScanSpec != null && nextScanSpec != null) { + nodeScanSpec.mergeScanSpec(functionName, nextScanSpec); + } else { + allExpressionsConverted = false; + if ("booleanAnd".equals(functionName)) { + nodeScanSpec = nodeScanSpec == null ? nextScanSpec : nodeScanSpec; + } + } + } + break; + } + } + + if (nodeScanSpec == null) { + allExpressionsConverted = false; + } + + return nodeScanSpec; + } + + private void setIsCondition(QueryCondition c, + String str, + QueryCondition.Op op, + Value v) { + switch (v.getType()) { + case BOOLEAN: + c.is(str, op, v.getBoolean()); + break; + case STRING: + c.is(str, op, v.getString()); + break; + case BYTE: + c.is(str, op, v.getByte()); + break; + case SHORT: + c.is(str, op, v.getShort()); + break; + case INT: + c.is(str, op, v.getInt()); + break; + case LONG: + c.is(str, op, v.getLong()); + break; + case FLOAT: + c.is(str, op, v.getFloat()); + break; + case DOUBLE: + c.is(str, op, v.getDouble()); + break; + case DECIMAL: + c.is(str, op, v.getDecimal()); + break; + case DATE: + c.is(str, op, v.getDate()); + break; + case TIME: + c.is(str, op, v.getTime()); + break; + case TIMESTAMP: + c.is(str, op, v.getTimestamp()); + break; + case BINARY: + c.is(str, op, v.getBinary()); + break; + // XXX/TODO: Map, Array? + default: + break; + } + } + + private JsonScanSpec createJsonScanSpec(FunctionCall call, + CompareFunctionsProcessor processor) { + String functionName = processor.getFunctionName(); + SchemaPath field = processor.getPath(); + Value fieldValue = processor.getValue(); + + boolean isRowKey = field.getAsUnescapedPath().equals(ID_KEY); + + QueryCondition cond = null; + switch (functionName) { + case "equal": + cond = MapRDB.newCondition(); + setIsCondition(cond, field.getAsUnescapedPath(), Op.EQUAL, fieldValue); + cond.build(); + break; + + case "not_equal": + cond = MapRDB.newCondition(); + setIsCondition(cond, field.getAsUnescapedPath(), Op.NOT_EQUAL, fieldValue); + cond.build(); + break; + + case "less_than": + cond = MapRDB.newCondition(); + setIsCondition(cond, field.getAsUnescapedPath(), Op.LESS, fieldValue); + cond.build(); + break; + + case "less_than_or_equal_to": + cond = MapRDB.newCondition(); + setIsCondition(cond, field.getAsUnescapedPath(), Op.LESS_OR_EQUAL, fieldValue); + cond.build(); + break; + + case "greater_than": + cond = MapRDB.newCondition(); + setIsCondition(cond, field.getAsUnescapedPath(), Op.GREATER, fieldValue); + cond.build(); + break; + + case "greater_than_or_equal_to": + cond = MapRDB.newCondition(); + setIsCondition(cond, field.getAsUnescapedPath(), Op.GREATER_OR_EQUAL, fieldValue); + cond.build(); + break; + + case "isnull": + cond = MapRDB.newCondition().notExists(field.getAsUnescapedPath()).build(); + break; + + case "isnotnull": + cond = MapRDB.newCondition().exists(field.getAsUnescapedPath()).build(); + break; + + case "istrue": + cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.EQUAL, true).build(); + break; + + case "isnotfalse": + cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.NOT_EQUAL, false).build(); + break; + + case "isfalse": + cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.EQUAL, false).build(); + break; + + case "isnottrue": + cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.NOT_EQUAL, true).build(); + break; + + case "like": + cond = MapRDB.newCondition().like(field.getAsUnescapedPath(), fieldValue.getString()).build(); + break; + + default: + } + + if (cond != null) { + return new JsonScanSpec(groupScan.getTableName(), cond); + } + + return null; + } +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java new file mode 100644 index 00000000000..f278dd4aa56 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java @@ -0,0 +1,94 @@ +package org.apache.drill.exec.store.maprdb.json; + +import org.apache.drill.exec.store.hbase.HBaseUtils; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.Bytes; +import org.ojai.store.QueryCondition; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.mapr.db.MapRDB; +import com.mapr.db.impl.ConditionImpl; + +public class JsonScanSpec { + protected String tableName; + protected QueryCondition condition; + + @JsonCreator + public JsonScanSpec(@JsonProperty("tableName") String tableName, + @JsonProperty("condition") QueryCondition condition) { + this.tableName = tableName; + this.condition = condition; + } + + public String getTableName() { + return this.tableName; + } + + public byte[] getStartRow() { + if (condition == null) { + return HConstants.EMPTY_START_ROW; + } + return ((ConditionImpl)this.condition).getRowkeyRanges().get(0).getStartRow(); + } + + public byte[] getStopRow() { + if (condition == null) { + return HConstants.EMPTY_END_ROW; + } + + return ((ConditionImpl)this.condition).getRowkeyRanges().get(0).getStopRow(); + } + + public Object getSerializedFilter() { + if (this.condition != null) { + return ((ConditionImpl)this.condition).getDescriptor().getSerialized(); + } + + return null; + } + + public void setCondition(QueryCondition condition) { + this.condition = condition; + } + + @JsonIgnore + public QueryCondition getCondition() { + return this.condition; + } + + public void mergeScanSpec(String functionName, JsonScanSpec scanSpec) { + + if (this.condition != null && scanSpec.getCondition() != null) { + QueryCondition newCond = MapRDB.newCondition(); + switch (functionName) { + case "booleanAnd": + newCond.and(); + break; + case "booleanOr": + newCond.or(); + break; + default: + assert(false); + } + + newCond.condition(this.condition) + .condition(scanSpec.getCondition()) + .close() + .build(); + + this.condition = newCond; + } else if (scanSpec.getCondition() != null){ + this.condition = scanSpec.getCondition(); + } + } + + @Override + public String toString() { + return "JsonScanSpec [tableName=" + tableName + + ", condition=" + (condition == null ? null : condition.toString()) + + "]"; + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java new file mode 100644 index 00000000000..936002d48b3 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java @@ -0,0 +1,94 @@ +package org.apache.drill.exec.store.maprdb.json; + +import java.nio.ByteBuffer; + +import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; +import org.apache.hadoop.hbase.HConstants; +import org.bouncycastle.util.Arrays; +import org.ojai.DocumentConstants; +import org.ojai.Value; +import org.ojai.store.QueryCondition; +import org.ojai.store.QueryCondition.Op; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.mapr.db.MapRDB; +import com.mapr.db.impl.ConditionImpl; +import com.mapr.db.impl.IdCodec; + +public class JsonSubScanSpec extends MapRDBSubScanSpec { + + protected QueryCondition condition; + + @JsonCreator + public JsonSubScanSpec(@JsonProperty("tableName") String tableName, + @JsonProperty("regionServer") String regionServer, + @JsonProperty("startRow") byte[] startRow, + @JsonProperty("stopRow") byte[] stopRow, + @JsonProperty("cond") QueryCondition cond) { + super(tableName, regionServer, null, null, null, null); + + this.condition = MapRDB.newCondition().and(); + + if (cond != null) { + this.condition.condition(cond); + } + + if (startRow != null && + Arrays.areEqual(startRow, HConstants.EMPTY_START_ROW) == false) { + Value startVal = IdCodec.decode(startRow); + + switch(startVal.getType()) { + case BINARY: + this.condition.is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, startVal.getBinary()); + break; + case STRING: + this.condition.is(DocumentConstants.ID_FIELD, Op.LESS, startVal.getString()); + break; + default: + throw new IllegalStateException("Encountered an unsupported type " + startVal.getType() + + " for _id"); + } + } + + if (stopRow != null && + Arrays.areEqual(stopRow, HConstants.EMPTY_END_ROW) == false) { + Value stopVal = IdCodec.decode(stopRow); + + switch(stopVal.getType()) { + case BINARY: + this.condition.is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, stopVal.getBinary()); + break; + case STRING: + this.condition.is(DocumentConstants.ID_FIELD, Op.LESS, stopVal.getString()); + break; + default: + throw new IllegalStateException("Encountered an unsupported type " + stopVal.getType() + + " for _id"); + } + } + + this.condition.close().build(); + } + + public void setCondition(QueryCondition cond) { + condition = cond; + } + + @JsonIgnore + public QueryCondition getCondition() { + return this.condition; + } + + public byte[] getSerializedFilter() { + if (this.condition != null) { + ByteBuffer bbuf = ((ConditionImpl)this.condition).getDescriptor().getSerialized(); + byte[] serFilter = new byte[bbuf.limit() - bbuf.position()]; + bbuf.get(serFilter); + return serFilter; + } + + return null; + } +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java index e798c52e0c4..e7231798fed 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java @@ -21,15 +21,20 @@ import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.NavigableMap; import java.util.TreeMap; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.PhysicalOperatorSetupException; +import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.base.ScanStats; import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.dfs.FileSystemConfig; import org.apache.drill.exec.store.dfs.FileSystemPlugin; @@ -37,9 +42,10 @@ import org.apache.drill.exec.store.maprdb.MapRDBFormatPluginConfig; import org.apache.drill.exec.store.maprdb.MapRDBGroupScan; import org.apache.drill.exec.store.maprdb.MapRDBSubScan; -import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; import org.apache.drill.exec.store.maprdb.MapRDBTableStats; import org.apache.drill.exec.store.maprdb.TabletFragmentInfo; +import org.apache.drill.exec.store.maprdb.json.JsonScanSpec; +import org.apache.drill.exec.store.maprdb.json.JsonSubScanSpec; import org.apache.hadoop.conf.Configuration; import org.codehaus.jackson.annotate.JsonCreator; @@ -61,11 +67,11 @@ public class JsonTableGroupScan extends MapRDBGroupScan { private MapRDBTableStats tableStats; - private MapRDBSubScanSpec subscanSpec; + private JsonScanSpec scanSpec; @JsonCreator public JsonTableGroupScan(@JsonProperty("userName") final String userName, - @JsonProperty("subscanSpec") MapRDBSubScanSpec subscanSpec, + @JsonProperty("scanSpec") JsonScanSpec scanSpec, @JsonProperty("storage") FileSystemConfig storagePluginConfig, @JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig, @JsonProperty("columns") List columns, @@ -73,13 +79,13 @@ public JsonTableGroupScan(@JsonProperty("userName") final String userName, this (userName, (FileSystemPlugin) pluginRegistry.getPlugin(storagePluginConfig), (MapRDBFormatPlugin) pluginRegistry.getFormatPlugin(storagePluginConfig, formatPluginConfig), - subscanSpec, columns); + scanSpec, columns); } public JsonTableGroupScan(String userName, FileSystemPlugin storagePlugin, - MapRDBFormatPlugin formatPlugin, MapRDBSubScanSpec subscanSpec, List columns) { + MapRDBFormatPlugin formatPlugin, JsonScanSpec scanSpec, List columns) { super(storagePlugin, formatPlugin, columns, userName); - this.subscanSpec = subscanSpec; + this.scanSpec = scanSpec; init(); } @@ -89,7 +95,7 @@ public JsonTableGroupScan(String userName, FileSystemPlugin storagePlugin, */ private JsonTableGroupScan(JsonTableGroupScan that) { super(that); - this.subscanSpec = that.subscanSpec; + this.scanSpec = that.scanSpec; this.endpointFragmentMapping = that.endpointFragmentMapping; this.tableStats = that.tableStats; } @@ -105,40 +111,40 @@ private void init() { logger.debug("Getting tablet locations"); try { Configuration conf = new Configuration(); - Table t = MapRDB.getTable(subscanSpec.getTableName()); - TabletInfo[] tabletInfos = t.getTabletInfos(); - tableStats = new MapRDBTableStats(conf, subscanSpec.getTableName()); + Table t = MapRDB.getTable(scanSpec.getTableName()); + TabletInfo[] tabletInfos = t.getTabletInfos(scanSpec.getCondition()); + tableStats = new MapRDBTableStats(conf, scanSpec.getTableName()); boolean foundStartRegion = false; regionsToScan = new TreeMap(); for (TabletInfo tabletInfo : tabletInfos) { TabletInfoImpl tabletInfoImpl = (TabletInfoImpl) tabletInfo; if (!foundStartRegion - && !isNullOrEmpty(subscanSpec.getStartRow()) - && !tabletInfoImpl.containsRow(subscanSpec.getStartRow())) { + && !isNullOrEmpty(scanSpec.getStartRow()) + && !tabletInfoImpl.containsRow(scanSpec.getStartRow())) { continue; } foundStartRegion = true; regionsToScan.put(new TabletFragmentInfo(tabletInfoImpl), tabletInfo.getLocations()[0]); - if (!isNullOrEmpty(subscanSpec.getStopRow()) - && tabletInfoImpl.containsRow(subscanSpec.getStopRow())) { + if (!isNullOrEmpty(scanSpec.getStopRow()) + && tabletInfoImpl.containsRow(scanSpec.getStopRow())) { break; } } } catch (Exception e) { - throw new DrillRuntimeException("Error getting region info for table: " + subscanSpec.getTableName(), e); + throw new DrillRuntimeException("Error getting region info for table: " + scanSpec.getTableName(), e); } } - protected MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) { - MapRDBSubScanSpec spec = subscanSpec; - MapRDBSubScanSpec subScanSpec = new MapRDBSubScanSpec( + protected JsonSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) { + // XXX/TODO check filter/Condition + JsonScanSpec spec = scanSpec; + JsonSubScanSpec subScanSpec = new JsonSubScanSpec( spec.getTableName(), regionsToScan.get(tfi), (!isNullOrEmpty(spec.getStartRow()) && tfi.containsRow(spec.getStartRow())) ? spec.getStartRow() : tfi.getStartKey(), (!isNullOrEmpty(spec.getStopRow()) && tfi.containsRow(spec.getStopRow())) ? spec.getStopRow() : tfi.getEndKey(), - spec.getSerializedFilter(), - null); + spec.getCondition()); return subScanSpec; } @@ -154,7 +160,7 @@ public MapRDBSubScan getSpecificScan(int minorFragmentId) { @Override public ScanStats getScanStats() { //TODO: look at stats for this. - long rowCount = (long) ((subscanSpec.getSerializedFilter() != null ? .5 : 1) * tableStats.getNumRows()); + long rowCount = (long) ((scanSpec.getSerializedFilter() != null ? .5 : 1) * tableStats.getNumRows()); int avgColumnSize = 10; int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size(); return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount); @@ -169,18 +175,17 @@ public PhysicalOperator getNewWithChildren(List children) { @JsonIgnore public String getTableName() { - return subscanSpec.getTableName(); + return scanSpec.getTableName(); } @Override public String toString() { return "JsonTableGroupScan [ScanSpec=" - + subscanSpec + ", columns=" + + scanSpec + ", columns=" + columns + "]"; } - public MapRDBSubScanSpec getSubscanSpec() { - return subscanSpec; + public JsonScanSpec getScanSpec() { + return scanSpec; } - } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 590c6e398b0..8044e40f4df 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -63,6 +63,7 @@ import com.mapr.db.Table.TableOption; import com.mapr.db.exceptions.DBException; import com.mapr.db.impl.IdCodec; +import com.mapr.db.ojai.DBDocumentReader; import com.mapr.db.ojai.DBDocumentReaderBase; import com.mapr.db.util.ByteBufs; import com.mapr.org.apache.hadoop.hbase.util.Bytes; @@ -80,9 +81,6 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private OperatorContext operatorContext; private VectorContainerWriter writer; - @SuppressWarnings("unused") - private boolean idOnly; - private DrillBuf buffer; private DocumentStream documentStream; @@ -93,13 +91,7 @@ public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, List projectedColumns, FragmentContext context) { buffer = context.getManagedBuffer(); tableName = Preconditions.checkNotNull(subScanSpec, "MapRDB reader needs a sub-scan spec").getTableName(); - condition = MapRDB.newCondition().and(); - addKeyCondition(condition, Op.GREATER_OR_EQUAL, subScanSpec.getStartRow()); - addKeyCondition(condition, Op.LESS, subScanSpec.getStopRow()); - if (subScanSpec.getSerializedFilter() != null) { - condition.condition(com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter()))); - } - condition.close().build(); + condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter())); setColumns(projectedColumns); } @@ -122,20 +114,19 @@ private void addKeyCondition(QueryCondition condition, Op op, byte[] key) { @Override protected Collection transformColumns(Collection columns) { Set transformed = Sets.newLinkedHashSet(); - idOnly = true; // TODO: handle the case when only ID is requested. if (!isStarQuery()) { ArrayList projectedFieldsList = Lists.newArrayList(); for (SchemaPath column : columns) { if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) { transformed.add(ID_PATH); - continue; + projectedFieldsList.add(ID_FIELD); + } else { + transformed.add(SchemaPath.getSimplePath(column.getRootSegment().getPath())); + projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath())); } - idOnly = false; - projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath())); } projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]); } else { - idOnly = false; transformed.add(ID_PATH); } diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java new file mode 100644 index 00000000000..525b034f387 --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.binary; + +import org.apache.drill.hbase.TestHBaseCFAsJSONString; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +/** + * This class does not define any test method but includes all test methods + * defined in the parent class, all of which are tested against MapRDB instead + * of HBase. + */ +@Category(ClusterTest.class) +public class TestMapRDBCFAsJSONString extends TestHBaseCFAsJSONString { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java new file mode 100644 index 00000000000..59d7a513c3a --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.binary; + +import org.apache.drill.hbase.TestHBaseProjectPushDown; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +/** + * This class does not define any test method but includes all test methods + * defined in the parent class, all of which are tested against MapRDB instead + * of HBase. + */ +@Category(ClusterTest.class) +public class TestMapRDBProjectPushDown extends TestHBaseProjectPushDown { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java new file mode 100644 index 00000000000..69e04a5ff48 --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.binary; + +import org.apache.drill.hbase.TestHBaseQueries; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +/** + * This class does not define any test method but includes all test methods + * defined in the parent class, all of which are tested against MapRDB instead + * of HBase. + */ +@Category(ClusterTest.class) +public class TestMapRDBQueries extends TestHBaseQueries { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index c92fc448b26..f4c7e8911d8 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.drill.BaseTestQuery; +import org.apache.drill.PlanTestBase; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.junit.AfterClass; @@ -55,6 +56,314 @@ public void testMe() throws Exception { runSQLAndVerifyCount(sql, 10); } + @Test + public void testPushdownStringEqual() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " name = 'Sprint'" + ; + runSQLAndVerifyCount(sql, 1); + + final String[] expectedPlan = {"condition=\\(name = \"Sprint\"\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushdownStringLike() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " name LIKE 'S%'" + ; + runSQLAndVerifyCount(sql, 3); + + final String[] expectedPlan = {"condition=\\(name MATCHES \"\\^\\\\\\\\QS\\\\\\\\E\\.\\*\\$\"\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushdownStringNotEqual() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " name <> 'Sprint'" + ; + runSQLAndVerifyCount(sql, 9); + + final String[] expectedPlan = {"condition=\\(name != \"Sprint\"\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushdownLongEqual() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " zip = 85260" + ; + runSQLAndVerifyCount(sql, 1); + + final String[] expectedPlan = {"condition=\\(zip = \\{\"\\$numberLong\":85260\\}\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testCompositePredicate() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " zip = 85260\n" + + " OR\n" + + " city = 'Las Vegas'" + ; + runSQLAndVerifyCount(sql, 4); + + final String[] expectedPlan = {"condition=\\(\\(zip = \\{\"\\$numberLong\":85260\\}\\) or \\(city = \"Las Vegas\"\\)\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPruneScanRange() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " _id = 'jFTZmywe7StuZ2hEjxyA'" + ; + runSQLAndVerifyCount(sql, 1); + + final String[] expectedPlan = {"condition=\\(_id = \"jFTZmywe7StuZ2hEjxyA\"\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPruneScanRangeAndPushDownCondition() throws Exception { + // XXX/TODO: + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " _id = 'jFTZmywe7StuZ2hEjxyA' AND\n" + + " name = 'Subway'" + ; + runSQLAndVerifyCount(sql, 1); + + final String[] expectedPlan = {"condition=\\(\\(_id = \"jFTZmywe7StuZ2hEjxyA\"\\) and \\(name = \"Subway\"\\)\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushDownOnSubField1() throws Exception { + setColumnWidths(new int[] {25, 120, 20}); + final String sql = "SELECT\n" + + " _id, name, b.attributes.Ambience.touristy attributes\n" + + "FROM\n" + + " hbase.`business` b\n" + + "WHERE\n" + + " b.`attributes.Ambience.casual` = false" + ; + runSQLAndVerifyCount(sql, 1); + + final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual = false\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushDownOnSubField2() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, b.attributes.Attire attributes\n" + + "FROM\n" + + " hbase.`business` b\n" + + "WHERE\n" + + " b.`attributes.Attire` = 'casual'" + ; + runSQLAndVerifyCount(sql, 4); + + final String[] expectedPlan = {"condition=\\(attributes.Attire = \"casual\"\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + @Test + public void testPushDownIsNull() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + + final String sql = "SELECT\n" + + " _id, name, attributes\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " business.`attributes.Ambience.casual` IS NULL" + ; + runSQLAndVerifyCount(sql, 7); + + final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual = null\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushDownIsNotNull() throws Exception { + setColumnWidths(new int[] {25, 75, 75, 50}); + + final String sql = "SELECT\n" + + " _id, name, b.attributes.Parking\n" + + "FROM\n" + + " hbase.`business` b\n" + + "WHERE\n" + + " b.`attributes.Ambience.casual` IS NOT NULL" + ; + runSQLAndVerifyCount(sql, 3); + + final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual != null\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushDownOnSubField3() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, b.attributes.`Accepts Credit Cards` attributes\n" + + "FROM\n" + + " hbase.`business` b\n" + + "WHERE\n" + + " b.`attributes.Accepts Credit Cards` IS NULL" + ; + runSQLAndVerifyCount(sql, 3); + + final String[] expectedPlan = {"condition=\\(attributes.Accepts Credit Cards = null\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushDownLong() throws Exception { + final String sql = "SELECT\n" + + " *\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " stars > 4.0" + ; + runSQLAndVerifyCount(sql, 2); + + final String[] expectedPlan = {"condition=\\(stars > 4\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + @Test + public void testPushDownSubField4() throws Exception { + final String sql = "SELECT\n" + + " *\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " business.`attributes.Good For.lunch` = true AND" + + " stars > 4.1" + ; + runSQLAndVerifyCount(sql, 1); + + final String[] expectedPlan = {"condition=\\(\\(attributes.Good For.lunch = true\\) and \\(stars > 4.1\\)\\)"}; + final String[] excludedPlan ={}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + + /* + @Test + public void testPushDownSubField5() throws Exception { + final String sql = "SELECT\n" + + " *\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " business.`hours.Tuesday.open` < TIME '10:30:00'" + ; + runSQLAndVerifyCount(sql, 1); + } + + @Test + public void testPushDownSubField6() throws Exception { + final String sql = "SELECT\n" + + " *\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " business.`hours.Sunday.close` > TIME '20:30:00'" + ; + runSQLAndVerifyCount(sql, 4); + } + + @Test + public void testPushDownSubField7() throws Exception { + setColumnWidths(new int[] {25, 40, 25, 45}); + final String sql = "SELECT\n" + + " _id, name, start_date, last_update\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " business.`start_date` = DATE '2012-07-14'" + ; + runSQLAndVerifyCount(sql, 1); + } + + @Test + public void testPushDownSubField8() throws Exception { + setColumnWidths(new int[] {25, 40, 25, 45}); + final String sql = "SELECT\n" + + " _id, name, start_date, last_update\n" + + "FROM\n" + + " hbase.`business` business\n" + + "WHERE\n" + + " business.`last_update` = TIMESTAMP '2012-10-20 07:42:46'" + ; + runSQLAndVerifyCount(sql, 1); + } + */ + protected List runHBaseSQLlWithResults(String sql) throws Exception { System.out.println("Running query:\n" + sql); return testSqlWithResults(sql); diff --git a/contrib/format-maprdb/src/test/resources/json/business.json b/contrib/format-maprdb/src/test/resources/json/business.json index e1d46ac13c0..1121aa3d318 100644 --- a/contrib/format-maprdb/src/test/resources/json/business.json +++ b/contrib/format-maprdb/src/test/resources/json/business.json @@ -1,10 +1,10 @@ -{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} -{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business", "start_date":"2011-07-14"} +{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":"2012-07-14"} +{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2013-07-14"} +{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-07-14"} +{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business", "start_date":"2011-04-14"} +{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business", "start_date":"2013-02-15"} +{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business", "start_date":"2013-01-21"} +{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business", "start_date":"2015-03-21"} +{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-02-13"} +{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":"2014-02-17"} From 8d72fcc1dde66266ce08dc40e9c10e26b2d7231e Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 21 Dec 2015 00:50:19 -0800 Subject: [PATCH 32/58] Updating with the new DocumentReader APIs. --- .../exec/store/maprdb/json/MaprDBJsonRecordReader.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 8044e40f4df..9080e8e7f27 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -19,7 +19,6 @@ import static org.ojai.DocumentConstants.ID_FIELD; import static org.ojai.DocumentConstants.ID_KEY; -import io.netty.buffer.DrillBuf; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -63,11 +62,12 @@ import com.mapr.db.Table.TableOption; import com.mapr.db.exceptions.DBException; import com.mapr.db.impl.IdCodec; -import com.mapr.db.ojai.DBDocumentReader; import com.mapr.db.ojai.DBDocumentReaderBase; import com.mapr.db.util.ByteBufs; import com.mapr.org.apache.hadoop.hbase.util.Bytes; +import io.netty.buffer.DrillBuf; + public class MaprDBJsonRecordReader extends AbstractRecordReader { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MaprDBJsonRecordReader.class); @@ -199,10 +199,8 @@ private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { outside: while (true) { EventType event = reader.next(); if (event == null) break outside; + fieldName = reader.getFieldName(); switch (event) { - case FIELD_NAME: - fieldName = reader.getFieldName(); - break; case NULL: map.varChar(fieldName).write(null); // treat as VARCHAR for now case BINARY: @@ -268,8 +266,6 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { EventType event = reader.next(); if (event == null) break outside; switch (event) { - case FIELD_NAME: - throw new IllegalStateException("Shouldn't get a field name inside a list"); case NULL: list.varChar().write(null); // treat as VARCHAR for now case BINARY: From 1f853f6a5117a90831ad592d250a1e25a781f9fd Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Mon, 4 Jan 2016 16:30:43 -0800 Subject: [PATCH 33/58] Update the json-support branch version in pom.xml + Fix for column name mangling in drill output. + Comment out Date, Time, Timestamp types since we are still deciding these on OJAI front. --- .../maprdb/json/CompareFunctionsProcessor.java | 4 ++-- .../maprdb/json/MaprDBJsonRecordReader.java | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java index 924c93f7a09..836361bb5b1 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java @@ -139,7 +139,7 @@ public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) thro this.path = path; return true; } - +/* if (valueArg instanceof DateExpression) { this.value = KeyValueBuilder.initFrom(new Date(((DateExpression)valueArg).getDate())); this.path = path; @@ -157,7 +157,7 @@ public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) thro this.path = path; return true; } - +*/ return false; } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 9080e8e7f27..e38ad2df991 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -232,12 +232,12 @@ private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { break; case DECIMAL: throw new UnsupportedOperationException("Decimals are currently not supported."); - case DATE: - map.date(fieldName).writeDate(reader.getDate().getTime()); - break; - case TIME: - map.time(fieldName).writeTime(reader.getTimeInt()); - break; + //case DATE: + // map.date(fieldName).writeDate(reader.getDate().getTime()); + // break; + //case TIME: + // map.time(fieldName).writeTime(reader.getTimeInt()); + // break; case TIMESTAMP: map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); break; @@ -297,9 +297,9 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { break; case DECIMAL: throw new UnsupportedOperationException("Decimals are currently not supported."); - case DATE: - list.date().writeDate(reader.getDate().getTime()); - break; + //case DATE: + // list.date().writeDate(reader.getDate().getTime()); + // break; case TIME: list.time().writeTime(reader.getTimeInt()); break; From 1887de29990dc1144544b285e262970c1d2ae99a Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Fri, 8 Jan 2016 10:00:57 -0800 Subject: [PATCH 34/58] MD-578: Query with three-way join fails with IllegalStateException --- .../exec/store/maprdb/json/MaprDBJsonRecordReader.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index e38ad2df991..fbb67def035 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -86,11 +86,14 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private DocumentStream documentStream; private Iterator documentReaderIterators; + + private boolean includeId; public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, List projectedColumns, FragmentContext context) { buffer = context.getManagedBuffer(); tableName = Preconditions.checkNotNull(subScanSpec, "MapRDB reader needs a sub-scan spec").getTableName(); + includeId = false; condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter())); setColumns(projectedColumns); } @@ -120,6 +123,7 @@ protected Collection transformColumns(Collection columns if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) { transformed.add(ID_PATH); projectedFieldsList.add(ID_FIELD); + includeId = true; } else { transformed.add(SchemaPath.getSimplePath(column.getRootSegment().getPath())); projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath())); @@ -128,6 +132,7 @@ protected Collection transformColumns(Collection columns projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]); } else { transformed.add(ID_PATH); + includeId = true; } return transformed; @@ -167,7 +172,7 @@ public int next() { } try { MapWriter map = writer.rootAsMap(); - if (reader.getId() != null) { + if (includeId && reader.getId() != null) { switch (reader.getId().getType()) { case BINARY: writeBinary(map.varBinary(ID_KEY), reader.getId().getBinary()); From 004aad9db375972556980cb608f439d13382a7dd Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 3 Jan 2016 21:23:06 -0800 Subject: [PATCH 35/58] Updated with the OJAI APIs changes in Date/Time/Document classes. --- .../json/CompareFunctionsProcessor.java | 19 +++++----------- .../maprdb/json/MaprDBJsonRecordReader.java | 22 +++++++++---------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java index 836361bb5b1..2e191a19433 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java @@ -1,11 +1,5 @@ package org.apache.drill.exec.store.maprdb.json; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.Calendar; -import java.util.TimeZone; - import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; @@ -16,16 +10,15 @@ import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; import org.apache.drill.common.expression.ValueExpressions.FloatExpression; import org.apache.drill.common.expression.ValueExpressions.IntExpression; -import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression; -import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression; import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.expression.ValueExpressions.TimeExpression; import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; import org.ojai.Value; - -import static org.ojai.util.Constants.MILLISECONDSPERDAY; +import org.ojai.types.ODate; +import org.ojai.types.OTime; +import org.ojai.types.OTimestamp; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -141,19 +134,19 @@ public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) thro } /* if (valueArg instanceof DateExpression) { - this.value = KeyValueBuilder.initFrom(new Date(((DateExpression)valueArg).getDate())); + this.value = KeyValueBuilder.initFrom(new ODate(((DateExpression)valueArg).getDate())); this.path = path; return true; } if (valueArg instanceof TimeExpression) { - this.value = KeyValueBuilder.initFrom(new Time(((TimeExpression)valueArg).getTime())); + this.value = KeyValueBuilder.initFrom(new OTime(((TimeExpression)valueArg).getTime())); this.path = path; return true; } if (valueArg instanceof TimeStampExpression) { - this.value = KeyValueBuilder.initFrom(new Timestamp(((TimeStampExpression)valueArg).getTimeStamp())); + this.value = KeyValueBuilder.initFrom(new OTimestamp(((TimeStampExpression)valueArg).getTimeStamp())); this.path = path; return true; } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index fbb67def035..fd8bf9369ec 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -44,6 +44,7 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter; import org.apache.drill.exec.vector.complex.writer.VarCharWriter; +import org.ojai.Document; import org.ojai.DocumentReader; import org.ojai.DocumentReader.EventType; import org.ojai.DocumentStream; @@ -56,7 +57,6 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.mapr.db.DBDocument; import com.mapr.db.MapRDB; import com.mapr.db.Table; import com.mapr.db.Table.TableOption; @@ -83,7 +83,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private DrillBuf buffer; - private DocumentStream documentStream; + private DocumentStream documentStream; private Iterator documentReaderIterators; @@ -237,12 +237,12 @@ private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { break; case DECIMAL: throw new UnsupportedOperationException("Decimals are currently not supported."); - //case DATE: - // map.date(fieldName).writeDate(reader.getDate().getTime()); - // break; - //case TIME: - // map.time(fieldName).writeTime(reader.getTimeInt()); - // break; + case DATE: + map.date(fieldName).writeDate(reader.getDate().toDate().getTime()); + break; + case TIME: + map.time(fieldName).writeTime(reader.getTimeInt()); + break; case TIMESTAMP: map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); break; @@ -302,9 +302,9 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { break; case DECIMAL: throw new UnsupportedOperationException("Decimals are currently not supported."); - //case DATE: - // list.date().writeDate(reader.getDate().getTime()); - // break; + case DATE: + list.date().writeDate(reader.getDate().toDate().getTime()); + break; case TIME: list.time().writeTime(reader.getTimeInt()); break; From 869cfbdf0599c708db62fb7a4d308e22ccbe2fb7 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 1 Feb 2016 17:43:42 -0800 Subject: [PATCH 36/58] MD-670: Querying MapR-DB JSON Tables returns no results * Use DocumentReader API to emit "_id" field instead of handling it as a special case. * Update the DrillBuf reference field when reallocation happen. * Catch the correct exception when schema change happens and include the field name in the warning message. + Get rid of unused code. --- .../maprdb/json/MaprDBJsonRecordReader.java | 94 ++++++------------- .../drill/maprdb/tests/MaprDBTestsSuite.java | 5 + .../maprdb/tests/json/TestSimpleJson.java | 62 ++++++------ 3 files changed, 66 insertions(+), 95 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index fd8bf9369ec..b5e4ceb69ec 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -37,7 +37,6 @@ import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.store.AbstractRecordReader; import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; -import org.apache.drill.exec.store.maprdb.util.CommonFns; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; @@ -49,9 +48,7 @@ import org.ojai.DocumentReader.EventType; import org.ojai.DocumentStream; import org.ojai.FieldPath; -import org.ojai.Value; import org.ojai.store.QueryCondition; -import org.ojai.store.QueryCondition.Op; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; @@ -86,9 +83,11 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private DocumentStream documentStream; private Iterator documentReaderIterators; - + private boolean includeId; + private String currentFieldName; + public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, List projectedColumns, FragmentContext context) { buffer = context.getManagedBuffer(); @@ -98,22 +97,6 @@ public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, setColumns(projectedColumns); } - private void addKeyCondition(QueryCondition condition, Op op, byte[] key) { - if (!CommonFns.isNullOrEmpty(key)) { - Value value = IdCodec.decode(key); - switch (value.getType()) { - case STRING: - condition.is(ID_FIELD, op, value.getString()); - return; - case BINARY: - condition.is(ID_FIELD, op, value.getBinary()); - return; - default: - throw new UnsupportedOperationException(""); - } - } - } - @Override protected Collection transformColumns(Collection columns) { Set transformed = Sets.newLinkedHashSet(); @@ -145,7 +128,7 @@ public void setup(OperatorContext context, OutputMutator output) throws Executio try { table = MapRDB.getTable(tableName); - table.setOption(TableOption.EXCLUDEID, true); + table.setOption(TableOption.EXCLUDEID, !includeId); documentStream = table.find(condition, projectedFields); documentReaderIterators = documentStream.documentReaders().iterator(); } catch (DBException e) { @@ -171,25 +154,11 @@ public int next() { throw new IllegalStateException("The document did not start with START_MAP!"); } try { - MapWriter map = writer.rootAsMap(); - if (includeId && reader.getId() != null) { - switch (reader.getId().getType()) { - case BINARY: - writeBinary(map.varBinary(ID_KEY), reader.getId().getBinary()); - break; - case STRING: - writeString(map.varChar(ID_KEY), reader.getId().getString()); - break; - default: - throw new UnsupportedOperationException(reader.getId().getType() + - " is not a supported type for _id field."); - } - } - writeToMap(reader, map); + writeToMap(reader, writer.rootAsMap()); recordCount++; - } catch (IllegalStateException e) { - logger.warn(String.format("Possible schema change at _id: %s", - IdCodec.asString(reader.getId())), e); + } catch (IllegalArgumentException e) { + logger.warn(String.format("Possible schema change at _id: '%s', field: '%s'", + IdCodec.asString(reader.getId()), currentFieldName), e); } } @@ -199,62 +168,60 @@ public int next() { } private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { - String fieldName = null; map.start(); outside: while (true) { EventType event = reader.next(); - if (event == null) break outside; - fieldName = reader.getFieldName(); + if (event == null || event == EventType.END_MAP) break outside; + + currentFieldName = reader.getFieldName(); switch (event) { case NULL: - map.varChar(fieldName).write(null); // treat as VARCHAR for now + map.varChar(currentFieldName).write(null); // treat as VARCHAR for now case BINARY: - writeBinary(map.varBinary(fieldName), reader.getBinary()); + writeBinary(map.varBinary(currentFieldName), reader.getBinary()); break; case BOOLEAN: - map.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); + map.bit(currentFieldName).writeBit(reader.getBoolean() ? 1 : 0); break; case STRING: - writeString(map.varChar(fieldName), reader.getString()); + writeString(map.varChar(currentFieldName), reader.getString()); break; case BYTE: - map.tinyInt(fieldName).writeTinyInt(reader.getByte()); + map.tinyInt(currentFieldName).writeTinyInt(reader.getByte()); break; case SHORT: - map.smallInt(fieldName).writeSmallInt(reader.getShort()); + map.smallInt(currentFieldName).writeSmallInt(reader.getShort()); break; case INT: - map.integer(fieldName).writeInt(reader.getInt()); + map.integer(currentFieldName).writeInt(reader.getInt()); break; case LONG: - map.bigInt(fieldName).writeBigInt(reader.getLong()); + map.bigInt(currentFieldName).writeBigInt(reader.getLong()); break; case FLOAT: - map.float4(fieldName).writeFloat4(reader.getFloat()); + map.float4(currentFieldName).writeFloat4(reader.getFloat()); break; case DOUBLE: - map.float8(fieldName).writeFloat8(reader.getDouble()); + map.float8(currentFieldName).writeFloat8(reader.getDouble()); break; case DECIMAL: throw new UnsupportedOperationException("Decimals are currently not supported."); case DATE: - map.date(fieldName).writeDate(reader.getDate().toDate().getTime()); + map.date(currentFieldName).writeDate(reader.getDate().toDate().getTime()); break; case TIME: - map.time(fieldName).writeTime(reader.getTimeInt()); + map.time(currentFieldName).writeTime(reader.getTimeInt()); break; case TIMESTAMP: - map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); + map.timeStamp(currentFieldName).writeTimeStamp(reader.getTimestampLong()); break; case INTERVAL: throw new UnsupportedOperationException("Interval is currently not supported."); case START_MAP: - writeToMap(reader, map.map(fieldName)); + writeToMap(reader, map.map(currentFieldName)); break; - case END_MAP: - break outside; case START_ARRAY: - writeToList(reader, map.list(fieldName)); + writeToList(reader, map.list(currentFieldName)); break; case END_ARRAY: throw new IllegalStateException("Shouldn't get a END_ARRAY inside a map"); @@ -269,7 +236,8 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { list.startList(); outside: while (true) { EventType event = reader.next(); - if (event == null) break outside; + if (event == null || event == EventType.END_ARRAY) break outside; + switch (event) { case NULL: list.varChar().write(null); // treat as VARCHAR for now @@ -321,8 +289,6 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { case START_ARRAY: writeToList(reader, list.list()); break; - case END_ARRAY: - break outside; default: throw new UnsupportedOperationException("Unsupported type: " + event); } @@ -331,14 +297,14 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { } private void writeBinary(VarBinaryWriter binaryWriter, ByteBuffer buf) { - buffer.reallocIfNeeded(buf.remaining()); + buffer = buffer.reallocIfNeeded(buf.remaining()); buffer.setBytes(0, buf, buf.position(), buf.remaining()); binaryWriter.writeVarBinary(0, buf.remaining(), buffer); } private void writeString(VarCharWriter varCharWriter, String string) { final byte[] strBytes = Bytes.toBytes(string); - buffer.reallocIfNeeded(strBytes.length); + buffer = buffer.reallocIfNeeded(strBytes.length); buffer.setBytes(0, strBytes); varCharWriter.writeVarChar(0, strBytes.length, buffer); } diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java index 0f54796e142..e81aa093119 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java @@ -108,6 +108,11 @@ public static Configuration createPluginAndGetConf(DrillbitContext ctx) throws E " \"location\": \"/tmp\"," + " \"writable\": false," + " \"defaultInputFormat\": \"maprdb\"" + + " }," + + " \"root\": {" + + " \"location\": \"/\"," + + " \"writable\": false," + + " \"defaultInputFormat\": \"maprdb\"" + " }" + " }," + " \"formats\": {" + diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index f4c7e8911d8..f05b87a797d 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -67,10 +67,10 @@ public void testPushdownStringEqual() throws Exception { + " name = 'Sprint'" ; runSQLAndVerifyCount(sql, 1); - + final String[] expectedPlan = {"condition=\\(name = \"Sprint\"\\)"}; final String[] excludedPlan = {}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -85,13 +85,13 @@ public void testPushdownStringLike() throws Exception { + " name LIKE 'S%'" ; runSQLAndVerifyCount(sql, 3); - + final String[] expectedPlan = {"condition=\\(name MATCHES \"\\^\\\\\\\\QS\\\\\\\\E\\.\\*\\$\"\\)"}; final String[] excludedPlan = {}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } - + @Test public void testPushdownStringNotEqual() throws Exception { setColumnWidths(new int[] {25, 40, 40, 40}); @@ -103,10 +103,10 @@ public void testPushdownStringNotEqual() throws Exception { + " name <> 'Sprint'" ; runSQLAndVerifyCount(sql, 9); - + final String[] expectedPlan = {"condition=\\(name != \"Sprint\"\\)"}; final String[] excludedPlan = {}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -121,10 +121,10 @@ public void testPushdownLongEqual() throws Exception { + " zip = 85260" ; runSQLAndVerifyCount(sql, 1); - + final String[] expectedPlan = {"condition=\\(zip = \\{\"\\$numberLong\":85260\\}\\)"}; final String[] excludedPlan = {}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -141,10 +141,10 @@ public void testCompositePredicate() throws Exception { + " city = 'Las Vegas'" ; runSQLAndVerifyCount(sql, 4); - + final String[] expectedPlan = {"condition=\\(\\(zip = \\{\"\\$numberLong\":85260\\}\\) or \\(city = \"Las Vegas\"\\)\\)"}; final String[] excludedPlan = {}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -159,10 +159,10 @@ public void testPruneScanRange() throws Exception { + " _id = 'jFTZmywe7StuZ2hEjxyA'" ; runSQLAndVerifyCount(sql, 1); - + final String[] expectedPlan = {"condition=\\(_id = \"jFTZmywe7StuZ2hEjxyA\"\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -179,10 +179,10 @@ public void testPruneScanRangeAndPushDownCondition() throws Exception { + " name = 'Subway'" ; runSQLAndVerifyCount(sql, 1); - + final String[] expectedPlan = {"condition=\\(\\(_id = \"jFTZmywe7StuZ2hEjxyA\"\\) and \\(name = \"Subway\"\\)\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -197,10 +197,10 @@ public void testPushDownOnSubField1() throws Exception { + " b.`attributes.Ambience.casual` = false" ; runSQLAndVerifyCount(sql, 1); - + final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual = false\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -215,10 +215,10 @@ public void testPushDownOnSubField2() throws Exception { + " b.`attributes.Attire` = 'casual'" ; runSQLAndVerifyCount(sql, 4); - + final String[] expectedPlan = {"condition=\\(attributes.Attire = \"casual\"\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @Test @@ -233,10 +233,10 @@ public void testPushDownIsNull() throws Exception { + " business.`attributes.Ambience.casual` IS NULL" ; runSQLAndVerifyCount(sql, 7); - + final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual = null\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -252,10 +252,10 @@ public void testPushDownIsNotNull() throws Exception { + " b.`attributes.Ambience.casual` IS NOT NULL" ; runSQLAndVerifyCount(sql, 3); - + final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual != null\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -270,10 +270,10 @@ public void testPushDownOnSubField3() throws Exception { + " b.`attributes.Accepts Credit Cards` IS NULL" ; runSQLAndVerifyCount(sql, 3); - + final String[] expectedPlan = {"condition=\\(attributes.Accepts Credit Cards = null\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -287,10 +287,10 @@ public void testPushDownLong() throws Exception { + " stars > 4.0" ; runSQLAndVerifyCount(sql, 2); - + final String[] expectedPlan = {"condition=\\(stars > 4\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @@ -305,13 +305,13 @@ public void testPushDownSubField4() throws Exception { + " stars > 4.1" ; runSQLAndVerifyCount(sql, 1); - + final String[] expectedPlan = {"condition=\\(\\(attributes.Good For.lunch = true\\) and \\(stars > 4.1\\)\\)"}; final String[] excludedPlan ={}; - + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } - + /* @Test public void testPushDownSubField5() throws Exception { @@ -363,7 +363,7 @@ public void testPushDownSubField8() throws Exception { runSQLAndVerifyCount(sql, 1); } */ - + protected List runHBaseSQLlWithResults(String sql) throws Exception { System.out.println("Running query:\n" + sql); return testSqlWithResults(sql); From 48d195c3d27fba6c6efef6b624e684bcdecdcca5 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 2 Feb 2016 02:04:22 -0800 Subject: [PATCH 37/58] MAPR-22405: DocumentStream interface in OJAI API need not be parametrized --- .../drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java | 3 +-- .../java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index b5e4ceb69ec..f033de48815 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -43,7 +43,6 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter; import org.apache.drill.exec.vector.complex.writer.VarCharWriter; -import org.ojai.Document; import org.ojai.DocumentReader; import org.ojai.DocumentReader.EventType; import org.ojai.DocumentStream; @@ -80,7 +79,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private DrillBuf buffer; - private DocumentStream documentStream; + private DocumentStream documentStream; private Iterator documentReaderIterators; diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java index e81aa093119..18b7c3158d1 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java @@ -147,7 +147,7 @@ public static void createJsonTables() throws IOException { try (Table table = admin.createTable(TMP_BUSINESS_TABLE); InputStream in = getJsonStream("json/business.json"); - DocumentStream stream = Json.newDocumentStream(in)) { + DocumentStream stream = Json.newDocumentStream(in)) { for (Document document : stream) { table.insert(document, "business_id"); } From 16d4ab896df5cb368988220bc90f2834d9651d02 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 3 Feb 2016 17:46:44 -0800 Subject: [PATCH 38/58] Enhance exception handling. --- .../maprdb/json/MaprDBJsonRecordReader.java | 163 ++++++++++-------- 1 file changed, 91 insertions(+), 72 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index f033de48815..2e0e0c16155 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -28,8 +28,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; @@ -85,8 +85,6 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private boolean includeId; - private String currentFieldName; - public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, List projectedColumns, FragmentContext context) { buffer = context.getManagedBuffer(); @@ -144,20 +142,24 @@ public int next() { writer.reset(); int recordCount = 0; + DBDocumentReaderBase reader = null; while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION) { - DBDocumentReaderBase reader = nextDocumentReader(); - if (reader == null) break; - writer.setPosition(recordCount); - if (reader.next() != EventType.START_MAP) { - throw new IllegalStateException("The document did not start with START_MAP!"); - } try { + reader = nextDocumentReader(); + if (reader == null) break; + writer.setPosition(recordCount); + if (reader.next() != EventType.START_MAP) { + throw dataReadError("The document did not start with START_MAP!"); + } writeToMap(reader, writer.rootAsMap()); recordCount++; - } catch (IllegalArgumentException e) { - logger.warn(String.format("Possible schema change at _id: '%s', field: '%s'", - IdCodec.asString(reader.getId()), currentFieldName), e); + } catch (UserException e) { + throw UserException.unsupportedError(e) + .addContext(String.format("Table: %s, document id: '%s'", + table.getPath(), + reader == null ? null : IdCodec.asString(reader.getId()))) + .build(logger); } } @@ -172,60 +174,65 @@ private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { EventType event = reader.next(); if (event == null || event == EventType.END_MAP) break outside; - currentFieldName = reader.getFieldName(); - switch (event) { - case NULL: - map.varChar(currentFieldName).write(null); // treat as VARCHAR for now - case BINARY: - writeBinary(map.varBinary(currentFieldName), reader.getBinary()); - break; - case BOOLEAN: - map.bit(currentFieldName).writeBit(reader.getBoolean() ? 1 : 0); - break; - case STRING: - writeString(map.varChar(currentFieldName), reader.getString()); - break; - case BYTE: - map.tinyInt(currentFieldName).writeTinyInt(reader.getByte()); - break; - case SHORT: - map.smallInt(currentFieldName).writeSmallInt(reader.getShort()); - break; - case INT: - map.integer(currentFieldName).writeInt(reader.getInt()); - break; - case LONG: - map.bigInt(currentFieldName).writeBigInt(reader.getLong()); - break; - case FLOAT: - map.float4(currentFieldName).writeFloat4(reader.getFloat()); - break; - case DOUBLE: - map.float8(currentFieldName).writeFloat8(reader.getDouble()); - break; - case DECIMAL: - throw new UnsupportedOperationException("Decimals are currently not supported."); - case DATE: - map.date(currentFieldName).writeDate(reader.getDate().toDate().getTime()); - break; - case TIME: - map.time(currentFieldName).writeTime(reader.getTimeInt()); - break; - case TIMESTAMP: - map.timeStamp(currentFieldName).writeTimeStamp(reader.getTimestampLong()); - break; - case INTERVAL: - throw new UnsupportedOperationException("Interval is currently not supported."); - case START_MAP: - writeToMap(reader, map.map(currentFieldName)); - break; - case START_ARRAY: - writeToList(reader, map.list(currentFieldName)); - break; - case END_ARRAY: - throw new IllegalStateException("Shouldn't get a END_ARRAY inside a map"); - default: - throw new UnsupportedOperationException("Unsupported type: " + event); + String fieldName = reader.getFieldName(); + try { + switch (event) { + case NULL: + break; // not setting the field will leave it as null + case BINARY: + writeBinary(map.varBinary(fieldName), reader.getBinary()); + break; + case BOOLEAN: + map.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); + break; + case STRING: + writeString(map.varChar(fieldName), reader.getString()); + break; + case BYTE: + map.tinyInt(fieldName).writeTinyInt(reader.getByte()); + break; + case SHORT: + map.smallInt(fieldName).writeSmallInt(reader.getShort()); + break; + case INT: + map.integer(fieldName).writeInt(reader.getInt()); + break; + case LONG: + map.bigInt(fieldName).writeBigInt(reader.getLong()); + break; + case FLOAT: + map.float4(fieldName).writeFloat4(reader.getFloat()); + break; + case DOUBLE: + map.float8(fieldName).writeFloat8(reader.getDouble()); + break; + case DECIMAL: + throw unsupportedError("Decimal type is currently not supported."); + case DATE: + map.date(fieldName).writeDate(reader.getDate().toDate().getTime()); + break; + case TIME: + map.time(fieldName).writeTime(reader.getTimeInt()); + break; + case TIMESTAMP: + map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); + break; + case INTERVAL: + throw unsupportedError("Interval type is currently not supported."); + case START_MAP: + writeToMap(reader, map.map(fieldName)); + break; + case START_ARRAY: + writeToList(reader, map.list(fieldName)); + break; + case END_ARRAY: + throw dataReadError("Encountered an END_ARRAY event inside a map."); + default: + throw unsupportedError("Unsupported type: %s encountered during the query.", event); + } + } catch (IllegalArgumentException e) { + logger.warn(String.format("Possible schema change at _id: '%s', field: '%s'", + IdCodec.asString(reader.getId()), fieldName), e); } } map.end(); @@ -239,7 +246,7 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { switch (event) { case NULL: - list.varChar().write(null); // treat as VARCHAR for now + throw unsupportedError("Null values are not supported in lists."); case BINARY: writeBinary(list.varBinary(), reader.getBinary()); break; @@ -268,7 +275,7 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { list.float8().writeFloat8(reader.getDouble()); break; case DECIMAL: - throw new UnsupportedOperationException("Decimals are currently not supported."); + throw unsupportedError("Decimals are currently not supported."); case DATE: list.date().writeDate(reader.getDate().toDate().getTime()); break; @@ -279,17 +286,17 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { list.timeStamp().writeTimeStamp(reader.getTimestampLong()); break; case INTERVAL: - throw new UnsupportedOperationException("Interval is currently not supported."); + throw unsupportedError("Interval is currently not supported."); case START_MAP: writeToMap(reader, list.map()); break; case END_MAP: - throw new IllegalStateException("Shouldn't get a END_MAP inside a list"); + throw dataReadError("Encountered an END_MAP event inside a list."); case START_ARRAY: writeToList(reader, list.list()); break; default: - throw new UnsupportedOperationException("Unsupported type: " + event); + throw unsupportedError("Unsupported type: %s encountered during the query.%s", event); } } list.endList(); @@ -308,6 +315,18 @@ private void writeString(VarCharWriter varCharWriter, String string) { varCharWriter.writeVarChar(0, strBytes.length, buffer); } + private UserException unsupportedError(String format, Object... args) { + return UserException.unsupportedError() + .message(String.format(format, args)) + .build(logger); + } + + private UserException dataReadError(String format, Object... args) { + return UserException.dataReadError() + .message(String.format(format, args)) + .build(logger); + } + private DBDocumentReaderBase nextDocumentReader() { final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats(); try { @@ -326,7 +345,7 @@ private DBDocumentReaderBase nextDocumentReader() { } } } catch (DBException e) { - throw new DrillRuntimeException(e); + throw UserException.dataReadError(e).build(logger); } } From 5d36f8013829406f698610bc4a4f7185698365e6 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 4 Feb 2016 16:43:27 -0800 Subject: [PATCH 39/58] MD-703: Cannot select `_id` field from MapR-DB JSON Table + Included Smidth's fix for column shuffling. + Fix the private-drill-mapr-plugin build. --- contrib/format-maprdb/pom.xml | 6 +++++ .../exec/store/maprdb/MapRDBGroupScan.java | 2 +- .../maprdb/MapRDBPushFilterIntoScan.java | 4 ++-- .../maprdb/json/MaprDBJsonRecordReader.java | 24 ++++++++++++------- .../maprdb/tests/json/TestSimpleJson.java | 4 ++-- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index bcb6c295972..db0572f1534 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -35,6 +35,7 @@ ${project.parent.version} ${project.version} 0.98.12-mapr-1506 + 18.0 **/MaprDBTestsSuite.class @@ -117,6 +118,11 @@ + + com.google.guava + guava + ${guava.version} + org.apache.drill.exec drill-java-exec diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java index cbfb18c2767..4d303993bf9 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -61,7 +61,7 @@ public abstract class MapRDBGroupScan extends AbstractGroupScan { private boolean filterPushedDown = false; - private Stopwatch watch = new Stopwatch(); + private Stopwatch watch = Stopwatch.createUnstarted(); private static final Comparator> LIST_SIZE_COMPARATOR = new Comparator>() { @Override diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index 714221fae50..8a0902e34b6 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -90,11 +90,11 @@ public void onMatch(RelOptRuleCall call) { if (scan.getGroupScan() instanceof BinaryTableGroupScan) { BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); - doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition); + doPushFilterIntoBinaryGroupScan(call, filter, project, scan, groupScan, condition); } else { assert(scan.getGroupScan() instanceof JsonTableGroupScan); JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan(); - doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition); + doPushFilterIntoJsonGroupScan(call, filter, project, scan, groupScan, condition); } } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 2e0e0c16155..b69f4c4c003 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -17,11 +17,9 @@ */ package org.apache.drill.exec.store.maprdb.json; -import static org.ojai.DocumentConstants.ID_FIELD; import static org.ojai.DocumentConstants.ID_KEY; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -51,7 +49,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.mapr.db.MapRDB; import com.mapr.db.Table; @@ -98,18 +95,27 @@ public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, protected Collection transformColumns(Collection columns) { Set transformed = Sets.newLinkedHashSet(); if (!isStarQuery()) { - ArrayList projectedFieldsList = Lists.newArrayList(); + Set projectedFieldsSet = Sets.newTreeSet(); for (SchemaPath column : columns) { if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) { + /* + * we do not include _id field in the set of projected fields + * because the DB currently can not return a document if only + * the _id field was projected. This should really be fixed in + * the DB client (Bug 21708) to avoid transferring the entire + * document when only _id is requested. + */ + // projectedFieldsList.add(ID_FIELD); transformed.add(ID_PATH); - projectedFieldsList.add(ID_FIELD); includeId = true; } else { transformed.add(SchemaPath.getSimplePath(column.getRootSegment().getPath())); - projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath())); + projectedFieldsSet.add(FieldPath.parseFrom(column.getAsUnescapedPath())); } } - projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]); + if (projectedFieldsSet.size() > 0) { + projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]); + } } else { transformed.add(ID_PATH); includeId = true; @@ -135,7 +141,7 @@ public void setup(OperatorContext context, OutputMutator output) throws Executio @Override public int next() { - Stopwatch watch = new Stopwatch(); + Stopwatch watch = Stopwatch.createUnstarted(); watch.start(); writer.allocate(); @@ -230,7 +236,7 @@ private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { default: throw unsupportedError("Unsupported type: %s encountered during the query.", event); } - } catch (IllegalArgumentException e) { + } catch (IllegalStateException | IllegalArgumentException e) { logger.warn(String.format("Possible schema change at _id: '%s', field: '%s'", IdCodec.asString(reader.getId()), fieldName), e); } diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index f05b87a797d..414b823f8a5 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -48,9 +48,9 @@ public static void tearDownAfterClass() throws Exception { @Test public void testMe() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); + setColumnWidths(new int[] {23}); final String sql = "SELECT\n" - + " _id, name, categories, full_address\n" + + " _id\n" + "FROM\n" + " hbase.`business` business"; runSQLAndVerifyCount(sql, 10); From d41bfb3e111b3eeec503fdf921d8a3360614c3c5 Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Wed, 17 Feb 2016 23:51:26 +0000 Subject: [PATCH 40/58] MD-719: Fix creating QueryConditions for JsonSubScanSpec --- .../apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java index 936002d48b3..adb80140557 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java @@ -44,7 +44,7 @@ public JsonSubScanSpec(@JsonProperty("tableName") String tableName, this.condition.is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, startVal.getBinary()); break; case STRING: - this.condition.is(DocumentConstants.ID_FIELD, Op.LESS, startVal.getString()); + this.condition.is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, startVal.getString()); break; default: throw new IllegalStateException("Encountered an unsupported type " + startVal.getType() @@ -58,7 +58,7 @@ public JsonSubScanSpec(@JsonProperty("tableName") String tableName, switch(stopVal.getType()) { case BINARY: - this.condition.is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, stopVal.getBinary()); + this.condition.is(DocumentConstants.ID_FIELD, Op.LESS, stopVal.getBinary()); break; case STRING: this.condition.is(DocumentConstants.ID_FIELD, Op.LESS, stopVal.getString()); From 891380f0e61daa865ae030521d8ef684cb62edca Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Fri, 19 Feb 2016 18:41:56 +0000 Subject: [PATCH 41/58] Support for `_id` only projection. --- .../maprdb/json/MaprDBJsonRecordReader.java | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index b69f4c4c003..42944f5dbc6 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -46,6 +46,7 @@ import org.ojai.DocumentStream; import org.ojai.FieldPath; import org.ojai.store.QueryCondition; +import org.ojai.Value; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; @@ -81,12 +82,16 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private Iterator documentReaderIterators; private boolean includeId; + private boolean idOnly; public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, List projectedColumns, FragmentContext context) { buffer = context.getManagedBuffer(); + projectedFields = null; tableName = Preconditions.checkNotNull(subScanSpec, "MapRDB reader needs a sub-scan spec").getTableName(); + documentReaderIterators = null; includeId = false; + idOnly = false; condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter())); setColumns(projectedColumns); } @@ -121,6 +126,10 @@ protected Collection transformColumns(Collection columns includeId = true; } + /* + * (Bug 21708) if we are projecting only the id field, save that condition here. + */ + idOnly = !isStarQuery() && (projectedFields == null); return transformed; } @@ -155,11 +164,35 @@ public int next() { reader = nextDocumentReader(); if (reader == null) break; writer.setPosition(recordCount); - if (reader.next() != EventType.START_MAP) { - throw dataReadError("The document did not start with START_MAP!"); + if (idOnly) { + Value id = reader.getId(); + MapWriter map = writer.rootAsMap(); + + try { + switch(id.getType()) { + case STRING: + writeString(map.varChar(ID_KEY), id.getString()); + recordCount++; + break; + case BINARY: + writeBinary(map.varBinary(ID_KEY), id.getBinary()); + recordCount++; + break; + default: + throw new UnsupportedOperationException(id.getType() + + " is not a supported type for _id field."); + } + } catch (IllegalStateException | IllegalArgumentException e) { + logger.warn(String.format("Possible schema change at _id: '%s'", + IdCodec.asString(id)), e); + } + } else { + if (reader.next() != EventType.START_MAP) { + throw dataReadError("The document did not start with START_MAP!"); + } + writeToMap(reader, writer.rootAsMap()); + recordCount++; } - writeToMap(reader, writer.rootAsMap()); - recordCount++; } catch (UserException e) { throw UserException.unsupportedError(e) .addContext(String.format("Table: %s, document id: '%s'", From c62361e870fbd4b99d649d58cc740ca09bb0e366 Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Mon, 22 Feb 2016 17:09:53 -0800 Subject: [PATCH 42/58] MD-735: Handle CastExpression in filter push-down. --- .../exec/store/maprdb/json/CompareFunctionsProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java index 2e191a19433..024fa2bd21a 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java @@ -1,5 +1,6 @@ package org.apache.drill.exec.store.maprdb.json; +import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; @@ -41,6 +42,11 @@ public static boolean isCompareFunction(String functionName) { return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName); } + @Override + public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) throws RuntimeException { + return false; + } + public static CompareFunctionsProcessor process(FunctionCall call) { String functionName = call.getName(); LogicalExpression nameArg = call.args.get(0); From 6988254f08dd4e2519eb17b5863b827a7803264b Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 23 Feb 2016 23:15:36 -0800 Subject: [PATCH 43/58] MD-548: Remove reference to mapr-release pom. * Bumped the Drill+plugin version to 1.6.0-SNAPSHOT. * Updated MapR client artifacts to 5.1.0. * Added missing Apache license headers to few files. * Added a base test class BaseJsonTest for all OJAI Unit tests. * Invoke GuavaPatcher from BaseJsonTest. --- contrib/format-maprdb/pom.xml | 145 +++++++++--------- .../json/CompareFunctionsProcessor.java | 21 ++- .../maprdb/json/JsonConditionBuilder.java | 25 ++- .../exec/store/maprdb/json/JsonScanSpec.java | 19 ++- .../store/maprdb/json/JsonSubScanSpec.java | 17 ++ .../store/maprdb/json/JsonTableGroupScan.java | 7 - .../drill/maprdb/tests/json/BaseJsonTest.java | 47 ++++++ .../maprdb/tests/json/TestSimpleJson.java | 17 +- 8 files changed, 194 insertions(+), 104 deletions(-) create mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index db0572f1534..4dfc4a70680 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,24 +21,67 @@ 4.0.0 - com.mapr - mapr-release - 5.1.0-mapr-SNAPSHOT + drill-root + org.apache.drill + 1.6.0-SNAPSHOT drill-storage-maprdb maprdb-storage-plugin - 1.5.0-SNAPSHOT - ${project.parent.version} - ${project.version} - 0.98.12-mapr-1506 - 18.0 + 5.1.0-mapr + 0.98.12-mapr-1506 + 2.7.0-mapr-1602 **/MaprDBTestsSuite.class + + + + com.mapr.hadoop + maprfs + ${maprdb-storage-plugin.mapr.version} + + + com.mapr.fs + mapr-hbase + ${maprdb-storage-plugin.mapr.version} + + + com.mapr.db + maprdb + ${maprdb-storage-plugin.mapr.version} + + + org.apache.hbase + hbase-client + ${maprdb-storage-plugin.hbase.version} + + + org.apache.hbase + hbase-server + ${maprdb-storage-plugin.hbase.version} + + + org.apache.hadoop + hadoop-annotations + ${maprdb-storage-plugin.hadoop.version} + + + org.apache.hadoop + hadoop-client + ${maprdb-storage-plugin.hadoop.version} + + + org.apache.hadoop + hadoop-common + ${maprdb-storage-plugin.hadoop.version} + + + + @@ -60,6 +103,7 @@ + com.mapr.hadoop maprfs @@ -68,6 +112,10 @@ commons-logging commons-logging + + log4j + log4j + slf4j-log4j12 org.slf4j @@ -79,54 +127,27 @@ mapr-hbase - slf4j-log4j12 - org.slf4j - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-json - - - commons-logging commons-logging + commons-logging - asm - asm + log4j + log4j - org.mortbay.jetty servlet-api-2.5 + org.mortbay.jetty com.mapr.db maprdb - - - slf4j-log4j12 - org.slf4j - - - - - com.google.guava - guava - ${guava.version} org.apache.drill.exec drill-java-exec - ${drill.version} + ${project.version} log4j-over-slf4j @@ -137,57 +158,38 @@ org.apache.drill.contrib drill-storage-hbase - ${drill.version} + ${project.version} log4j-over-slf4j org.slf4j - - hbase-client - org.apache.hbase - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - de.huxhorn.lilith - de.huxhorn.lilith.logback.appender.multiplex-classic - 0.9.44 - test - - com.mapr mapr-java-utils + ${maprdb-storage-plugin.mapr.version} tests - - - slf4j-log4j12 - org.slf4j - - + + + com.mapr + mapr-test-annotations + ${maprdb-storage-plugin.mapr.version} org.apache.drill.exec drill-java-exec - ${drill.version} + ${project.version} tests test org.apache.drill drill-common - ${drill.version} + ${project.version} tests test @@ -200,13 +202,14 @@ org.apache.drill.contrib drill-storage-hbase - ${drill.version} + ${project.version} tests test org.apache.hbase hbase-server + ${maprdb-storage-plugin.hbase.version} test tests @@ -226,6 +229,10 @@ slf4j-log4j12 org.slf4j + + log4j + log4j + diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java index 024fa2bd21a..ba44145e6d9 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.drill.exec.store.maprdb.json; import org.apache.drill.common.expression.CastExpression; @@ -14,12 +31,8 @@ import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.expression.ValueExpressions.TimeExpression; -import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; import org.ojai.Value; -import org.ojai.types.ODate; -import org.ojai.types.OTime; -import org.ojai.types.OTimestamp; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java index a48d784b0b5..ef32436de1a 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java @@ -1,23 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.drill.exec.store.maprdb.json; +import static org.ojai.DocumentConstants.ID_KEY; + import org.apache.drill.common.expression.BooleanOperator; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; import org.apache.drill.exec.store.hbase.DrillHBaseConstants; -import org.apache.hadoop.hbase.HConstants; -import org.bouncycastle.util.Arrays; import org.ojai.Value; - -import static org.ojai.DocumentConstants.ID_KEY; - import org.ojai.store.QueryCondition; import org.ojai.store.QueryCondition.Op; import com.google.common.collect.ImmutableList; import com.mapr.db.MapRDB; -import com.mapr.db.impl.IdCodec; public class JsonConditionBuilder extends AbstractExprVisitor implements DrillHBaseConstants { diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java index f278dd4aa56..7763273772e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java @@ -1,8 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.drill.exec.store.maprdb.json; -import org.apache.drill.exec.store.hbase.HBaseUtils; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.util.Bytes; import org.ojai.store.QueryCondition; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java index adb80140557..996f658fb23 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.drill.exec.store.maprdb.json; import java.nio.ByteBuffer; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java index e7231798fed..db2713795a9 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java @@ -21,20 +21,15 @@ import java.io.IOException; import java.util.List; -import java.util.Map; -import java.util.NavigableMap; import java.util.TreeMap; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.physical.PhysicalOperatorSetupException; -import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.base.ScanStats; import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.dfs.FileSystemConfig; import org.apache.drill.exec.store.dfs.FileSystemPlugin; @@ -44,8 +39,6 @@ import org.apache.drill.exec.store.maprdb.MapRDBSubScan; import org.apache.drill.exec.store.maprdb.MapRDBTableStats; import org.apache.drill.exec.store.maprdb.TabletFragmentInfo; -import org.apache.drill.exec.store.maprdb.json.JsonScanSpec; -import org.apache.drill.exec.store.maprdb.json.JsonSubScanSpec; import org.apache.hadoop.conf.Configuration; import org.codehaus.jackson.annotate.JsonCreator; diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java new file mode 100644 index 00000000000..5e105288867 --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.json; + +import org.apache.drill.BaseTestQuery; +import org.apache.drill.hbase.GuavaPatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; + +public class BaseJsonTest extends BaseTestQuery { + + @BeforeClass + public static void setupDefaultTestCluster() throws Exception { + // Invoke the Guava patcher before any code + GuavaPatcher.patch(); + + // Since we override the class initializer of parent class, + // invoke it explicitly. This will setup a Drill cluster. + BaseTestQuery.setupDefaultTestCluster(); + + MaprDBTestsSuite.setupTests(); + MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index 414b823f8a5..84a73ce7f32 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -19,32 +19,17 @@ import java.util.List; -import org.apache.drill.BaseTestQuery; import org.apache.drill.PlanTestBase; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.rpc.user.QueryDataBatch; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; import com.mapr.tests.annotations.ClusterTest; @Category(ClusterTest.class) -public class TestSimpleJson extends BaseTestQuery { - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - MaprDBTestsSuite.setupTests(); - MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - MaprDBTestsSuite.cleanupTests(); - } +public class TestSimpleJson extends BaseJsonTest { @Test public void testMe() throws Exception { From 8844a73eff13115dc0a6b7a01720ace99e06797c Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 25 Feb 2016 02:34:30 -0800 Subject: [PATCH 44/58] MD-741: Cannot select array elements by index in MapR-DB JSON Tables --- .../maprdb/json/MaprDBJsonRecordReader.java | 34 +++++++++-- .../drill/maprdb/tests/json/BaseJsonTest.java | 23 ++++++++ .../maprdb/tests/json/TestSimpleJson.java | 58 ++++++++++++------- .../src/test/resources/json/business.json | 20 +++---- 4 files changed, 99 insertions(+), 36 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 42944f5dbc6..0f0166293cb 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -24,10 +24,12 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.Stack; import java.util.concurrent.TimeUnit; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; @@ -45,8 +47,9 @@ import org.ojai.DocumentReader.EventType; import org.ojai.DocumentStream; import org.ojai.FieldPath; -import org.ojai.store.QueryCondition; +import org.ojai.FieldSegment; import org.ojai.Value; +import org.ojai.store.QueryCondition; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; @@ -111,18 +114,17 @@ protected Collection transformColumns(Collection columns * document when only _id is requested. */ // projectedFieldsList.add(ID_FIELD); - transformed.add(ID_PATH); includeId = true; } else { - transformed.add(SchemaPath.getSimplePath(column.getRootSegment().getPath())); - projectedFieldsSet.add(FieldPath.parseFrom(column.getAsUnescapedPath())); + projectedFieldsSet.add(getFieldPathForProjection(column)); } + transformed.add(column); } if (projectedFieldsSet.size() > 0) { projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]); } } else { - transformed.add(ID_PATH); + transformed.add(AbstractRecordReader.STAR_COLUMN); includeId = true; } @@ -388,6 +390,28 @@ private DBDocumentReaderBase nextDocumentReader() { } } + /* + * Extracts contiguous named segments from the SchemaPath, starting from the + * root segment and build the FieldPath from it for projection. + * + * This is due to bug 22726 and 22727, which cause DB's DocumentReaders to + * behave incorrectly for sparse lists, hence we avoid projecting beyond the + * first encountered ARRAY field and let Drill handle the projection. + */ + private static FieldPath getFieldPathForProjection(SchemaPath column) { + Stack pathSegments = new Stack(); + PathSegment seg = column.getRootSegment(); + while (seg != null && seg.isNamed()) { + pathSegments.push((PathSegment.NameSegment) seg); + seg = seg.getChild(); + } + FieldSegment.NameSegment child = null; + while (!pathSegments.isEmpty()) { + child = new FieldSegment.NameSegment(pathSegments.pop().getPath(), child, false); + } + return new FieldPath(child); + } + @Override public void close() { if (documentStream != null) { diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java index 5e105288867..6aafed322db 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java @@ -17,9 +17,14 @@ */ package com.mapr.drill.maprdb.tests.json; +import java.util.List; + import org.apache.drill.BaseTestQuery; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.apache.drill.hbase.GuavaPatcher; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; @@ -44,4 +49,22 @@ public static void tearDownAfterClass() throws Exception { MaprDBTestsSuite.cleanupTests(); } + + protected List runHBaseSQLlWithResults(String sql) throws Exception { + System.out.println("Running query:\n" + sql); + return testSqlWithResults(sql); + } + + protected void runSQLAndVerifyCount(String sql, int expectedRowCount) throws Exception{ + List results = runHBaseSQLlWithResults(sql); + printResultAndVerifyRowCount(results, expectedRowCount); + } + + private void printResultAndVerifyRowCount(List results, int expectedRowCount) throws SchemaChangeException { + int rowCount = printResult(results); + if (expectedRowCount != -1) { + Assert.assertEquals(expectedRowCount, rowCount); + } + } + } diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index 84a73ce7f32..b74b1c516ee 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -17,15 +17,22 @@ */ package com.mapr.drill.maprdb.tests.json; -import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import org.apache.drill.PlanTestBase; +import org.apache.drill.SingleRowListener; import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.proto.UserBitShared.QueryType; +import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.rpc.user.QueryDataBatch; -import org.junit.Assert; +import org.apache.drill.exec.util.VectorUtil; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.ojai.Document; +import com.mapr.db.MapRDB; import com.mapr.tests.annotations.ClusterTest; @Category(ClusterTest.class) @@ -45,13 +52,39 @@ public void testMe() throws Exception { public void testPushdownStringEqual() throws Exception { setColumnWidths(new int[] {25, 40, 40, 40}); final String sql = "SELECT\n" - + " _id, name, categories, full_address\n" + + " _id, name, business.hours.Monday.`open`, categories[1], years[2], full_address\n" + "FROM\n" + " hbase.`business` business\n" + "WHERE\n" + " name = 'Sprint'" ; - runSQLAndVerifyCount(sql, 1); + + final Document queryResult = MapRDB.newDocument(); + SingleRowListener listener = new SingleRowListener() { + @Override + protected void rowArrived(QueryDataBatch result) { + try { + final RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); + loader.load(result.getHeader().getDef(), result.getData()); + StringBuilder sb = new StringBuilder(); + VectorUtil.appendVectorAccessibleContent(loader, sb, "|", false); + loader.clear(); + queryResult.set("result", sb.toString()); + } catch (SchemaChangeException e) { + queryResult.set("error", "true"); + } + } + }; + testWithListener(QueryType.SQL, sql, listener); + listener.waitForCompletion(); + + assertNull(queryResult.getString("error")); + assertNotNull(queryResult.getString("result")); + + String[] fields = queryResult.getString("result").split("\\|"); + assertEquals("1970-01-01T11:00:00.000", fields[2]); + assertEquals("Mobile Phones", fields[3]); + assertEquals("2016.0", fields[4]); final String[] expectedPlan = {"condition=\\(name = \"Sprint\"\\)"}; final String[] excludedPlan = {}; @@ -349,21 +382,4 @@ public void testPushDownSubField8() throws Exception { } */ - protected List runHBaseSQLlWithResults(String sql) throws Exception { - System.out.println("Running query:\n" + sql); - return testSqlWithResults(sql); - } - - protected void runSQLAndVerifyCount(String sql, int expectedRowCount) throws Exception{ - List results = runHBaseSQLlWithResults(sql); - printResultAndVerifyRowCount(results, expectedRowCount); - } - - private void printResultAndVerifyRowCount(List results, int expectedRowCount) throws SchemaChangeException { - int rowCount = printResult(results); - if (expectedRowCount != -1) { - Assert.assertEquals(expectedRowCount, rowCount); - } - } - } diff --git a/contrib/format-maprdb/src/test/resources/json/business.json b/contrib/format-maprdb/src/test/resources/json/business.json index 1121aa3d318..0010ea17a96 100644 --- a/contrib/format-maprdb/src/test/resources/json/business.json +++ b/contrib/format-maprdb/src/test/resources/json/business.json @@ -1,10 +1,10 @@ -{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business", "start_date":"2011-07-14"} -{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":"2012-07-14"} -{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2013-07-14"} -{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-07-14"} -{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business", "start_date":"2011-04-14"} -{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business", "start_date":"2013-02-15"} -{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business", "start_date":"2013-01-21"} -{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business", "start_date":"2015-03-21"} -{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-02-13"} -{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":"2014-02-17"} +{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business", "start_date":"2011-07-14"} +{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":"2012-07-14"} +{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2013-07-14"} +{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-07-14"} +{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business", "start_date":"2011-04-14"} +{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business", "start_date":"2013-02-15"} +{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business", "start_date":"2013-01-21", "years":[2014,2015,2016]} +{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business", "start_date":"2015-03-21"} +{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-02-13"} +{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":"2014-02-17"} From b1218f318806c96a8e4e28f6298d17e180fd09eb Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Thu, 25 Feb 2016 16:46:38 +0000 Subject: [PATCH 45/58] MD-711: Enable support for UNION types in Drill-MapRDB plugin --- .../drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 0f0166293cb..1d1ef441824 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -31,6 +31,7 @@ import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.ops.OperatorStats; @@ -86,6 +87,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private boolean includeId; private boolean idOnly; + private boolean unionEnabled; public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, List projectedColumns, FragmentContext context) { @@ -97,6 +99,7 @@ public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, idOnly = false; condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter())); setColumns(projectedColumns); + unionEnabled = context.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE); } @Override @@ -137,7 +140,7 @@ protected Collection transformColumns(Collection columns @Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { - this.writer = new VectorContainerWriter(output); + this.writer = new VectorContainerWriter(output, unionEnabled); this.operatorContext = context; try { From c327f113b7c32bbe210b3047591f4f3a2339edd1 Mon Sep 17 00:00:00 2001 From: Smidth Panchamia Date: Wed, 2 Mar 2016 22:32:37 -0800 Subject: [PATCH 46/58] MD-726: Add support for `read_numbers_as_double` and `all_text_mode` options + MD-773: Add support to push-down filters on Date, Time and Timestamp + Fix the offsets in Date and Time data types in returned values + Updated support for DATE, TIME, and TIMESTAMP types + Modified unit-tests to check for operation push-down --- .../maprdb/MapRDBFormatPluginConfig.java | 52 ++++- .../exec/store/maprdb/MapRDBGroupScan.java | 4 + .../maprdb/MapRDBPushFilterIntoScan.java | 11 +- .../store/maprdb/MapRDBScanBatchCreator.java | 2 +- .../exec/store/maprdb/MapRDBSubScan.java | 11 +- .../maprdb/binary/BinaryTableGroupScan.java | 2 +- .../maprdb/binary/MapRDBFilterBuilder.java | 1 - .../json/CompareFunctionsProcessor.java | 21 +- .../maprdb/json/JsonConditionBuilder.java | 6 +- .../store/maprdb/json/JsonSubScanSpec.java | 21 +- .../store/maprdb/json/JsonTableGroupScan.java | 2 +- .../maprdb/json/MaprDBJsonRecordReader.java | 197 +++++++++++++++--- .../maprdb/tests/json/TestSimpleJson.java | 26 ++- .../src/test/resources/json/business.json | 18 +- 14 files changed, 297 insertions(+), 77 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java index 7eff4e4536b..eb341d98e40 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java @@ -17,12 +17,19 @@ */ package org.apache.drill.exec.store.maprdb; -import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.drill.common.logical.FormatPluginConfig; -@JsonTypeName("maprdb") +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName("maprdb") @JsonInclude(Include.NON_DEFAULT) public class MapRDBFormatPluginConfig implements FormatPluginConfig { + private boolean allTextMode = false; + private boolean readAllNumbersAsDouble = false; + @Override public int hashCode() { return 53; @@ -30,7 +37,46 @@ public int hashCode() { @Override public boolean equals(Object obj) { - return obj instanceof MapRDBFormatPluginConfig; + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + MapRDBFormatPluginConfig other = (MapRDBFormatPluginConfig)obj; + + if (readAllNumbersAsDouble != other.readAllNumbersAsDouble) { + return false; + } + + if (allTextMode != other.allTextMode) { + return false; + } + + return true; + } + + public boolean isReadAllNumbersAsDouble() { + return readAllNumbersAsDouble; + } + + public boolean isAllTextMode() { + return allTextMode; + } + + @JsonProperty("allTextMode") + public void setAllTextMode(boolean mode) { + allTextMode = mode; } + @JsonProperty("readAllNumbersAsDouble") + public void setReadAllNumbersAsDouble(boolean read) { + readAllNumbersAsDouble = read; + } } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java index 4d303993bf9..393bfe56057 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -53,6 +53,8 @@ public abstract class MapRDBGroupScan extends AbstractGroupScan { private MapRDBFormatPlugin formatPlugin; + protected MapRDBFormatPluginConfig formatPluginConfig; + protected List columns; protected Map> endpointFragmentMapping; @@ -76,6 +78,7 @@ public MapRDBGroupScan(MapRDBGroupScan that) { super(that); this.columns = that.columns; this.formatPlugin = that.formatPlugin; + this.formatPluginConfig = that.formatPluginConfig; this.storagePlugin = that.storagePlugin; this.regionsToScan = that.regionsToScan; this.filterPushedDown = that.filterPushedDown; @@ -86,6 +89,7 @@ public MapRDBGroupScan(FileSystemPlugin storagePlugin, super(userName); this.storagePlugin = storagePlugin; this.formatPlugin = formatPlugin; + this.formatPluginConfig = (MapRDBFormatPluginConfig)formatPlugin.getConfig(); this.columns = columns; } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index 8a0902e34b6..634bf9a8bdc 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -17,6 +17,11 @@ */ package org.apache.drill.exec.store.maprdb; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.exec.planner.logical.DrillOptiq; import org.apache.drill.exec.planner.logical.DrillParseContext; @@ -32,12 +37,6 @@ import org.apache.drill.exec.store.maprdb.json.JsonConditionBuilder; import org.apache.drill.exec.store.maprdb.json.JsonScanSpec; import org.apache.drill.exec.store.maprdb.json.JsonTableGroupScan; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptRuleOperand; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rex.RexNode; -import org.ojai.store.QueryCondition; import com.google.common.collect.ImmutableList; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java index 058de615980..1cd33cad585 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java @@ -48,7 +48,7 @@ public ScanBatch getBatch(FragmentContext context, MapRDBSubScan subScan, List regionScanSpecList; private final List columns; @@ -54,11 +55,13 @@ public class MapRDBSubScan extends AbstractBase implements SubScan { @JsonCreator public MapRDBSubScan(@JacksonInject StoragePluginRegistry registry, @JsonProperty("userName") String userName, + @JsonProperty("formatPluginConfig") MapRDBFormatPluginConfig formatPluginConfig, @JsonProperty("storage") StoragePluginConfig storage, @JsonProperty("regionScanSpecList") List regionScanSpecList, @JsonProperty("columns") List columns, @JsonProperty("tableType") String tableType) throws ExecutionSetupException { super(userName); + this.fsFormatPluginConfig = formatPluginConfig; this.fsStoragePlugin = (FileSystemPlugin) registry.getPlugin(storage); this.regionScanSpecList = regionScanSpecList; this.storage = storage; @@ -66,9 +69,10 @@ public MapRDBSubScan(@JacksonInject StoragePluginRegistry registry, this.tableType = tableType; } - public MapRDBSubScan(String userName, FileSystemPlugin storagePlugin, StoragePluginConfig config, + public MapRDBSubScan(String userName, MapRDBFormatPluginConfig formatPluginConfig, FileSystemPlugin storagePlugin, StoragePluginConfig config, List maprSubScanSpecs, List columns, String tableType) { super(userName); + fsFormatPluginConfig = formatPluginConfig; fsStoragePlugin = storagePlugin; storage = config; this.regionScanSpecList = maprSubScanSpecs; @@ -97,7 +101,7 @@ public T accept(PhysicalVisitor physicalVis @Override public PhysicalOperator getNewWithChildren(List children) { Preconditions.checkArgument(children.isEmpty()); - return new MapRDBSubScan(getUserName(), fsStoragePlugin, storage, regionScanSpecList, columns, tableType); + return new MapRDBSubScan(getUserName(), fsFormatPluginConfig, fsStoragePlugin, storage, regionScanSpecList, columns, tableType); } @Override @@ -114,4 +118,7 @@ public String getTableType() { return tableType; } + public MapRDBFormatPluginConfig getFormatPluginConfig() { + return fsFormatPluginConfig; + } } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java index 69fda9c5efd..59d0d01a58e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java @@ -171,7 +171,7 @@ public MapRDBSubScan getSpecificScan(int minorFragmentId) { assert minorFragmentId < endpointFragmentMapping.size() : String.format( "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), minorFragmentId); - return new MapRDBSubScan(getUserName(), getStoragePlugin(), getStoragePlugin().getConfig(), + return new MapRDBSubScan(getUserName(), formatPluginConfig, getStoragePlugin(), getStoragePlugin().getConfig(), endpointFragmentMapping.get(minorFragmentId), columns, TABLE_BINARY); } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java index 07c33648110..c4de6bbc121 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.NullComparator; -import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java index ba44145e6d9..dc5c2b7d477 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.maprdb.json; -import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; @@ -31,8 +30,13 @@ import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.expression.ValueExpressions.TimeExpression; +import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.joda.time.LocalTime; import org.ojai.Value; +import org.ojai.types.ODate; +import org.ojai.types.OTime; +import org.ojai.types.OTimestamp; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -59,7 +63,7 @@ public static boolean isCompareFunction(String functionName) { public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) throws RuntimeException { return false; } - + public static CompareFunctionsProcessor process(FunctionCall call) { String functionName = call.getName(); LogicalExpression nameArg = call.args.get(0); @@ -151,15 +155,20 @@ public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) thro this.path = path; return true; } -/* + if (valueArg instanceof DateExpression) { - this.value = KeyValueBuilder.initFrom(new ODate(((DateExpression)valueArg).getDate())); + long d = ((DateExpression)valueArg).getDate(); + final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; + int daysSinceEpoch = (int)(d / MILLISECONDS_IN_A_DAY); + this.value = KeyValueBuilder.initFrom(ODate.fromDaysSinceEpoch(daysSinceEpoch)); this.path = path; return true; } if (valueArg instanceof TimeExpression) { - this.value = KeyValueBuilder.initFrom(new OTime(((TimeExpression)valueArg).getTime())); + int t = ((TimeExpression)valueArg).getTime(); + LocalTime lT = LocalTime.fromMillisOfDay(t); + this.value = KeyValueBuilder.initFrom(new OTime(lT.getHourOfDay(), lT.getMinuteOfHour(), lT.getSecondOfMinute(), lT.getMillisOfSecond())); this.path = path; return true; } @@ -169,7 +178,7 @@ public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) thro this.path = path; return true; } -*/ + return false; } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java index ef32436de1a..889f5a919d6 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java @@ -17,8 +17,6 @@ */ package org.apache.drill.exec.store.maprdb.json; -import static org.ojai.DocumentConstants.ID_KEY; - import org.apache.drill.common.expression.BooleanOperator; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; @@ -164,8 +162,6 @@ private JsonScanSpec createJsonScanSpec(FunctionCall call, SchemaPath field = processor.getPath(); Value fieldValue = processor.getValue(); - boolean isRowKey = field.getAsUnescapedPath().equals(ID_KEY); - QueryCondition cond = null; switch (functionName) { case "equal": @@ -231,7 +227,7 @@ private JsonScanSpec createJsonScanSpec(FunctionCall call, case "like": cond = MapRDB.newCondition().like(field.getAsUnescapedPath(), fieldValue.getString()).build(); break; - + default: } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java index 996f658fb23..aa5375a1cfa 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java @@ -18,10 +18,10 @@ package org.apache.drill.exec.store.maprdb.json; import java.nio.ByteBuffer; +import java.util.Arrays; import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; import org.apache.hadoop.hbase.HConstants; -import org.bouncycastle.util.Arrays; import org.ojai.DocumentConstants; import org.ojai.Value; import org.ojai.store.QueryCondition; @@ -45,17 +45,17 @@ public JsonSubScanSpec(@JsonProperty("tableName") String tableName, @JsonProperty("stopRow") byte[] stopRow, @JsonProperty("cond") QueryCondition cond) { super(tableName, regionServer, null, null, null, null); - + this.condition = MapRDB.newCondition().and(); - + if (cond != null) { this.condition.condition(cond); } - + if (startRow != null && - Arrays.areEqual(startRow, HConstants.EMPTY_START_ROW) == false) { + Arrays.equals(startRow, HConstants.EMPTY_START_ROW) == false) { Value startVal = IdCodec.decode(startRow); - + switch(startVal.getType()) { case BINARY: this.condition.is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, startVal.getBinary()); @@ -68,11 +68,11 @@ public JsonSubScanSpec(@JsonProperty("tableName") String tableName, + " for _id"); } } - + if (stopRow != null && - Arrays.areEqual(stopRow, HConstants.EMPTY_END_ROW) == false) { + Arrays.equals(stopRow, HConstants.EMPTY_END_ROW) == false) { Value stopVal = IdCodec.decode(stopRow); - + switch(stopVal.getType()) { case BINARY: this.condition.is(DocumentConstants.ID_FIELD, Op.LESS, stopVal.getBinary()); @@ -85,7 +85,7 @@ public JsonSubScanSpec(@JsonProperty("tableName") String tableName, + " for _id"); } } - + this.condition.close().build(); } @@ -98,6 +98,7 @@ public QueryCondition getCondition() { return this.condition; } + @Override public byte[] getSerializedFilter() { if (this.condition != null) { ByteBuffer bbuf = ((ConditionImpl)this.condition).getDescriptor().getSerialized(); diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java index db2713795a9..6d896aac119 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java @@ -146,7 +146,7 @@ public MapRDBSubScan getSpecificScan(int minorFragmentId) { assert minorFragmentId < endpointFragmentMapping.size() : String.format( "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), minorFragmentId); - return new MapRDBSubScan(getUserName(), getStoragePlugin(), getStoragePlugin().getConfig(), + return new MapRDBSubScan(getUserName(), formatPluginConfig, getStoragePlugin(), getStoragePlugin().getConfig(), endpointFragmentMapping.get(minorFragmentId), columns, TABLE_JSON); } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 1d1ef441824..8f22a2a6293 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -18,8 +18,10 @@ package org.apache.drill.exec.store.maprdb.json; import static org.ojai.DocumentConstants.ID_KEY; +import io.netty.buffer.DrillBuf; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -37,6 +39,7 @@ import org.apache.drill.exec.ops.OperatorStats; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.store.AbstractRecordReader; +import org.apache.drill.exec.store.maprdb.MapRDBFormatPluginConfig; import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; @@ -51,6 +54,7 @@ import org.ojai.FieldSegment; import org.ojai.Value; import org.ojai.store.QueryCondition; +import org.ojai.types.OTime; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; @@ -64,8 +68,6 @@ import com.mapr.db.util.ByteBufs; import com.mapr.org.apache.hadoop.hbase.util.Bytes; -import io.netty.buffer.DrillBuf; - public class MaprDBJsonRecordReader extends AbstractRecordReader { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MaprDBJsonRecordReader.class); @@ -75,7 +77,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private QueryCondition condition; private FieldPath[] projectedFields; - private String tableName; + private final String tableName; private OperatorContext operatorContext; private VectorContainerWriter writer; @@ -87,9 +89,13 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private boolean includeId; private boolean idOnly; - private boolean unionEnabled; + private final boolean unionEnabled; + private final boolean readNumbersAsDouble; + private final boolean allTextMode; + private final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, + MapRDBFormatPluginConfig formatPluginConfig, List projectedColumns, FragmentContext context) { buffer = context.getManagedBuffer(); projectedFields = null; @@ -97,9 +103,17 @@ public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, documentReaderIterators = null; includeId = false; idOnly = false; - condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter())); + byte[] serializedFilter = subScanSpec.getSerializedFilter(); + condition = null; + + if (serializedFilter != null) { + condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(serializedFilter)); + } + setColumns(projectedColumns); unionEnabled = context.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE); + readNumbersAsDouble = formatPluginConfig.isReadAllNumbersAsDouble(); + allTextMode = formatPluginConfig.isAllTextMode(); } @Override @@ -180,7 +194,11 @@ public int next() { recordCount++; break; case BINARY: - writeBinary(map.varBinary(ID_KEY), id.getBinary()); + if (allTextMode) { + writeString(map.varChar(ID_KEY), new String(id.getBinary().array(), Charset.forName("UTF-8"))); + } else { + writeBinary(map.varBinary(ID_KEY), id.getBinary()); + } recordCount++; break; default: @@ -224,42 +242,104 @@ private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { case NULL: break; // not setting the field will leave it as null case BINARY: - writeBinary(map.varBinary(fieldName), reader.getBinary()); + if (allTextMode) { + writeString(map.varChar(fieldName), new String(reader.getBinary().array(), Charset.forName("UTF-8"))); + } else { + writeBinary(map.varBinary(fieldName), reader.getBinary()); + } break; case BOOLEAN: - map.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); + if (allTextMode) { + writeString(map.varChar(fieldName), String.valueOf(reader.getBoolean())); + } else { + map.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); + } break; case STRING: writeString(map.varChar(fieldName), reader.getString()); break; case BYTE: - map.tinyInt(fieldName).writeTinyInt(reader.getByte()); + if (allTextMode) { + writeString(map.varChar(fieldName), String.valueOf(reader.getByte())); + } else if (readNumbersAsDouble) { + map.float8(fieldName).writeFloat8(reader.getByte()); + } else { + map.tinyInt(fieldName).writeTinyInt(reader.getByte()); + } break; case SHORT: - map.smallInt(fieldName).writeSmallInt(reader.getShort()); + if (allTextMode) { + writeString(map.varChar(fieldName), String.valueOf(reader.getShort())); + } else if (readNumbersAsDouble) { + map.float8(fieldName).writeFloat8(reader.getShort()); + } else { + map.smallInt(fieldName).writeSmallInt(reader.getShort()); + } break; case INT: - map.integer(fieldName).writeInt(reader.getInt()); + if (allTextMode) { + writeString(map.varChar(fieldName), String.valueOf(reader.getInt())); + } else if (readNumbersAsDouble) { + map.float8(fieldName).writeFloat8(reader.getInt()); + } else { + map.integer(fieldName).writeInt(reader.getInt()); + } break; case LONG: - map.bigInt(fieldName).writeBigInt(reader.getLong()); + if (allTextMode) { + writeString(map.varChar(fieldName), String.valueOf(reader.getLong())); + } else if (readNumbersAsDouble) { + map.float8(fieldName).writeFloat8(reader.getLong()); + } else { + map.bigInt(fieldName).writeBigInt(reader.getLong()); + } break; case FLOAT: - map.float4(fieldName).writeFloat4(reader.getFloat()); + if (allTextMode) { + writeString(map.varChar(fieldName), String.valueOf(reader.getFloat())); + } else if (readNumbersAsDouble) { + map.float8(fieldName).writeFloat8(reader.getFloat()); + } else { + map.float4(fieldName).writeFloat4(reader.getFloat()); + } break; case DOUBLE: - map.float8(fieldName).writeFloat8(reader.getDouble()); + if (allTextMode) { + writeString(map.varChar(fieldName), String.valueOf(reader.getDouble())); + } else { + map.float8(fieldName).writeFloat8(reader.getDouble()); + } break; case DECIMAL: throw unsupportedError("Decimal type is currently not supported."); case DATE: - map.date(fieldName).writeDate(reader.getDate().toDate().getTime()); + if (allTextMode) { + writeString(map.varChar(fieldName), reader.getDate().toString()); + } else { + + long milliSecondsSinceEpoch = reader.getDate().toDaysSinceEpoch() * MILLISECONDS_IN_A_DAY; + map.date(fieldName).writeDate(milliSecondsSinceEpoch); + } break; case TIME: - map.time(fieldName).writeTime(reader.getTimeInt()); + if (allTextMode) { + writeString(map.varChar(fieldName), reader.getTime().toString()); + } else { + OTime t = reader.getTime(); + int h = t.getHour(); + int m = t.getMinute(); + int s = t.getSecond(); + int ms = t.getMilliSecond(); + int millisOfDay = ms + (s + ((m + (h * 60)) * 60)) * 1000; + map.time(fieldName).writeTime(millisOfDay); + } break; case TIMESTAMP: - map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); + if (allTextMode) { + writeString(map.varChar(fieldName), reader.getTimestamp().toString()); + } else { + map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); + } break; case INTERVAL: throw unsupportedError("Interval type is currently not supported."); @@ -292,42 +372,103 @@ private void writeToList(DBDocumentReaderBase reader, ListWriter list) { case NULL: throw unsupportedError("Null values are not supported in lists."); case BINARY: - writeBinary(list.varBinary(), reader.getBinary()); + if (allTextMode) { + writeString(list.varChar(), new String(reader.getBinary().array(), Charset.forName("UTF-8"))); + } else { + writeBinary(list.varBinary(), reader.getBinary()); + } break; case BOOLEAN: - list.bit().writeBit(reader.getBoolean() ? 1 : 0); + if (allTextMode) { + writeString(list.varChar(), String.valueOf(reader.getBoolean())); + } else { + list.bit().writeBit(reader.getBoolean() ? 1 : 0); + } break; case STRING: writeString(list.varChar(), reader.getString()); break; case BYTE: - list.tinyInt().writeTinyInt(reader.getByte()); + if (allTextMode) { + writeString(list.varChar(), String.valueOf(reader.getByte())); + } else if (readNumbersAsDouble) { + list.float8().writeFloat8(reader.getByte()); + } else { + list.tinyInt().writeTinyInt(reader.getByte()); + } break; case SHORT: - list.smallInt().writeSmallInt(reader.getShort()); + if (allTextMode) { + writeString(list.varChar(), String.valueOf(reader.getShort())); + } else if (readNumbersAsDouble) { + list.float8().writeFloat8(reader.getShort()); + } else { + list.smallInt().writeSmallInt(reader.getShort()); + } break; case INT: - list.integer().writeInt(reader.getInt()); + if (allTextMode) { + writeString(list.varChar(), String.valueOf(reader.getInt())); + } else if (readNumbersAsDouble) { + list.float8().writeFloat8(reader.getInt()); + } else { + list.integer().writeInt(reader.getInt()); + } break; case LONG: - list.bigInt().writeBigInt(reader.getLong()); + if (allTextMode) { + writeString(list.varChar(), String.valueOf(reader.getLong())); + } else if (readNumbersAsDouble) { + list.float8().writeFloat8(reader.getLong()); + } else { + list.bigInt().writeBigInt(reader.getLong()); + } break; case FLOAT: - list.float4().writeFloat4(reader.getFloat()); + if (allTextMode) { + writeString(list.varChar(), String.valueOf(reader.getFloat())); + } else if (readNumbersAsDouble) { + list.float8().writeFloat8(reader.getFloat()); + } else { + list.float4().writeFloat4(reader.getFloat()); + } break; case DOUBLE: - list.float8().writeFloat8(reader.getDouble()); + if (allTextMode) { + writeString(list.varChar(), String.valueOf(reader.getDouble())); + } else { + list.float8().writeFloat8(reader.getDouble()); + } break; case DECIMAL: throw unsupportedError("Decimals are currently not supported."); case DATE: - list.date().writeDate(reader.getDate().toDate().getTime()); + if (allTextMode) { + writeString(list.varChar(), reader.getDate().toString()); + } else { + long milliSecondsSinceEpoch = reader.getDate().toDaysSinceEpoch() * MILLISECONDS_IN_A_DAY; + list.date().writeDate(milliSecondsSinceEpoch); + } break; case TIME: - list.time().writeTime(reader.getTimeInt()); + if (allTextMode) { + writeString(list.varChar(), reader.getTime().toString()); + } else { + OTime t = reader.getTime(); + int h = t.getHour(); + int m = t.getMinute(); + int s = t.getSecond(); + int ms = t.getMilliSecond(); + int millisOfDay = ms + (s + ((m + (h * 60)) * 60)) * 1000; + list.time().writeTime(millisOfDay); + } break; case TIMESTAMP: - list.timeStamp().writeTimeStamp(reader.getTimestampLong()); + if (allTextMode) { + writeString(list.varChar(), reader.getTimestamp().toString()); + } else { + list.timeStamp().writeTimeStamp(reader.getTimestampLong()); + } break; case INTERVAL: throw unsupportedError("Interval is currently not supported."); diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index b74b1c516ee..f8d4ac48890 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -330,7 +330,7 @@ public void testPushDownSubField4() throws Exception { PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } - /* + @Test public void testPushDownSubField5() throws Exception { final String sql = "SELECT\n" @@ -341,6 +341,11 @@ public void testPushDownSubField5() throws Exception { + " business.`hours.Tuesday.open` < TIME '10:30:00'" ; runSQLAndVerifyCount(sql, 1); + + final String[] expectedPlan = {"condition=\\(hours.Tuesday.open < \\{\"\\$time\":\"10:30:00\"\\}\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @Test @@ -352,7 +357,12 @@ public void testPushDownSubField6() throws Exception { + "WHERE\n" + " business.`hours.Sunday.close` > TIME '20:30:00'" ; - runSQLAndVerifyCount(sql, 4); + runSQLAndVerifyCount(sql, 3); + + final String[] expectedPlan = {"condition=\\(hours.Sunday.close > \\{\"\\$time\":\"20:30:00\"\\}\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @Test @@ -366,6 +376,11 @@ public void testPushDownSubField7() throws Exception { + " business.`start_date` = DATE '2012-07-14'" ; runSQLAndVerifyCount(sql, 1); + + final String[] expectedPlan = {"condition=\\(start_date = \\{\"\\$dateDay\":\"2012-07-14\"\\}\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } @Test @@ -379,7 +394,10 @@ public void testPushDownSubField8() throws Exception { + " business.`last_update` = TIMESTAMP '2012-10-20 07:42:46'" ; runSQLAndVerifyCount(sql, 1); - } - */ + final String[] expectedPlan = {"condition=\\(last_update = \\{\"\\$date\":\"2012-10-20T07:42:46.000Z\"\\}\\)"}; + final String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } } diff --git a/contrib/format-maprdb/src/test/resources/json/business.json b/contrib/format-maprdb/src/test/resources/json/business.json index 0010ea17a96..6cc54dfba2a 100644 --- a/contrib/format-maprdb/src/test/resources/json/business.json +++ b/contrib/format-maprdb/src/test/resources/json/business.json @@ -1,10 +1,10 @@ -{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business", "start_date":"2011-07-14"} -{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":"2012-07-14"} -{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2013-07-14"} -{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-07-14"} -{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business", "start_date":"2011-04-14"} -{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business", "start_date":"2013-02-15"} -{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business", "start_date":"2013-01-21", "years":[2014,2015,2016]} -{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business", "start_date":"2015-03-21"} +{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business", "start_date":{"$dateDay":"2011-07-14"}, "last_update":{"$date":"2012-10-20T07:42:46.000Z"}} +{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":{"$dateDay":"2012-07-14"}} +{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":{"$dateDay":"2013-07-14"}} +{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":{"$dateDay":"2014-07-14"}} +{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business", "start_date":{"$dateDay":"2011-04-14"}} +{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business", "start_date":{"$dateDay":"2013-02-15"}} +{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business", "start_date":{"$dateDay":"2013-01-21"}, "years":[2014,2015,2016]} +{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"14:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business", "start_date":{"$dateDay":"2015-03-21"}} {"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-02-13"} -{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":"2014-02-17"} +{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":{"$dateDay":"2014-02-17"}} From 58e95ca27a570accb115882bf2f1b9d74caf3223 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 8 Mar 2016 01:40:39 -0800 Subject: [PATCH 47/58] MD-771: Query with nested functions within a sub-query on MapR-DB JSON Tables fail with ClassCastException Fix for failing test cases. --- .../exec/store/maprdb/MapRDBPushFilterIntoScan.java | 9 ++++++++- .../com/mapr/drill/maprdb/tests/json/TestSimpleJson.java | 9 +++++++++ .../format-maprdb/src/test/resources/json/business.json | 4 ++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index 634bf9a8bdc..c0a33bf4c95 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -122,7 +122,14 @@ protected void doPushFilterIntoJsonGroupScan(RelOptRuleCall call, return; } - final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + LogicalExpression conditionExp = null; + try { + conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + } catch (ClassCastException e) { + // MD-771 bug in DrillOptiq.toDrill() causes filter condition on ITEM operator to throw ClassCastException + // For such cases, we return without pushdown + return; + } final JsonConditionBuilder jsonConditionBuilder = new JsonConditionBuilder(groupScan, conditionExp); final JsonScanSpec newScanSpec = jsonConditionBuilder.parseTree(); if (newScanSpec == null) { diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index f8d4ac48890..225fb2f7c0b 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -48,6 +48,15 @@ public void testMe() throws Exception { runSQLAndVerifyCount(sql, 10); } + @Test + public void testKVGen() throws Exception { + setColumnWidths(new int[] {21, 10, 6}); + final String sql = "select _id, t.parking[0].`key` K, t.parking[0].`value` V from" + + " (select _id, kvgen(b.attributes.Parking) as parking from hbase.business b)" + + " as t where t.parking[0].`key` = 'garage' AND t.parking[0].`value` = true"; + runSQLAndVerifyCount(sql, 1); + } + @Test public void testPushdownStringEqual() throws Exception { setColumnWidths(new int[] {25, 40, 40, 40}); diff --git a/contrib/format-maprdb/src/test/resources/json/business.json b/contrib/format-maprdb/src/test/resources/json/business.json index 6cc54dfba2a..ab1326fef64 100644 --- a/contrib/format-maprdb/src/test/resources/json/business.json +++ b/contrib/format-maprdb/src/test/resources/json/business.json @@ -5,6 +5,6 @@ {"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business", "start_date":{"$dateDay":"2011-04-14"}} {"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business", "start_date":{"$dateDay":"2013-02-15"}} {"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business", "start_date":{"$dateDay":"2013-01-21"}, "years":[2014,2015,2016]} -{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"14:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business", "start_date":{"$dateDay":"2015-03-21"}} -{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":"2014-02-13"} +{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"14:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":true,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business", "start_date":{"$dateDay":"2015-03-21"}} +{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":{"$dateDay":"2014-02-13"}} {"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":{"$dateDay":"2014-02-17"}} From c74d75ce4aebc2fe188793d77dc5bb22542c9c4a Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 4 Mar 2016 15:17:36 -0800 Subject: [PATCH 48/58] Refactoring code for better organization. + Adding skeleton streams plugin. --- .../TableFormatMatcher.java} | 22 +++-- .../TableFormatPlugin.java} | 69 ++++------------ .../store/mapr/TableFormatPluginConfig.java | 38 +++++++++ .../store/mapr/db/MapRDBFormatMatcher.java | 42 ++++++++++ .../store/mapr/db/MapRDBFormatPlugin.java | 82 +++++++++++++++++++ .../db}/MapRDBFormatPluginConfig.java | 26 ++---- .../{maprdb => mapr/db}/MapRDBGroupScan.java | 2 +- .../db}/MapRDBPushFilterIntoScan.java | 14 ++-- .../db}/MapRDBScanBatchCreator.java | 6 +- .../{maprdb => mapr/db}/MapRDBSubScan.java | 7 +- .../db}/MapRDBSubScanSpec.java | 2 +- .../{maprdb => mapr/db}/MapRDBTableStats.java | 2 +- .../db}/TabletFragmentInfo.java | 2 +- .../db}/binary/BinaryTableGroupScan.java | 22 +++-- .../db}/binary/CompareFunctionsProcessor.java | 2 +- .../db}/binary/MapRDBFilterBuilder.java | 2 +- .../db}/json/CompareFunctionsProcessor.java | 2 +- .../db}/json/JsonConditionBuilder.java | 2 +- .../db}/json/JsonScanSpec.java | 2 +- .../db}/json/JsonSubScanSpec.java | 4 +- .../db}/json/JsonTableGroupScan.java | 21 +++-- .../db}/json/MaprDBJsonRecordReader.java | 6 +- .../{maprdb => mapr/db}/util/CommonFns.java | 2 +- .../mapr/streams/StreamsFormatMatcher.java | 42 ++++++++++ .../mapr/streams/StreamsFormatPlugin.java | 79 ++++++++++++++++++ .../streams/StreamsFormatPluginConfig.java | 39 +++++++++ .../drill/maprdb/tests/MaprDBTestsSuite.java | 9 +- .../maprdb/tests/binary/TestMapRDBSimple.java | 2 +- 28 files changed, 417 insertions(+), 133 deletions(-) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb/MapRDBFormatMatcher.java => mapr/TableFormatMatcher.java} (75%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb/MapRDBFormatPlugin.java => mapr/TableFormatPlugin.java} (61%) create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatMatcher.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/MapRDBFormatPluginConfig.java (81%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/MapRDBGroupScan.java (99%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/MapRDBPushFilterIntoScan.java (95%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/MapRDBScanBatchCreator.java (93%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/MapRDBSubScan.java (96%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/MapRDBSubScanSpec.java (98%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/MapRDBTableStats.java (97%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/TabletFragmentInfo.java (98%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/binary/BinaryTableGroupScan.java (93%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/binary/CompareFunctionsProcessor.java (99%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/binary/MapRDBFilterBuilder.java (99%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/json/CompareFunctionsProcessor.java (99%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/json/JsonConditionBuilder.java (99%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/json/JsonScanSpec.java (98%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/json/JsonSubScanSpec.java (97%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/json/JsonTableGroupScan.java (92%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/json/MaprDBJsonRecordReader.java (99%) rename contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/{maprdb => mapr/db}/util/CommonFns.java (94%) create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatMatcher.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java create mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPluginConfig.java diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatMatcher.java similarity index 75% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatMatcher.java index fa084f0c3d1..192e57d6fb0 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatMatcher.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatMatcher.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr; import java.io.IOException; @@ -25,17 +25,16 @@ import org.apache.drill.exec.store.dfs.FileSelection; import org.apache.drill.exec.store.dfs.FileSystemPlugin; import org.apache.drill.exec.store.dfs.FormatMatcher; -import org.apache.drill.exec.store.dfs.FormatPlugin; import org.apache.drill.exec.store.dfs.FormatSelection; import org.apache.hadoop.fs.FileStatus; import com.mapr.fs.MapRFileStatus; -public class MapRDBFormatMatcher extends FormatMatcher { +public abstract class TableFormatMatcher extends FormatMatcher { - private final FormatPlugin plugin; + private final TableFormatPlugin plugin; - public MapRDBFormatMatcher(FormatPlugin plugin) { + public TableFormatMatcher(TableFormatPlugin plugin) { this.plugin = plugin; } @@ -58,11 +57,20 @@ public DrillTable isReadable(DrillFileSystem fs, @Override public boolean isFileReadable(DrillFileSystem fs, FileStatus status) throws IOException { - return (status instanceof MapRFileStatus) && ((MapRFileStatus) status).isTable(); + return (status instanceof MapRFileStatus) + && ((MapRFileStatus) status).isTable() + && isSupportedTable((MapRFileStatus) status); } @Override - public FormatPlugin getFormatPlugin() { + public TableFormatPlugin getFormatPlugin() { return plugin; } + + /** + * Returns true if the path pointed by the MapRFileStatus is a supported table + * by this format plugin. The path must point to a MapR table. + */ + protected abstract boolean isSupportedTable(MapRFileStatus status) throws IOException; + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPlugin.java similarity index 61% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPlugin.java index 0694f5b2590..b0131fda7be 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPlugin.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr; import static com.mapr.fs.jni.MapRConstants.MAPRFS_PREFIX; @@ -26,61 +26,44 @@ import java.util.Set; import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.FormatPluginConfig; import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.AbstractWriter; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.StoragePluginOptimizerRule; -import org.apache.drill.exec.store.dfs.FileSelection; import org.apache.drill.exec.store.dfs.FileSystemConfig; import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.dfs.FormatMatcher; import org.apache.drill.exec.store.dfs.FormatPlugin; -import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan; -import org.apache.drill.exec.store.maprdb.json.JsonScanSpec; -import org.apache.drill.exec.store.maprdb.json.JsonTableGroupScan; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; import com.mapr.fs.MapRFileSystem; -import com.mapr.fs.tables.TableProperties; -public class MapRDBFormatPlugin implements FormatPlugin { +public abstract class TableFormatPlugin implements FormatPlugin { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory - .getLogger(MapRDBFormatPlugin.class); + .getLogger(TableFormatPlugin.class); private final FileSystemConfig storageConfig; - private final MapRDBFormatPluginConfig config; - private final MapRDBFormatMatcher matcher; + private final TableFormatPluginConfig config; private final Configuration fsConf; private final DrillbitContext context; private final String name; private volatile FileSystemPlugin storagePlugin; - private volatile MapRFileSystem maprfs; + private final MapRFileSystem maprfs; - public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, - StoragePluginConfig storageConfig) { - this(name, context, fsConf, storageConfig, new MapRDBFormatPluginConfig()); - } - - public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, - StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig) { + protected TableFormatPlugin(String name, DrillbitContext context, Configuration fsConf, + StoragePluginConfig storageConfig, TableFormatPluginConfig formatConfig) { this.context = context; this.config = formatConfig; - this.matcher = new MapRDBFormatMatcher(this); this.storageConfig = (FileSystemConfig) storageConfig; this.fsConf = fsConf; this.name = name == null ? "maprdb" : name; try { this.maprfs = new MapRFileSystem(); - maprfs.initialize(new URI(MAPRFS_PREFIX), fsConf); + getMaprFS().initialize(new URI(MAPRFS_PREFIX), fsConf); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); } @@ -101,42 +84,19 @@ public boolean supportsAutoPartitioning() { return false; } - @Override - public FormatMatcher getMatcher() { - return matcher; - } - public Configuration getFsConf() { return fsConf; } @Override - public AbstractWriter getWriter(PhysicalOperator child, String location, - List partitionColumns) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - @JsonIgnore public Set getOptimizerRules() { - return ImmutableSet.of(MapRDBPushFilterIntoScan.FILTER_ON_SCAN, MapRDBPushFilterIntoScan.FILTER_ON_PROJECT); + return ImmutableSet.of(); } @Override - public AbstractGroupScan getGroupScan(String userName, FileSelection selection, - List columns) throws IOException { - List files = selection.getFiles(); - assert (files.size() == 1); - String tableName = files.get(0); - TableProperties props = maprfs.getTableProperties(new Path(tableName)); - - if (props.getAttr().getJson()) { - JsonScanSpec scanSpec = new JsonScanSpec(tableName, null/*condition*/); - return new JsonTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns); - } else { - HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); - return new BinaryTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns); - } + public AbstractWriter getWriter(PhysicalOperator child, String location, + List partitionColumns) throws IOException { + throw new UnsupportedOperationException(); } @Override @@ -170,4 +130,9 @@ public synchronized FileSystemPlugin getStoragePlugin() { return storagePlugin; } + @JsonIgnore + public MapRFileSystem getMaprFS() { + return maprfs; + } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java new file mode 100644 index 00000000000..904cdb98a5c --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.mapr; + +import org.apache.drill.common.logical.FormatPluginConfig; + +public abstract class TableFormatPluginConfig implements FormatPluginConfig { + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } else if (obj == null) { + return false; + } else if (getClass() != obj.getClass()) { + return false; + } + return impEquals(obj); + } + + protected abstract boolean impEquals(Object obj); + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatMatcher.java new file mode 100644 index 00000000000..4a5d118ce54 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatMatcher.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.mapr.db; + +import java.io.IOException; + +import org.apache.drill.exec.store.mapr.TableFormatMatcher; +import org.apache.drill.exec.store.mapr.TableFormatPlugin; + +import com.mapr.fs.MapRFileStatus; + +public class MapRDBFormatMatcher extends TableFormatMatcher { + + public MapRDBFormatMatcher(TableFormatPlugin plugin) { + super(plugin); + } + + @Override + protected boolean isSupportedTable(MapRFileStatus status) throws IOException { + return !getFormatPlugin() + .getMaprFS() + .getTableProperties(status.getPath()) + .getAttr() + .getIsMarlinTable(); + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java new file mode 100644 index 00000000000..9fe16e4601d --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.mapr.db; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.dfs.FormatMatcher; +import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.drill.exec.store.mapr.TableFormatPlugin; +import org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan; +import org.apache.drill.exec.store.mapr.db.json.JsonScanSpec; +import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.ImmutableSet; +import com.mapr.fs.tables.TableProperties; + +public class MapRDBFormatPlugin extends TableFormatPlugin { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBFormatPlugin.class); + + private final MapRDBFormatMatcher matcher; + + public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, + StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig) { + super(name, context, fsConf, storageConfig, formatConfig); + matcher = new MapRDBFormatMatcher(this); + } + + @Override + public FormatMatcher getMatcher() { + return matcher; + } + + @Override + @JsonIgnore + public Set getOptimizerRules() { + return ImmutableSet.of(MapRDBPushFilterIntoScan.FILTER_ON_SCAN, MapRDBPushFilterIntoScan.FILTER_ON_PROJECT); + } + + @Override + public AbstractGroupScan getGroupScan(String userName, FileSelection selection, + List columns) throws IOException { + List files = selection.getFiles(); + assert (files.size() == 1); + String tableName = files.get(0); + TableProperties props = getMaprFS().getTableProperties(new Path(tableName)); + + if (props.getAttr().getJson()) { + JsonScanSpec scanSpec = new JsonScanSpec(tableName, null/*condition*/); + return new JsonTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns); + } else { + HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); + return new BinaryTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns); + } + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java similarity index 81% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java index eb341d98e40..82b360c0f2f 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPluginConfig.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr.db; -import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.exec.store.mapr.TableFormatPluginConfig; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; @@ -25,7 +25,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName; @JsonTypeName("maprdb") @JsonInclude(Include.NON_DEFAULT) -public class MapRDBFormatPluginConfig implements FormatPluginConfig { +public class MapRDBFormatPluginConfig extends TableFormatPluginConfig { private boolean allTextMode = false; private boolean readAllNumbersAsDouble = false; @@ -36,26 +36,11 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null) { - return false; - } - - if (getClass() != obj.getClass()) { - return false; - } - + protected boolean impEquals(Object obj) { MapRDBFormatPluginConfig other = (MapRDBFormatPluginConfig)obj; - if (readAllNumbersAsDouble != other.readAllNumbersAsDouble) { return false; - } - - if (allTextMode != other.allTextMode) { + } else if (allTextMode != other.allTextMode) { return false; } @@ -79,4 +64,5 @@ public void setAllTextMode(boolean mode) { public void setReadAllNumbersAsDouble(boolean read) { readAllNumbersAsDouble = read; } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java similarity index 99% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java index 393bfe56057..8563b786f1a 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr.db; import java.util.ArrayList; import java.util.Collections; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java similarity index 95% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java index c0a33bf4c95..7292182c6ad 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr.db; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; @@ -32,11 +32,11 @@ import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan; -import org.apache.drill.exec.store.maprdb.binary.MapRDBFilterBuilder; -import org.apache.drill.exec.store.maprdb.json.JsonConditionBuilder; -import org.apache.drill.exec.store.maprdb.json.JsonScanSpec; -import org.apache.drill.exec.store.maprdb.json.JsonTableGroupScan; +import org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan; +import org.apache.drill.exec.store.mapr.db.binary.MapRDBFilterBuilder; +import org.apache.drill.exec.store.mapr.db.json.JsonConditionBuilder; +import org.apache.drill.exec.store.mapr.db.json.JsonScanSpec; +import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; import com.google.common.collect.ImmutableList; @@ -85,7 +85,7 @@ public void onMatch(RelOptRuleCall call) { final FilterPrel filter = (FilterPrel) call.rel(0); // convert the filter to one that references the child of the project - final RexNode condition = RelOptUtil.pushFilterPastProject(filter.getCondition(), project); + final RexNode condition = RelOptUtil.pushPastProject(filter.getCondition(), project); if (scan.getGroupScan() instanceof BinaryTableGroupScan) { BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java similarity index 93% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java index 1cd33cad585..1d512235cf7 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBScanBatchCreator.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr.db; import java.util.List; @@ -27,8 +27,8 @@ import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.hbase.HBaseRecordReader; import org.apache.drill.exec.store.hbase.HBaseSubScan.HBaseSubScanSpec; -import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan; -import org.apache.drill.exec.store.maprdb.json.MaprDBJsonRecordReader; +import org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan; +import org.apache.drill.exec.store.mapr.db.json.MaprDBJsonRecordReader; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java similarity index 96% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java index 7ea4cbf880a..dea6867b228 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr.db; import java.util.Iterator; import java.util.List; @@ -36,7 +36,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterators; +import com.google.common.collect.ImmutableSet; // Class containing information for reading a single HBase region @JsonTypeName("maprdb-sub-scan") @@ -106,7 +106,7 @@ public PhysicalOperator getNewWithChildren(List children) { @Override public Iterator iterator() { - return Iterators.emptyIterator(); + return ImmutableSet.of().iterator(); } @Override @@ -121,4 +121,5 @@ public String getTableType() { public MapRDBFormatPluginConfig getFormatPluginConfig() { return fsFormatPluginConfig; } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScanSpec.java similarity index 98% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScanSpec.java index cc8bc5df436..3ffe47c840b 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBSubScanSpec.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScanSpec.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr.db; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableStats.java similarity index 97% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableStats.java index d2b1453c62e..162776c4cbf 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBTableStats.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableStats.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr.db; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.mapr.TableMappingRulesFactory; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/TabletFragmentInfo.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/TabletFragmentInfo.java similarity index 98% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/TabletFragmentInfo.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/TabletFragmentInfo.java index 389f00dfc70..e71c67c0f17 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/TabletFragmentInfo.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/TabletFragmentInfo.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb; +package org.apache.drill.exec.store.mapr.db; import org.apache.hadoop.hbase.HRegionInfo; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java similarity index 93% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java index 59d0d01a58e..a59799598b9 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/BinaryTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java @@ -15,7 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.binary; +package org.apache.drill.exec.store.mapr.db.binary; + +import static org.apache.drill.exec.store.mapr.db.util.CommonFns.isNullOrEmpty; import java.io.IOException; import java.util.List; @@ -35,13 +37,13 @@ import org.apache.drill.exec.store.dfs.FileSystemPlugin; import org.apache.drill.exec.store.hbase.DrillHBaseConstants; import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.maprdb.MapRDBFormatPlugin; -import org.apache.drill.exec.store.maprdb.MapRDBFormatPluginConfig; -import org.apache.drill.exec.store.maprdb.MapRDBGroupScan; -import org.apache.drill.exec.store.maprdb.MapRDBSubScan; -import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; -import org.apache.drill.exec.store.maprdb.MapRDBTableStats; -import org.apache.drill.exec.store.maprdb.TabletFragmentInfo; +import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; +import org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig; +import org.apache.drill.exec.store.mapr.db.MapRDBGroupScan; +import org.apache.drill.exec.store.mapr.db.MapRDBSubScan; +import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; +import org.apache.drill.exec.store.mapr.db.MapRDBTableStats; +import org.apache.drill.exec.store.mapr.db.TabletFragmentInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; @@ -162,10 +164,6 @@ protected MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) { return subScanSpec; } - private boolean isNullOrEmpty(byte[] key) { - return key == null || key.length == 0; - } - @Override public MapRDBSubScan getSpecificScan(int minorFragmentId) { assert minorFragmentId < endpointFragmentMapping.size() : String.format( diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java similarity index 99% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/CompareFunctionsProcessor.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java index f06786dcd00..0c901d75f42 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.binary; +package org.apache.drill.exec.store.mapr.db.binary; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java similarity index 99% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java index c4de6bbc121..3aba1e7de82 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.binary; +package org.apache.drill.exec.store.mapr.db.binary; import java.util.Arrays; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java similarity index 99% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java index dc5c2b7d477..827fa8c73ee 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.json; +package org.apache.drill.exec.store.mapr.db.json; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java similarity index 99% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java index 889f5a919d6..16802add760 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.json; +package org.apache.drill.exec.store.mapr.db.json; import org.apache.drill.common.expression.BooleanOperator; import org.apache.drill.common.expression.FunctionCall; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java similarity index 98% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java index 7763273772e..f316eebb580 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.json; +package org.apache.drill.exec.store.mapr.db.json; import org.apache.hadoop.hbase.HConstants; import org.ojai.store.QueryCondition; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonSubScanSpec.java similarity index 97% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonSubScanSpec.java index aa5375a1cfa..3e5dce7367e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonSubScanSpec.java @@ -15,12 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.json; +package org.apache.drill.exec.store.mapr.db.json; import java.nio.ByteBuffer; import java.util.Arrays; -import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; +import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; import org.apache.hadoop.hbase.HConstants; import org.ojai.DocumentConstants; import org.ojai.Value; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java similarity index 92% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java index 6d896aac119..9e23af77165 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.json; +package org.apache.drill.exec.store.mapr.db.json; -import static org.apache.drill.exec.store.maprdb.util.CommonFns.isNullOrEmpty; +import static org.apache.drill.exec.store.mapr.db.util.CommonFns.isNullOrEmpty; import java.io.IOException; import java.util.List; @@ -33,12 +33,12 @@ import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.dfs.FileSystemConfig; import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.maprdb.MapRDBFormatPlugin; -import org.apache.drill.exec.store.maprdb.MapRDBFormatPluginConfig; -import org.apache.drill.exec.store.maprdb.MapRDBGroupScan; -import org.apache.drill.exec.store.maprdb.MapRDBSubScan; -import org.apache.drill.exec.store.maprdb.MapRDBTableStats; -import org.apache.drill.exec.store.maprdb.TabletFragmentInfo; +import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; +import org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig; +import org.apache.drill.exec.store.mapr.db.MapRDBGroupScan; +import org.apache.drill.exec.store.mapr.db.MapRDBSubScan; +import org.apache.drill.exec.store.mapr.db.MapRDBTableStats; +import org.apache.drill.exec.store.mapr.db.TabletFragmentInfo; import org.apache.hadoop.conf.Configuration; import org.codehaus.jackson.annotate.JsonCreator; @@ -173,12 +173,11 @@ public String getTableName() { @Override public String toString() { - return "JsonTableGroupScan [ScanSpec=" - + scanSpec + ", columns=" - + columns + "]"; + return "JsonTableGroupScan [ScanSpec=" + scanSpec + ", columns=" + columns + "]"; } public JsonScanSpec getScanSpec() { return scanSpec; } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java similarity index 99% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java index 8f22a2a6293..7fbcd1b2ca3 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.json; +package org.apache.drill.exec.store.mapr.db.json; import static org.ojai.DocumentConstants.ID_KEY; import io.netty.buffer.DrillBuf; @@ -39,8 +39,8 @@ import org.apache.drill.exec.ops.OperatorStats; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.store.maprdb.MapRDBFormatPluginConfig; -import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; +import org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig; +import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/CommonFns.java similarity index 94% rename from contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java rename to contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/CommonFns.java index 894e5bd4ea9..a7b8cd1614d 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/CommonFns.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.maprdb.util; +package org.apache.drill.exec.store.mapr.db.util; public class CommonFns { diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatMatcher.java new file mode 100644 index 00000000000..47e9927cc87 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatMatcher.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.mapr.streams; + +import java.io.IOException; + +import org.apache.drill.exec.store.mapr.TableFormatMatcher; +import org.apache.drill.exec.store.mapr.TableFormatPlugin; + +import com.mapr.fs.MapRFileStatus; + +public class StreamsFormatMatcher extends TableFormatMatcher { + + public StreamsFormatMatcher(TableFormatPlugin plugin) { + super(plugin); + } + + @Override + protected boolean isSupportedTable(MapRFileStatus status) throws IOException { + return getFormatPlugin() + .getMaprFS() + .getTableProperties(status.getPath()) + .getAttr() + .getIsMarlinTable(); + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java new file mode 100644 index 00000000000..811e245b7c9 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.mapr.streams; + +import java.io.IOException; +import java.util.List; + +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.physical.base.AbstractWriter; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.dfs.FormatMatcher; +import org.apache.drill.exec.store.mapr.TableFormatPlugin; +import org.apache.hadoop.conf.Configuration; + +public class StreamsFormatPlugin extends TableFormatPlugin { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StreamsFormatPlugin.class); + private StreamsFormatMatcher matcher; + + public StreamsFormatPlugin(String name, DrillbitContext context, Configuration fsConf, + StoragePluginConfig storageConfig, StreamsFormatPluginConfig formatConfig) { + super(name, context, fsConf, storageConfig, formatConfig); + matcher = new StreamsFormatMatcher(this); + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public boolean supportsWrite() { + return false; + } + + @Override + public boolean supportsAutoPartitioning() { + return false; + } + + @Override + public FormatMatcher getMatcher() { + return matcher; + } + + @Override + public AbstractWriter getWriter(PhysicalOperator child, String location, + List partitionColumns) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public AbstractGroupScan getGroupScan(String userName, FileSelection selection, + List columns) throws IOException { + List files = selection.getFiles(); + assert (files.size() == 1); + //TableProperties props = getMaprFS().getTableProperties(new Path(files.get(0))); + throw new UnsupportedOperationException("not implemented"); + } + +} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPluginConfig.java new file mode 100644 index 00000000000..b061f03b15b --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPluginConfig.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.mapr.streams; + +import org.apache.drill.exec.store.mapr.TableFormatPluginConfig; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName("streams") @JsonInclude(Include.NON_DEFAULT) +public class StreamsFormatPluginConfig extends TableFormatPluginConfig { + + @Override + public int hashCode() { + return 47; + } + + @Override + protected boolean impEquals(Object obj) { + return true; // TODO: compare custom properties once added + } + +} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java index 18b7c3158d1..702aa5bcf87 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java @@ -63,7 +63,7 @@ public static void setupTests() throws Exception { if (initCount.get() == 0) { synchronized (MaprDBTestsSuite.class) { if (initCount.get() == 0) { - HBaseTestsSuite.configure(false, true); + HBaseTestsSuite.configure(false /*manageHBaseCluster*/, true /*createTables*/); HBaseTestsSuite.initCluster(); createJsonTables(); @@ -117,7 +117,12 @@ public static Configuration createPluginAndGetConf(DrillbitContext ctx) throws E " }," + " \"formats\": {" + " \"maprdb\": {" + - " \"type\": \"maprdb\"" + + " \"type\": \"maprdb\"," + + " \"allTextMode\": false," + + " \"readAllNumbersAsDouble\": false" + + " }," + + " \"streams\": {" + + " \"type\": \"streams\"" + " }" + " }" + "}"; diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java index 894e64d0860..6271fb641aa 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java @@ -47,7 +47,7 @@ public void testMe() throws Exception { + " *\n" + "FROM\n" + " hbase.`[TABLE_NAME]` tableName"; - runHBaseSQLVerifyCount(sql, 7); + runHBaseSQLVerifyCount(sql, 8); } } From 156819d0470b95a2822c03a498ed5c5b872d2022 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 8 Mar 2016 16:57:52 -0800 Subject: [PATCH 49/58] MD-572: Column names in MapR-DB JSON tables are case-sensitive Disable pushdown of both filter and projects (by default set to false). This will allow Drill to handle both of these operators in a case-insensitive way. --- .../mapr/db/MapRDBFormatPluginConfig.java | 16 +- .../mapr/db/MapRDBPushFilterIntoScan.java | 4 +- .../mapr/db/json/JsonTableGroupScan.java | 11 +- .../mapr/db/json/MaprDBJsonRecordReader.java | 382 +++++++----------- .../maprdb/tests/json/TestSimpleJson.java | 32 +- 5 files changed, 200 insertions(+), 245 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java index 82b360c0f2f..72952659cee 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java @@ -27,8 +27,9 @@ @JsonTypeName("maprdb") @JsonInclude(Include.NON_DEFAULT) public class MapRDBFormatPluginConfig extends TableFormatPluginConfig { - private boolean allTextMode = false; - private boolean readAllNumbersAsDouble = false; + public boolean allTextMode = false; + public boolean readAllNumbersAsDouble = false; + public boolean enablePushdown = true; @Override public int hashCode() { @@ -42,6 +43,8 @@ protected boolean impEquals(Object obj) { return false; } else if (allTextMode != other.allTextMode) { return false; + } else if (enablePushdown != other.enablePushdown) { + return false; } return true; @@ -65,4 +68,13 @@ public void setReadAllNumbersAsDouble(boolean read) { readAllNumbersAsDouble = read; } + public boolean isEnablePushdown() { + return enablePushdown; + } + + @JsonProperty("enablePushdown") + public void setEnablePushdown(boolean enablePushdown) { + this.enablePushdown = enablePushdown; + } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java index 7292182c6ad..6a286a8e2b7 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java @@ -112,7 +112,8 @@ protected void doPushFilterIntoJsonGroupScan(RelOptRuleCall call, FilterPrel filter, final ProjectPrel project, ScanPrel scan, JsonTableGroupScan groupScan, RexNode condition) { - if (groupScan.isFilterPushedDown()) { + if (groupScan.isDisablePushdown() // Do not pushdown filter if it is disabled in plugin configuration + || groupScan.isFilterPushedDown()) { // see below /* * The rule can get triggered again due to the transformed "scan => filter" sequence * created by the earlier execution of this rule when we could not do a complete @@ -202,4 +203,5 @@ protected void doPushFilterIntoBinaryGroupScan(final RelOptRuleCall call, call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel))); } } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java index 9e23af77165..0c8ffda9711 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java @@ -112,7 +112,7 @@ private void init() { regionsToScan = new TreeMap(); for (TabletInfo tabletInfo : tabletInfos) { TabletInfoImpl tabletInfoImpl = (TabletInfoImpl) tabletInfo; - if (!foundStartRegion + if (!foundStartRegion && !isNullOrEmpty(scanSpec.getStartRow()) && !tabletInfoImpl.containsRow(scanSpec.getStartRow())) { continue; @@ -171,6 +171,15 @@ public String getTableName() { return scanSpec.getTableName(); } + public boolean isDisablePushdown() { + return !formatPluginConfig.isEnablePushdown(); + } + + @JsonIgnore + public boolean canPushdownProjects(List columns) { + return formatPluginConfig.isEnablePushdown(); + } + @Override public String toString() { return "JsonTableGroupScan [ScanSpec=" + scanSpec + ", columns=" + columns + "]"; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java index 7fbcd1b2ca3..cb86e32c3eb 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java @@ -18,10 +18,8 @@ package org.apache.drill.exec.store.mapr.db.json; import static org.ojai.DocumentConstants.ID_KEY; -import io.netty.buffer.DrillBuf; import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -42,11 +40,8 @@ import org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig; import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; import org.apache.drill.exec.vector.BaseValueVector; +import org.apache.drill.exec.vector.complex.impl.MapOrListWriterImpl; import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; -import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; -import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; -import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter; -import org.apache.drill.exec.vector.complex.writer.VarCharWriter; import org.ojai.DocumentReader; import org.ojai.DocumentReader.EventType; import org.ojai.DocumentStream; @@ -54,7 +49,6 @@ import org.ojai.FieldSegment; import org.ojai.Value; import org.ojai.store.QueryCondition; -import org.ojai.types.OTime; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; @@ -68,10 +62,13 @@ import com.mapr.db.util.ByteBufs; import com.mapr.org.apache.hadoop.hbase.util.Bytes; +import io.netty.buffer.DrillBuf; + public class MaprDBJsonRecordReader extends AbstractRecordReader { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MaprDBJsonRecordReader.class); public static final SchemaPath ID_PATH = SchemaPath.getSimplePath(ID_KEY); + private final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; private Table table; private QueryCondition condition; @@ -79,7 +76,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private final String tableName; private OperatorContext operatorContext; - private VectorContainerWriter writer; + private VectorContainerWriter vectorWriter; private DrillBuf buffer; @@ -91,8 +88,8 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private boolean idOnly; private final boolean unionEnabled; private final boolean readNumbersAsDouble; + private boolean disablePushdown; private final boolean allTextMode; - private final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, MapRDBFormatPluginConfig formatPluginConfig, @@ -114,12 +111,13 @@ public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, unionEnabled = context.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE); readNumbersAsDouble = formatPluginConfig.isReadAllNumbersAsDouble(); allTextMode = formatPluginConfig.isAllTextMode(); + disablePushdown = !formatPluginConfig.isEnablePushdown(); } @Override protected Collection transformColumns(Collection columns) { Set transformed = Sets.newLinkedHashSet(); - if (!isStarQuery()) { + if (!isStarQuery() && !disablePushdown) { Set projectedFieldsSet = Sets.newTreeSet(); for (SchemaPath column : columns) { if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) { @@ -154,7 +152,7 @@ protected Collection transformColumns(Collection columns @Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { - this.writer = new VectorContainerWriter(output, unionEnabled); + this.vectorWriter = new VectorContainerWriter(output, unionEnabled); this.operatorContext = context; try { @@ -172,8 +170,8 @@ public int next() { Stopwatch watch = Stopwatch.createUnstarted(); watch.start(); - writer.allocate(); - writer.reset(); + vectorWriter.allocate(); + vectorWriter.reset(); int recordCount = 0; DBDocumentReaderBase reader = null; @@ -182,24 +180,18 @@ public int next() { try { reader = nextDocumentReader(); if (reader == null) break; - writer.setPosition(recordCount); + + vectorWriter.setPosition(recordCount); + MapOrListWriterImpl writer = new MapOrListWriterImpl(vectorWriter.rootAsMap()); if (idOnly) { Value id = reader.getId(); - MapWriter map = writer.rootAsMap(); - try { switch(id.getType()) { case STRING: - writeString(map.varChar(ID_KEY), id.getString()); - recordCount++; + writeString(writer, ID_KEY, id.getString()); break; case BINARY: - if (allTextMode) { - writeString(map.varChar(ID_KEY), new String(id.getBinary().array(), Charset.forName("UTF-8"))); - } else { - writeBinary(map.varBinary(ID_KEY), id.getBinary()); - } - recordCount++; + writeBinary(writer, ID_KEY, id.getBinary()); break; default: throw new UnsupportedOperationException(id.getType() + @@ -213,9 +205,9 @@ public int next() { if (reader.next() != EventType.START_MAP) { throw dataReadError("The document did not start with START_MAP!"); } - writeToMap(reader, writer.rootAsMap()); - recordCount++; + writeToListOrMap(writer, reader); } + recordCount++; } catch (UserException e) { throw UserException.unsupportedError(e) .addContext(String.format("Table: %s, document id: '%s'", @@ -225,132 +217,74 @@ public int next() { } } - writer.setValueCount(recordCount); + vectorWriter.setValueCount(recordCount); logger.debug("Took {} ms to get {} records", watch.elapsed(TimeUnit.MILLISECONDS), recordCount); return recordCount; } - private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { - map.start(); + private void writeToListOrMap(MapOrListWriterImpl writer, DBDocumentReaderBase reader) { + String fieldName = null; + writer.start(); outside: while (true) { EventType event = reader.next(); - if (event == null || event == EventType.END_MAP) break outside; + if (event == null + || event == EventType.END_MAP + || event == EventType.END_ARRAY) { + break outside; + } else if (reader.inMap()) { + fieldName = reader.getFieldName(); + } - String fieldName = reader.getFieldName(); try { switch (event) { case NULL: break; // not setting the field will leave it as null case BINARY: - if (allTextMode) { - writeString(map.varChar(fieldName), new String(reader.getBinary().array(), Charset.forName("UTF-8"))); - } else { - writeBinary(map.varBinary(fieldName), reader.getBinary()); - } + writeBinary(writer, fieldName, reader.getBinary()); break; case BOOLEAN: - if (allTextMode) { - writeString(map.varChar(fieldName), String.valueOf(reader.getBoolean())); - } else { - map.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); - } + writeBoolean(writer, fieldName, reader); break; case STRING: - writeString(map.varChar(fieldName), reader.getString()); + writeString(writer, fieldName, reader.getString()); break; case BYTE: - if (allTextMode) { - writeString(map.varChar(fieldName), String.valueOf(reader.getByte())); - } else if (readNumbersAsDouble) { - map.float8(fieldName).writeFloat8(reader.getByte()); - } else { - map.tinyInt(fieldName).writeTinyInt(reader.getByte()); - } + writeByte(writer, fieldName, reader); break; case SHORT: - if (allTextMode) { - writeString(map.varChar(fieldName), String.valueOf(reader.getShort())); - } else if (readNumbersAsDouble) { - map.float8(fieldName).writeFloat8(reader.getShort()); - } else { - map.smallInt(fieldName).writeSmallInt(reader.getShort()); - } + writeShort(writer, fieldName, reader); break; case INT: - if (allTextMode) { - writeString(map.varChar(fieldName), String.valueOf(reader.getInt())); - } else if (readNumbersAsDouble) { - map.float8(fieldName).writeFloat8(reader.getInt()); - } else { - map.integer(fieldName).writeInt(reader.getInt()); - } + writeInt(writer, fieldName, reader); break; case LONG: - if (allTextMode) { - writeString(map.varChar(fieldName), String.valueOf(reader.getLong())); - } else if (readNumbersAsDouble) { - map.float8(fieldName).writeFloat8(reader.getLong()); - } else { - map.bigInt(fieldName).writeBigInt(reader.getLong()); - } + writeLong(writer, fieldName, reader); break; case FLOAT: - if (allTextMode) { - writeString(map.varChar(fieldName), String.valueOf(reader.getFloat())); - } else if (readNumbersAsDouble) { - map.float8(fieldName).writeFloat8(reader.getFloat()); - } else { - map.float4(fieldName).writeFloat4(reader.getFloat()); - } + writeFloat(writer, fieldName, reader); break; case DOUBLE: - if (allTextMode) { - writeString(map.varChar(fieldName), String.valueOf(reader.getDouble())); - } else { - map.float8(fieldName).writeFloat8(reader.getDouble()); - } + writeDouble(writer, fieldName, reader); break; case DECIMAL: throw unsupportedError("Decimal type is currently not supported."); case DATE: - if (allTextMode) { - writeString(map.varChar(fieldName), reader.getDate().toString()); - } else { - - long milliSecondsSinceEpoch = reader.getDate().toDaysSinceEpoch() * MILLISECONDS_IN_A_DAY; - map.date(fieldName).writeDate(milliSecondsSinceEpoch); - } + writeDate(writer, fieldName, reader); break; case TIME: - if (allTextMode) { - writeString(map.varChar(fieldName), reader.getTime().toString()); - } else { - OTime t = reader.getTime(); - int h = t.getHour(); - int m = t.getMinute(); - int s = t.getSecond(); - int ms = t.getMilliSecond(); - int millisOfDay = ms + (s + ((m + (h * 60)) * 60)) * 1000; - map.time(fieldName).writeTime(millisOfDay); - } + writeTime(writer, fieldName, reader); break; case TIMESTAMP: - if (allTextMode) { - writeString(map.varChar(fieldName), reader.getTimestamp().toString()); - } else { - map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); - } + writeTimeStamp(writer, fieldName, reader); break; case INTERVAL: throw unsupportedError("Interval type is currently not supported."); case START_MAP: - writeToMap(reader, map.map(fieldName)); + writeToListOrMap((MapOrListWriterImpl) (reader.inMap() ? writer.map(fieldName) : writer.listoftmap(fieldName)), reader); break; case START_ARRAY: - writeToList(reader, map.list(fieldName)); + writeToListOrMap((MapOrListWriterImpl) writer.list(fieldName), reader); break; - case END_ARRAY: - throw dataReadError("Encountered an END_ARRAY event inside a map."); default: throw unsupportedError("Unsupported type: %s encountered during the query.", event); } @@ -359,145 +293,115 @@ private void writeToMap(DBDocumentReaderBase reader, MapWriter map) { IdCodec.asString(reader.getId()), fieldName), e); } } - map.end(); + writer.end(); } - private void writeToList(DBDocumentReaderBase reader, ListWriter list) { - list.startList(); - outside: while (true) { - EventType event = reader.next(); - if (event == null || event == EventType.END_ARRAY) break outside; - - switch (event) { - case NULL: - throw unsupportedError("Null values are not supported in lists."); - case BINARY: - if (allTextMode) { - writeString(list.varChar(), new String(reader.getBinary().array(), Charset.forName("UTF-8"))); - } else { - writeBinary(list.varBinary(), reader.getBinary()); - } - break; - case BOOLEAN: - if (allTextMode) { - writeString(list.varChar(), String.valueOf(reader.getBoolean())); - } else { - list.bit().writeBit(reader.getBoolean() ? 1 : 0); - } - break; - case STRING: - writeString(list.varChar(), reader.getString()); - break; - case BYTE: - if (allTextMode) { - writeString(list.varChar(), String.valueOf(reader.getByte())); - } else if (readNumbersAsDouble) { - list.float8().writeFloat8(reader.getByte()); - } else { - list.tinyInt().writeTinyInt(reader.getByte()); - } - break; - case SHORT: - if (allTextMode) { - writeString(list.varChar(), String.valueOf(reader.getShort())); - } else if (readNumbersAsDouble) { - list.float8().writeFloat8(reader.getShort()); - } else { - list.smallInt().writeSmallInt(reader.getShort()); - } - break; - case INT: - if (allTextMode) { - writeString(list.varChar(), String.valueOf(reader.getInt())); - } else if (readNumbersAsDouble) { - list.float8().writeFloat8(reader.getInt()); - } else { - list.integer().writeInt(reader.getInt()); - } - break; - case LONG: - if (allTextMode) { - writeString(list.varChar(), String.valueOf(reader.getLong())); - } else if (readNumbersAsDouble) { - list.float8().writeFloat8(reader.getLong()); - } else { - list.bigInt().writeBigInt(reader.getLong()); - } - break; - case FLOAT: - if (allTextMode) { - writeString(list.varChar(), String.valueOf(reader.getFloat())); - } else if (readNumbersAsDouble) { - list.float8().writeFloat8(reader.getFloat()); - } else { - list.float4().writeFloat4(reader.getFloat()); - } - break; - case DOUBLE: - if (allTextMode) { - writeString(list.varChar(), String.valueOf(reader.getDouble())); - } else { - list.float8().writeFloat8(reader.getDouble()); - } - break; - case DECIMAL: - throw unsupportedError("Decimals are currently not supported."); - case DATE: - if (allTextMode) { - writeString(list.varChar(), reader.getDate().toString()); - } else { - long milliSecondsSinceEpoch = reader.getDate().toDaysSinceEpoch() * MILLISECONDS_IN_A_DAY; - list.date().writeDate(milliSecondsSinceEpoch); - } - break; - case TIME: - if (allTextMode) { - writeString(list.varChar(), reader.getTime().toString()); - } else { - OTime t = reader.getTime(); - int h = t.getHour(); - int m = t.getMinute(); - int s = t.getSecond(); - int ms = t.getMilliSecond(); - int millisOfDay = ms + (s + ((m + (h * 60)) * 60)) * 1000; - list.time().writeTime(millisOfDay); - } - break; - case TIMESTAMP: - if (allTextMode) { - writeString(list.varChar(), reader.getTimestamp().toString()); - } else { - list.timeStamp().writeTimeStamp(reader.getTimestampLong()); - } - break; - case INTERVAL: - throw unsupportedError("Interval is currently not supported."); - case START_MAP: - writeToMap(reader, list.map()); - break; - case END_MAP: - throw dataReadError("Encountered an END_MAP event inside a list."); - case START_ARRAY: - writeToList(reader, list.list()); - break; - default: - throw unsupportedError("Unsupported type: %s encountered during the query.%s", event); - } + private void writeTimeStamp(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, fieldName, reader.getTimestamp().toUTCString()); + } else { + ((writer.map != null) ? writer.map.timeStamp(fieldName) : writer.list.timeStamp()).writeTimeStamp(reader.getTimestampLong()); + } + } + + private void writeTime(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, reader.getTime().toTimeStr(), fieldName); + } else { + ((writer.map != null) ? writer.map.time(fieldName) : writer.list.time()).writeTime(reader.getTimeInt()); + } + } + + private void writeDate(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, reader.getDate().toDateStr(), fieldName); + } else { + long milliSecondsSinceEpoch = reader.getDateInt() * MILLISECONDS_IN_A_DAY; + ((writer.map != null) ? writer.map.date(fieldName) : writer.list.date()).writeDate(milliSecondsSinceEpoch); + } + } + + private void writeDouble(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, String.valueOf(reader.getDouble()), fieldName); + } else { + writer.float8(fieldName).writeFloat8(reader.getDouble()); + } + } + + private void writeFloat(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, String.valueOf(reader.getFloat()), fieldName); + } else if (readNumbersAsDouble) { + writer.float8(fieldName).writeFloat8(reader.getFloat()); + } else { + writer.float4(fieldName).writeFloat4(reader.getFloat()); + } + } + + private void writeLong(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, String.valueOf(reader.getLong()), fieldName); + } else if (readNumbersAsDouble) { + writer.float8(fieldName).writeFloat8(reader.getLong()); + } else { + writer.bigInt(fieldName).writeBigInt(reader.getLong()); + } + } + + private void writeInt(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, String.valueOf(reader.getInt()), fieldName); + } else if (readNumbersAsDouble) { + writer.float8(fieldName).writeFloat8(reader.getInt()); + } else { + writer.integer(fieldName).writeInt(reader.getInt()); + } + } + + private void writeShort(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, String.valueOf(reader.getShort()), fieldName); + } else if (readNumbersAsDouble) { + writer.float8(fieldName).writeFloat8(reader.getShort()); + } else { + ((writer.map != null) ? writer.map.smallInt(fieldName) : writer.list.smallInt()).writeSmallInt(reader.getShort()); } - list.endList(); } - private void writeBinary(VarBinaryWriter binaryWriter, ByteBuffer buf) { - buffer = buffer.reallocIfNeeded(buf.remaining()); - buffer.setBytes(0, buf, buf.position(), buf.remaining()); - binaryWriter.writeVarBinary(0, buf.remaining(), buffer); + private void writeByte(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, String.valueOf(reader.getByte()), fieldName); + } else if (readNumbersAsDouble) { + writer.float8(fieldName).writeFloat8(reader.getByte()); + } else { + ((writer.map != null) ? writer.map.tinyInt(fieldName) : writer.list.tinyInt()).writeTinyInt(reader.getByte()); + } + } + + private void writeBoolean(MapOrListWriterImpl writer, String fieldName, DBDocumentReaderBase reader) { + if (allTextMode) { + writeString(writer, String.valueOf(reader.getBoolean()), fieldName); + } else { + writer.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); + } + } + + private void writeBinary(MapOrListWriterImpl writer, String fieldName, ByteBuffer buf) { + if (allTextMode) { + writeString(writer, fieldName, Bytes.toString(buf)); + } else { + buffer = buffer.reallocIfNeeded(buf.remaining()); + buffer.setBytes(0, buf, buf.position(), buf.remaining()); + writer.binary(fieldName).writeVarBinary(0, buf.remaining(), buffer); + } } - private void writeString(VarCharWriter varCharWriter, String string) { - final byte[] strBytes = Bytes.toBytes(string); + private void writeString(MapOrListWriterImpl writer, String fieldName, String value) { + final byte[] strBytes = Bytes.toBytes(value); buffer = buffer.reallocIfNeeded(strBytes.length); buffer.setBytes(0, strBytes); - varCharWriter.writeVarChar(0, strBytes.length, buffer); + writer.varChar(fieldName).writeVarChar(0, strBytes.length, buffer); } private UserException unsupportedError(String format, Object... args) { diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index 225fb2f7c0b..2bf2c319762 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -39,7 +39,16 @@ public class TestSimpleJson extends BaseJsonTest { @Test - public void testMe() throws Exception { + public void testSelectStar() throws Exception { + final String sql = "SELECT\n" + + " *\n" + + "FROM\n" + + " hbase.`business` business"; + runSQLAndVerifyCount(sql, 10); + } + + @Test + public void testSelectId() throws Exception { setColumnWidths(new int[] {23}); final String sql = "SELECT\n" + " _id\n" @@ -57,6 +66,24 @@ public void testKVGen() throws Exception { runSQLAndVerifyCount(sql, 1); } + @Test + public void testPushdownDisabled() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " table(hbase.`business`(type => 'maprdb', enablePushdown => false)) business\n" + + "WHERE\n" + + " name <> 'Sprint'" + ; + runSQLAndVerifyCount(sql, 9); + + final String[] expectedPlan = {"condition=null", "columns=\\[`\\*`\\]"}; + final String[] excludedPlan = {"condition=\\(name != \"Sprint\"\\)", "columns=\\[`name`, `_id`, `categories`, `full_address`\\]"}; + + PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); + } + @Test public void testPushdownStringEqual() throws Exception { setColumnWidths(new int[] {25, 40, 40, 40}); @@ -131,7 +158,7 @@ public void testPushdownStringNotEqual() throws Exception { ; runSQLAndVerifyCount(sql, 9); - final String[] expectedPlan = {"condition=\\(name != \"Sprint\"\\)"}; + final String[] expectedPlan = {"condition=\\(name != \"Sprint\"\\)", "columns=\\[`name`, `_id`, `categories`, `full_address`\\]"}; final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); @@ -409,4 +436,5 @@ public void testPushDownSubField8() throws Exception { PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } + } From e5bef16f09986ec46a71176f2e54298f18245e42 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Thu, 10 Mar 2016 17:37:34 -0800 Subject: [PATCH 50/58] Updated plugin version to 1.7.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index 4dfc4a70680..c2e6cb7410b 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -23,7 +23,7 @@ drill-root org.apache.drill - 1.6.0-SNAPSHOT + 1.7.0-SNAPSHOT From 391027a7f30934f6f0f77065908381f555df7aa6 Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 11 Mar 2016 02:23:35 -0800 Subject: [PATCH 51/58] MD-789: Query with condition involving addition of DATE and INTERVAL types returns no results [MapR-DB JSON Tables] + Added `enablePushdown` option to enable/disable all filter pushdown, enabled by default. --- .../store/mapr/db/json/CompareFunctionsProcessor.java | 8 +++++--- .../com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java | 3 ++- .../com/mapr/drill/maprdb/tests/json/TestSimpleJson.java | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java index 827fa8c73ee..0faa8880f6d 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java @@ -174,9 +174,11 @@ public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) thro } if (valueArg instanceof TimeStampExpression) { - this.value = KeyValueBuilder.initFrom(new OTimestamp(((TimeStampExpression)valueArg).getTimeStamp())); - this.path = path; - return true; + // disable pushdown of TimeStampExpression type until bug 22824 is fixed. + // + // this.value = KeyValueBuilder.initFrom(new OTimestamp(((TimeStampExpression)valueArg).getTimeStamp())); + // this.path = path; + // return true; } return false; diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java index 702aa5bcf87..a26ddebadbc 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java @@ -119,7 +119,8 @@ public static Configuration createPluginAndGetConf(DrillbitContext ctx) throws E " \"maprdb\": {" + " \"type\": \"maprdb\"," + " \"allTextMode\": false," + - " \"readAllNumbersAsDouble\": false" + + " \"readAllNumbersAsDouble\": false," + + " \"enablePushdown\": true" + " }," + " \"streams\": {" + " \"type\": \"streams\"" + diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index 2bf2c319762..0d9a991d560 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -431,8 +431,8 @@ public void testPushDownSubField8() throws Exception { ; runSQLAndVerifyCount(sql, 1); - final String[] expectedPlan = {"condition=\\(last_update = \\{\"\\$date\":\"2012-10-20T07:42:46.000Z\"\\}\\)"}; - final String[] excludedPlan = {}; + final String[] expectedPlan = {"condition=null"}; + final String[] excludedPlan = {"condition=\\(last_update = \\{\"\\$date\":\"2012-10-20T07:42:46.000Z\"\\}\\)"}; PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); } From 12cbd27ef8cf23db15645453d815a631926bd8cd Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 16 Mar 2016 01:58:56 -0700 Subject: [PATCH 52/58] MD-813: Improve count(*) queries against MapR-DB Json tables. + Fail query on schema change. + Added a configuration option 'ignoreSchemaChange', which when enabled, drops the rows from the result --- .../mapr/db/MapRDBFormatPluginConfig.java | 14 ++- .../mapr/db/json/MaprDBJsonRecordReader.java | 90 ++++++++++++------- .../mapr/streams/StreamsFormatPlugin.java | 3 +- 3 files changed, 73 insertions(+), 34 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java index 72952659cee..1bb07ed0be6 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java @@ -28,8 +28,9 @@ public class MapRDBFormatPluginConfig extends TableFormatPluginConfig { public boolean allTextMode = false; - public boolean readAllNumbersAsDouble = false; public boolean enablePushdown = true; + public boolean ignoreSchemaChange = false; + public boolean readAllNumbersAsDouble = false; @Override public int hashCode() { @@ -43,10 +44,11 @@ protected boolean impEquals(Object obj) { return false; } else if (allTextMode != other.allTextMode) { return false; + } else if (isIgnoreSchemaChange() != other.isIgnoreSchemaChange()) { + return false; } else if (enablePushdown != other.enablePushdown) { return false; } - return true; } @@ -77,4 +79,12 @@ public void setEnablePushdown(boolean enablePushdown) { this.enablePushdown = enablePushdown; } + public boolean isIgnoreSchemaChange() { + return ignoreSchemaChange; + } + + public void setIgnoreSchemaChange(boolean ignoreSchemaChange) { + this.ignoreSchemaChange = ignoreSchemaChange; + } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java index cb86e32c3eb..9bb7daa579c 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java @@ -32,10 +32,12 @@ import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.ops.OperatorStats; import org.apache.drill.exec.physical.impl.OutputMutator; +import org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType; import org.apache.drill.exec.store.AbstractRecordReader; import org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig; import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; @@ -90,6 +92,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { private final boolean readNumbersAsDouble; private boolean disablePushdown; private final boolean allTextMode; + private final boolean ignoreSchemaChange; public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, MapRDBFormatPluginConfig formatPluginConfig, @@ -111,6 +114,7 @@ public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, unionEnabled = context.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE); readNumbersAsDouble = formatPluginConfig.isReadAllNumbersAsDouble(); allTextMode = formatPluginConfig.isAllTextMode(); + ignoreSchemaChange = formatPluginConfig.isIgnoreSchemaChange(); disablePushdown = !formatPluginConfig.isEnablePushdown(); } @@ -177,35 +181,23 @@ public int next() { DBDocumentReaderBase reader = null; while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION) { + vectorWriter.setPosition(recordCount); try { reader = nextDocumentReader(); - if (reader == null) break; - - vectorWriter.setPosition(recordCount); - MapOrListWriterImpl writer = new MapOrListWriterImpl(vectorWriter.rootAsMap()); - if (idOnly) { - Value id = reader.getId(); - try { - switch(id.getType()) { - case STRING: - writeString(writer, ID_KEY, id.getString()); - break; - case BINARY: - writeBinary(writer, ID_KEY, id.getBinary()); - break; - default: - throw new UnsupportedOperationException(id.getType() + - " is not a supported type for _id field."); - } - } catch (IllegalStateException | IllegalArgumentException e) { - logger.warn(String.format("Possible schema change at _id: '%s'", - IdCodec.asString(id)), e); - } + if (reader == null) { + break; // no more documents for this scanner + } else if (isSkipQuery()) { + vectorWriter.rootAsMap().bit("count").writeBit(1); } else { - if (reader.next() != EventType.START_MAP) { - throw dataReadError("The document did not start with START_MAP!"); + MapOrListWriterImpl writer = new MapOrListWriterImpl(vectorWriter.rootAsMap()); + if (idOnly) { + writeId(writer, reader.getId()); + } else { + if (reader.next() != EventType.START_MAP) { + throw dataReadError("The document did not start with START_MAP!"); + } + writeToListOrMap(writer, reader); } - writeToListOrMap(writer, reader); } recordCount++; } catch (UserException e) { @@ -214,6 +206,13 @@ public int next() { table.getPath(), reader == null ? null : IdCodec.asString(reader.getId()))) .build(logger); + } catch (SchemaChangeException e) { + if (ignoreSchemaChange) { + logger.warn("{}. Dropping the row from result.", e.getMessage()); + logger.debug("Stack trace:", e); + } else { + throw dataReadError(e); + } } } @@ -222,7 +221,25 @@ public int next() { return recordCount; } - private void writeToListOrMap(MapOrListWriterImpl writer, DBDocumentReaderBase reader) { + private void writeId(MapOrListWriterImpl writer, Value id) throws SchemaChangeException { + try { + switch(id.getType()) { + case STRING: + writeString(writer, ID_KEY, id.getString()); + break; + case BINARY: + writeBinary(writer, ID_KEY, id.getBinary()); + break; + default: + throw new UnsupportedOperationException(id.getType() + + " is not a supported type for _id field."); + } + } catch (IllegalStateException | IllegalArgumentException e) { + throw schemaChangeException(e, "Possible schema change at _id: '%s'", IdCodec.asString(id)); + } + } + + private void writeToListOrMap(MapOrListWriterImpl writer, DBDocumentReaderBase reader) throws SchemaChangeException { String fieldName = null; writer.start(); outside: while (true) { @@ -289,8 +306,7 @@ private void writeToListOrMap(MapOrListWriterImpl writer, DBDocumentReaderBase r throw unsupportedError("Unsupported type: %s encountered during the query.", event); } } catch (IllegalStateException | IllegalArgumentException e) { - logger.warn(String.format("Possible schema change at _id: '%s', field: '%s'", - IdCodec.asString(reader.getId()), fieldName), e); + throw schemaChangeException(e, "Possible schema change at _id: '%s', field: '%s'", IdCodec.asString(reader.getId()), fieldName); } } writer.end(); @@ -410,12 +426,24 @@ private UserException unsupportedError(String format, Object... args) { .build(logger); } + private UserException dataReadError(Throwable t) { + return dataReadError(t, null); + } + private UserException dataReadError(String format, Object... args) { - return UserException.dataReadError() - .message(String.format(format, args)) + return dataReadError(null, format, args); + } + + private UserException dataReadError(Throwable t, String format, Object... args) { + return UserException.dataReadError(t) + .message(format == null ? null : String.format(format, args)) .build(logger); } + private SchemaChangeException schemaChangeException(Throwable t, String format, Object... args) { + return new SchemaChangeException(format, t, args); + } + private DBDocumentReaderBase nextDocumentReader() { final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats(); try { @@ -434,7 +462,7 @@ private DBDocumentReaderBase nextDocumentReader() { } } } catch (DBException e) { - throw UserException.dataReadError(e).build(logger); + throw dataReadError(e); } } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java index 811e245b7c9..f7c76b5cfb7 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.List; +import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.physical.base.AbstractGroupScan; @@ -73,7 +74,7 @@ public AbstractGroupScan getGroupScan(String userName, FileSelection selection, List files = selection.getFiles(); assert (files.size() == 1); //TableProperties props = getMaprFS().getTableProperties(new Path(files.get(0))); - throw new UnsupportedOperationException("not implemented"); + throw UserException.unsupportedError().message("MapR streams can not be querried at this time.").build(logger); } } From 5fa9ba3f2a9b5c6db25fd468ad9e2d5fa354ee87 Mon Sep 17 00:00:00 2001 From: Patrick Wong Date: Thu, 24 Mar 2016 16:36:38 -0700 Subject: [PATCH 53/58] Update MapR v5.1.0 artifacts version --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index c2e6cb7410b..d14a2c5a41b 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -31,7 +31,7 @@ maprdb-storage-plugin - 5.1.0-mapr + 5.1.0.37817-mapr 0.98.12-mapr-1506 2.7.0-mapr-1602 **/MaprDBTestsSuite.class From 1882d938f0ed94679f1a2766181eca2a02eb555a Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 30 Mar 2016 17:55:59 -0700 Subject: [PATCH 54/58] DRILL-4199: Add Support for HBase 1.X --- contrib/format-maprdb/pom.xml | 2 +- .../store/mapr/db/MapRDBFormatPlugin.java | 20 +++++++++- .../exec/store/mapr/db/MapRDBGroupScan.java | 4 +- .../store/mapr/db/MapRDBScanBatchCreator.java | 6 +-- .../exec/store/mapr/db/MapRDBSubScan.java | 39 +++++++++++-------- .../mapr/db/binary/BinaryTableGroupScan.java | 30 +++++++------- .../db/binary/CompareFunctionsProcessor.java | 12 +++--- .../drill/maprdb/tests/json/BaseJsonTest.java | 2 +- 8 files changed, 67 insertions(+), 48 deletions(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index d14a2c5a41b..fb018180bff 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -32,7 +32,7 @@ 5.1.0.37817-mapr - 0.98.12-mapr-1506 + 1.1.1-mapr-1602-m7-5.1.0 2.7.0-mapr-1602 **/MaprDBTestsSuite.class diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java index 9fe16e4601d..755ae4f751e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java @@ -35,6 +35,9 @@ import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; @@ -44,11 +47,16 @@ public class MapRDBFormatPlugin extends TableFormatPlugin { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBFormatPlugin.class); private final MapRDBFormatMatcher matcher; + private final Configuration hbaseConf; + private final Connection connection; public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, - StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig) { + StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig) throws IOException { super(name, context, fsConf, storageConfig, formatConfig); matcher = new MapRDBFormatMatcher(this); + hbaseConf = HBaseConfiguration.create(fsConf); + hbaseConf.set(ConnectionFactory.DEFAULT_DB, ConnectionFactory.MAPR_ENGINE2); + connection = ConnectionFactory.createConnection(hbaseConf); } @Override @@ -79,4 +87,14 @@ public AbstractGroupScan getGroupScan(String userName, FileSelection selection, } } + @JsonIgnore + public Configuration getHBaseConf() { + return hbaseConf; + } + + @JsonIgnore + public Connection getConnection() { + return connection; + } + } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java index 8563b786f1a..e6c71e0c463 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java @@ -49,9 +49,9 @@ public abstract class MapRDBGroupScan extends AbstractGroupScan { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBGroupScan.class); - private FileSystemPlugin storagePlugin; + protected FileSystemPlugin storagePlugin; - private MapRDBFormatPlugin formatPlugin; + protected MapRDBFormatPlugin formatPlugin; protected MapRDBFormatPluginConfig formatPluginConfig; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java index 1d512235cf7..c989bb041ec 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java @@ -29,8 +29,6 @@ import org.apache.drill.exec.store.hbase.HBaseSubScan.HBaseSubScanSpec; import org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan; import org.apache.drill.exec.store.mapr.db.json.MaprDBJsonRecordReader; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -42,11 +40,11 @@ public class MapRDBScanBatchCreator implements BatchCreator{ public ScanBatch getBatch(FragmentContext context, MapRDBSubScan subScan, List children) throws ExecutionSetupException { Preconditions.checkArgument(children.isEmpty()); List readers = Lists.newArrayList(); - Configuration conf = HBaseConfiguration.create(); for(MapRDBSubScanSpec scanSpec : subScan.getRegionScanSpecList()){ try { if (BinaryTableGroupScan.TABLE_BINARY.equals(subScan.getTableType())) { - readers.add(new HBaseRecordReader(conf, getHBaseSubScanSpec(scanSpec), subScan.getColumns(), context)); + readers.add(new HBaseRecordReader(subScan.getFormatPlugin().getConnection(), + getHBaseSubScanSpec(scanSpec), subScan.getColumns(), context)); } else { readers.add(new MaprDBJsonRecordReader(scanSpec, subScan.getFormatPluginConfig(), subScan.getColumns(), context)); } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java index dea6867b228..794141cf691 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java @@ -44,37 +44,37 @@ public class MapRDBSubScan extends AbstractBase implements SubScan { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBSubScan.class); @JsonProperty - public final StoragePluginConfig storage; + public final StoragePluginConfig storageConfig; @JsonIgnore - private final MapRDBFormatPluginConfig fsFormatPluginConfig; - private final FileSystemPlugin fsStoragePlugin; + private final MapRDBFormatPluginConfig formatPluginConfig; + private final FileSystemPlugin storagePlugin; private final List regionScanSpecList; private final List columns; private final String tableType; + private final MapRDBFormatPlugin formatPlugin; + @JsonCreator public MapRDBSubScan(@JacksonInject StoragePluginRegistry registry, @JsonProperty("userName") String userName, @JsonProperty("formatPluginConfig") MapRDBFormatPluginConfig formatPluginConfig, - @JsonProperty("storage") StoragePluginConfig storage, + @JsonProperty("storageConfig") StoragePluginConfig storage, @JsonProperty("regionScanSpecList") List regionScanSpecList, @JsonProperty("columns") List columns, @JsonProperty("tableType") String tableType) throws ExecutionSetupException { - super(userName); - this.fsFormatPluginConfig = formatPluginConfig; - this.fsStoragePlugin = (FileSystemPlugin) registry.getPlugin(storage); - this.regionScanSpecList = regionScanSpecList; - this.storage = storage; - this.columns = columns; - this.tableType = tableType; + this(userName, formatPluginConfig, + (FileSystemPlugin) registry.getPlugin(storage), + storage, regionScanSpecList, columns, tableType); } - public MapRDBSubScan(String userName, MapRDBFormatPluginConfig formatPluginConfig, FileSystemPlugin storagePlugin, StoragePluginConfig config, + public MapRDBSubScan(String userName, MapRDBFormatPluginConfig formatPluginConfig, FileSystemPlugin storagePlugin, StoragePluginConfig storageConfig, List maprSubScanSpecs, List columns, String tableType) { super(userName); - fsFormatPluginConfig = formatPluginConfig; - fsStoragePlugin = storagePlugin; - storage = config; + this.storageConfig = storageConfig; + this.storagePlugin = storagePlugin; + this.formatPluginConfig = formatPluginConfig; + this.formatPlugin = (MapRDBFormatPlugin) storagePlugin.getFormatPlugin(formatPluginConfig); + this.regionScanSpecList = maprSubScanSpecs; this.columns = columns; this.tableType = tableType; @@ -101,7 +101,7 @@ public T accept(PhysicalVisitor physicalVis @Override public PhysicalOperator getNewWithChildren(List children) { Preconditions.checkArgument(children.isEmpty()); - return new MapRDBSubScan(getUserName(), fsFormatPluginConfig, fsStoragePlugin, storage, regionScanSpecList, columns, tableType); + return new MapRDBSubScan(getUserName(), formatPluginConfig, storagePlugin, storageConfig, regionScanSpecList, columns, tableType); } @Override @@ -119,7 +119,12 @@ public String getTableType() { } public MapRDBFormatPluginConfig getFormatPluginConfig() { - return fsFormatPluginConfig; + return formatPluginConfig; + } + + @JsonIgnore + public MapRDBFormatPlugin getFormatPlugin() { + return formatPlugin; } } diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java index a59799598b9..4eaeee7f1d3 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java @@ -21,8 +21,6 @@ import java.io.IOException; import java.util.List; -import java.util.Map.Entry; -import java.util.NavigableMap; import java.util.TreeMap; import org.apache.drill.common.exceptions.DrillRuntimeException; @@ -45,11 +43,12 @@ import org.apache.drill.exec.store.mapr.db.MapRDBTableStats; import org.apache.drill.exec.store.mapr.db.TabletFragmentInfo; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.RegionLocator; import org.codehaus.jackson.annotate.JsonCreator; import com.fasterxml.jackson.annotation.JacksonInject; @@ -112,23 +111,22 @@ public GroupScan clone(List columns) { private void init() { logger.debug("Getting region locations"); - try { - Configuration conf = HBaseConfiguration.create(); - HTable table = new HTable(conf, hbaseScanSpec.getTableName()); - tableStats = new MapRDBTableStats(conf, hbaseScanSpec.getTableName()); - this.hTableDesc = table.getTableDescriptor(); - NavigableMap regionsMap = table.getRegionLocations(); - table.close(); + TableName tableName = TableName.valueOf(hbaseScanSpec.getTableName()); + try (Admin admin = formatPlugin.getConnection().getAdmin(); + RegionLocator locator = formatPlugin.getConnection().getRegionLocator(tableName)) { + hTableDesc = admin.getTableDescriptor(tableName); + tableStats = new MapRDBTableStats(getHBaseConf(), hbaseScanSpec.getTableName()); boolean foundStartRegion = false; regionsToScan = new TreeMap(); - for (Entry mapEntry : regionsMap.entrySet()) { - HRegionInfo regionInfo = mapEntry.getKey(); + List regionLocations = locator.getAllRegionLocations(); + for (HRegionLocation regionLocation : regionLocations) { + HRegionInfo regionInfo = regionLocation.getRegionInfo(); if (!foundStartRegion && hbaseScanSpec.getStartRow() != null && hbaseScanSpec.getStartRow().length != 0 && !regionInfo.containsRow(hbaseScanSpec.getStartRow())) { continue; } foundStartRegion = true; - regionsToScan.put(new TabletFragmentInfo(regionInfo), mapEntry.getValue().getHostname()); + regionsToScan.put(new TabletFragmentInfo(regionInfo), regionLocation.getHostname()); if (hbaseScanSpec.getStopRow() != null && hbaseScanSpec.getStopRow().length != 0 && regionInfo.containsRow(hbaseScanSpec.getStopRow())) { break; } @@ -191,7 +189,7 @@ public PhysicalOperator getNewWithChildren(List children) { @JsonIgnore public Configuration getHBaseConf() { - return HBaseConfiguration.create(); + return getFormatPlugin().getHBaseConf(); } @JsonIgnore diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java index 0c901d75f42..a83abf3a832 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java @@ -40,7 +40,7 @@ import org.apache.drill.common.expression.visitors.AbstractExprVisitor; import org.apache.hadoop.hbase.util.Order; import org.apache.hadoop.hbase.util.PositionedByteRange; -import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedMutableByteRange; import org.apache.drill.exec.store.hbase.DrillHBaseConstants; import org.apache.hadoop.hbase.HConstants; @@ -256,7 +256,7 @@ public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression val case "DOUBLE_OBD": if (valueArg instanceof DoubleExpression) { bb = newByteBuf(9, true); - PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, 9); + PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br, ((DoubleExpression)valueArg).getDouble(), Order.DESCENDING); @@ -271,7 +271,7 @@ public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression val case "FLOAT_OBD": if (valueArg instanceof FloatExpression) { bb = newByteBuf(5, true); - PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, 5); + PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br, ((FloatExpression)valueArg).getFloat(), Order.DESCENDING); @@ -286,7 +286,7 @@ public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression val case "BIGINT_OBD": if (valueArg instanceof LongExpression) { bb = newByteBuf(9, true); - PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, 9); + PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br, ((LongExpression)valueArg).getLong(), Order.DESCENDING); @@ -301,7 +301,7 @@ public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression val case "INT_OBD": if (valueArg instanceof IntExpression) { bb = newByteBuf(5, true); - PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, 5); + PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br, ((IntExpression)valueArg).getInt(), Order.DESCENDING); @@ -317,7 +317,7 @@ public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression val if (valueArg instanceof QuotedString) { int stringLen = ((QuotedString) valueArg).value.getBytes(Charsets.UTF_8).length; bb = newByteBuf(stringLen + 2, true); - PositionedByteRange br = new SimplePositionedByteRange(bb.array(), 0, stringLen + 2); + PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, stringLen + 2); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br, ((QuotedString)valueArg).value, Order.DESCENDING); diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java index 6aafed322db..b503b00cee8 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java @@ -22,7 +22,7 @@ import org.apache.drill.BaseTestQuery; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.rpc.user.QueryDataBatch; -import org.apache.drill.hbase.GuavaPatcher; +import org.apache.drill.exec.util.GuavaPatcher; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; From e42fb30b6c235adc7a73d86a225995f86efbaa54 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 8 Sep 2016 15:30:40 -0700 Subject: [PATCH 55/58] Updated plugin version to 1.9.0-SNAPSHOT --- contrib/format-maprdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index fb018180bff..db1c55f1900 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -23,7 +23,7 @@ drill-root org.apache.drill - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT From ac462c510261217a9f7674c279bc7b7b2c995a7f Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Fri, 9 Sep 2016 15:53:06 -0700 Subject: [PATCH 56/58] Explicitly specify `hbase-server` dependency in Hive storage plugin. Hive's HBaseStorageHandler uses HBase's TableInputFormat which is in hbase-server module. --- contrib/storage-hive/core/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index da462e0cac7..43e9cf92baf 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -58,6 +58,10 @@ org.apache.hive hive-hbase-handler + + org.apache.hbase + hbase-server + org.apache.hive hive-contrib From 373272c5e1c8e50c102c397e6326708fec4d3a41 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Sun, 11 Sep 2016 00:19:25 -0700 Subject: [PATCH 57/58] Exclude 'drill-memory-base' and 'tpch-sample-data' jars from jars/3rdparty folder. --- distribution/src/assemble/bin.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index aad468e0cc8..a9be086e7b0 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -118,7 +118,9 @@ org.apache.drill org.apache.drill.exec org.apache.drill.contrib + org.apache.drill.contrib.data org.apache.drill.contrib.storage-hive + org.apache.drill.memory org.apache.hive:hive-common org.apache.hive:hive-serde From 5a4ad2a88331dfe7561ee76fc87e882afe170681 Mon Sep 17 00:00:00 2001 From: Aditya Kishore Date: Sun, 11 Sep 2016 13:23:23 -0700 Subject: [PATCH 58/58] DRILL-4886: Modifying projects POMs to align with Drill's build and distribution. The default build/test/packaging behavior for mapr-format-plugin module are 1. BUILD of mapr-format-plugin is ENABLED. 2. Unit tests of mapr-format-plugin module are DISABLED (use `-Pmapr` to enable). 3. Packaging of mapr-format-plugin is DISABLED (use `-Pmapr` to enable). Please see LEGAL-251 for discussion/conclusion regarding inclusion of source code with non-open-source dependency. --- contrib/format-maprdb/pom.xml | 246 ++++++++++++++------------ contrib/pom.xml | 1 + distribution/pom.xml | 18 +- distribution/src/assemble/bin.xml | 1 + pom.xml | 280 ++++++++++++++++++++---------- 5 files changed, 332 insertions(+), 214 deletions(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index db1c55f1900..3a5499a587e 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -21,63 +21,26 @@ 4.0.0 - drill-root - org.apache.drill + drill-contrib-parent + org.apache.drill.contrib 1.9.0-SNAPSHOT - - drill-storage-maprdb - maprdb-storage-plugin + drill-format-mapr + contrib/mapr-format-plugin - 5.1.0.37817-mapr - 1.1.1-mapr-1602-m7-5.1.0 - 2.7.0-mapr-1602 - **/MaprDBTestsSuite.class + 1.1.1-mapr-1602-m7-5.2.0 + **/MaprDBTestsSuite.class + true - - com.mapr.hadoop - maprfs - ${maprdb-storage-plugin.mapr.version} - - - com.mapr.fs - mapr-hbase - ${maprdb-storage-plugin.mapr.version} - - - com.mapr.db - maprdb - ${maprdb-storage-plugin.mapr.version} - org.apache.hbase hbase-client - ${maprdb-storage-plugin.hbase.version} - - - org.apache.hbase - hbase-server - ${maprdb-storage-plugin.hbase.version} - - - org.apache.hadoop - hadoop-annotations - ${maprdb-storage-plugin.hadoop.version} - - - org.apache.hadoop - hadoop-client - ${maprdb-storage-plugin.hadoop.version} - - - org.apache.hadoop - hadoop-common - ${maprdb-storage-plugin.hadoop.version} + ${mapr-format-plugin.hbase.version} @@ -88,8 +51,9 @@ org.apache.maven.plugins maven-surefire-plugin + ${mapr.skip.tests} - ${maprdb.TestSuite} + ${mapr.TestSuite} @@ -107,38 +71,10 @@ com.mapr.hadoop maprfs - - - commons-logging - commons-logging - - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - - com.mapr.fs mapr-hbase - - - commons-logging - commons-logging - - - log4j - log4j - - - servlet-api-2.5 - org.mortbay.jetty - - com.mapr.db @@ -171,13 +107,14 @@ com.mapr mapr-java-utils - ${maprdb-storage-plugin.mapr.version} + ${mapr.release.version} + test tests com.mapr mapr-test-annotations - ${maprdb-storage-plugin.mapr.version} + ${mapr.release.version} org.apache.drill.exec @@ -206,45 +143,128 @@ tests test - - org.apache.hbase - hbase-server - ${maprdb-storage-plugin.hbase.version} - test - tests - - - commons-logging - commons-logging - - - servlet-api-2.5 - org.mortbay.jetty - - - servlet-api - javax.servlet - - - slf4j-log4j12 - org.slf4j - - - log4j - log4j - - - - - - mapr-releases - http://repository.mapr.com/nexus/content/repositories/releases - true - true - - + + + mapr + + false + + + + + + + simple-tests-default + + !tests + + + + + org.apache.maven.plugins + maven-surefire-plugin + + com.mapr.tests.annotations.ClusterTest + + + + + + + + simple-tests + + testssimple + + + + + org.apache.maven.plugins + maven-surefire-plugin + + com.mapr.tests.annotations.ClusterTest + + + + + + + + cluster-tests + + testscluster + + + + + org.apache.maven.plugins + maven-surefire-plugin + + com.mapr.tests.annotations.StressTest + + + + + + + + stress-tests + + testsstress + + + + + org.apache.maven.plugins + maven-surefire-plugin + + com.mapr.tests.annotations.IntegrationTest + + + + + + + + all-tests + + testsall + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + selected-tests + + + test + + + + + org.apache.maven.plugins + maven-surefire-plugin + + com.mapr.tests.annotations.AlwaysExclude + + + + + + + + diff --git a/contrib/pom.xml b/contrib/pom.xml index eb09da90972..d5485e328ab 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -33,6 +33,7 @@ storage-hbase + format-maprdb storage-hive storage-mongo storage-jdbc diff --git a/distribution/pom.xml b/distribution/pom.xml index 3733055936d..fbadb28f8de 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -225,25 +225,17 @@ mapr - - - com.mapr.hadoop - maprfs - - com.mapr.fs - mapr-hbase + org.apache.drill.contrib + drill-format-mapr + ${project.version} - log4j - log4j + com.mapr + mapr-test-annotations - - com.mapr.db - maprdb - org.apache.hadoop hadoop-winutils diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index a9be086e7b0..e428e6c29df 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -96,6 +96,7 @@ org.apache.drill.contrib.data:tpch-sample-data:jar org.apache.drill.contrib:drill-mongo-storage org.apache.drill.contrib:drill-storage-hbase + org.apache.drill.contrib:drill-format-mapr org.apache.drill.contrib:drill-jdbc-storage org.apache.drill.contrib:drill-kudu-storage org.apache.drill.contrib:drill-gis diff --git a/pom.xml b/pom.xml index 3108528a7f1..decfcbc1ef5 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,8 @@ 1.8.1-drill-r0 1.1.9-drill-r7 2.7.1 + 5.2.0-mapr + 1.1 1.2.1 2.7.1 + 1.1.3 0.9.15 2.3.21 @@ -101,6 +104,17 @@ + + mapr-releases + http://repository.mapr.com/maven/ + + true + + + false + + + mapr-drill-optiq-snapshots @@ -787,6 +801,88 @@ + + com.mapr.db + maprdb + ${mapr.release.version} + + + com.mapr.fs + mapr-hbase + ${mapr.release.version} + + + io.netty + netty-all + + + log4j + log4j + + + slf4j-log4j12 + org.slf4j + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + commons-logging + commons-logging + + + asm + asm + + + org.mortbay.jetty + servlet-api-2.5 + + + + + com.mapr.hadoop + maprfs + ${mapr.release.version} + + + org.yaml + snakeyaml + + + commons-logging + commons-logging + + + log4j + log4j + + + slf4j-log4j12 + org.slf4j + + + + + org.ojai + ojai + ${ojai.version} + + + org.apache.hadoop + hadoop-common + + + @@ -1127,7 +1223,7 @@ org.apache.hbase hbase-client - 1.1.3 + ${hbase.version} netty-all @@ -1225,6 +1321,37 @@ zookeeper org.apache.zookeeper + + stax-api + stax + + + + + org.apache.hbase + hbase-server + ${hbase.version} + + + io.netty + netty-all + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + javax.servlet + servlet-api + + + servlet-api-2.5 + org.mortbay.jetty + jsp-2.1 org.mortbay.jetty @@ -1234,8 +1361,40 @@ org.mortbay.jetty - stax-api - stax + jetty-sslengine + org.mortbay.jetty + + + jamon-runtime + org.jamon + + + commons-logging + commons-logging + + + slf4j-log4j12 + org.slf4j + + + log4j + log4j + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-client @@ -1243,7 +1402,7 @@ org.apache.hbase hbase-testing-util tests - 1.1.3 + ${hbase.version} test @@ -1398,9 +1557,8 @@ mapr true 1.2.0-mapr-1608 - 1.1.1-mapr-1602-m7-5.1.0 + 1.1.1-mapr-1602-m7-5.2.0 2.7.0-mapr-1607 - 5.2.0-mapr @@ -1591,76 +1749,6 @@ - - com.mapr.fs - mapr-hbase - ${mapr.core.version} - - - io.netty - netty-all - - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-json - - - commons-logging - commons-logging - - - asm - asm - - - org.mortbay.jetty - servlet-api-2.5 - - - - - com.mapr.hadoop - maprfs - ${mapr.core.version} - - - com.mapr.db - maprdb - ${mapr.core.version} - - - com.mapr - mapr-java-utils - - - - - org.ojai - ojai - 1.0 - - - org.apache.hadoop - hadoop-common - - - org.antlr antlr4-runtime @@ -1778,6 +1866,18 @@ hbase-server ${hbase.version} + + io.netty + netty-all + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + javax.servlet servlet-api @@ -1786,6 +1886,22 @@ servlet-api-2.5 org.mortbay.jetty + + jsp-2.1 + org.mortbay.jetty + + + jsp-api-2.1 + org.mortbay.jetty + + + jetty-sslengine + org.mortbay.jetty + + + jamon-runtime + org.jamon + commons-logging commons-logging @@ -1888,18 +2004,6 @@ - - - mapr-releases - http://repository.mapr.com/maven/ - - true - - - true - - - cdh