From bebbe8c5ad9134859066fce6e76d952a63e0db88 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Mon, 3 Aug 2015 11:20:17 +0530 Subject: [PATCH 01/15] Apache Solr Storage plugin Basic storage plugin integration of apache solr search engine with drill. --- contrib/storage-solr/pom.xml | 50 +++++ .../exec/store/solr/SolrClientAPIExec.java | 160 ++++++++++++++++ .../exec/store/solr/SolrClientOptions.java | 32 ++++ .../drill/exec/store/solr/SolrGroupScan.java | 121 ++++++++++++ .../exec/store/solr/SolrQueryBuilder.java | 31 ++++ .../exec/store/solr/SolrQueryFilterRule.java | 37 ++++ .../exec/store/solr/SolrRecordReader.java | 174 ++++++++++++++++++ .../exec/store/solr/SolrScanBatchCreator.java | 64 +++++++ .../drill/exec/store/solr/SolrScanSpec.java | 33 ++++ .../exec/store/solr/SolrStoragePlugin.java | 101 ++++++++++ .../store/solr/SolrStoragePluginConfig.java | 65 +++++++ .../drill/exec/store/solr/SolrSubScan.java | 129 +++++++++++++ .../exec/store/solr/schema/SolrSchema.java | 97 ++++++++++ .../store/solr/schema/SolrSchemaFactory.java | 76 ++++++++ .../resources/bootstrap-storage-plugins.json | 10 + .../src/main/resources/drill-module.conf | 28 +++ 16 files changed, 1208 insertions(+) create mode 100644 contrib/storage-solr/pom.xml create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientOptions.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanBatchCreator.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePluginConfig.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrSubScan.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java create mode 100644 contrib/storage-solr/src/main/resources/bootstrap-storage-plugins.json create mode 100644 contrib/storage-solr/src/main/resources/drill-module.conf diff --git a/contrib/storage-solr/pom.xml b/contrib/storage-solr/pom.xml new file mode 100644 index 00000000000..b88329b324a --- /dev/null +++ b/contrib/storage-solr/pom.xml @@ -0,0 +1,50 @@ + + + + 4.0.0 + + org.apache.drill.contrib + drill-contrib-parent + 1.2.0-SNAPSHOT + + + drill-storage-solr + + contrib/solr-storage-plugin + + + + + org.apache.drill.exec + drill-java-exec + ${project.version} + + + hadoop-common + org.apache.hadoop + + + hadoop-client + org.apache.hadoop + + + + + + org.apache.solr + solr-solrj + 5.2.1 + + + + \ No newline at end of file diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java new file mode 100644 index 00000000000..6aec35e04be --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.drill.exec.store.solr.schema.CVSchema; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.SolrStream; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; +import org.codehaus.jackson.map.ObjectMapper; + +import com.beust.jcommander.internal.Lists; + +public class SolrClientAPIExec { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrClientAPIExec.class); + private SolrClient solrClient; + + public SolrClient getSolrClient() { + return solrClient; + } + + public void setSolrClient(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec() { + + } + + public Set getSolrCoreList() { + // Request core list + logger.debug("getting cores from solr.."); + CoreAdminRequest request = new CoreAdminRequest(); + request.setAction(CoreAdminAction.STATUS); + Set coreList = null; + try { + CoreAdminResponse cores = request.process(solrClient); + coreList = new HashSet(cores.getCoreStatus().size()); + for (int i = 0; i < cores.getCoreStatus().size(); i++) { + coreList.add(cores.getCoreStatus().getName(i)); + } + } catch (SolrServerException | IOException e) { + logger.error("error getting core info from solr server..."); + } + return coreList; + } + + public CVSchema getSchemaForCore(String coreName) { + String schemaUrl = "http://dm2perf:20000/servlets/collection?type=GETSCHEMAFIELDS&corename={0}"; + schemaUrl = MessageFormat.format(schemaUrl, coreName); + HttpClient client = new DefaultHttpClient(); + HttpGet request = new HttpGet(schemaUrl); + CVSchema oCVSchema = null; + request.setHeader("Content-Type", "application/json"); + try { + logger.info("getting schema details for core... " + coreName); + HttpResponse response = client.execute(request); + BufferedReader rd = new BufferedReader(new InputStreamReader(response + .getEntity().getContent())); + StringBuffer result = new StringBuffer(); + String line = ""; + while ((line = rd.readLine()) != null) { + result.append(line); + } + ObjectMapper mapper = new ObjectMapper(); + oCVSchema = mapper.readValue(result.toString(), CVSchema.class); + logger.info("schema info... " + oCVSchema.getDefaultSearchField()); + } catch (Exception e) { + logger.info("exception occured while fetching schema details..." + + e.getMessage()); + } + return oCVSchema; + } + public SolrDocumentList getSolrDocs(String solrServer,String solrCoreName){ + SolrClient solrClient= new HttpSolrClient(solrServer+solrCoreName); + SolrDocumentList sList=null; + SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive").setRows(100); + solrQuery.setParam("q", "*:*"); + try { + QueryResponse rsp = solrClient.query(solrQuery); + sList = rsp.getResults(); + } catch (SolrServerException | IOException e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } + return sList; + } + public List getSolrResponse(String solrServer,SolrClient solrClient, String solrCoreName, + Map solrParams) { + logger.info("sending request to solr server "+solrServer+" on core "+solrCoreName); + SolrStream solrStream = new SolrStream(solrServer, solrParams); + List resultTuple = Lists.newArrayList(); + try { + solrStream.open(); + + Tuple tuple = null; + while (true) { + tuple = solrStream.read(); + if (tuple.EOF) { + break; + } + resultTuple.add(tuple); + } + } catch (Exception e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } finally { + try { + solrStream.close(); + } catch (IOException e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } + + } + return resultTuple; + } + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientOptions.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientOptions.java new file mode 100644 index 00000000000..59500255990 --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientOptions.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +public class SolrClientOptions { + private String solrServer; + + public SolrClientOptions(String solrServer) { + this.solrServer=solrServer; + } + public SolrClientOptions(SolrStoragePluginConfig solrStoragePlugin){ + this.solrServer=solrStoragePlugin.getSolrServer(); + } + public String getSolrServer(){ + return this.solrServer; + } +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java new file mode 100644 index 00000000000..e5ac04aadc8 --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.PhysicalOperatorSetupException; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.base.SubScan; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.beust.jcommander.internal.Lists; +import com.google.common.base.Preconditions; + +public class SolrGroupScan extends AbstractGroupScan { + protected SolrStoragePlugin solrPlugin; + protected SolrStoragePluginConfig solrPluginConfig; + protected SolrScanSpec solrScanSpec; + protected List scanList=Lists.newArrayList(); + protected List columns; + static final Logger logger = LoggerFactory.getLogger(SolrGroupScan.class); + + public SolrGroupScan(SolrGroupScan that) { + super(that); + this.solrPlugin = that.solrPlugin; + this.solrPluginConfig = that.solrPlugin.getSolrStorageConfig(); + this.solrScanSpec = that.solrScanSpec; + this.columns = that.columns; + this.scanList.add(this.solrScanSpec); + logger.info("SolrGroupScan :: default constructor :: "); + } + + public SolrGroupScan(String userName, SolrStoragePlugin solrStoragePlugin, + SolrScanSpec scanSpec, List columns) { + super(userName); + this.solrPlugin = solrStoragePlugin; + this.solrPluginConfig = solrStoragePlugin.getSolrStorageConfig(); + this.solrScanSpec = scanSpec; + this.columns = columns; + this.scanList.add(this.solrScanSpec); + logger.info("SolrGroupScan :: param constructor :: "+columns); + + } + + @Override + public GroupScan clone(List columns) { + logger.info("SolrGroupScan :: clone :: "+columns); + SolrGroupScan clone = new SolrGroupScan(this); + clone.columns = columns; + return clone; + } + + @Override + public void applyAssignments(List endpoints) + throws PhysicalOperatorSetupException { + // TODO Auto-generated method stub + logger.info("SolrGroupScan :: applyAssignments"); + } + + @Override + public SubScan getSpecificScan(int minorFragmentId) + throws ExecutionSetupException { + // TODO Auto-generated method stub + logger.info("SolrGroupScan :: getSpecificScan :: "+columns); + return new SolrSubScan(this); + + + } + + @Override + public int getMaxParallelizationWidth() { + // TODO Auto-generated method stub + logger.info("SolrGroupScan :: getMaxParallelizationWidth"); + return -1; + } + + @Override + public String getDigest() { + // TODO Auto-generated method stub + logger.info("SolrGroupScan :: getDigest"); + return toString(); + } + + @Override + public ScanStats getScanStats() { + // TODO Auto-generated method stub + return ScanStats.TRIVIAL_TABLE; + } + + @Override + public PhysicalOperator getNewWithChildren(List children) + throws ExecutionSetupException { + // TODO Auto-generated method stub + logger.info("SolrGroupScan :: getNewWithChildren"); + Preconditions.checkArgument(children.isEmpty()); + return new SolrGroupScan(this); + } + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java new file mode 100644 index 00000000000..2f182a8143a --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SolrQueryBuilder extends + AbstractExprVisitor { + static final Logger logger = LoggerFactory.getLogger(SolrQueryBuilder.class); + final SolrGroupScan groupScan = null; + final LogicalExpression le =null; + private boolean allExpressionsConverted = true; +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java new file mode 100644 index 00000000000..9d6ebb9c92e --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + // TODO Auto-generated constructor stub + } + + @Override + public void onMatch(RelOptRuleCall call) { + // TODO Auto-generated method stub + + } + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java new file mode 100644 index 00000000000..2e6a5b9a081 --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.Charsets; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.memory.OutOfMemoryException; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.ops.OperatorContext; +import org.apache.drill.exec.physical.impl.OutputMutator; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.MaterializedField.Key; +import org.apache.drill.exec.store.AbstractRecordReader; +import org.apache.drill.exec.vector.AllocationHelper; +import org.apache.drill.exec.vector.NullableVarCharVector; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.TypeProtos.MajorType; + +import com.google.common.collect.Lists; + +public class SolrRecordReader extends AbstractRecordReader { + static final Logger logger = LoggerFactory.getLogger(SolrRecordReader.class); + + private FragmentContext fc; + protected List vectors = Lists.newArrayList(); + protected String solrServerUrl; + protected SolrClient solrClient; + protected List scanList; + protected SolrClientAPIExec solrClientApiExec; + protected OutputMutator outputMutator; + // protected List resultTuple; + protected SolrDocumentList solrDocList; + protected Iterator resultIter; + private MajorType.Builder t; + + public SolrRecordReader(FragmentContext context, SolrSubScan config) { + fc = context; + solrServerUrl = config.getSolrPlugin().getSolrStorageConfig() + .getSolrServer(); + scanList = config.getScanList(); + solrClientApiExec = config.getSolrPlugin().getSolrClientApiExec(); + solrClient = config.getSolrPlugin().getSolrClient(); + String solrCoreName = scanList.get(0).getSolrCoreName(); + + Map solrParams = new HashMap(); + solrParams.put("q", "*:*"); + solrParams.put("rows", "100"); + solrParams.put("qt", "/select"); + solrDocList = solrClientApiExec.getSolrDocs(solrServerUrl, solrCoreName); + resultIter=solrDocList.iterator(); + logger.info("SolrRecordReader:: solrDocList:: " + solrDocList.size()); + } + + @Override + public void setup(OperatorContext context, OutputMutator output) + throws ExecutionSetupException { + logger.info("SolrRecordReader :: setup"); + if (solrDocList != null) { + SolrDocument solrDocument = solrDocList.get(0); + + Collection fieldNames = solrDocument.getFieldNames(); + for (String field : fieldNames) { + MaterializedField m_field = null; + logger.debug("solr column is " + field); + t = MajorType.newBuilder().setMinorType(TypeProtos.MinorType.VARCHAR); + m_field = MaterializedField.create(field, t.build()); + try { + vectors.add(output.addField(m_field, NullableVarCharVector.class)); + } catch (SchemaChangeException e) { + logger.debug("error while creating result vector " + + e.getLocalizedMessage()); + } + } + this.outputMutator = output; + } + } + + // @Override + // public void allocate(Map vectorMap) + // throws OutOfMemoryException { + // logger.info("SolrRecordReader :: allocate"); + // int prec = 0; + // try { + // for (ValueVector vv : vectorMap.values()) { + // if (vv.getClass().equals(NullableVarCharVector.class)) { + // NullableVarCharVector v = (NullableVarCharVector) vv; + // if(prec > 0) { + // AllocationHelper.allocate(v, 65536, prec); + // } else { + // AllocationHelper.allocate(v, 65536, 2000); + // } + // } + // } + // } catch (NullPointerException e) { + // throw new OutOfMemoryException(); + // } + // } + + @Override + public int next() { + int counter = 0; + logger.info("SolrRecordReader :: next"); + try { + while(counter <= solrDocList.getNumFound() && resultIter.hasNext()){ + SolrDocument solrDocument =resultIter.next(); + for (ValueVector vv : vectors) { + String solrField=vv.getField().getPath().toString().replaceAll("`", ""); //re-think ?? + Object fieldValue = solrDocument.get(solrField); + String fieldValueStr = "NULL"; + if (fieldValue != null) { + fieldValueStr = fieldValue.toString(); + } + byte[] record = fieldValueStr.getBytes(Charsets.UTF_8); + if (vv.getClass().equals(NullableVarCharVector.class)) { + NullableVarCharVector v = (NullableVarCharVector) vv; + v.getMutator().setSafe(counter, record, 0, record.length); + v.getMutator().setValueLengthSafe(counter, record.length); + } else { + NullableVarCharVector v = (NullableVarCharVector) vv; + v.getMutator().setSafe(counter, record, 0, record.length); + v.getMutator().setValueLengthSafe(counter, record.length); + } + } + counter++; + } + + } catch (Exception e) { + + throw new DrillRuntimeException(e); + } + + for (ValueVector vv : vectors) { + vv.getMutator().setValueCount(counter > 0 ? counter : 0); + } + return counter > 0 ? counter : 0; + } + + @Override + public void cleanup() { + // TODO Auto-generated method stub + + } + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanBatchCreator.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanBatchCreator.java new file mode 100644 index 00000000000..8e399dc5852 --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanBatchCreator.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.impl.BatchCreator; +import org.apache.drill.exec.physical.impl.ScanBatch; +import org.apache.drill.exec.record.CloseableRecordBatch; +import org.apache.drill.exec.record.RecordBatch; +import org.apache.drill.exec.store.RecordReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +public class SolrScanBatchCreator implements BatchCreator { + static final Logger logger = LoggerFactory + .getLogger(SolrScanBatchCreator.class); + + @Override + public CloseableRecordBatch getBatch(FragmentContext context, + SolrSubScan solrSubScan, List children) + throws ExecutionSetupException { + // TODO Auto-generated method stub + logger.info("SolrScanBatchCreator :: CloseableRecordBatch"); + Preconditions.checkArgument(children.isEmpty()); + List readers = Lists.newArrayList(); + List columns = null; + try { + if ((columns = solrSubScan.getColumns()) == null) { + logger.info("solrSubScan :: " + solrSubScan.getColumns()); + columns = GroupScan.ALL_COLUMNS; + } + readers.add(new SolrRecordReader(context, solrSubScan)); + } catch (Exception e) { + logger.info("SolrScanBatchCreator creation failed for subScan: " + + solrSubScan + "."); + throw new ExecutionSetupException(e); + } + return new ScanBatch(solrSubScan, context, readers.iterator()); + } + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java new file mode 100644 index 00000000000..f2ebc0a1b7a --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +public class SolrScanSpec { + private String solrCoreName; + + public SolrScanSpec(String solrCoreName) { + super(); + this.solrCoreName = solrCoreName; + } + + public String getSolrCoreName() { + return solrCoreName; + } + + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java new file mode 100644 index 00000000000..3432780d83f --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; +import java.util.List; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.solr.schema.SolrSchemaFactory; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class SolrStoragePlugin extends AbstractStoragePlugin { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrStoragePlugin.class); + + private final SolrStoragePluginConfig solrStorageConfig; + private final SolrClient solrClient; + private final DrillbitContext context; + private final SolrSchemaFactory schemaFactory; + private SolrClientAPIExec solrClientApiExec; + + public SolrStoragePlugin(SolrStoragePluginConfig solrStoragePluginConfig, + DrillbitContext context, String name) { + logger.debug("initializing solr storage plugin...."); + this.context = context; + this.solrStorageConfig = solrStoragePluginConfig; + this.solrClient = new HttpSolrClient(solrStorageConfig.getSolrServer()); + solrClientApiExec = new SolrClientAPIExec(solrClient); + this.schemaFactory = new SolrSchemaFactory(this, name); + logger.info("solr storage plugin name :: " + name); + } + + public SolrClientAPIExec getSolrClientApiExec() { + return solrClientApiExec; + } + + public SolrStoragePluginConfig getSolrStorageConfig() { + return solrStorageConfig; + } + + public SolrClient getSolrClient() { + return this.solrClient; + } + + public DrillbitContext getContext() { + return this.context; + } + + @Override + public StoragePluginConfig getConfig() { + return this.solrStorageConfig; + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) + throws IOException { + schemaFactory.registerSchemas(schemaConfig, parent); + } + + @Override + public AbstractGroupScan getPhysicalScan(String userName, + JSONOptions selection, List columns) throws IOException { + logger.info("SolrStoragePlugin :: getPhysicalScan" + " userName : " + + userName + " columns ::" + columns); + SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(), + new TypeReference() { + }); + return new SolrGroupScan(userName, this, solrScanSpec, columns); + } +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePluginConfig.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePluginConfig.java new file mode 100644 index 00000000000..4ed8b62024f --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePluginConfig.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import org.apache.drill.common.logical.StoragePluginConfig; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName(SolrStoragePluginConfig.NAME) +public class SolrStoragePluginConfig extends StoragePluginConfig { + public static final String NAME = "solr"; + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrStoragePluginConfig.class); + + @JsonProperty + private String solrServer=""; + + + @JsonCreator + public SolrStoragePluginConfig(@JsonProperty("solrServer") String solrServer) { + logger.debug("Initializing SOLR StoragePlugin configuration with solr server :: "+solrServer); + this.solrServer = solrServer; + + } + + @JsonProperty("solrServer") + public String getSolrServer() { + return this.solrServer; + } + + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } else if (that == null || getClass() != that.getClass()) { + return false; + } + SolrStoragePluginConfig thatConfig = (SolrStoragePluginConfig) that; + return this.solrServer.equals(thatConfig.solrServer); + + } + + @Override + public int hashCode() { + return this.solrServer != null ? this.solrServer.hashCode() : 0; + } +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrSubScan.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrSubScan.java new file mode 100644 index 00000000000..2f0639cd449 --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrSubScan.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.util.Iterator; +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.base.AbstractBase; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.PhysicalVisitor; +import org.apache.drill.exec.physical.base.SubScan; +import org.apache.drill.exec.store.StoragePluginRegistry; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Iterators; + +public class SolrSubScan extends AbstractBase implements SubScan { + @JsonIgnore + private SolrStoragePlugin solrPlugin; + @JsonProperty + private SolrStoragePluginConfig solrPluginConfig; + private SolrScanSpec solrScanSpec; + private List columns; + private List scanList; + + private String userName; + + public SolrSubScan(SolrGroupScan that) { + super(that); + this.columns = that.columns; + this.solrPlugin = that.solrPlugin; + this.solrPluginConfig = that.solrPluginConfig; + this.solrScanSpec = that.solrScanSpec; + this.scanList=that.scanList; + logger.info("SolrSubScan : constructor ::" + columns); + } + + @JsonCreator + public SolrSubScan(@JacksonInject StoragePluginRegistry registry, + String userName, + @JsonProperty("solrPluginConfig") SolrStoragePluginConfig pluginConfig, + @JsonProperty("solrScanSpec") List scanList, + @JsonProperty("columns") List columns) + throws ExecutionSetupException { + super(userName); + logger.info("SolrSubScan : constructor11 ::" + columns); + this.solrPlugin = (SolrStoragePlugin) registry.getPlugin(pluginConfig); + this.solrPluginConfig = pluginConfig; + this.columns = columns; + this.scanList = scanList; + this.userName = userName; + } + + public SolrSubScan(SolrStoragePlugin plugin, String userName, + SolrStoragePluginConfig pluginConfig, List scanList, + List columns) { + super(userName); + this.columns = columns; + this.solrPlugin = plugin; + this.solrPluginConfig = pluginConfig; + this.scanList = scanList; + logger.info("SolrSubScan : constructor ::" + userName); + } + + @Override + public T accept( + PhysicalVisitor physicalVisitor, X value) throws E { + return physicalVisitor.visitSubScan(this, value); + } + + @Override + public PhysicalOperator getNewWithChildren(List children) + throws ExecutionSetupException { + logger.info("SolrSubScan : getNewWithChildren ::"); + return new SolrSubScan(solrPlugin, null, solrPluginConfig, scanList, + columns); + } + + @Override + public int getOperatorType() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } + + @JsonProperty("columns") + public List getColumns() { + return columns; + } + + @Override + public boolean isExecutable() { + return false; + } + + public SolrStoragePlugin getSolrPlugin() { + return solrPlugin; + } + public List getScanList() { + return scanList; + } + public SolrScanSpec getSolrScanSpec() { + return solrScanSpec; + } +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java new file mode 100644 index 00000000000..2cdbd722b8d --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr.schema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; + +import javax.inject.Inject; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DynamicDrillTable; +import org.apache.drill.exec.store.AbstractSchema; +import org.apache.drill.exec.store.solr.SolrClientAPIExec; +import org.apache.drill.exec.store.solr.SolrScanSpec; +import org.apache.drill.exec.store.solr.SolrStoragePlugin; +import org.apache.drill.exec.store.solr.SolrStoragePluginConfig; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +public class SolrSchema extends AbstractSchema { + + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrSchema.class); + private final Set availableSolrCores; + private String currentSchema = "root"; + private final Map drillTables = Maps.newHashMap(); + private final SolrStoragePlugin solrStoragePlugin; + + public SolrSchema(List schemaPath,String currentSchema, SolrStoragePlugin solrStoragePlugin) { + super(schemaPath, currentSchema); + this.solrStoragePlugin = solrStoragePlugin; + availableSolrCores = solrStoragePlugin.getSolrClientApiExec().getSolrCoreList(); + } + + @Override + public String getTypeName() { + return SolrStoragePluginConfig.NAME; + } + + void setHolder(SchemaPlus plusOfThis) { + for (String solrCore : availableSolrCores) { + plusOfThis.add("root", getSubSchema(solrCore)); + } + + } + + @Override + public Table getTable(String coreName) { + logger.info("SolrSchema :: getTable"); + if (!availableSolrCores.contains(coreName)) { // table does not exist + return null; + } + + if (!drillTables.containsKey(coreName)) { + drillTables.put(coreName, this.getDrillTable(this.name, coreName)); + } + + return drillTables.get(coreName); + } + + DrillTable getDrillTable(String dbName, String collectionName) { + logger.info("SolrSchema :: getDrillTable"); + SolrScanSpec solrScanSpec = new SolrScanSpec(collectionName); + return new DynamicDrillTable(solrStoragePlugin, SolrStoragePluginConfig.NAME, null, solrScanSpec); + } + + @Override + public Set getTableNames() { + logger.info("SolrSchema :: getTableNames"); + return availableSolrCores; + } + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java new file mode 100644 index 00000000000..3c00d0052ce --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr.schema; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.SchemaFactory; +import org.apache.drill.exec.store.solr.SolrClientAPIExec; +import org.apache.drill.exec.store.solr.SolrStoragePlugin; +import org.apache.drill.exec.store.solr.SolrStoragePluginConfig; +import org.apache.solr.client.solrj.SolrClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Lists; + +public class SolrSchemaFactory implements SchemaFactory { + static final Logger logger = LoggerFactory.getLogger(SolrSchemaFactory.class); + + private final SolrStoragePlugin solrStorage; + private final SolrClient solrClient; + private final String storageName; + private List solrCoreLst; + + public SolrSchemaFactory(SolrStoragePlugin solrStorage, String storageName) { + this.solrStorage = solrStorage; + solrClient = solrStorage.getSolrClient(); + this.storageName = storageName; + logger.info("available solr cores are..." + solrCoreLst); + } + + + /* + * (non-Javadoc) + * + * @see + * org.apache.drill.exec.store.SchemaFactory#registerSchemas(org.apache.drill + * .exec.store.SchemaConfig, org.apache.calcite.schema.SchemaPlus) + */ + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) + throws IOException { + logger.info("registering schema...."); + List schemaPath = Lists.newArrayList(); + schemaPath.add(SolrStoragePluginConfig.NAME); + SolrSchema schema = new SolrSchema(schemaPath,"root", solrStorage); + SchemaPlus hPlus = parent.add(SolrStoragePluginConfig.NAME, schema); + + } + +} diff --git a/contrib/storage-solr/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-solr/src/main/resources/bootstrap-storage-plugins.json new file mode 100644 index 00000000000..a1bd549a1c3 --- /dev/null +++ b/contrib/storage-solr/src/main/resources/bootstrap-storage-plugins.json @@ -0,0 +1,10 @@ +{ + "storage":{ + solr : { + type:"solr", + enabled: true, + solrServer: "http://dm2perf:20000/" + } + } +} +drill.exec.storage.packages += org.apache.drill.exec.store.solr \ No newline at end of file diff --git a/contrib/storage-solr/src/main/resources/drill-module.conf b/contrib/storage-solr/src/main/resources/drill-module.conf new file mode 100644 index 00000000000..dded68e0d09 --- /dev/null +++ b/contrib/storage-solr/src/main/resources/drill-module.conf @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file tells Drill to consider this module when class path scanning. +// This file can also include any supplementary configuration information. +// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. + +drill.exec: { + + sys.store.provider: { + solr: { + "solrServer": "http://dm2perf:20000/" + } + } + +} \ No newline at end of file From 892ee85fffbbb33aabfc44116c243a32b1c6ca9d Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Tue, 4 Aug 2015 11:13:42 +0530 Subject: [PATCH 02/15] Apache Solr Storage plugin couple of classes for solr schema mapping --- .../exec/store/solr/schema/CVSchema.java | 182 +++++++ .../exec/store/solr/schema/CVSchemaField.java | 488 ++++++++++++++++++ .../resources/bootstrap-storage-plugins.json | 2 +- .../src/main/resources/drill-module.conf | 2 +- 4 files changed, 672 insertions(+), 2 deletions(-) create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java new file mode 100644 index 00000000000..0d93e045eb6 --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java @@ -0,0 +1,182 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr.schema; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.AbstractSchema; + +import com.google.common.collect.ImmutableList; + +public class CVSchema { + + protected List schemaFields = null; + + protected List dynSchemaFields = null; + + protected List fieldTypes = null; + + protected String uniqueKey = null; + + protected String defaultSearchField = null; + + protected Error errorObj = null; + + public List getSchemaFields(boolean init) { + if (init) { + return getSchemaFields(); + } else { + return this.schemaFields; + } + } + + public List getSchemaFields() { + if (schemaFields == null) { + this.schemaFields = new ArrayList(); + } + return this.schemaFields; + } + + public void setSchemaFields(List fields) { + this.schemaFields = fields; + + } + + public List getDynSchemaFields(boolean init) { + if (init) { + return getDynSchemaFields(); + } else { + return this.dynSchemaFields; + } + } + + public List getDynSchemaFields() { + if (dynSchemaFields == null) { + this.dynSchemaFields = new ArrayList(); + } + return this.dynSchemaFields; + } + + public void setDynSchemaFields(List dynSchemaFields) { + this.dynSchemaFields = dynSchemaFields; + + } + + public List getFieldTypes(boolean init) { + if (init) { + return getFieldTypes(); + } else { + return this.fieldTypes; + } + } + + public List getFieldTypes() { + if (fieldTypes == null) { + this.fieldTypes = new ArrayList(); + } + return this.fieldTypes; + } + + public void setFieldTypes(List fieldTypes) { + this.fieldTypes = fieldTypes; + + } + + public String getUniqueKey(boolean init) { + if (init) { + return getUniqueKey(); + } else { + return this.uniqueKey; + } + } + + public String getUniqueKey() { + if (uniqueKey == null) { + this.uniqueKey = ""; + } + return this.uniqueKey; + } + + public void setUniqueKey(String uniqueKey) { + this.uniqueKey = uniqueKey; + + } + + public String getDefaultSearchField(boolean init) { + if (init) { + return getDefaultSearchField(); + } else { + return this.defaultSearchField; + } + } + + public String getDefaultSearchField() { + if (defaultSearchField == null) { + this.defaultSearchField = ""; + } + return this.defaultSearchField; + } + + public void setDefaultSearchField(String defaultSearchField) { + this.defaultSearchField = defaultSearchField; + + } + + public Error getErrorObj(boolean init) { + if (init) { + return getErrorObj(); + } else { + return this.errorObj; + } + } + + public Error getErrorObj() { + if (errorObj == null) { + this.errorObj = new Error(); + } + return this.errorObj; + } + + public void setErrorObj(Error errorObj) { + this.errorObj = errorObj; + + } + + public static List getAllFields() { + ArrayList list = new ArrayList(); + list.add(Fields.SCHEMA_FIELDS); + list.add(Fields.DYN_SCHEMA_FIELDS); + list.add(Fields.FIELD_TYPES); + list.add(Fields.UNIQUE_KEY); + list.add(Fields.DEFAULT_SEARCH_FIELD); + list.add(Fields.ERROR_OBJ); + return list; + } + + public static interface Fields { + String SCHEMA_FIELDS = "fields"; + String DYN_SCHEMA_FIELDS = "dynSchemaFields"; + String FIELD_TYPES = "fieldTypes"; + String UNIQUE_KEY = "uniqueKey"; + String DEFAULT_SEARCH_FIELD = "defaultSearchField"; + String ERROR_OBJ = "errorObj"; + } + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java new file mode 100644 index 00000000000..7da9fe0030e --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java @@ -0,0 +1,488 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr.schema; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +public class CVSchemaField implements Serializable { + protected String fieldName = null; + + protected String type = null; + + protected Boolean indexed = null; + + protected Boolean stored = null; + + protected Boolean multiValued = null; + + protected Boolean required = null; + + protected Boolean termVectors = null; + + protected Boolean omitNorms = null; + + protected Boolean omitPositions = null; + + protected Boolean omitTermFreqAndPositions = null; + + protected Boolean skipdelete = null; + + protected Boolean dynamicField = null; + + /* + * *will copy to default field + */ + protected Boolean searchDefault = null; + + /* + * *will copy to autocomplete field + */ + protected Boolean autocomplete = null; + + /* + * *will copy to spell field + */ + protected Boolean spellcheck = null; + + protected List copyFields = null; + + public CVSchemaField() { + } + + /** + * copy constructors + **/ + public CVSchemaField(CVSchemaField obj) { + if (obj == null) + return; + if (obj.fieldName != null) { + this.fieldName = new String(obj.fieldName); + } + if (obj.type != null) { + this.type = new String(obj.type); + } + if (obj.indexed != null) { + this.indexed = new Boolean(obj.indexed); + } + if (obj.stored != null) { + this.stored = new Boolean(obj.stored); + } + if (obj.multiValued != null) { + this.multiValued = new Boolean(obj.multiValued); + } + if (obj.required != null) { + this.required = new Boolean(obj.required); + } + if (obj.termVectors != null) { + this.termVectors = new Boolean(obj.termVectors); + } + if (obj.omitNorms != null) { + this.omitNorms = new Boolean(obj.omitNorms); + } + if (obj.omitPositions != null) { + this.omitPositions = new Boolean(obj.omitPositions); + } + if (obj.omitTermFreqAndPositions != null) { + this.omitTermFreqAndPositions = new Boolean(obj.omitTermFreqAndPositions); + } + if (obj.skipdelete != null) { + this.skipdelete = new Boolean(obj.skipdelete); + } + if (obj.dynamicField != null) { + this.dynamicField = new Boolean(obj.dynamicField); + } + if (obj.searchDefault != null) { + this.searchDefault = new Boolean(obj.searchDefault); + } + if (obj.autocomplete != null) { + this.autocomplete = new Boolean(obj.autocomplete); + } + if (obj.spellcheck != null) { + this.spellcheck = new Boolean(obj.spellcheck); + } + if (obj.copyFields != null) { + this.copyFields = new ArrayList(obj.getCopyFields().size()); + for (String listObj : obj.getCopyFields()) { + this.copyFields.add(new String(listObj)); + } + } + } + + public String getFieldName(boolean init) { + if (init) { + return getFieldName(); + } else { + return this.fieldName; + } + } + + public String getFieldName() { + if (fieldName == null) { + this.fieldName = ""; + } + return this.fieldName; + } + + public void setFieldName(String name) { + this.fieldName = name; + + } + + public String getType(boolean init) { + if (init) { + return getType(); + } else { + return this.type; + } + } + + public String getType() { + if (type == null) { + this.type = ""; + } + return this.type; + } + + public void setType(String type) { + this.type = type; + + } + + public Boolean getIndexed(boolean init) { + if (init) { + return getIndexed(); + } else { + return this.indexed; + } + } + + public Boolean getIndexed() { + if (indexed == null) { + this.indexed = Boolean.valueOf(false); + } + return this.indexed; + } + + public void setIndexed(Boolean indexed) { + this.indexed = indexed; + + } + + public Boolean getStored(boolean init) { + if (init) { + return getStored(); + } else { + return this.stored; + } + } + + public Boolean getStored() { + if (stored == null) { + this.stored = Boolean.valueOf(false); + } + return this.stored; + } + + public void setStored(Boolean stored) { + this.stored = stored; + + } + + public Boolean getMultiValued(boolean init) { + if (init) { + return getMultiValued(); + } else { + return this.multiValued; + } + } + + public Boolean getMultiValued() { + if (multiValued == null) { + this.multiValued = Boolean.valueOf(false); + } + return this.multiValued; + } + + public void setMultiValued(Boolean multiValued) { + this.multiValued = multiValued; + + } + + public Boolean getRequired(boolean init) { + if (init) { + return getRequired(); + } else { + return this.required; + } + } + + public Boolean getRequired() { + if (required == null) { + this.required = Boolean.valueOf(false); + } + return this.required; + } + + public void setRequired(Boolean required) { + this.required = required; + + } + + public Boolean getTermVectors(boolean init) { + if (init) { + return getTermVectors(); + } else { + return this.termVectors; + } + } + + public Boolean getTermVectors() { + if (termVectors == null) { + this.termVectors = Boolean.valueOf(false); + } + return this.termVectors; + } + + public void setTermVectors(Boolean termVectors) { + this.termVectors = termVectors; + + } + + public Boolean getOmitNorms(boolean init) { + if (init) { + return getOmitNorms(); + } else { + return this.omitNorms; + } + } + + public Boolean getOmitNorms() { + if (omitNorms == null) { + this.omitNorms = Boolean.valueOf(false); + } + return this.omitNorms; + } + + public void setOmitNorms(Boolean omitNorms) { + this.omitNorms = omitNorms; + + } + + public Boolean getOmitPositions(boolean init) { + if (init) { + return getOmitPositions(); + } else { + return this.omitPositions; + } + } + + public Boolean getOmitPositions() { + if (omitPositions == null) { + this.omitPositions = Boolean.valueOf(false); + } + return this.omitPositions; + } + + public void setOmitPositions(Boolean omitPositions) { + this.omitPositions = omitPositions; + + } + + public Boolean getOmitTermFreqAndPositions(boolean init) { + if (init) { + return getOmitTermFreqAndPositions(); + } else { + return this.omitTermFreqAndPositions; + } + } + + public Boolean getOmitTermFreqAndPositions() { + if (omitTermFreqAndPositions == null) { + this.omitTermFreqAndPositions = Boolean.valueOf(false); + } + return this.omitTermFreqAndPositions; + } + + public void setOmitTermFreqAndPositions(Boolean omitTermFreqAndPositions) { + this.omitTermFreqAndPositions = omitTermFreqAndPositions; + + } + + public Boolean getSkipdelete(boolean init) { + if (init) { + return getSkipdelete(); + } else { + return this.skipdelete; + } + } + + public Boolean getSkipdelete() { + if (skipdelete == null) { + this.skipdelete = Boolean.valueOf(false); + } + return this.skipdelete; + } + + public void setSkipdelete(Boolean skipdelete) { + this.skipdelete = skipdelete; + + } + + public Boolean getDynamicField(boolean init) { + if (init) { + return getDynamicField(); + } else { + return this.dynamicField; + } + } + + public Boolean getDynamicField() { + if (dynamicField == null) { + this.dynamicField = Boolean.valueOf(false); + } + return this.dynamicField; + } + + public void setDynamicField(Boolean dynamicField) { + this.dynamicField = dynamicField; + + } + + public Boolean getSearchDefault(boolean init) { + if (init) { + return getSearchDefault(); + } else { + return this.searchDefault; + } + } + + public Boolean getSearchDefault() { + if (searchDefault == null) { + this.searchDefault = Boolean.valueOf(false); + } + return this.searchDefault; + } + + public void setSearchDefault(Boolean searchDefault) { + this.searchDefault = searchDefault; + + } + + public Boolean getAutocomplete(boolean init) { + if (init) { + return getAutocomplete(); + } else { + return this.autocomplete; + } + } + + public Boolean getAutocomplete() { + if (autocomplete == null) { + this.autocomplete = Boolean.valueOf(false); + } + return this.autocomplete; + } + + public void setAutocomplete(Boolean autocomplete) { + this.autocomplete = autocomplete; + + } + + public Boolean getSpellcheck(boolean init) { + if (init) { + return getSpellcheck(); + } else { + return this.spellcheck; + } + } + + public Boolean getSpellcheck() { + if (spellcheck == null) { + this.spellcheck = Boolean.valueOf(false); + } + return this.spellcheck; + } + + public void setSpellcheck(Boolean spellcheck) { + this.spellcheck = spellcheck; + + } + + public List getCopyFields(boolean init) { + if (init) { + return getCopyFields(); + } else { + return this.copyFields; + } + } + + public List getCopyFields() { + if (copyFields == null) { + this.copyFields = new ArrayList(); + } + return this.copyFields; + } + + public void setCopyFields(List copyFields) { + this.copyFields = copyFields; + + } + + public static List getAllFields() { + ArrayList list = new ArrayList(); + list.add(Fields.FIELD_NAME); + list.add(Fields.TYPE); + list.add(Fields.INDEXED); + list.add(Fields.STORED); + list.add(Fields.MULTI_VALUED); + list.add(Fields.REQUIRED); + list.add(Fields.TERM_VECTORS); + list.add(Fields.OMIT_NORMS); + list.add(Fields.OMIT_POSITIONS); + list.add(Fields.OMIT_TERM_FREQ_AND_POSITIONS); + list.add(Fields.SKIPDELETE); + list.add(Fields.DYNAMIC_FIELD); + list.add(Fields.SEARCH_DEFAULT); + list.add(Fields.AUTOCOMPLETE); + list.add(Fields.SPELLCHECK); + list.add(Fields.COPY_FIELDS); + return list; + } + + public static interface Fields { + String FIELD_NAME = "name"; + String TYPE = "type"; + String INDEXED = "indexed"; + String STORED = "stored"; + String MULTI_VALUED = "multiValued"; + String REQUIRED = "required"; + String TERM_VECTORS = "termVectors"; + String OMIT_NORMS = "omitNorms"; + String OMIT_POSITIONS = "omitPositions"; + String OMIT_TERM_FREQ_AND_POSITIONS = "omitTermFreqAndPositions"; + String SKIPDELETE = "skipdelete"; + String DYNAMIC_FIELD = "dynamicField"; + String SEARCH_DEFAULT = "searchDefault"; + String AUTOCOMPLETE = "autocomplete"; + String SPELLCHECK = "spellcheck"; + String COPY_FIELDS = "copyFields"; + } + +} diff --git a/contrib/storage-solr/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-solr/src/main/resources/bootstrap-storage-plugins.json index a1bd549a1c3..4e9683d43ea 100644 --- a/contrib/storage-solr/src/main/resources/bootstrap-storage-plugins.json +++ b/contrib/storage-solr/src/main/resources/bootstrap-storage-plugins.json @@ -3,7 +3,7 @@ solr : { type:"solr", enabled: true, - solrServer: "http://dm2perf:20000/" + solrServer: "" } } } diff --git a/contrib/storage-solr/src/main/resources/drill-module.conf b/contrib/storage-solr/src/main/resources/drill-module.conf index dded68e0d09..cbff454bf8c 100644 --- a/contrib/storage-solr/src/main/resources/drill-module.conf +++ b/contrib/storage-solr/src/main/resources/drill-module.conf @@ -21,7 +21,7 @@ drill.exec: { sys.store.provider: { solr: { - "solrServer": "http://dm2perf:20000/" + "solrServer": "" } } From 26ae6f7b40e552aeb22f107dd4984c683ca60a12 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Thu, 6 Aug 2015 21:44:18 +0530 Subject: [PATCH 03/15] Apache solr storage plugin --- .../exec/store/solr/SolrClientAPIExec.java | 30 +- .../solr/SolrCompareFunctionProcessor.java | 403 ++++++++++++++++++ .../drill/exec/store/solr/SolrCompareOp.java | 22 + .../drill/exec/store/solr/SolrFilters.java | 62 +++ .../drill/exec/store/solr/SolrGroupScan.java | 36 +- .../exec/store/solr/SolrQueryBuilder.java | 194 ++++++++- .../exec/store/solr/SolrQueryFilterRule.java | 42 +- .../exec/store/solr/SolrRecordReader.java | 32 +- .../drill/exec/store/solr/SolrScanSpec.java | 44 +- .../exec/store/solr/SolrStoragePlugin.java | 8 + .../exec/store/solr/schema/CVSchema.java | 2 +- .../exec/store/solr/schema/CVSchemaField.java | 24 +- .../exec/store/solr/schema/SolrSchema.java | 2 +- .../store/solr/schema/SolrSchemaFactory.java | 10 +- 14 files changed, 871 insertions(+), 40 deletions(-) create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrCompareFunctionProcessor.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrCompareOp.java create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java index 6aec35e04be..fa575ac1adf 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java @@ -46,6 +46,7 @@ import org.codehaus.jackson.map.ObjectMapper; import com.beust.jcommander.internal.Lists; +import com.google.common.base.Joiner; public class SolrClientAPIExec { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory @@ -94,7 +95,8 @@ public CVSchema getSchemaForCore(String coreName) { CVSchema oCVSchema = null; request.setHeader("Content-Type", "application/json"); try { - logger.info("getting schema details for core... " + coreName); + logger.info("getting schema details for core... " + coreName + + " schemaUrl :" + schemaUrl); HttpResponse response = client.execute(request); BufferedReader rd = new BufferedReader(new InputStreamReader(response .getEntity().getContent())); @@ -103,20 +105,28 @@ public CVSchema getSchemaForCore(String coreName) { while ((line = rd.readLine()) != null) { result.append(line); } + logger.info("schema info... " + result); ObjectMapper mapper = new ObjectMapper(); oCVSchema = mapper.readValue(result.toString(), CVSchema.class); - logger.info("schema info... " + oCVSchema.getDefaultSearchField()); } catch (Exception e) { logger.info("exception occured while fetching schema details..." + e.getMessage()); } return oCVSchema; } - public SolrDocumentList getSolrDocs(String solrServer,String solrCoreName){ - SolrClient solrClient= new HttpSolrClient(solrServer+solrCoreName); - SolrDocumentList sList=null; - SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive").setRows(100); + + public SolrDocumentList getSolrDocs(String solrServer, String solrCoreName, + List fields) { + SolrClient solrClient = new HttpSolrClient(solrServer + solrCoreName); + SolrDocumentList sList = null; + SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive") + .setRows(Integer.MAX_VALUE); solrQuery.setParam("q", "*:*"); + if (fields != null) { + logger.info("solr fields are " + fields); + String fieldStr = Joiner.on(",").join(fields); + solrQuery.setParam("fl", fieldStr); + } try { QueryResponse rsp = solrClient.query(solrQuery); sList = rsp.getResults(); @@ -126,9 +136,11 @@ public SolrDocumentList getSolrDocs(String solrServer,String solrCoreName){ } return sList; } - public List getSolrResponse(String solrServer,SolrClient solrClient, String solrCoreName, - Map solrParams) { - logger.info("sending request to solr server "+solrServer+" on core "+solrCoreName); + + public List getSolrResponse(String solrServer, SolrClient solrClient, + String solrCoreName, Map solrParams) { + logger.info("sending request to solr server " + solrServer + " on core " + + solrCoreName); SolrStream solrStream = new SolrStream(solrServer, solrParams); List resultTuple = Lists.newArrayList(); try { diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrCompareFunctionProcessor.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrCompareFunctionProcessor.java new file mode 100644 index 00000000000..359dd5a3c4d --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrCompareFunctionProcessor.java @@ -0,0 +1,403 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +import java.nio.ByteOrder; + +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.CastExpression; +import org.apache.drill.common.expression.ConvertExpression; +import org.apache.drill.common.expression.ErrorCollector; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.FunctionHolderExpression; +import org.apache.drill.common.expression.IfExpression; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.NullExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedNullConstant; +import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; +import org.apache.drill.common.expression.ValueExpressions.DateExpression; +import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression; +import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; +import org.apache.drill.common.expression.ValueExpressions.FloatExpression; +import org.apache.drill.common.expression.ValueExpressions.IntExpression; +import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression; +import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression; +import org.apache.drill.common.expression.ValueExpressions.LongExpression; +import org.apache.drill.common.expression.ValueExpressions.QuotedString; +import org.apache.drill.common.expression.ValueExpressions.TimeExpression; +import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.drill.common.expression.visitors.AggregateChecker; +import org.apache.drill.common.expression.visitors.ExprVisitor; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +public class SolrCompareFunctionProcessor implements + ExprVisitor { + private Object value; + private boolean success; + private boolean isEqualityFn; + private SchemaPath path; + private String functionName; + public static final AggregateChecker INSTANCE = new AggregateChecker(); + + public static boolean isAggregating(LogicalExpression e, ErrorCollector errors) { + return e.accept(INSTANCE, errors); + } + public static boolean isCompareFunction(String functionName) { + return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName); + } + + public static SolrCompareFunctionProcessor process(FunctionCall call) { + String functionName = call.getName(); + LogicalExpression nameArg = call.args.get(0); + LogicalExpression valueArg = call.args.size() == 2 ? call.args.get(1) + : null; + SolrCompareFunctionProcessor evaluator = new SolrCompareFunctionProcessor( + functionName); + + if (valueArg != null) { // binary function + if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) { + LogicalExpression swapArg = valueArg; + valueArg = nameArg; + nameArg = swapArg; + evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP + .get(functionName); + } + evaluator.success = nameArg.accept(evaluator, valueArg); + } else if (call.args.get(0) instanceof SchemaPath) { + evaluator.success = true; + evaluator.path = (SchemaPath) nameArg; + } + + return evaluator; + } + + public SolrCompareFunctionProcessor(String functionName) { + this.success = false; + this.functionName = functionName; + this.isEqualityFn = COMPARE_FUNCTIONS_TRANSPOSE_MAP + .containsKey(functionName) + && COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName).equals( + functionName); + } + + public Object getValue() { + return value; + } + + public boolean isSuccess() { + return success; + } + + public SchemaPath getPath() { + return path; + } + + public String getFunctionName() { + return functionName; + } + + @Override + public Boolean visitCastExpression(CastExpression e, + LogicalExpression valueArg) throws RuntimeException { + if (e.getInput() instanceof CastExpression + || e.getInput() instanceof SchemaPath) { + return e.getInput().accept(this, valueArg); + } + return false; + } + + @Override + public Boolean visitConvertExpression(ConvertExpression e, + LogicalExpression valueArg) throws RuntimeException { + if (e.getConvertFunction() == ConvertExpression.CONVERT_FROM + && e.getInput() instanceof SchemaPath) { + String encodingType = e.getEncodingType(); + switch (encodingType) { + case "INT_BE": + case "INT": + case "UINT_BE": + case "UINT": + case "UINT4_BE": + case "UINT4": + if (valueArg instanceof IntExpression + && (isEqualityFn || encodingType.startsWith("U"))) { + this.value = ((IntExpression) valueArg).getInt(); + } + break; + case "BIGINT_BE": + case "BIGINT": + case "UINT8_BE": + case "UINT8": + if (valueArg instanceof LongExpression + && (isEqualityFn || encodingType.startsWith("U"))) { + this.value = ((LongExpression) valueArg).getLong(); + } + break; + case "FLOAT": + if (valueArg instanceof FloatExpression && isEqualityFn) { + this.value = ((FloatExpression) valueArg).getFloat(); + } + break; + case "DOUBLE": + if (valueArg instanceof DoubleExpression && isEqualityFn) { + this.value = ((DoubleExpression) valueArg).getDouble(); + } + break; + case "TIME_EPOCH": + case "TIME_EPOCH_BE": + if (valueArg instanceof TimeExpression) { + this.value = ((TimeExpression) valueArg).getTime(); + } + break; + case "DATE_EPOCH": + case "DATE_EPOCH_BE": + if (valueArg instanceof DateExpression) { + this.value = ((DateExpression) valueArg).getDate(); + } + break; + case "BOOLEAN_BYTE": + if (valueArg instanceof BooleanExpression) { + this.value = ((BooleanExpression) valueArg).getBoolean(); + } + break; + case "UTF8": + // let visitSchemaPath() handle this. + return e.getInput().accept(this, valueArg); + } + + if (value != null) { + this.path = (SchemaPath) e.getInput(); + return true; + } + } + return false; + } + + @Override + public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) + throws RuntimeException { + return false; + } + + @Override + public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) + throws RuntimeException { + if (valueArg instanceof QuotedString) { + this.value = ((QuotedString) valueArg).value; + this.path = path; + return true; + } + + if (valueArg instanceof IntExpression) { + this.value = ((IntExpression) valueArg).getInt(); + this.path = path; + return true; + } + + if (valueArg instanceof LongExpression) { + this.value = ((LongExpression) valueArg).getLong(); + this.path = path; + return true; + } + + if (valueArg instanceof FloatExpression) { + this.value = ((FloatExpression) valueArg).getFloat(); + this.path = path; + return true; + } + + if (valueArg instanceof DoubleExpression) { + this.value = ((DoubleExpression) valueArg).getDouble(); + this.path = path; + return true; + } + + if (valueArg instanceof BooleanExpression) { + this.value = ((BooleanExpression) valueArg).getBoolean(); + this.path = path; + return true; + } + + return false; + } + + private static final ImmutableSet> VALUE_EXPRESSION_CLASSES; + static { + ImmutableSet.Builder> builder = ImmutableSet + .builder(); + VALUE_EXPRESSION_CLASSES = builder.add(BooleanExpression.class) + .add(DateExpression.class).add(DoubleExpression.class) + .add(FloatExpression.class).add(IntExpression.class) + .add(LongExpression.class).add(QuotedString.class) + .add(TimeExpression.class).build(); + } + private static final ImmutableMap COMPARE_FUNCTIONS_TRANSPOSE_MAP; + static { + ImmutableMap.Builder builder = ImmutableMap.builder(); + COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder + // unary functions + .put("isnotnull", "isnotnull") + .put("isNotNull", "isNotNull") + .put("is not null", "is not null") + .put("isnull", "isnull") + .put("isNull", "isNull") + .put("is null", "is null") + // binary functions + .put("equal", "equal").put("not_equal", "not_equal") + .put("greater_than_or_equal_to", "less_than_or_equal_to") + .put("greater_than", "less_than") + .put("less_than_or_equal_to", "greater_than_or_equal_to") + .put("less_than", "greater_than").build(); + } + @Override + public Boolean visitFunctionCall(FunctionCall call, LogicalExpression value) + throws RuntimeException { + return false; + } + + @Override + public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitIfExpression(IfExpression ifExpr, LogicalExpression value) + throws RuntimeException { + return false; + } + + @Override + public Boolean visitBooleanOperator(BooleanOperator call, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitIntConstant(IntExpression intExpr, LogicalExpression value) + throws RuntimeException { + return false; + } + + @Override + public Boolean visitFloatConstant(FloatExpression fExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitLongConstant(LongExpression intExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitDateConstant(DateExpression intExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitTimeConstant(TimeExpression intExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitTimeStampConstant(TimeStampExpression intExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitIntervalYearConstant(IntervalYearExpression intExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitIntervalDayConstant(IntervalDayExpression intExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitDecimal9Constant(Decimal9Expression decExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitDecimal18Constant(Decimal18Expression decExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitDecimal28Constant(Decimal28Expression decExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitDecimal38Constant(Decimal38Expression decExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitDoubleConstant(DoubleExpression dExpr, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitBooleanConstant(BooleanExpression e, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitQuotedStringConstant(QuotedString e, + LogicalExpression value) throws RuntimeException { + return false; + } + + @Override + public Boolean visitNullConstant(TypedNullConstant e, LogicalExpression value) + throws RuntimeException { + return false; + } + + @Override + public Boolean visitNullExpression(NullExpression e, LogicalExpression value) + throws RuntimeException { + return false; + } +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrCompareOp.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrCompareOp.java new file mode 100644 index 00000000000..137ada05105 --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrCompareOp.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +public class SolrCompareOp { + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java new file mode 100644 index 00000000000..cc4c0c9c82e --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import org.apache.drill.exec.store.solr.SolrScanSpec.SolrFilterParam; + +public class SolrFilters { + private SolrFilterParam leftFilterParam; + private String operator; + private SolrFilterParam rightFilterParam; + + public SolrFilters(SolrFilterParam leftFilterParam, String operator, + SolrFilterParam rightFilterParam) { + super(); + this.leftFilterParam = leftFilterParam; + this.operator = operator; + this.rightFilterParam = rightFilterParam; + } + + public SolrFilters() { + super(); + } + + public SolrFilterParam getLeftFilterParam() { + return leftFilterParam; + } + + public void setLeftFilterParam(SolrFilterParam leftFilterParam) { + this.leftFilterParam = leftFilterParam; + } + + public String getOperator() { + return operator; + } + + public void setOperator(String operator) { + this.operator = operator; + } + + public SolrFilterParam getRightFilterParam() { + return rightFilterParam; + } + + public void setRightFilterParam(SolrFilterParam rightFilterParam) { + this.rightFilterParam = rightFilterParam; + } +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java index e5ac04aadc8..a7670efd714 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java @@ -32,13 +32,17 @@ import org.slf4j.LoggerFactory; import com.beust.jcommander.internal.Lists; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; +@JsonTypeName("solr-scan") public class SolrGroupScan extends AbstractGroupScan { protected SolrStoragePlugin solrPlugin; protected SolrStoragePluginConfig solrPluginConfig; protected SolrScanSpec solrScanSpec; - protected List scanList=Lists.newArrayList(); + protected List scanList = Lists.newArrayList(); protected List columns; static final Logger logger = LoggerFactory.getLogger(SolrGroupScan.class); @@ -60,13 +64,13 @@ public SolrGroupScan(String userName, SolrStoragePlugin solrStoragePlugin, this.solrScanSpec = scanSpec; this.columns = columns; this.scanList.add(this.solrScanSpec); - logger.info("SolrGroupScan :: param constructor :: "+columns); + logger.info("SolrGroupScan :: param constructor :: " + columns); } @Override public GroupScan clone(List columns) { - logger.info("SolrGroupScan :: clone :: "+columns); + logger.info("SolrGroupScan :: clone :: " + columns); SolrGroupScan clone = new SolrGroupScan(this); clone.columns = columns; return clone; @@ -83,9 +87,8 @@ public void applyAssignments(List endpoints) public SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException { // TODO Auto-generated method stub - logger.info("SolrGroupScan :: getSpecificScan :: "+columns); + logger.info("SolrGroupScan :: getSpecificScan :: " + columns); return new SolrSubScan(this); - } @@ -99,7 +102,6 @@ public int getMaxParallelizationWidth() { @Override public String getDigest() { // TODO Auto-generated method stub - logger.info("SolrGroupScan :: getDigest"); return toString(); } @@ -108,7 +110,7 @@ public ScanStats getScanStats() { // TODO Auto-generated method stub return ScanStats.TRIVIAL_TABLE; } - + @JsonIgnore @Override public PhysicalOperator getNewWithChildren(List children) throws ExecutionSetupException { @@ -118,4 +120,24 @@ public PhysicalOperator getNewWithChildren(List children) return new SolrGroupScan(this); } + @JsonProperty + public SolrScanSpec getSolrScanSpec() { + return solrScanSpec; + } + + @JsonProperty + public SolrStoragePluginConfig getSolrPluginConfig() { + return solrPluginConfig; + } + + @JsonIgnore + public SolrStoragePlugin getSolrPlugin() { + return solrPlugin; + } + + @Override + public String toString() { + return "SolrGroupScan [SolrScanSpec=" + solrScanSpec + ", columns=" + + columns + "]"; + } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java index 2f182a8143a..f8f8cd34fa6 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java @@ -17,15 +17,205 @@ */ package org.apache.drill.exec.store.solr; +import java.util.List; +import java.util.Queue; + +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.drill.exec.store.solr.SolrScanSpec.SolrFilters; +import org.apache.solr.client.solrj.SolrQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.beust.jcommander.internal.Lists; +import com.google.common.collect.ImmutableList; + public class SolrQueryBuilder extends AbstractExprVisitor { static final Logger logger = LoggerFactory.getLogger(SolrQueryBuilder.class); - final SolrGroupScan groupScan = null; - final LogicalExpression le =null; + final SolrGroupScan groupScan; + final LogicalExpression le; private boolean allExpressionsConverted = true; + + public SolrQueryBuilder(SolrGroupScan solrGroupScan, + LogicalExpression conditionExp) { + this.groupScan = solrGroupScan; + this.le = conditionExp; + logger.info("SolrQueryBuilder :: constructor"); + } + + public SolrScanSpec parseTree() { + logger.info("SolrQueryBuilder :: parseTree"); + SolrScanSpec parsedSpec = le.accept(this, null); + if (parsedSpec != null) { + logger.info("parsedSpec : " + parsedSpec); + parsedSpec = mergeScanSpecs("booleanAnd", + this.groupScan.getSolrScanSpec(), parsedSpec); + } + return parsedSpec; + } + + public SolrScanSpec mergeScanSpecs(String functionName, + SolrScanSpec leftScanSpec, SolrScanSpec rightScanSpec) { + List solrFilterList; + logger.info("mergeScanSpecs : init"); + switch (functionName) { + case "booleanAnd": + if (leftScanSpec.getFilter() != null && rightScanSpec.getFilter() != null) { + logger.info("mergeScanSpecs : 1"); + solrFilterList = leftScanSpec.getFilter(); + } else if (leftScanSpec.getFilter() != null) { + logger.info("mergeScanSpecs : 2"); + } else { + logger.info("mergeScanSpecs : 3"); + } + break; + case "booleanOr": + + } + return new SolrScanSpec(groupScan.getSolrScanSpec().getSolrCoreName()); + } + + @Override + public SolrScanSpec visitUnknown(LogicalExpression e, Void valueArg) + throws RuntimeException { + logger.info("SolrQueryBuilder :: visitUnknown"); + allExpressionsConverted = false; + return null; + } + + public boolean isAllExpressionsConverted() { + return allExpressionsConverted; + } + + @Override + public SolrScanSpec visitFunctionHolderExpression( + FunctionHolderExpression fhe, Void valueArg) { + logger.info("SolrQueryBuilder :: visitFunctionHolderExpression"); + + return null; + + } + + @Override + public SolrScanSpec visitBooleanOperator(BooleanOperator op, Void valueArg) { + logger.info("SolrQueryBuilder :: visitBooleanOperator"); + List args = op.args; + String functionName = op.getName(); + SolrScanSpec nodeScanSpec = null; + logger.info("functionName :: " + functionName); + for (int i = 0; i < args.size(); ++i) { + logger.info(" args " + args.get(i)); + switch (functionName) { + case "booleanAnd": + case "booleanOr": + if (nodeScanSpec == null) { + nodeScanSpec = args.get(i).accept(this, valueArg); + } else { + SolrScanSpec scanSpec = args.get(i).accept(this, valueArg); + if (scanSpec != null) { + nodeScanSpec = mergeScanSpecs(functionName, nodeScanSpec, scanSpec); + } else { + allExpressionsConverted = false; + } + } + logger.info(" expression converted!"); + break; + } + } + logger.info("nodeScanSpec :: " + nodeScanSpec); + return nodeScanSpec; + } + + @Override + public SolrScanSpec visitFunctionCall(FunctionCall call, Void valueArg) + throws RuntimeException { + logger.info("SolrQueryBuilder :: visitFunctionCall"); + SolrScanSpec nodeScanSpec = null; + String functionName = call.getName(); + ImmutableList args = call.args; + LogicalExpression nameVal = call.args.get(0); + LogicalExpression valueVal = null; + StringBuilder strBuilder = new StringBuilder(); + if (call.args.size() >= 2) { + valueVal = call.args.get(1); + } + if (SolrCompareFunctionProcessor.isCompareFunction(functionName)) { + SolrCompareFunctionProcessor evaluator = SolrCompareFunctionProcessor + .process(call); + if (evaluator.isSuccess()) { + try { + nodeScanSpec = createSolrScanSpec(evaluator.getFunctionName(), + evaluator.getPath(), evaluator.getValue()); + + } catch (Exception e) { + logger.debug("Failed to create filters ", e); + } + } + } else { + switch (functionName) { + case "booleanAnd": + case "booleanOr": + SolrScanSpec leftScanSpec = args.get(0).accept(this, null); + SolrScanSpec rightScanSpec = args.get(1).accept(this, null); + if (leftScanSpec != null && rightScanSpec != null) { + nodeScanSpec = mergeScanSpecs(functionName, leftScanSpec, + rightScanSpec); + } else { + allExpressionsConverted = false; + if ("booleanAnd".equals(functionName)) { + nodeScanSpec = leftScanSpec == null ? rightScanSpec : leftScanSpec; + } + } + break; + } + } + logger.info("functionName:" + functionName); + logger.info("Name Val:" + nameVal.toString()); + logger.info("Value Val:" + valueVal.toString()); + + if (nodeScanSpec == null) { + allExpressionsConverted = false; + } + + return nodeScanSpec; + + } + + public SolrScanSpec createSolrScanSpec(String functionName, SchemaPath field, + Object fieldValue) { + // extract the field name + String fieldName = field.getAsUnescapedPath(); + List solrFilters=Lists.newArrayList(); + SolrFilters solrFilter = new SolrFilters(); + switch (functionName) { + case "equal": + break; + case "not_equal": + break; + case "greater_than_or_equal_to": + break; + case "greater_than": + break; + case "less_than_or_equal_to": + break; + case "less_than": + break; + case "isnull": + case "isNull": + case "is null": + break; + case "isnotnull": + case "isNotNull": + case "is not null": + break; + } + logger.info("createSolrScanSpec :: fieldName " + fieldName + + " :: functionName " + functionName); + return new SolrScanSpec(this.groupScan.getSolrScanSpec().getSolrCoreName()); + } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java index 9d6ebb9c92e..0c1183ed5e9 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java @@ -19,19 +19,53 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule INSTANCE = new SolrQueryFilterRule(); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { super(operand, description); - // TODO Auto-generated constructor stub + logger.info("SolrQueryFilterRule :: contructor"); + } + + public SolrQueryFilterRule() { + super( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule"); } @Override public void onMatch(RelOptRuleCall call) { - // TODO Auto-generated method stub - + logger.info("SolrQueryFilterRule :: onMatch"); + final ScanPrel scan = (ScanPrel) call.rel(1); + final FilterPrel filter = (FilterPrel) call.rel(0); + final RexNode condition = filter.getCondition(); + SolrGroupScan solrGroupScan=(SolrGroupScan) scan.getGroupScan(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + logger.info("conditionExp " + conditionExp); + SolrQueryBuilder sQueryBuilder=new SolrQueryBuilder(solrGroupScan, conditionExp); + sQueryBuilder.parseTree(); + } + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; } - } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java index 2e6a5b9a081..8e31a909f6a 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java @@ -22,10 +22,12 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.io.Charsets; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.memory.OutOfMemoryException; import org.apache.drill.exec.ops.FragmentContext; @@ -47,6 +49,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; public class SolrRecordReader extends AbstractRecordReader { static final Logger logger = LoggerFactory.getLogger(SolrRecordReader.class); @@ -61,6 +64,7 @@ public class SolrRecordReader extends AbstractRecordReader { // protected List resultTuple; protected SolrDocumentList solrDocList; protected Iterator resultIter; + protected List fields; private MajorType.Builder t; public SolrRecordReader(FragmentContext context, SolrSubScan config) { @@ -71,16 +75,36 @@ public SolrRecordReader(FragmentContext context, SolrSubScan config) { solrClientApiExec = config.getSolrPlugin().getSolrClientApiExec(); solrClient = config.getSolrPlugin().getSolrClient(); String solrCoreName = scanList.get(0).getSolrCoreName(); - + List colums=config.getColumns(); + setColumns(colums); Map solrParams = new HashMap(); solrParams.put("q", "*:*"); - solrParams.put("rows", "100"); + solrParams.put("rows", String.valueOf(Integer.MAX_VALUE)); solrParams.put("qt", "/select"); - solrDocList = solrClientApiExec.getSolrDocs(solrServerUrl, solrCoreName); + + solrDocList = solrClientApiExec.getSolrDocs(solrServerUrl, solrCoreName,this.fields); + //solrClientApiExec.getSchemaForCore(solrCoreName); resultIter=solrDocList.iterator(); logger.info("SolrRecordReader:: solrDocList:: " + solrDocList.size()); + + } + @Override + protected Collection transformColumns( + Collection projectedColumns) { + Set transformed = Sets.newLinkedHashSet(); + if (!isStarQuery()) { + logger.debug(" This is not a start query, restring response to "); + fields = Lists.newArrayListWithExpectedSize(projectedColumns.size()); + for (SchemaPath column : projectedColumns) { + String fieldName = column.getRootSegment().getPath(); + transformed.add(SchemaPath.getSimplePath(fieldName)); + this.fields.add(fieldName); + } + } else { + transformed.add(AbstractRecordReader.STAR_COLUMN); + } + return transformed; } - @Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java index f2ebc0a1b7a..166a1abc624 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java @@ -17,11 +17,30 @@ */ package org.apache.drill.exec.store.solr; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.common.params.SolrParams; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + public class SolrScanSpec { private String solrCoreName; + private List filter; + + @JsonCreator + public SolrScanSpec(@JsonProperty("solrCoreName") String solrCoreName) { + this.solrCoreName = solrCoreName; + } - public SolrScanSpec(String solrCoreName) { - super(); + @JsonCreator + public SolrScanSpec(@JsonProperty("solrCoreName") String solrCoreName, + @JsonProperty("filter") String filter) { this.solrCoreName = solrCoreName; } @@ -29,5 +48,24 @@ public String getSolrCoreName() { return solrCoreName; } - + public List getFilter() { + return filter; + } + + class SolrFilterParam { + private String filterName; + private String operator; + private String filterValue; + + public SolrFilterParam(String filterName, String operator, + String filterValue) { + this.filterName = filterName; + this.operator = operator; + this.filterValue = filterValue; + } + } + @Override + public String toString(){ + return "SolrScanSpec [solrCoreName=" + solrCoreName + ", filter=" + filter + "]"; + } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java index 3432780d83f..ba1b779e91b 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.common.JSONOptions; @@ -28,12 +29,14 @@ import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.apache.drill.exec.store.solr.schema.SolrSchemaFactory; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; public class SolrStoragePlugin extends AbstractStoragePlugin { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory @@ -98,4 +101,9 @@ public AbstractGroupScan getPhysicalScan(String userName, }); return new SolrGroupScan(userName, this, solrScanSpec, columns); } + @Override + public Set getOptimizerRules(){ + logger.info("SolrStoragePlugin :: getOptimizerRules"); + return ImmutableSet.of(SolrQueryFilterRule.INSTANCE); + } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java index 0d93e045eb6..b4f80e16db9 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java @@ -36,7 +36,7 @@ public class CVSchema { protected String uniqueKey = null; protected String defaultSearchField = null; - + protected Error errorObj = null; public List getSchemaFields(boolean init) { diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java index 7da9fe0030e..a7b3cf2a745 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java @@ -63,8 +63,7 @@ public class CVSchemaField implements Serializable { protected List copyFields = null; - public CVSchemaField() { - } + protected Boolean idxCopyField = null; /** * copy constructors @@ -123,6 +122,12 @@ public CVSchemaField(CVSchemaField obj) { this.copyFields.add(new String(listObj)); } } + if (obj.idxCopyField != null) { + this.idxCopyField = new Boolean(obj.idxCopyField); + } + } + + public CVSchemaField() { } public String getFieldName(boolean init) { @@ -133,6 +138,19 @@ public String getFieldName(boolean init) { } } + public Boolean getIdxCopyField() { + return idxCopyField; + } + + public Boolean getIdxCopyField(boolean init) { + if (init) { + return getIdxCopyField(); + } else { + + return this.idxCopyField; + } + } + public String getFieldName() { if (fieldName == null) { this.fieldName = ""; @@ -463,6 +481,7 @@ public static List getAllFields() { list.add(Fields.AUTOCOMPLETE); list.add(Fields.SPELLCHECK); list.add(Fields.COPY_FIELDS); + list.add(Fields.IDX_COPY_FIELD); return list; } @@ -483,6 +502,7 @@ public static interface Fields { String AUTOCOMPLETE = "autocomplete"; String SPELLCHECK = "spellcheck"; String COPY_FIELDS = "copyFields"; + String IDX_COPY_FIELD = "idxCopyField"; } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java index 2cdbd722b8d..82b8ce0bcbc 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java @@ -85,7 +85,7 @@ public Table getTable(String coreName) { DrillTable getDrillTable(String dbName, String collectionName) { logger.info("SolrSchema :: getDrillTable"); SolrScanSpec solrScanSpec = new SolrScanSpec(collectionName); - return new DynamicDrillTable(solrStoragePlugin, SolrStoragePluginConfig.NAME, null, solrScanSpec); + return new DynamicDrillTable(solrStoragePlugin, SolrStoragePluginConfig.NAME, solrScanSpec); } @Override diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java index 3c00d0052ce..891fae3d0ee 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java @@ -43,15 +43,11 @@ public class SolrSchemaFactory implements SchemaFactory { static final Logger logger = LoggerFactory.getLogger(SolrSchemaFactory.class); private final SolrStoragePlugin solrStorage; - private final SolrClient solrClient; private final String storageName; - private List solrCoreLst; public SolrSchemaFactory(SolrStoragePlugin solrStorage, String storageName) { this.solrStorage = solrStorage; - solrClient = solrStorage.getSolrClient(); this.storageName = storageName; - logger.info("available solr cores are..." + solrCoreLst); } @@ -67,9 +63,9 @@ public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { logger.info("registering schema...."); List schemaPath = Lists.newArrayList(); - schemaPath.add(SolrStoragePluginConfig.NAME); - SolrSchema schema = new SolrSchema(schemaPath,"root", solrStorage); - SchemaPlus hPlus = parent.add(SolrStoragePluginConfig.NAME, schema); + //schemaPath.add(SolrStoragePluginConfig.NAME); + SolrSchema schema = new SolrSchema(schemaPath,"root", solrStorage); + SchemaPlus hPlus = parent.add(this.storageName, schema); } From 320983462bd6074d3295584161b3e1493715e4b2 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Thu, 6 Aug 2015 21:48:44 +0530 Subject: [PATCH 04/15] Apache solr storage plugin --- .../1d/d09b7982563c0015185183732c172ff5 | 221 ++++++++++++++++++ .../exec/store/solr/SolrQueryBuilder.java | 2 +- 2 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/1d/d09b7982563c0015185183732c172ff5 diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/1d/d09b7982563c0015185183732c172ff5 b/.metadata/.plugins/org.eclipse.core.resources/.history/1d/d09b7982563c0015185183732c172ff5 new file mode 100644 index 00000000000..f8f8cd34fa6 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/1d/d09b7982563c0015185183732c172ff5 @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.util.List; +import java.util.Queue; + +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.FunctionHolderExpression; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.drill.exec.store.solr.SolrScanSpec.SolrFilters; +import org.apache.solr.client.solrj.SolrQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.beust.jcommander.internal.Lists; +import com.google.common.collect.ImmutableList; + +public class SolrQueryBuilder extends + AbstractExprVisitor { + static final Logger logger = LoggerFactory.getLogger(SolrQueryBuilder.class); + final SolrGroupScan groupScan; + final LogicalExpression le; + private boolean allExpressionsConverted = true; + + public SolrQueryBuilder(SolrGroupScan solrGroupScan, + LogicalExpression conditionExp) { + this.groupScan = solrGroupScan; + this.le = conditionExp; + logger.info("SolrQueryBuilder :: constructor"); + } + + public SolrScanSpec parseTree() { + logger.info("SolrQueryBuilder :: parseTree"); + SolrScanSpec parsedSpec = le.accept(this, null); + if (parsedSpec != null) { + logger.info("parsedSpec : " + parsedSpec); + parsedSpec = mergeScanSpecs("booleanAnd", + this.groupScan.getSolrScanSpec(), parsedSpec); + } + return parsedSpec; + } + + public SolrScanSpec mergeScanSpecs(String functionName, + SolrScanSpec leftScanSpec, SolrScanSpec rightScanSpec) { + List solrFilterList; + logger.info("mergeScanSpecs : init"); + switch (functionName) { + case "booleanAnd": + if (leftScanSpec.getFilter() != null && rightScanSpec.getFilter() != null) { + logger.info("mergeScanSpecs : 1"); + solrFilterList = leftScanSpec.getFilter(); + } else if (leftScanSpec.getFilter() != null) { + logger.info("mergeScanSpecs : 2"); + } else { + logger.info("mergeScanSpecs : 3"); + } + break; + case "booleanOr": + + } + return new SolrScanSpec(groupScan.getSolrScanSpec().getSolrCoreName()); + } + + @Override + public SolrScanSpec visitUnknown(LogicalExpression e, Void valueArg) + throws RuntimeException { + logger.info("SolrQueryBuilder :: visitUnknown"); + allExpressionsConverted = false; + return null; + } + + public boolean isAllExpressionsConverted() { + return allExpressionsConverted; + } + + @Override + public SolrScanSpec visitFunctionHolderExpression( + FunctionHolderExpression fhe, Void valueArg) { + logger.info("SolrQueryBuilder :: visitFunctionHolderExpression"); + + return null; + + } + + @Override + public SolrScanSpec visitBooleanOperator(BooleanOperator op, Void valueArg) { + logger.info("SolrQueryBuilder :: visitBooleanOperator"); + List args = op.args; + String functionName = op.getName(); + SolrScanSpec nodeScanSpec = null; + logger.info("functionName :: " + functionName); + for (int i = 0; i < args.size(); ++i) { + logger.info(" args " + args.get(i)); + switch (functionName) { + case "booleanAnd": + case "booleanOr": + if (nodeScanSpec == null) { + nodeScanSpec = args.get(i).accept(this, valueArg); + } else { + SolrScanSpec scanSpec = args.get(i).accept(this, valueArg); + if (scanSpec != null) { + nodeScanSpec = mergeScanSpecs(functionName, nodeScanSpec, scanSpec); + } else { + allExpressionsConverted = false; + } + } + logger.info(" expression converted!"); + break; + } + } + logger.info("nodeScanSpec :: " + nodeScanSpec); + return nodeScanSpec; + } + + @Override + public SolrScanSpec visitFunctionCall(FunctionCall call, Void valueArg) + throws RuntimeException { + logger.info("SolrQueryBuilder :: visitFunctionCall"); + SolrScanSpec nodeScanSpec = null; + String functionName = call.getName(); + ImmutableList args = call.args; + LogicalExpression nameVal = call.args.get(0); + LogicalExpression valueVal = null; + StringBuilder strBuilder = new StringBuilder(); + if (call.args.size() >= 2) { + valueVal = call.args.get(1); + } + if (SolrCompareFunctionProcessor.isCompareFunction(functionName)) { + SolrCompareFunctionProcessor evaluator = SolrCompareFunctionProcessor + .process(call); + if (evaluator.isSuccess()) { + try { + nodeScanSpec = createSolrScanSpec(evaluator.getFunctionName(), + evaluator.getPath(), evaluator.getValue()); + + } catch (Exception e) { + logger.debug("Failed to create filters ", e); + } + } + } else { + switch (functionName) { + case "booleanAnd": + case "booleanOr": + SolrScanSpec leftScanSpec = args.get(0).accept(this, null); + SolrScanSpec rightScanSpec = args.get(1).accept(this, null); + if (leftScanSpec != null && rightScanSpec != null) { + nodeScanSpec = mergeScanSpecs(functionName, leftScanSpec, + rightScanSpec); + } else { + allExpressionsConverted = false; + if ("booleanAnd".equals(functionName)) { + nodeScanSpec = leftScanSpec == null ? rightScanSpec : leftScanSpec; + } + } + break; + } + } + logger.info("functionName:" + functionName); + logger.info("Name Val:" + nameVal.toString()); + logger.info("Value Val:" + valueVal.toString()); + + if (nodeScanSpec == null) { + allExpressionsConverted = false; + } + + return nodeScanSpec; + + } + + public SolrScanSpec createSolrScanSpec(String functionName, SchemaPath field, + Object fieldValue) { + // extract the field name + String fieldName = field.getAsUnescapedPath(); + List solrFilters=Lists.newArrayList(); + SolrFilters solrFilter = new SolrFilters(); + switch (functionName) { + case "equal": + break; + case "not_equal": + break; + case "greater_than_or_equal_to": + break; + case "greater_than": + break; + case "less_than_or_equal_to": + break; + case "less_than": + break; + case "isnull": + case "isNull": + case "is null": + break; + case "isnotnull": + case "isNotNull": + case "is not null": + break; + } + logger.info("createSolrScanSpec :: fieldName " + fieldName + + " :: functionName " + functionName); + return new SolrScanSpec(this.groupScan.getSolrScanSpec().getSolrCoreName()); + } +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java index f8f8cd34fa6..4789143f8c5 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java @@ -26,7 +26,6 @@ import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; -import org.apache.drill.exec.store.solr.SolrScanSpec.SolrFilters; import org.apache.solr.client.solrj.SolrQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -194,6 +193,7 @@ public SolrScanSpec createSolrScanSpec(String functionName, SchemaPath field, SolrFilters solrFilter = new SolrFilters(); switch (functionName) { case "equal": + break; case "not_equal": break; From 9b76c12fcb5a72842367372e2788cab9b2825413 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Mon, 10 Aug 2015 10:09:26 +0530 Subject: [PATCH 05/15] Apache Solr Storage plugin Initial code check-ins --- .../exec/store/solr/SolrClientAPIExec.java | 19 +++++++--- .../exec/store/solr/SolrFilterParam.java | 33 +++++++++++++++++ .../drill/exec/store/solr/SolrFilters.java | 37 ++++--------------- .../drill/exec/store/solr/SolrGroupScan.java | 4 ++ .../exec/store/solr/SolrQueryBuilder.java | 29 +++++++++++---- .../exec/store/solr/SolrQueryFilterRule.java | 35 ++++++++++++++++-- .../exec/store/solr/SolrRecordReader.java | 31 +++++++++++----- .../exec/store/solr/SolrStoragePlugin.java | 5 ++- 8 files changed, 135 insertions(+), 58 deletions(-) create mode 100644 contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilterParam.java diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java index fa575ac1adf..d23dcf03305 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java @@ -82,7 +82,7 @@ public Set getSolrCoreList() { coreList.add(cores.getCoreStatus().getName(i)); } } catch (SolrServerException | IOException e) { - logger.error("error getting core info from solr server..."); + logger.info("error getting core info from solr server..."); } return coreList; } @@ -116,22 +116,29 @@ public CVSchema getSchemaForCore(String coreName) { } public SolrDocumentList getSolrDocs(String solrServer, String solrCoreName, - List fields) { + List fields,StringBuilder filters) { + logger.debug("getSolrDocs :: "+solrCoreName); SolrClient solrClient = new HttpSolrClient(solrServer + solrCoreName); SolrDocumentList sList = null; - SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive") + SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive").setQuery("*:*") .setRows(Integer.MAX_VALUE); - solrQuery.setParam("q", "*:*"); + if (fields != null) { - logger.info("solr fields are " + fields); + logger.debug("solr fields are " + fields); String fieldStr = Joiner.on(",").join(fields); solrQuery.setParam("fl", fieldStr); } + if(filters.length()>0){ + logger.info("adding filter query :: "+filters.toString()); + solrQuery.setParam("fq", filters.toString()); + } try { + logger.debug("setting solrquery.."); QueryResponse rsp = solrClient.query(solrQuery); + logger.debug("response recieved from " + solrServer +" core "+solrCoreName); sList = rsp.getResults(); } catch (SolrServerException | IOException e) { - logger.info("error occured while fetching results from solr server " + logger.debug("error occured while fetching results from solr server " + e.getMessage()); } return sList; diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilterParam.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilterParam.java new file mode 100644 index 00000000000..2ea33f06070 --- /dev/null +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilterParam.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.util.LinkedList; + +public class SolrFilterParam extends LinkedList { + + public SolrFilterParam(String filterName, String operator, String filterValue) { + add(filterName); + add(operator); + add(filterValue); + } + + public SolrFilterParam() { + super(); + } +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java index cc4c0c9c82e..dd87ab24d7d 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java @@ -17,46 +17,25 @@ */ package org.apache.drill.exec.store.solr; -import org.apache.drill.exec.store.solr.SolrScanSpec.SolrFilterParam; public class SolrFilters { - private SolrFilterParam leftFilterParam; - private String operator; - private SolrFilterParam rightFilterParam; - - public SolrFilters(SolrFilterParam leftFilterParam, String operator, - SolrFilterParam rightFilterParam) { + private SolrFilterParam filterParam; + + public SolrFilters(SolrFilterParam filterParam) { super(); - this.leftFilterParam = leftFilterParam; - this.operator = operator; - this.rightFilterParam = rightFilterParam; + this.filterParam = filterParam; } public SolrFilters() { super(); } - public SolrFilterParam getLeftFilterParam() { - return leftFilterParam; - } - - public void setLeftFilterParam(SolrFilterParam leftFilterParam) { - this.leftFilterParam = leftFilterParam; - } - - public String getOperator() { - return operator; - } - - public void setOperator(String operator) { - this.operator = operator; + public SolrFilterParam getFilterParam() { + return filterParam; } - public SolrFilterParam getRightFilterParam() { - return rightFilterParam; + public void setFilterParam(SolrFilterParam filterParam) { + this.filterParam = filterParam; } - public void setRightFilterParam(SolrFilterParam rightFilterParam) { - this.rightFilterParam = rightFilterParam; - } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java index a7670efd714..0804cc575fd 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java @@ -135,6 +135,10 @@ public SolrStoragePlugin getSolrPlugin() { return solrPlugin; } + public List getColumns() { + return columns; + } + @Override public String toString() { return "SolrGroupScan [SolrScanSpec=" + solrScanSpec + ", columns=" diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java index 4789143f8c5..0d0f44b79cb 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java @@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory; import com.beust.jcommander.internal.Lists; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; public class SolrQueryBuilder extends @@ -60,23 +61,31 @@ public SolrScanSpec parseTree() { public SolrScanSpec mergeScanSpecs(String functionName, SolrScanSpec leftScanSpec, SolrScanSpec rightScanSpec) { - List solrFilterList; + SolrFilterParam solrFilter = new SolrFilterParam(); logger.info("mergeScanSpecs : init"); switch (functionName) { case "booleanAnd": if (leftScanSpec.getFilter() != null && rightScanSpec.getFilter() != null) { logger.info("mergeScanSpecs : 1"); - solrFilterList = leftScanSpec.getFilter(); + solrFilter.add(Joiner.on("").join(leftScanSpec.getFilter())); + solrFilter.add(" AND "); + solrFilter.add(Joiner.on("").join(rightScanSpec.getFilter())); + } else if (leftScanSpec.getFilter() != null) { logger.info("mergeScanSpecs : 2"); + solrFilter = leftScanSpec.getFilter(); } else { logger.info("mergeScanSpecs : 3"); + solrFilter = rightScanSpec.getFilter(); } break; case "booleanOr": - + solrFilter.add(Joiner.on("").join(leftScanSpec.getFilter())); + solrFilter.add(" OR "); + solrFilter.add(Joiner.on("").join(rightScanSpec.getFilter())); } - return new SolrScanSpec(groupScan.getSolrScanSpec().getSolrCoreName()); + return new SolrScanSpec(groupScan.getSolrScanSpec().getSolrCoreName(), + solrFilter); } @Override @@ -189,11 +198,11 @@ public SolrScanSpec createSolrScanSpec(String functionName, SchemaPath field, Object fieldValue) { // extract the field name String fieldName = field.getAsUnescapedPath(); - List solrFilters=Lists.newArrayList(); - SolrFilters solrFilter = new SolrFilters(); + + String operator = null; switch (functionName) { case "equal": - + operator = ":"; break; case "not_equal": break; @@ -214,6 +223,12 @@ public SolrScanSpec createSolrScanSpec(String functionName, SchemaPath field, case "is not null": break; } + if (operator != null) { + SolrFilterParam filterParam = new SolrFilterParam(fieldName,operator,fieldValue.toString()); + + return new SolrScanSpec(this.groupScan.getSolrScanSpec() + .getSolrCoreName(), filterParam); + } logger.info("createSolrScanSpec :: fieldName " + fieldName + " :: functionName " + functionName); return new SolrScanSpec(this.groupScan.getSolrScanSpec().getSolrCoreName()); diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java index 0c1183ed5e9..1ab550398cc 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java @@ -17,9 +17,13 @@ */ package org.apache.drill.exec.store.solr; +import java.io.IOException; + import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.exec.planner.logical.DrillOptiq; import org.apache.drill.exec.planner.logical.DrillParseContext; @@ -31,6 +35,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableList; + public class SolrQueryFilterRule extends StoragePluginOptimizerRule { public static final StoragePluginOptimizerRule INSTANCE = new SolrQueryFilterRule(); static final Logger logger = LoggerFactory @@ -49,17 +55,38 @@ public SolrQueryFilterRule() { @Override public void onMatch(RelOptRuleCall call) { - logger.info("SolrQueryFilterRule :: onMatch"); + logger.debug("SolrQueryFilterRule :: onMatch"); final ScanPrel scan = (ScanPrel) call.rel(1); final FilterPrel filter = (FilterPrel) call.rel(0); final RexNode condition = filter.getCondition(); - SolrGroupScan solrGroupScan=(SolrGroupScan) scan.getGroupScan(); + + SolrGroupScan solrGroupScan = (SolrGroupScan) scan.getGroupScan(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + logger.info("conditionExp " + conditionExp); - SolrQueryBuilder sQueryBuilder=new SolrQueryBuilder(solrGroupScan, conditionExp); - sQueryBuilder.parseTree(); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: "+scan.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), + newGroupScan, scan.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filter.copy(filter.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } } + @Override public boolean matches(RelOptRuleCall call) { final ScanPrel scan = (ScanPrel) call.rel(1); diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java index 8e31a909f6a..5dfb55dfee7 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java @@ -75,19 +75,28 @@ public SolrRecordReader(FragmentContext context, SolrSubScan config) { solrClientApiExec = config.getSolrPlugin().getSolrClientApiExec(); solrClient = config.getSolrPlugin().getSolrClient(); String solrCoreName = scanList.get(0).getSolrCoreName(); - List colums=config.getColumns(); + List colums = config.getColumns(); + SolrFilterParam filters = config.getSolrScanSpec().getFilter(); + StringBuilder sb=new StringBuilder(); + if(filters!=null){ + for (String filter : filters) { + sb.append(filter); + } + } setColumns(colums); Map solrParams = new HashMap(); solrParams.put("q", "*:*"); solrParams.put("rows", String.valueOf(Integer.MAX_VALUE)); solrParams.put("qt", "/select"); - - solrDocList = solrClientApiExec.getSolrDocs(solrServerUrl, solrCoreName,this.fields); - //solrClientApiExec.getSchemaForCore(solrCoreName); - resultIter=solrDocList.iterator(); + + solrDocList = solrClientApiExec.getSolrDocs(solrServerUrl, solrCoreName, + this.fields,sb); + // solrClientApiExec.getSchemaForCore(solrCoreName); + resultIter = solrDocList.iterator(); logger.info("SolrRecordReader:: solrDocList:: " + solrDocList.size()); - + } + @Override protected Collection transformColumns( Collection projectedColumns) { @@ -105,11 +114,12 @@ protected Collection transformColumns( } return transformed; } + @Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { logger.info("SolrRecordReader :: setup"); - if (solrDocList != null) { + if (!solrDocList.isEmpty()) { SolrDocument solrDocument = solrDocList.get(0); Collection fieldNames = solrDocument.getFieldNames(); @@ -155,10 +165,11 @@ public int next() { int counter = 0; logger.info("SolrRecordReader :: next"); try { - while(counter <= solrDocList.getNumFound() && resultIter.hasNext()){ - SolrDocument solrDocument =resultIter.next(); + while (counter <= solrDocList.getNumFound() && resultIter.hasNext()) { + SolrDocument solrDocument = resultIter.next(); for (ValueVector vv : vectors) { - String solrField=vv.getField().getPath().toString().replaceAll("`", ""); //re-think ?? + String solrField = vv.getField().getPath().toString() + .replaceAll("`", ""); // re-think ?? Object fieldValue = solrDocument.get(solrField); String fieldValueStr = "NULL"; if (fieldValue != null) { diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java index ba1b779e91b..a2c07939b4f 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.store.solr; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -94,7 +95,7 @@ public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) @Override public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection, List columns) throws IOException { - logger.info("SolrStoragePlugin :: getPhysicalScan" + " userName : " + logger.debug("SolrStoragePlugin :: getPhysicalScan" + " userName : " + userName + " columns ::" + columns); SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(), new TypeReference() { @@ -103,7 +104,7 @@ public AbstractGroupScan getPhysicalScan(String userName, } @Override public Set getOptimizerRules(){ - logger.info("SolrStoragePlugin :: getOptimizerRules"); + logger.debug("SolrStoragePlugin :: getOptimizerRules"); return ImmutableSet.of(SolrQueryFilterRule.INSTANCE); } } From 232c13b66abe634c851f81c1ec2950747e58f3b1 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Mon, 10 Aug 2015 10:09:56 +0530 Subject: [PATCH 06/15] Apache Solr Storage Plugin Initial code check-in --- .../drill/exec/store/solr/SolrScanSpec.java | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java index 166a1abc624..88431415d61 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java @@ -31,7 +31,7 @@ public class SolrScanSpec { private String solrCoreName; - private List filter; + private SolrFilterParam filter; @JsonCreator public SolrScanSpec(@JsonProperty("solrCoreName") String solrCoreName) { @@ -43,27 +43,18 @@ public SolrScanSpec(@JsonProperty("solrCoreName") String solrCoreName, @JsonProperty("filter") String filter) { this.solrCoreName = solrCoreName; } - + public SolrScanSpec(@JsonProperty("solrCoreName") String solrCoreName, + @JsonProperty("filter") SolrFilterParam filter) { + this.solrCoreName = solrCoreName; + this.filter=filter; + } public String getSolrCoreName() { return solrCoreName; } - public List getFilter() { + public SolrFilterParam getFilter() { return filter; } - - class SolrFilterParam { - private String filterName; - private String operator; - private String filterValue; - - public SolrFilterParam(String filterName, String operator, - String filterValue) { - this.filterName = filterName; - this.operator = operator; - this.filterValue = filterValue; - } - } @Override public String toString(){ return "SolrScanSpec [solrCoreName=" + solrCoreName + ", filter=" + filter + "]"; From 30ab01ae88a0962313beb6956826f55d9d5ee3c2 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Mon, 10 Aug 2015 16:20:40 +0530 Subject: [PATCH 07/15] Apache solr Storage plugin Initial code check-in --- distribution/pom.xml | 5 +++++ distribution/src/assemble/bin.xml | 1 + 2 files changed, 6 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index c08efc71bc6..ff7dadadc15 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -162,6 +162,11 @@ drill-storage-hive-core ${project.version} + + org.apache.drill.contrib + drill-storage-solr + ${project.version} + diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index 244230fdec5..de682054f5d 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -92,6 +92,7 @@ org.apache.drill.contrib.data:tpch-sample-data:jar org.apache.drill.contrib:drill-mongo-storage org.apache.drill.contrib:drill-storage-hbase + org.apache.drill.contrib:drill-storage-solr org.apache.drill.contrib.storage-hive:drill-storage-hive-core:jar:tests From ee491c973acc8bee37d79ec9b732dfc3563d05d1 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Mon, 10 Aug 2015 16:22:12 +0530 Subject: [PATCH 08/15] Apache Solr Storage Initial code check-in --- contrib/pom.xml | 68 +++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/contrib/pom.xml b/contrib/pom.xml index 8c00e762d48..67dca47203b 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -1,41 +1,37 @@ - + + 4.0.0 + + drill-root + org.apache.drill + 1.2.0-SNAPSHOT + - http://www.apache.org/licenses/LICENSE-2.0 + org.apache.drill.contrib + drill-contrib-parent + contrib/Parent Pom + pom - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - - 4.0.0 - - drill-root - org.apache.drill - 1.2.0-SNAPSHOT - + + - org.apache.drill.contrib - drill-contrib-parent - contrib/Parent Pom - pom - - - - - - storage-hbase - storage-hive - storage-mongo - sqlline - data - + + storage-hbase + storage-hive + storage-mongo + storage-solr + sqlline + data + From 39498c905fda8173f6433d4154f151f9ec9b67f5 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Mon, 10 Aug 2015 18:53:10 +0530 Subject: [PATCH 09/15] Apache Solr Storage plugin initial code changes. --- .../1a/005544a2573f001514edec3e43ba2377 | 106 ++++++++++++++ .../1f/205b372c5a3f001514edec3e43ba2377 | 135 ++++++++++++++++++ .../21/f0f6ed4f5a3f001514edec3e43ba2377 | 135 ++++++++++++++++++ .../28/10b0bd3a583f001514edec3e43ba2377 | 111 ++++++++++++++ .../28/505800715b3f001512449b35bb2b24ab | 134 +++++++++++++++++ .../33/000b124e5a3f001514edec3e43ba2377 | 135 ++++++++++++++++++ .../33/e06bf6ea5b3f001512449b35bb2b24ab | 108 ++++++++++++++ .../46/30b2fa2b583f001514edec3e43ba2377 | 104 ++++++++++++++ .../4f/b0f8c3aa583f001514edec3e43ba2377 | 111 ++++++++++++++ .../53/80af88ed5b3f001512449b35bb2b24ab | 108 ++++++++++++++ .../5a/3021dc645b3f001512449b35bb2b24ab | 135 ++++++++++++++++++ .../5b/70059d5b573f001514edec3e43ba2377 | 99 +++++++++++++ .../6c/f04740d6623f001512449b35bb2b24ab | 135 ++++++++++++++++++ .../72/505b61d44d3f001514edec3e43ba2377 | 42 ++++++ .../7c/000473cd623f001512449b35bb2b24ab | 135 ++++++++++++++++++ .../7d/70036cfc573f001514edec3e43ba2377 | 108 ++++++++++++++ .../80/60325d0a583f001514edec3e43ba2377 | 104 ++++++++++++++ .../89/202f9830583f001514edec3e43ba2377 | 110 ++++++++++++++ .../99/4055547a573f001514edec3e43ba2377 | 106 ++++++++++++++ .../9f/00fcb4a3573f001514edec3e43ba2377 | 108 ++++++++++++++ .../a/40adfb275a3f001514edec3e43ba2377 | 113 +++++++++++++++ .../c1/b03e017f5b3f001512449b35bb2b24ab | 134 +++++++++++++++++ .../c7/205d492e583f001514edec3e43ba2377 | 109 ++++++++++++++ .../d6/40fa9bf7563f001514edec3e43ba2377 | 60 ++++++++ .../d7/60c16faa5c3f001512449b35bb2b24ab | 135 ++++++++++++++++++ .../e/d0cb8ed8623f001512449b35bb2b24ab | 135 ++++++++++++++++++ .../e8/6033c3b1583f001514edec3e43ba2377 | 112 +++++++++++++++ .../f0/50c24713583f001514edec3e43ba2377 | 108 ++++++++++++++ .../.projects/contrib/.indexes/history.index | Bin 0 -> 41 bytes .../org.eclipse.jdt.ui/jdt-images/2.png | Bin 0 -> 220 bytes .../org.eclipse.jdt.ui/jdt-images/3.png | Bin 0 -> 281 bytes .../org.eclipse.jdt.ui/jdt-images/4.png | Bin 0 -> 478 bytes .../org.eclipse.jdt.ui/jdt-images/5.png | Bin 0 -> 283 bytes .../1e8a3cc803639e6c266275d8b92f05af/_n_1.del | Bin 0 -> 9 bytes .../1e8a3cc803639e6c266275d8b92f05af/_p.cfs | Bin 0 -> 303 bytes .../segments_p | Bin 0 -> 208 bytes .../26522e0d83a422eed93329ece7565cfc/_c_1.del | Bin 0 -> 9 bytes .../26522e0d83a422eed93329ece7565cfc/_d.cfs | Bin 0 -> 297 bytes .../segments_e | Bin 0 -> 148 bytes .../830bc118332e77292949ed1e6d2fabe0/_1o.cfs | Bin 0 -> 6003 bytes .../_1o_3.del | Bin 0 -> 11 bytes .../830bc118332e77292949ed1e6d2fabe0/_1p.cfs | Bin 0 -> 719 bytes .../830bc118332e77292949ed1e6d2fabe0/_1q.cfs | Bin 0 -> 782 bytes .../_1q_1.del | Bin 0 -> 9 bytes .../830bc118332e77292949ed1e6d2fabe0/_1r.cfs | Bin 0 -> 282 bytes .../830bc118332e77292949ed1e6d2fabe0/_1s.cfs | Bin 0 -> 782 bytes .../segments_1o | Bin 0 -> 183 bytes .../exec/store/solr/SolrQueryFilterRule.java | 100 ++++++++----- 48 files changed, 3243 insertions(+), 32 deletions(-) create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/1a/005544a2573f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/1f/205b372c5a3f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/21/f0f6ed4f5a3f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/28/10b0bd3a583f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/28/505800715b3f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/33/000b124e5a3f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/33/e06bf6ea5b3f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/46/30b2fa2b583f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/4f/b0f8c3aa583f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/53/80af88ed5b3f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/5a/3021dc645b3f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/5b/70059d5b573f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/6c/f04740d6623f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/72/505b61d44d3f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/7c/000473cd623f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/7d/70036cfc573f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/80/60325d0a583f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/89/202f9830583f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/99/4055547a573f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/9f/00fcb4a3573f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/a/40adfb275a3f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/c1/b03e017f5b3f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/c7/205d492e583f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/d6/40fa9bf7563f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/d7/60c16faa5c3f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/e/d0cb8ed8623f001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/e8/6033c3b1583f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/f0/50c24713583f001514edec3e43ba2377 create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.projects/contrib/.indexes/history.index create mode 100644 .metadata/.plugins/org.eclipse.jdt.ui/jdt-images/2.png create mode 100644 .metadata/.plugins/org.eclipse.jdt.ui/jdt-images/3.png create mode 100644 .metadata/.plugins/org.eclipse.jdt.ui/jdt-images/4.png create mode 100644 .metadata/.plugins/org.eclipse.jdt.ui/jdt-images/5.png create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/1e8a3cc803639e6c266275d8b92f05af/_n_1.del create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/1e8a3cc803639e6c266275d8b92f05af/_p.cfs create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/1e8a3cc803639e6c266275d8b92f05af/segments_p create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/_c_1.del create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/_d.cfs create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/segments_e create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1o.cfs create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1o_3.del create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1p.cfs create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1q.cfs create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1q_1.del create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1r.cfs create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1s.cfs create mode 100644 .metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/segments_1o diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/1a/005544a2573f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/1a/005544a2573f001514edec3e43ba2377 new file mode 100644 index 00000000000..96ec7c5fcdd --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/1a/005544a2573f001514edec3e43ba2377 @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule INSTANCE = new SolrQueryFilterRule(); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + public SolrQueryFilterRule() { + super( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + final RexNode condition = filter.getCondition(); + + SolrGroupScan solrGroupScan = (SolrGroupScan) scan.getGroupScan(); + doOnMatch(call, filter, projectRel, scan); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: "+scan.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), + newGroupScan, scan.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filter.copy(filter.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, DrillProjectRel projectRel, DrillScanRel scanRel) { + + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/1f/205b372c5a3f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/1f/205b372c5a3f001514edec3e43ba2377 new file mode 100644 index 00000000000..de99729fb99 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/1f/205b372c5a3f001514edec3e43ba2377 @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + } + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "SolrAggOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + } + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/21/f0f6ed4f5a3f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/21/f0f6ed4f5a3f001514edec3e43ba2377 new file mode 100644 index 00000000000..37ed3e07286 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/21/f0f6ed4f5a3f001514edec3e43ba2377 @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "SolrAggOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/28/10b0bd3a583f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/28/10b0bd3a583f001514edec3e43ba2377 new file mode 100644 index 00000000000..26e01b76deb --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/28/10b0bd3a583f001514edec3e43ba2377 @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"); + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), "SolrAggOptimizerRule"); + + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/28/505800715b3f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/28/505800715b3f001512449b35bb2b24ab new file mode 100644 index 00000000000..b223b0ff5c5 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/28/505800715b3f001512449b35bb2b24ab @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filter = (FilterPrel) call.rel(0); + final ScanPrel scan = (ScanPrel) call.rel(1); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "SolrAggOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, FilterPrel filterRel, + DrillProjectRel projectRel, ScanPrel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/33/000b124e5a3f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/33/000b124e5a3f001514edec3e43ba2377 new file mode 100644 index 00000000000..7d14199aaa1 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/33/000b124e5a3f001514edec3e43ba2377 @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "SolrAggOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/33/e06bf6ea5b3f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/33/e06bf6ea5b3f001512449b35bb2b24ab new file mode 100644 index 00000000000..e05c1877271 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/33/e06bf6ea5b3f001512449b35bb2b24ab @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.apache.drill.exec.store.solr.schema.SolrSchemaFactory; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; + +public class SolrStoragePlugin extends AbstractStoragePlugin { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrStoragePlugin.class); + + private final SolrStoragePluginConfig solrStorageConfig; + private final SolrClient solrClient; + private final DrillbitContext context; + private final SolrSchemaFactory schemaFactory; + private SolrClientAPIExec solrClientApiExec; + + public SolrStoragePlugin(SolrStoragePluginConfig solrStoragePluginConfig, + DrillbitContext context, String name) { + logger.debug("initializing solr storage plugin...."); + this.context = context; + this.solrStorageConfig = solrStoragePluginConfig; + this.solrClient = new HttpSolrClient(solrStorageConfig.getSolrServer()); + solrClientApiExec = new SolrClientAPIExec(solrClient); + this.schemaFactory = new SolrSchemaFactory(this, name); + logger.info("solr storage plugin name :: " + name); + } + + public SolrClientAPIExec getSolrClientApiExec() { + return solrClientApiExec; + } + + public SolrStoragePluginConfig getSolrStorageConfig() { + return solrStorageConfig; + } + + public SolrClient getSolrClient() { + return this.solrClient; + } + + public DrillbitContext getContext() { + return this.context; + } + + @Override + public StoragePluginConfig getConfig() { + return this.solrStorageConfig; + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) + throws IOException { + schemaFactory.registerSchemas(schemaConfig, parent); + } + + @Override + public AbstractGroupScan getPhysicalScan(String userName, + JSONOptions selection, List columns) throws IOException { + logger.debug("SolrStoragePlugin :: getPhysicalScan" + " userName : " + + userName + " columns ::" + columns+" selection "+selection); + SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(),new TypeReference() {}); + return new SolrGroupScan(userName, this, solrScanSpec, columns); + } + @Override + public Set getOptimizerRules(){ + logger.debug("SolrStoragePlugin :: getOptimizerRules"); + return ImmutableSet.of(SolrQueryFilterRule.FILTER_ON_SCAN,SolrAggOptimizerRule.INSTANCE); + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/46/30b2fa2b583f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/46/30b2fa2b583f001514edec3e43ba2377 new file mode 100644 index 00000000000..1ac9eb1cbb2 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/46/30b2fa2b583f001514edec3e43ba2377 @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/4f/b0f8c3aa583f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/4f/b0f8c3aa583f001514edec3e43ba2377 new file mode 100644 index 00000000000..7e56b1e6840 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/4f/b0f8c3aa583f001514edec3e43ba2377 @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"); + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), "SolrAggOptimizerRule:Filter_On_Project"); + + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/53/80af88ed5b3f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/53/80af88ed5b3f001512449b35bb2b24ab new file mode 100644 index 00000000000..b6cd72b0baf --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/53/80af88ed5b3f001512449b35bb2b24ab @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.apache.drill.exec.store.solr.schema.SolrSchemaFactory; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; + +public class SolrStoragePlugin extends AbstractStoragePlugin { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrStoragePlugin.class); + + private final SolrStoragePluginConfig solrStorageConfig; + private final SolrClient solrClient; + private final DrillbitContext context; + private final SolrSchemaFactory schemaFactory; + private SolrClientAPIExec solrClientApiExec; + + public SolrStoragePlugin(SolrStoragePluginConfig solrStoragePluginConfig, + DrillbitContext context, String name) { + logger.debug("initializing solr storage plugin...."); + this.context = context; + this.solrStorageConfig = solrStoragePluginConfig; + this.solrClient = new HttpSolrClient(solrStorageConfig.getSolrServer()); + solrClientApiExec = new SolrClientAPIExec(solrClient); + this.schemaFactory = new SolrSchemaFactory(this, name); + logger.info("solr storage plugin name :: " + name); + } + + public SolrClientAPIExec getSolrClientApiExec() { + return solrClientApiExec; + } + + public SolrStoragePluginConfig getSolrStorageConfig() { + return solrStorageConfig; + } + + public SolrClient getSolrClient() { + return this.solrClient; + } + + public DrillbitContext getContext() { + return this.context; + } + + @Override + public StoragePluginConfig getConfig() { + return this.solrStorageConfig; + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) + throws IOException { + schemaFactory.registerSchemas(schemaConfig, parent); + } + + @Override + public AbstractGroupScan getPhysicalScan(String userName, + JSONOptions selection, List columns) throws IOException { + logger.debug("SolrStoragePlugin :: getPhysicalScan" + " userName : " + + userName + " columns ::" + columns+" selection "+selection); + SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(),new TypeReference() {}); + return new SolrGroupScan(userName, this, solrScanSpec, columns); + } + @Override + public Set getOptimizerRules(){ + logger.debug("SolrStoragePlugin :: getOptimizerRules"); + return ImmutableSet.of(SolrQueryFilterRule.FILTER_ON_SCAN,SolrAggOptimizerRule.FILTER_ON_PROJECT); + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/5a/3021dc645b3f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/5a/3021dc645b3f001512449b35bb2b24ab new file mode 100644 index 00000000000..777c22723f4 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/5a/3021dc645b3f001512449b35bb2b24ab @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "SolrAggOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/5b/70059d5b573f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/5b/70059d5b573f001514edec3e43ba2377 new file mode 100644 index 00000000000..bee86a0ead7 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/5b/70059d5b573f001514edec3e43ba2377 @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule INSTANCE = new SolrQueryFilterRule(); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + public SolrQueryFilterRule() { + super( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final FilterPrel filter = (FilterPrel) call.rel(0); + final ScanPrel scan = (ScanPrel) call.rel(1); + final RexNode condition = filter.getCondition(); + + SolrGroupScan solrGroupScan = (SolrGroupScan) scan.getGroupScan(); + + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: "+scan.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), + newGroupScan, scan.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filter.copy(filter.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/6c/f04740d6623f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/6c/f04740d6623f001512449b35bb2b24ab new file mode 100644 index 00000000000..c12763418ea --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/6c/f04740d6623f001512449b35bb2b24ab @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ProjectPrel; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filter = (FilterPrel) call.rel(0); + final ScanPrel scan = (ScanPrel) call.rel(1); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + ProjectPrel.class, + RelOptHelper.some(ProjectPrel.class, + RelOptHelper.any(ScanPrel.class))), + "StoragePluginOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filterRel = (FilterPrel) call.rel(0); + final ProjectPrel projectRel = (ProjectPrel) call.rel(1); + final ScanPrel scanRel = (ScanPrel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, FilterPrel filterRel, + ProjectPrel projectRel, ScanPrel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/72/505b61d44d3f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/72/505b61d44d3f001514edec3e43ba2377 new file mode 100644 index 00000000000..88cc22f02f7 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/72/505b61d44d3f001514edec3e43ba2377 @@ -0,0 +1,42 @@ + + + + 4.0.0 + + drill-root + org.apache.drill + 1.2.0-SNAPSHOT + + + org.apache.drill.contrib + drill-contrib-parent + contrib/Parent Pom + pom + + + + + + + storage-solr + sqlline + data + + diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/7c/000473cd623f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/7c/000473cd623f001512449b35bb2b24ab new file mode 100644 index 00000000000..3f27f45bc47 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/7c/000473cd623f001512449b35bb2b24ab @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ProjectPrel; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filter = (FilterPrel) call.rel(0); + final ScanPrel scan = (ScanPrel) call.rel(1); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + ProjectPrel.class, + RelOptHelper.some(ProjectPrel.class, + RelOptHelper.any(ScanPrel.class))), + "SolrAggOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filterRel = (FilterPrel) call.rel(0); + final ProjectPrel projectRel = (ProjectPrel) call.rel(1); + final ScanPrel scanRel = (ScanPrel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, FilterPrel filterRel, + ProjectPrel projectRel, ScanPrel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/7d/70036cfc573f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/7d/70036cfc573f001514edec3e43ba2377 new file mode 100644 index 00000000000..b90eecadbf7 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/7d/70036cfc573f001514edec3e43ba2377 @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule INSTANCE = new SolrQueryFilterRule(); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + public SolrQueryFilterRule() { + super( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/80/60325d0a583f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/80/60325d0a583f001514edec3e43ba2377 new file mode 100644 index 00000000000..b61bb0e5f41 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/80/60325d0a583f001514edec3e43ba2377 @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule INSTANCE = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/89/202f9830583f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/89/202f9830583f001514edec3e43ba2377 new file mode 100644 index 00000000000..f9a36adf576 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/89/202f9830583f001514edec3e43ba2377 @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"); + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), "SolrAggOptimizerRule"); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/99/4055547a573f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/99/4055547a573f001514edec3e43ba2377 new file mode 100644 index 00000000000..498076d22cf --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/99/4055547a573f001514edec3e43ba2377 @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule INSTANCE = new SolrQueryFilterRule(); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + public SolrQueryFilterRule() { + super( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final FilterPrel filter = (FilterPrel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final ScanPrel scan = (ScanPrel) call.rel(2); + final RexNode condition = filter.getCondition(); + + SolrGroupScan solrGroupScan = (SolrGroupScan) scan.getGroupScan(); + doOnMatch(call, filter, projectRel, scan); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: "+scan.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), + newGroupScan, scan.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filter.copy(filter.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, DrillProjectRel projectRel, DrillScanRel scanRel) { + + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/9f/00fcb4a3573f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/9f/00fcb4a3573f001514edec3e43ba2377 new file mode 100644 index 00000000000..68940e73b36 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/9f/00fcb4a3573f001514edec3e43ba2377 @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule INSTANCE = new SolrQueryFilterRule(); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + public SolrQueryFilterRule() { + super( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, filterRel.getTraitSet(), + newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/a/40adfb275a3f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/a/40adfb275a3f001514edec3e43ba2377 new file mode 100644 index 00000000000..b8d860206c8 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/a/40adfb275a3f001514edec3e43ba2377 @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"); + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "SolrAggOptimizerRule:Filter_On_Project"); + + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/c1/b03e017f5b3f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/c1/b03e017f5b3f001512449b35bb2b24ab new file mode 100644 index 00000000000..e2b389dc812 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/c1/b03e017f5b3f001512449b35bb2b24ab @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filter = (FilterPrel) call.rel(0); + final ScanPrel scan = (ScanPrel) call.rel(1); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "SolrAggOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filterRel = (FilterPrel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final ScanPrel scanRel = (ScanPrel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, FilterPrel filterRel, + DrillProjectRel projectRel, ScanPrel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/c7/205d492e583f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/c7/205d492e583f001514edec3e43ba2377 new file mode 100644 index 00000000000..4dc06b35c1f --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/c7/205d492e583f001514edec3e43ba2377 @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"); + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule(RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), "SolrAggOptimizerRule"); + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/d6/40fa9bf7563f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/d6/40fa9bf7563f001514edec3e43ba2377 new file mode 100644 index 00000000000..994a2ee8741 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/d6/40fa9bf7563f001514edec3e43ba2377 @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.logical.partition.PruneScanRule; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; + +public class SolrAggOptimizerRule extends StoragePluginOptimizerRule { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrAggOptimizerRule.class); + public static final SolrAggOptimizerRule INSTANCE = new SolrAggOptimizerRule(); + + public SolrAggOptimizerRule(RelOptRuleOperand operand, String description) { + super(operand, description); + + } + + public SolrAggOptimizerRule() { + super(RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), "SolrAggOptimizerRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + + logger + .info(" SolrAggOptimizerRule :: " + call + " DrillProjectRel " + projectRel+" DrillScanRel "+scanRel); + + } + +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/d7/60c16faa5c3f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/d7/60c16faa5c3f001512449b35bb2b24ab new file mode 100644 index 00000000000..de1d41a119e --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/d7/60c16faa5c3f001512449b35bb2b24ab @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ProjectPrel; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filter = (FilterPrel) call.rel(0); + final ScanPrel scan = (ScanPrel) call.rel(1); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "SolrAggOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filterRel = (FilterPrel) call.rel(0); + final ProjectPrel projectRel = (ProjectPrel) call.rel(1); + final ScanPrel scanRel = (ScanPrel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, FilterPrel filterRel, + ProjectPrel projectRel, ScanPrel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/e/d0cb8ed8623f001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/e/d0cb8ed8623f001512449b35bb2b24ab new file mode 100644 index 00000000000..e9d722e3270 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/e/d0cb8ed8623f001512449b35bb2b24ab @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ProjectPrel; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filter = (FilterPrel) call.rel(0); + final ScanPrel scan = (ScanPrel) call.rel(1); + + doOnMatch(call, filter, null, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + ProjectPrel.class, + RelOptHelper.some(ProjectPrel.class, + RelOptHelper.any(ScanPrel.class))), + "StoragePluginOptimizerRule:Filter_On_Project"){ + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + final FilterPrel filterRel = (FilterPrel) call.rel(0); + final ProjectPrel projectRel = (ProjectPrel) call.rel(1); + final ScanPrel scanRel = (ScanPrel) call.rel(2); + + doOnMatch(call, filterRel, projectRel, scanRel); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + + + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + + + protected void doOnMatch(RelOptRuleCall call, FilterPrel filterRel, + ProjectPrel projectRel, ScanPrel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.debug("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/e8/6033c3b1583f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/e8/6033c3b1583f001514edec3e43ba2377 new file mode 100644 index 00000000000..83202bd0f89 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/e8/6033c3b1583f001514edec3e43ba2377 @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class SolrQueryFilterRule extends StoragePluginOptimizerRule { + + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "SolrQueryFilterRule:Filter_On_Scan"); + + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( + RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), "SolrAggOptimizerRule:Filter_On_Project"); + + static final Logger logger = LoggerFactory + .getLogger(SolrQueryFilterRule.class); + + public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { + super(operand, description); + logger.info("SolrQueryFilterRule :: contructor"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch"); + + final DrillFilterRel filter = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); + + doOnMatch(call, filter, projectRel, scan); + + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = (ScanPrel) call.rel(1); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( + PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); + + logger.info("conditionExp " + conditionExp); + + SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, + conditionExp); + SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); + if (newScanSpec == null) + return; + logger.debug(" field names :: " + scanRel.getRowType().getFieldNames()); + SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), + solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); + final ScanPrel newScanPrel = ScanPrel.create(scanRel, + filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + if (sQueryBuilder.isAllExpressionsConverted()) { + logger.info("all expressions converted.. "); + call.transformTo(newScanPrel); + } else { + call.transformTo(filterRel.copy(filterRel.getTraitSet(), + ImmutableList.of((RelNode) newScanPrel))); + } + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/f0/50c24713583f001514edec3e43ba2377 b/.metadata/.plugins/org.eclipse.core.resources/.history/f0/50c24713583f001514edec3e43ba2377 new file mode 100644 index 00000000000..a62f87796f3 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/f0/50c24713583f001514edec3e43ba2377 @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.apache.drill.exec.store.solr.schema.SolrSchemaFactory; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; + +public class SolrStoragePlugin extends AbstractStoragePlugin { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrStoragePlugin.class); + + private final SolrStoragePluginConfig solrStorageConfig; + private final SolrClient solrClient; + private final DrillbitContext context; + private final SolrSchemaFactory schemaFactory; + private SolrClientAPIExec solrClientApiExec; + + public SolrStoragePlugin(SolrStoragePluginConfig solrStoragePluginConfig, + DrillbitContext context, String name) { + logger.debug("initializing solr storage plugin...."); + this.context = context; + this.solrStorageConfig = solrStoragePluginConfig; + this.solrClient = new HttpSolrClient(solrStorageConfig.getSolrServer()); + solrClientApiExec = new SolrClientAPIExec(solrClient); + this.schemaFactory = new SolrSchemaFactory(this, name); + logger.info("solr storage plugin name :: " + name); + } + + public SolrClientAPIExec getSolrClientApiExec() { + return solrClientApiExec; + } + + public SolrStoragePluginConfig getSolrStorageConfig() { + return solrStorageConfig; + } + + public SolrClient getSolrClient() { + return this.solrClient; + } + + public DrillbitContext getContext() { + return this.context; + } + + @Override + public StoragePluginConfig getConfig() { + return this.solrStorageConfig; + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) + throws IOException { + schemaFactory.registerSchemas(schemaConfig, parent); + } + + @Override + public AbstractGroupScan getPhysicalScan(String userName, + JSONOptions selection, List columns) throws IOException { + logger.debug("SolrStoragePlugin :: getPhysicalScan" + " userName : " + + userName + " columns ::" + columns+" selection "+selection); + SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(),new TypeReference() {}); + return new SolrGroupScan(userName, this, solrScanSpec, columns); + } + @Override + public Set getOptimizerRules(){ + logger.debug("SolrStoragePlugin :: getOptimizerRules"); + return ImmutableSet.of(SolrQueryFilterRule.INSTANCE,SolrAggOptimizerRule.INSTANCE); + } +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.projects/contrib/.indexes/history.index b/.metadata/.plugins/org.eclipse.core.resources/.projects/contrib/.indexes/history.index new file mode 100644 index 0000000000000000000000000000000000000000..2336eafa9fa4bb8da3ed62eb9056111f770d493f GIT binary patch literal 41 xcmZQ#U|?Wm;LtC~&(*8Q&0$~+h)%rXYtJAm^7f6L^DgD`^AGp0@?&IR007qE46*}<%eM35~ZXNHW2vekiXhD}N(%n}3ci=H*Kv4vq54+X@-IPja~3) f1hGldJhT7+7j)wyXv7ug00000NkvXXu0mjf&~SmJ literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.jdt.ui/jdt-images/4.png b/.metadata/.plugins/org.eclipse.jdt.ui/jdt-images/4.png new file mode 100644 index 0000000000000000000000000000000000000000..38192cfe62a4950df3503f139a59fdcc7b7ddba6 GIT binary patch literal 478 zcmV<40U`d0P)jm==#8*aqA?ori>-||I61^z%hsq z0@eE_feirB7>0l}8#TE9Z`@q_f5U~<{}0{S|Nq#%!(e*B@!2T0W4Hq3Jdn+m>&pM{ zyS4j2E`Za3sM!Vo6BnfapMQK7T=DK3+y778*Ykh#r41;~hhY?Nxb?^V51JPDf8Mc$ zV8J`D?*31hmj=hkwxYWRY=Ci-?|*|jxBvb7ATBt3_vC-QTG#)&H7;Nb0tQ_HCKYsrNme_OS(Ese^ zRsZ|;LR|IX`^W#WK>46)3IB6eG@?e~jhA=-tCZRO51F0_jETnow}4@Q44!>_@xN!! zNZdKuUF&t->onFf5L*y|23Oh{ zW{7^R8_=0a|HEdb{0{?a0?{BoNF1+b)*aR>e1fKOSfO`SN#%e&}w%-3?%M1RmK0gx)&j5>o zwo(h z@&7w^HvaF}(fGd;h(T=o=?VWqas&+k0p~e6|Gj3X{-3_LfAhIP;?imobpa`b;ELb)VruZ?KTT}>D`~<`; z$;^as8QCG+Vj!0hs)EHY$oGE@NPvlffy*zoqO{mEFD12tkwMo`&!8qPGbhzb-`Pq( zv^ce>Sid;8G&>`;C@VEpUoY24zbLgJzc{lbzo=55fr-n-HP|`GGa$r2h>_jXCBoCs m&7Xns|NsAgpjLsbYye^bAm(CV0LuL52QnpaDnW7%$h`m!r9^)K literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.m2e.core/nexus/1e8a3cc803639e6c266275d8b92f05af/segments_p b/.metadata/.plugins/org.eclipse.m2e.core/nexus/1e8a3cc803639e6c266275d8b92f05af/segments_p new file mode 100644 index 0000000000000000000000000000000000000000..320fffa41cf0e8feec40b21dd8eb9a8a5daac376 GIT binary patch literal 208 zcmezW|Nl=021dVCN(W9cFfd2~F&k5SHjv^3Vh~{d4+M;00>ooviq8SkOdtUmfXgxf uWpjbDj6e(m5DAbnkO(7CHV;IDq<{cZHXkU8+vEb69xV2P%=4L2oDBe1V^uW( literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/_c_1.del b/.metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/_c_1.del new file mode 100644 index 0000000000000000000000000000000000000000..1b473bd5a71175377006feda10d8528a0f1eaa24 GIT binary patch literal 9 OcmZQzU|?hbVnzS}5M literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/_d.cfs b/.metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/_d.cfs new file mode 100644 index 0000000000000000000000000000000000000000..4a1364b9bda76194405c5b43d56341067ce4cd26 GIT binary patch literal 297 zcmd;JfPhN2_!Pa8%wi~aA&8q-lndo9198(*Dxlm=U~V2v_5_$)R0x$l2~k`E;WDy< zxCKQ}y^L%iZb@e5|NsC0Ks7J|=>{Mc0Ael%F2B@@(qhlNl++3app0LT@BbR01V|AG zurP7CxCT21c?N{|2QjjHxo8JU3k5xN-})D86vYBEYn3as??i&6{n^>P!- gQuFi@3lftvQuXqS()GatMhrL|1CsgA542VS050G`CjbBd literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/segments_e b/.metadata/.plugins/org.eclipse.m2e.core/nexus/26522e0d83a422eed93329ece7565cfc/segments_e new file mode 100644 index 0000000000000000000000000000000000000000..07330986b6ceb78b2bde528c92ed4cae7639d2a3 GIT binary patch literal 148 zcmezW|Nl=021dVCN(WanFfi}|F$+_CB9LMPVh~{b4+Ibb#A9TNPXf^(DIma=7!1RDj1p&Cp5KZg1@AUxyg=I7Q literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1o.cfs b/.metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1o.cfs new file mode 100644 index 0000000000000000000000000000000000000000..7b0792518fbb085beed17f387abef69c8f5de941 GIT binary patch literal 6003 zcmcgwUvt~W5y#;Gf*{0yR%BUmX;JaSY-dage@KGpOxrk?)5?>|itUH?sRNUU#DpXm zf|95SXPQpGK&Ky|FMaPrKSDo3U;5H-(67)Lb@vV+K@bcx(@bDVcYC{g_q*M_y*+0I z{{2Duo#8h=9}PK4D5oc1aaz1(^msDkH2oLn_d;6wI;5wZU-%EBr~N*si-nL5IDP;3 zj2=yf<=n}OH_u<6p1*zdnkc6yzd1eoJA1WE5GP6&f_DEe5h9P<=wsF z!QuWxr6%ucM6K+T+7VId@>3M0e5e+ce3|YMg;t8S%EJRSPi48L$-8?pl@(G0zM{w! z`0-DsrDBnQEPe_lwKP8JfqEJXhIfySG=LRNu}_|eWqMRB=Rqx>*8o^X+y!Xb-VuV+ z5Ew-qOPrL}%DdVUmkQhiz*>XNezi2vHO% z;}J?E=l~%ios<&!bwAYlN0VL?G8 z5)s37L|G&>LjWH8=-O6db8hUef&^+!a%GDQ%W|K*2P%!TsQjf_Sx^(XFySHJ9Edr1p+Ofc5muv_hP z3$}CvELg;qS1f0xgG0HEHcGDDHrj>`-m~ouYa4n~BTpjlA^du@m+|20qCZ-J^xHK^ zkuxJDoFU=VIP2ShSH5s3Vc0=jIUU`aH+3g6S}f*ltKI5or1sn&4*gN}$DjHa{jNKx zo)6s7)9S}=;8nfZc;W?k5d}XHh2tdb1wx!)J<`C6D_|_M<8(HGX|@bqBkwc!y>zeL z>I-nt)fASlz8HZ4JyEce>+>+!82AppZHV1>kmZ1 zNUk03eH=<`U)2IHY}`F;9)kBV-~GwQ&#SOLlmOMMpg+2(_Pnt->UpDX-wQ;+OhQ>P zL+{;puz=Mz9CMQeTDER$P>%oS%u0X0i(SQMjdpvn5APj9m|0eof4sr z?wA(jO^y60I#&>77k=VkiNepN=I+OVM>i*G+Wcq9Y}={%HXTPfE}RDCQuO$F1UU5rrmOyn{3oH zoVG?DE>`yklQp0HY#WmBA_zcv8yvV?JG!wkhMJZRaGpkR(p<*I@a)xtOSk9sMB!`` zzBKnG)wcp}>9*C{?7gOG#Df0v9c1D)18xiYMM$A`Dxjl9dE&G!UGKDwxnbIH#je#S z$T#hl1qtfh?SAP(HomC3qh9p{-XBn{@GSi-2)yCD!ENe25ffYo19f2T%$r>Wr;y}k zL?{^zeSYIlz6_w=@qT)I9J`JkQ@0$`Fd==xoopt-GIrDGIAFnhk;~(C)GszUc5#=n z0_;0ouua{v4O{P|WOa_iY#ER(DiID=)J@0T%X;7r#{-YmX`=AaCP=)rODe3}@9ysI zzr6NjwedI5FTrn45P)-6gzkfw2tHK5fnP2XGdxq?>BU?H1wSDg-h@zE;vJ6z*)q3K ztjNUSLWUQdtiQVmKdgk7j7%wcR!}^nE-r5*6$*D4c5)J4Axt8_lEm3iHoAHlrG!Zi z31ufLBQf}Kq?kn1(&ktLylRD8~9F_WtJ!jsUXY60euaD9~)3WCheIcgFLU~wuzLc8!GFbU2j$jlHp zix3)I#YdYGmLV!WJTg4I5EzkJLStPDn`WI1PxHLbO{L1X9PV8u@Ng_8LlWN7q?C=` z95TGWAIK~`Wl@$7cH-`q;h|e*`;sLvLxw+EvSmps3rcttvbp!l+(20|m*e)xmO+mj zBr72sLnRBZQ}#m1mMI1OaG@*(0SdvPp}oXWPs6<`VFp4%kkI$67y1eNp`WxI_mf6) zISxZ(eu|Y@8oYd09H%WDhZ9i>)fu1NW7R442?b|r;0{S`Yd?b~y*Ujfezgz-w|`Jaj#qMTNx3^DvU4gzrRLm{6$Q*;EofIy4Mk(I0A3 zCbI|;WJe*wR^uVU7lsgFoiIdrpN9xe#zRCSnT#N_(i8;*2`fa_!=3Ex;FjQW*^zg4 z;!n*yM%MTEj)kQkeULn^>)8=Kd+<>_LKCSz&f1vwah9haB(qogYEK`pOqpKL;e-!r z2`D@frO$eJxRL`m2pxoZFr8g}6wi$5GPV2ZMHd1tKf!zzf?g!;^m!@=5XI?l0Gz6N zFJ0|^C6FZGo-4mshSmsMlegJm zGdVwx!|YVnRe?54Hq2cr%sHHKi@vLLlh@U&ROT35-u> zQn$N%0243Hb~F2%|4esww}iO8DbGfvB{=2??~G@Yt>6#F0a*(E6nes0x?vm{t!M34uY8<+7zr|8SI0KHP+X{OXajJlg(ixAdsl)X+># zuMeGWfB4{zQl~5B)#t}yC69rwGG}mSGFOTNFNzmo!U!RHc4h@<@R+5!ZFk#UN@Gwc zpex#4YFLeyrCVmpY_^Tm(mTymXG2C6s&!1g(V?o|C_`0T#I?9b>Ho*dTLW3*9ZGb0 z@S1tkdph}RBC2FninWT`yBppP?Ec6dfMWNaZ0FuZum)HnzzF659>4~aISJR1Jpt7t X37&c5gszp%l*Bj;0O%LdoBL%yk7c6O literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1q.cfs b/.metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1q.cfs new file mode 100644 index 0000000000000000000000000000000000000000..27e62409f83ca407f4771c489305b993dfcbc5ea GIT binary patch literal 782 zcmbu7%}&BV5P)Y|popTzcrYVO01HPV5-~;#+ z#-z^vXf(u=({5+J`PrY@Rzb?Macj@aEKDWhi{OY&6~79;A1xHWsr*o~X2^J?^0k3v zHdFCH)n|&|1t)A#6vc=1!7k|~jcpn=*cq{KGHIk)%=}>7NaG2sAV3+`5D`k8xrG&( zAiR4q3rs)phYw>@l$*mi%2;?;<~0==sdC#;0Rd%^D@>hc_?56ODwD}P2qR&OmAUn` zvheDUKHK1_Cx2}rUp-H5n@!Vd3_7R1!DaV4@0@CUip}3{gZt2`31v&w6snp=ev&@K z8K+MFs>r!4NOEc+HA_h8M~k^{Nkg_6C{k<@?$vAU^TAo)>s@#IXn5@#uXE7_oFh&F zL%;yhM|?sfVDUWhMp*M%7T){AjBMj+%+fH95a~^5E2~0lV5tNL)TP{ZT5i*_TK19U xIJslFN4X^)9qKaHa-F8@9)cknjH*pdv;l1_my_)QZw#&%Bh>3PuJ2Lp_6<^8BLg;)2BFR4aXb zkV%XX;}}@}|Ns97$_9xw0I>iNb0M1nQjJp`NajC3P>Tc;my2t#bC731h<^|xyQfQp Kr=OcY10w+NS~{u# literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1s.cfs b/.metadata/.plugins/org.eclipse.m2e.core/nexus/830bc118332e77292949ed1e6d2fabe0/_1s.cfs new file mode 100644 index 0000000000000000000000000000000000000000..dcc0c543b835a2b5e896f46c9ee88ec80f4de578 GIT binary patch literal 782 zcmbtR%TB^T6uq|viXdu?3lifeP)i|V7cPisWFZh>;m%YZr3TtMErdlEy7LSC0RP92 zaP2Rc)On#2b>(fR_niBfJ9C-^VGr7MIn>AAO!61O$(`h{f)69XG2%QROUWNO_lWGr zCt1JBKXV?rp{Q5hBzFtB##!&Q+wAn)UDTS5t7hxG4LC=f0lI(=qK&vf%_06gXbv%F zlgJ(0gNR~nF(RSsdx-Q6@HeCLFqMLXTbsn?YQ-`Qt729Rvl>?o>nJw(qe4OEsz$A5 z9p`a>;Cm5qZw^C$N=jkm6WbY;g6Z6GJq1pD;^=l@4<;jhNZjdEm*p{MpY^OcF6*Xl zlzOdtr+3lr$E`Ymf`F<5pd$T%LP@K8nZ(SN)M9pMArz#NBzc#3kNSJMcIe8%?$22n z+LeY4ssp6hY=$r+4$q^-zYd;eJsEMOEW?_-5hrOribIv|uvU=gokAIo`DBJxlX+C7 wKVXxU{!crFZLzg3sm5Yxrq^=LG Date: Mon, 10 Aug 2015 18:53:34 +0530 Subject: [PATCH 10/15] Apache Solr Storage plugin initial code changes --- .../apache/drill/exec/store/solr/SolrStoragePlugin.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java index a2c07939b4f..5aa17f25aa5 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java @@ -96,15 +96,13 @@ public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection, List columns) throws IOException { logger.debug("SolrStoragePlugin :: getPhysicalScan" + " userName : " - + userName + " columns ::" + columns); - SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(), - new TypeReference() { - }); + + userName + " columns ::" + columns+" selection "+selection); + SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(),new TypeReference() {}); return new SolrGroupScan(userName, this, solrScanSpec, columns); } @Override public Set getOptimizerRules(){ logger.debug("SolrStoragePlugin :: getOptimizerRules"); - return ImmutableSet.of(SolrQueryFilterRule.INSTANCE); + return ImmutableSet.of(SolrQueryFilterRule.FILTER_ON_SCAN,SolrQueryFilterRule.FILTER_ON_PROJECT); } } From 1532706772c2c25ebf1c739ebb8e9e664d00796b Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Tue, 11 Aug 2015 14:44:10 +0530 Subject: [PATCH 11/15] Apache Solr Storage plugin Initial check-in --- contrib/storage-solr/pom.xml | 16 +++++++- .../drill/exec/store/solr/TestSolr.java | 41 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 contrib/storage-solr/src/test/java/org/apache/drill/exec/store/solr/TestSolr.java diff --git a/contrib/storage-solr/pom.xml b/contrib/storage-solr/pom.xml index b88329b324a..89078ff2bd6 100644 --- a/contrib/storage-solr/pom.xml +++ b/contrib/storage-solr/pom.xml @@ -45,6 +45,20 @@ solr-solrj 5.2.1 - + + + org.apache.drill.exec + drill-java-exec + ${project.version} + tests + test + + + org.apache.drill + drill-common + tests + ${project.version} + test + \ No newline at end of file diff --git a/contrib/storage-solr/src/test/java/org/apache/drill/exec/store/solr/TestSolr.java b/contrib/storage-solr/src/test/java/org/apache/drill/exec/store/solr/TestSolr.java new file mode 100644 index 00000000000..1992f4b80c6 --- /dev/null +++ b/contrib/storage-solr/src/test/java/org/apache/drill/exec/store/solr/TestSolr.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import org.apache.drill.BaseTestQuery; +import org.junit.Before; +import org.junit.Test; + +public class TestSolr extends BaseTestQuery { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(TestSolr.class); + private static final String solrServer = "http://sgdev1:20000/solr/"; + private static final String solrCoreName = "bootstrap_5"; + private SolrScanSpec solrScanSpec; + @Before + public void setUp(){ + solrScanSpec=new SolrScanSpec(solrCoreName); + + } + + @Test + public void solrBasicQuery() throws Exception { + testBuilder().sqlQuery("select * from solr.`bootstrap_5`").unOrdered() + .build().run(); + } +} From b5f5dacfdb97c43cc0f2f7de65f4c69cb19e2248 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Tue, 11 Aug 2015 15:00:55 +0530 Subject: [PATCH 12/15] Apache Solr Storage plugin --- .../ac/20631a3f0b40001512449b35bb2b24ab | 0 .../ba/90accc190b40001512449b35bb2b24ab | 57 ++++++++++++++++++ .../.launches/TestSolr.launch | 17 ++++++ .../org.eclipse.jdt.ui/jdt-images/13.png | Bin 0 -> 436 bytes .../org.eclipse.jdt.ui/jdt-images/14.png | Bin 0 -> 589 bytes .../org.eclipse.jdt.ui/jdt-images/15.png | Bin 0 -> 550 bytes pom.xml | 12 ++-- 7 files changed, 80 insertions(+), 6 deletions(-) rename contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java => .metadata/.plugins/org.eclipse.core.resources/.history/ac/20631a3f0b40001512449b35bb2b24ab (100%) create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/ba/90accc190b40001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.debug.core/.launches/TestSolr.launch create mode 100644 .metadata/.plugins/org.eclipse.jdt.ui/jdt-images/13.png create mode 100644 .metadata/.plugins/org.eclipse.jdt.ui/jdt-images/14.png create mode 100644 .metadata/.plugins/org.eclipse.jdt.ui/jdt-images/15.png diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java b/.metadata/.plugins/org.eclipse.core.resources/.history/ac/20631a3f0b40001512449b35bb2b24ab similarity index 100% rename from contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrFilters.java rename to .metadata/.plugins/org.eclipse.core.resources/.history/ac/20631a3f0b40001512449b35bb2b24ab diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/ba/90accc190b40001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/ba/90accc190b40001512449b35bb2b24ab new file mode 100644 index 00000000000..ac0ad7a0276 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/ba/90accc190b40001512449b35bb2b24ab @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.drill.exec.planner.logical.DrillFilterRel; +import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; + +public class SolrAggOptimizerRule extends StoragePluginOptimizerRule { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrAggOptimizerRule.class); + public static final SolrAggOptimizerRule INSTANCE = new SolrAggOptimizerRule(); + + public SolrAggOptimizerRule(RelOptRuleOperand operand, String description) { + super(operand, description); + + } + + public SolrAggOptimizerRule() { + super(RelOptHelper.some( + DrillProjectRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), "SolrAggOptimizerRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + + logger + .info(" SolrAggOptimizerRule :: " + call + " DrillProjectRel " + projectRel+" DrillScanRel "+scanRel); + + } + +} diff --git a/.metadata/.plugins/org.eclipse.debug.core/.launches/TestSolr.launch b/.metadata/.plugins/org.eclipse.debug.core/.launches/TestSolr.launch new file mode 100644 index 00000000000..0813e4dc221 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.debug.core/.launches/TestSolr.launch @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/.metadata/.plugins/org.eclipse.jdt.ui/jdt-images/13.png b/.metadata/.plugins/org.eclipse.jdt.ui/jdt-images/13.png new file mode 100644 index 0000000000000000000000000000000000000000..4768f9574b046bb13ba7d5e57a31af547462687a GIT binary patch literal 436 zcmV;l0ZaagP)W*ZgU5A$la1&J8)R;x#ymp*_pLi3~1KhV1J_yVUGrLd=WxZMOve7^qP=KH0h8i zoI^Z%!-zHN*zY?W7(a&X=rzcP_n^vcMyyfC{*NKgoe%JiD6qLyg?#b>jOeu9slFvv zsSo`<0k}s|ID^%N3T0}i!G>#<7p5au!evOt?|K8~y&B9o3lQW~Mjhqi6Vqm01xoby z1o#{ZxQEif*)rg?rYLo&xfau~KWUZ7Hjd0I@DP!+6T_fH$pYj@TI&IBTiUz=+*4pI z4?c%Xg0jF5XIOxyv}uhx(m$#|cF%AUuia0W7?2>ytFRWiFr7l}vP<<4xm@ToodQDI zZIC7AAToUlvE?fh1$TkGJrC&n^)g|Tx94EdUn11Lu|)O0!#eQJC)R5qxK8MY#pgln e_HX~f|HKzR&77BCUBCnY0000 z(MwZd6!%Y%Kp_wxO6n;KgE-D(2bliqCrKh}!fzrQj=OU|P$(u2XdjH=)N{@~=@2u`#`@P1=O zO_2PVW-F?HY$ltKkRmhKZ$EaoaNIHlWB+T^5gcoIlDo=Y#fx$)2X3$trobH{+ljte z5A&b4)=(-;Lf0LIPK=_oc~bj>q9mtn9NEY%G>i&p7!fd+dWR~9f)XK!lI9Qr@j?hk zn*!P-sqL?5mz`(~c0yi_VU4s`bkmOu;xiSBO%sRz8`O^QF%x4EgY1 bD)QMLBZtT9YhQb|00000NkvXXu0mjfLm?A4 literal 0 HcmV?d00001 diff --git a/.metadata/.plugins/org.eclipse.jdt.ui/jdt-images/15.png b/.metadata/.plugins/org.eclipse.jdt.ui/jdt-images/15.png new file mode 100644 index 0000000000000000000000000000000000000000..20c0fa855d627a4423a3f708f20fc2ea5944e2ca GIT binary patch literal 550 zcmV+>0@?kEP)jm==#8*aqA?ori>-||I61^z%hsq z0@eE_feirB7>0l}8#TE9Z`@q_f5U~<{}0{S|Nq#%!(e*B@!2T0W4Hq3Jdn+m>&pM{ zyS4j2E`Za3sM!Vo6BnfapMQK7T=DK3+y778*Ykh#r41;~hhY?Nxb?^V51JPDf8Mc$ zV8J`D?*31hmj=hkwxYWRY=Ci-?|*|jxBvb7ATBt3_vC-QTG#)&H7;Nb0tQ_HCKYsrNme_OS(Ese^ zRsZ|;LR|Ib=hy%0Im`d6WiS7K;=u*fD7^9V?thgsyZ<566M-?&`2Q9#43NRIk1zf! zr!V;*)VAw?Fc2$cEX7Pnb(<&r*Q;^+@75RoKVd=U|C&uL|66zTfa$9*uK$-yS^VF! zV#9yS@(usxQWs-Si6BGtYu$h`lJq}pR?7ddnJNGG-a7i@P&o{*InBE|2oinqVeK70N70v9j4|EkpKVy07*qoM6N<$g5R+rWB>pF literal 0 HcmV?d00001 diff --git a/pom.xml b/pom.xml index b661b93e096..955fed774b6 100644 --- a/pom.xml +++ b/pom.xml @@ -204,12 +204,12 @@ **/*.java UTF-8 - true - true - true - true + false + false + false + false **/*.properties,**/*.conf,**/*.json,**/*.xml - true + false src/main/resources/checkstyle-config.xml src/main/resources/checkstyle-suppressions.xml @@ -280,7 +280,7 @@ [3.0.4,4) - [1.7,1.8) + [1.7,1.8.0-51) From 04f288d15576207bc9a26971ad9f929b33bb3a61 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Thu, 27 Aug 2015 16:45:35 +0530 Subject: [PATCH 13/15] Solr Storage plugin initial code changes --- .../exec/store/solr/SolrClientAPIExec.java | 30 +- .../drill/exec/store/solr/SolrGroupScan.java | 1 + .../exec/store/solr/SolrQueryBuilder.java | 6 +- .../exec/store/solr/SolrRecordReader.java | 116 +++- .../drill/exec/store/solr/SolrScanSpec.java | 19 +- .../exec/store/solr/SolrStoragePlugin.java | 14 +- .../store/solr/SolrStoragePluginConfig.java | 12 +- .../drill/exec/store/solr/SolrSubScan.java | 8 +- .../exec/store/solr/schema/CVSchema.java | 159 +----- .../exec/store/solr/schema/CVSchemaField.java | 500 +++--------------- .../exec/store/solr/schema/SolrSchema.java | 25 +- .../store/solr/schema/SolrSchemaFactory.java | 18 +- .../drill/exec/store/solr/TestSolr.java | 17 +- 13 files changed, 243 insertions(+), 682 deletions(-) diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java index d23dcf03305..ced9f531b7b 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.text.MessageFormat; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -43,9 +42,10 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; -import org.codehaus.jackson.map.ObjectMapper; import com.beust.jcommander.internal.Lists; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; public class SolrClientAPIExec { @@ -87,9 +87,9 @@ public Set getSolrCoreList() { return coreList; } - public CVSchema getSchemaForCore(String coreName) { - String schemaUrl = "http://dm2perf:20000/servlets/collection?type=GETSCHEMAFIELDS&corename={0}"; - schemaUrl = MessageFormat.format(schemaUrl, coreName); + public CVSchema getSchemaForCore(String coreName, String solrServerUrl) { + String schemaUrl = "{0}{1}/schema/fields"; + schemaUrl = MessageFormat.format(schemaUrl, solrServerUrl, coreName); HttpClient client = new DefaultHttpClient(); HttpGet request = new HttpGet(schemaUrl); CVSchema oCVSchema = null; @@ -105,8 +105,9 @@ public CVSchema getSchemaForCore(String coreName) { while ((line = rd.readLine()) != null) { result.append(line); } - logger.info("schema info... " + result); ObjectMapper mapper = new ObjectMapper(); + mapper + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); oCVSchema = mapper.readValue(result.toString(), CVSchema.class); } catch (Exception e) { logger.info("exception occured while fetching schema details..." @@ -116,26 +117,27 @@ public CVSchema getSchemaForCore(String coreName) { } public SolrDocumentList getSolrDocs(String solrServer, String solrCoreName, - List fields,StringBuilder filters) { - logger.debug("getSolrDocs :: "+solrCoreName); + List fields, StringBuilder filters) { + logger.debug("getSolrDocs :: " + solrCoreName); SolrClient solrClient = new HttpSolrClient(solrServer + solrCoreName); SolrDocumentList sList = null; - SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive").setQuery("*:*") - .setRows(Integer.MAX_VALUE); - + SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive") + .setQuery("*:*").setRows(Integer.MAX_VALUE); + if (fields != null) { logger.debug("solr fields are " + fields); String fieldStr = Joiner.on(",").join(fields); solrQuery.setParam("fl", fieldStr); } - if(filters.length()>0){ - logger.info("adding filter query :: "+filters.toString()); + if (filters.length() > 0) { + logger.info("adding filter query :: " + filters.toString()); solrQuery.setParam("fq", filters.toString()); } try { logger.debug("setting solrquery.."); QueryResponse rsp = solrClient.query(solrQuery); - logger.debug("response recieved from " + solrServer +" core "+solrCoreName); + logger.debug("response recieved from " + solrServer + " core " + + solrCoreName); sList = rsp.getResults(); } catch (SolrServerException | IOException e) { logger.debug("error occured while fetching results from solr server " diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java index 0804cc575fd..d6e76e7d954 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrGroupScan.java @@ -110,6 +110,7 @@ public ScanStats getScanStats() { // TODO Auto-generated method stub return ScanStats.TRIVIAL_TABLE; } + @JsonIgnore @Override public PhysicalOperator getNewWithChildren(List children) diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java index 0d0f44b79cb..7d8abc24d55 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryBuilder.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.store.solr; import java.util.List; -import java.util.Queue; import org.apache.drill.common.expression.BooleanOperator; import org.apache.drill.common.expression.FunctionCall; @@ -26,11 +25,9 @@ import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; -import org.apache.solr.client.solrj.SolrQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.beust.jcommander.internal.Lists; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; @@ -224,7 +221,8 @@ public SolrScanSpec createSolrScanSpec(String functionName, SchemaPath field, break; } if (operator != null) { - SolrFilterParam filterParam = new SolrFilterParam(fieldName,operator,fieldValue.toString()); + SolrFilterParam filterParam = new SolrFilterParam(fieldName, operator, + fieldValue.toString()); return new SolrScanSpec(this.groupScan.getSolrScanSpec() .getSolrCoreName(), filterParam); diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java index 5dfb55dfee7..e69ec15d241 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrRecordReader.java @@ -28,25 +28,30 @@ import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.memory.OutOfMemoryException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.MaterializedField.Key; import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.vector.AllocationHelper; +import org.apache.drill.exec.store.solr.schema.CVSchema; +import org.apache.drill.exec.store.solr.schema.CVSchemaField; +import org.apache.drill.exec.vector.DateVector; +import org.apache.drill.exec.vector.Float8Vector; +import org.apache.drill.exec.vector.NullableBigIntVector; +import org.apache.drill.exec.vector.NullableIntVector; import org.apache.drill.exec.vector.NullableVarCharVector; +import org.apache.drill.exec.vector.TimeStampVector; import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VarCharVector; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.common.types.TypeProtos.MajorType; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -66,6 +71,7 @@ public class SolrRecordReader extends AbstractRecordReader { protected Iterator resultIter; protected List fields; private MajorType.Builder t; + private Map schemaFieldMap; public SolrRecordReader(FragmentContext context, SolrSubScan config) { fc = context; @@ -77,11 +83,11 @@ public SolrRecordReader(FragmentContext context, SolrSubScan config) { String solrCoreName = scanList.get(0).getSolrCoreName(); List colums = config.getColumns(); SolrFilterParam filters = config.getSolrScanSpec().getFilter(); - StringBuilder sb=new StringBuilder(); - if(filters!=null){ + StringBuilder sb = new StringBuilder(); + if (filters != null) { for (String filter : filters) { sb.append(filter); - } + } } setColumns(colums); Map solrParams = new HashMap(); @@ -90,8 +96,17 @@ public SolrRecordReader(FragmentContext context, SolrSubScan config) { solrParams.put("qt", "/select"); solrDocList = solrClientApiExec.getSolrDocs(solrServerUrl, solrCoreName, - this.fields,sb); - // solrClientApiExec.getSchemaForCore(solrCoreName); + this.fields, sb); // solr docs + CVSchema oCVSchema = solrClientApiExec.getSchemaForCore(solrCoreName, + solrServerUrl); // solr core schema + if (oCVSchema.getFields() != null) { + schemaFieldMap = new HashMap(oCVSchema.getFields() + .size()); + for (CVSchemaField cvSchemaField : oCVSchema.getFields()) { + schemaFieldMap.put(cvSchemaField.getName(), cvSchemaField); + } + } + resultIter = solrDocList.iterator(); logger.info("SolrRecordReader:: solrDocList:: " + solrDocList.size()); @@ -121,21 +136,60 @@ public void setup(OperatorContext context, OutputMutator output) logger.info("SolrRecordReader :: setup"); if (!solrDocList.isEmpty()) { SolrDocument solrDocument = solrDocList.get(0); - Collection fieldNames = solrDocument.getFieldNames(); - for (String field : fieldNames) { - MaterializedField m_field = null; - logger.debug("solr column is " + field); - t = MajorType.newBuilder().setMinorType(TypeProtos.MinorType.VARCHAR); - m_field = MaterializedField.create(field, t.build()); - try { - vectors.add(output.addField(m_field, NullableVarCharVector.class)); - } catch (SchemaChangeException e) { - logger.debug("error while creating result vector " - + e.getLocalizedMessage()); + try { + for (String field : fieldNames) { + MaterializedField m_field = null; + CVSchemaField cvSchemaField = schemaFieldMap.get(field); + if (cvSchemaField != null) { + logger.debug(" column field name ::" + cvSchemaField.getName() + + " type " + cvSchemaField.getType()); + switch (cvSchemaField.getType()) { + case "string": + t = MajorType.newBuilder().setMinorType( + TypeProtos.MinorType.VARCHAR); + m_field = MaterializedField.create(field, t.build()); + vectors + .add(output.addField(m_field, NullableVarCharVector.class)); + break; + case "double": + t = MajorType.newBuilder().setMinorType( + TypeProtos.MinorType.BIGINT); + m_field = MaterializedField.create(field, t.build()); + vectors.add(output.addField(m_field, NullableBigIntVector.class)); + break; + case "int": + t = MajorType.newBuilder().setMinorType(TypeProtos.MinorType.INT); + m_field = MaterializedField.create(field, t.build()); + vectors.add(output.addField(m_field, NullableIntVector.class)); + break; + case "float": + t = MajorType.newBuilder().setMinorType( + TypeProtos.MinorType.FLOAT8); + m_field = MaterializedField.create(field, t.build()); + vectors.add(output.addField(m_field, Float8Vector.class)); + break; + case "timestamp": + t = MajorType.newBuilder().setMinorType( + TypeProtos.MinorType.TIMESTAMP); + m_field = MaterializedField.create(field, t.build()); + vectors.add(output.addField(m_field, TimeStampVector.class)); + break; + default: + t = MajorType.newBuilder().setMinorType( + TypeProtos.MinorType.VARCHAR); + m_field = MaterializedField.create(field, t.build()); + vectors + .add(output.addField(m_field, NullableVarCharVector.class)); + break; + } + } } + this.outputMutator = output; + } catch (SchemaChangeException e) { + throw new ExecutionSetupException(e); } - this.outputMutator = output; + } } @@ -171,7 +225,7 @@ public int next() { String solrField = vv.getField().getPath().toString() .replaceAll("`", ""); // re-think ?? Object fieldValue = solrDocument.get(solrField); - String fieldValueStr = "NULL"; + String fieldValueStr = null; if (fieldValue != null) { fieldValueStr = fieldValue.toString(); } @@ -180,6 +234,20 @@ public int next() { NullableVarCharVector v = (NullableVarCharVector) vv; v.getMutator().setSafe(counter, record, 0, record.length); v.getMutator().setValueLengthSafe(counter, record.length); + } else if (vv.getClass().equals(VarCharVector.class)) { + VarCharVector v = (VarCharVector) vv; + v.getMutator().setSafe(counter, record, 0, record.length); + v.getMutator().setValueLengthSafe(counter, record.length); + } else if (vv.getClass().equals(NullableBigIntVector.class)) { + NullableBigIntVector v = (NullableBigIntVector) vv; + v.getMutator().setSafe(counter, Long.parseLong(fieldValueStr)); + } else if (vv.getClass().equals(NullableIntVector.class)) { + NullableIntVector v = (NullableIntVector) vv; + v.getMutator().setSafe(counter, Integer.parseInt(fieldValueStr)); + } else if (vv.getClass().equals(DateVector.class)) { + DateVector v = (DateVector) vv; + long dtime = DateTime.parse(fieldValueStr).toDate().getTime(); + v.getMutator().setSafe(counter, dtime); } else { NullableVarCharVector v = (NullableVarCharVector) vv; v.getMutator().setSafe(counter, record, 0, record.length); diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java index 88431415d61..c487cc9192f 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrScanSpec.java @@ -17,16 +17,7 @@ */ package org.apache.drill.exec.store.solr; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Queue; - -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.common.params.SolrParams; - import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; public class SolrScanSpec { @@ -43,11 +34,13 @@ public SolrScanSpec(@JsonProperty("solrCoreName") String solrCoreName, @JsonProperty("filter") String filter) { this.solrCoreName = solrCoreName; } + public SolrScanSpec(@JsonProperty("solrCoreName") String solrCoreName, @JsonProperty("filter") SolrFilterParam filter) { this.solrCoreName = solrCoreName; - this.filter=filter; + this.filter = filter; } + public String getSolrCoreName() { return solrCoreName; } @@ -55,8 +48,10 @@ public String getSolrCoreName() { public SolrFilterParam getFilter() { return filter; } + @Override - public String toString(){ - return "SolrScanSpec [solrCoreName=" + solrCoreName + ", filter=" + filter + "]"; + public String toString() { + return "SolrScanSpec [solrCoreName=" + solrCoreName + ", filter=" + filter + + "]"; } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java index 5aa17f25aa5..a96d87b9d04 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePlugin.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.store.solr; import java.io.IOException; -import java.util.HashSet; import java.util.List; import java.util.Set; @@ -96,13 +95,18 @@ public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection, List columns) throws IOException { logger.debug("SolrStoragePlugin :: getPhysicalScan" + " userName : " - + userName + " columns ::" + columns+" selection "+selection); - SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(),new TypeReference() {}); + + userName + " columns ::" + columns + " selection " + selection); + SolrScanSpec solrScanSpec = selection.getListWith(new ObjectMapper(), + new TypeReference() { + }); return new SolrGroupScan(userName, this, solrScanSpec, columns); } + @Override - public Set getOptimizerRules(){ + public Set getOptimizerRules() { logger.debug("SolrStoragePlugin :: getOptimizerRules"); - return ImmutableSet.of(SolrQueryFilterRule.FILTER_ON_SCAN,SolrQueryFilterRule.FILTER_ON_PROJECT); + return ImmutableSet.of(SolrQueryFilterRule.FILTER_ON_SCAN, + SolrQueryFilterRule.FILTER_ON_PROJECT, + SolrQueryFilterRule.AGG_PUSH_DOWN); } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePluginConfig.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePluginConfig.java index 4ed8b62024f..cbd4d08804a 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePluginConfig.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrStoragePluginConfig.java @@ -28,16 +28,17 @@ public class SolrStoragePluginConfig extends StoragePluginConfig { public static final String NAME = "solr"; static final org.slf4j.Logger logger = org.slf4j.LoggerFactory .getLogger(SolrStoragePluginConfig.class); - + @JsonProperty - private String solrServer=""; + private String solrServer = ""; - @JsonCreator public SolrStoragePluginConfig(@JsonProperty("solrServer") String solrServer) { - logger.debug("Initializing SOLR StoragePlugin configuration with solr server :: "+solrServer); + logger + .debug("Initializing SOLR StoragePlugin configuration with solr server :: " + + solrServer); this.solrServer = solrServer; - + } @JsonProperty("solrServer") @@ -45,7 +46,6 @@ public String getSolrServer() { return this.solrServer; } - @Override public boolean equals(Object that) { if (this == that) { diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrSubScan.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrSubScan.java index 2f0639cd449..730e49ff5e8 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrSubScan.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrSubScan.java @@ -17,6 +17,8 @@ */ package org.apache.drill.exec.store.solr; +import static org.apache.drill.common.graph.GraphValue.logger; + import java.util.Iterator; import java.util.List; @@ -41,7 +43,7 @@ public class SolrSubScan extends AbstractBase implements SubScan { private SolrStoragePluginConfig solrPluginConfig; private SolrScanSpec solrScanSpec; private List columns; - private List scanList; + private List scanList; private String userName; @@ -51,7 +53,7 @@ public SolrSubScan(SolrGroupScan that) { this.solrPlugin = that.solrPlugin; this.solrPluginConfig = that.solrPluginConfig; this.solrScanSpec = that.solrScanSpec; - this.scanList=that.scanList; + this.scanList = that.scanList; logger.info("SolrSubScan : constructor ::" + columns); } @@ -120,9 +122,11 @@ public boolean isExecutable() { public SolrStoragePlugin getSolrPlugin() { return solrPlugin; } + public List getScanList() { return scanList; } + public SolrScanSpec getSolrScanSpec() { return solrScanSpec; } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java index b4f80e16db9..bd6ae0094ef 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchema.java @@ -17,166 +17,17 @@ */ package org.apache.drill.exec.store.solr.schema; -import java.io.Serializable; -import java.util.ArrayList; import java.util.List; -import org.apache.drill.exec.store.AbstractSchema; - -import com.google.common.collect.ImmutableList; - public class CVSchema { + private List fields; - protected List schemaFields = null; - - protected List dynSchemaFields = null; - - protected List fieldTypes = null; - - protected String uniqueKey = null; - - protected String defaultSearchField = null; - - protected Error errorObj = null; - - public List getSchemaFields(boolean init) { - if (init) { - return getSchemaFields(); - } else { - return this.schemaFields; - } - } - - public List getSchemaFields() { - if (schemaFields == null) { - this.schemaFields = new ArrayList(); - } - return this.schemaFields; - } - - public void setSchemaFields(List fields) { - this.schemaFields = fields; - - } - - public List getDynSchemaFields(boolean init) { - if (init) { - return getDynSchemaFields(); - } else { - return this.dynSchemaFields; - } - } - - public List getDynSchemaFields() { - if (dynSchemaFields == null) { - this.dynSchemaFields = new ArrayList(); - } - return this.dynSchemaFields; - } - - public void setDynSchemaFields(List dynSchemaFields) { - this.dynSchemaFields = dynSchemaFields; - - } - - public List getFieldTypes(boolean init) { - if (init) { - return getFieldTypes(); - } else { - return this.fieldTypes; - } - } - - public List getFieldTypes() { - if (fieldTypes == null) { - this.fieldTypes = new ArrayList(); - } - return this.fieldTypes; - } - - public void setFieldTypes(List fieldTypes) { - this.fieldTypes = fieldTypes; - - } - - public String getUniqueKey(boolean init) { - if (init) { - return getUniqueKey(); - } else { - return this.uniqueKey; - } - } - - public String getUniqueKey() { - if (uniqueKey == null) { - this.uniqueKey = ""; - } - return this.uniqueKey; - } - - public void setUniqueKey(String uniqueKey) { - this.uniqueKey = uniqueKey; - - } - - public String getDefaultSearchField(boolean init) { - if (init) { - return getDefaultSearchField(); - } else { - return this.defaultSearchField; - } - } - - public String getDefaultSearchField() { - if (defaultSearchField == null) { - this.defaultSearchField = ""; - } - return this.defaultSearchField; - } - - public void setDefaultSearchField(String defaultSearchField) { - this.defaultSearchField = defaultSearchField; - - } - - public Error getErrorObj(boolean init) { - if (init) { - return getErrorObj(); - } else { - return this.errorObj; - } - } - - public Error getErrorObj() { - if (errorObj == null) { - this.errorObj = new Error(); - } - return this.errorObj; - } - - public void setErrorObj(Error errorObj) { - this.errorObj = errorObj; - - } - - public static List getAllFields() { - ArrayList list = new ArrayList(); - list.add(Fields.SCHEMA_FIELDS); - list.add(Fields.DYN_SCHEMA_FIELDS); - list.add(Fields.FIELD_TYPES); - list.add(Fields.UNIQUE_KEY); - list.add(Fields.DEFAULT_SEARCH_FIELD); - list.add(Fields.ERROR_OBJ); - return list; + public List getFields() { + return fields; } - public static interface Fields { - String SCHEMA_FIELDS = "fields"; - String DYN_SCHEMA_FIELDS = "dynSchemaFields"; - String FIELD_TYPES = "fieldTypes"; - String UNIQUE_KEY = "uniqueKey"; - String DEFAULT_SEARCH_FIELD = "defaultSearchField"; - String ERROR_OBJ = "errorObj"; + public void setFields(List fields) { + this.fields = fields; } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java index a7b3cf2a745..bee8f3dd4df 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/CVSchemaField.java @@ -1,6 +1,6 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation import java.util.List; +ore contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the @@ -17,492 +17,130 @@ */ package org.apache.drill.exec.store.solr.schema; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -public class CVSchemaField implements Serializable { - protected String fieldName = null; - - protected String type = null; - - protected Boolean indexed = null; - - protected Boolean stored = null; - - protected Boolean multiValued = null; - - protected Boolean required = null; - - protected Boolean termVectors = null; - - protected Boolean omitNorms = null; - - protected Boolean omitPositions = null; - - protected Boolean omitTermFreqAndPositions = null; - - protected Boolean skipdelete = null; - - protected Boolean dynamicField = null; - - /* - * *will copy to default field - */ - protected Boolean searchDefault = null; - - /* - * *will copy to autocomplete field - */ - protected Boolean autocomplete = null; - - /* - * *will copy to spell field - */ - protected Boolean spellcheck = null; - - protected List copyFields = null; - - protected Boolean idxCopyField = null; - - /** - * copy constructors - **/ - public CVSchemaField(CVSchemaField obj) { - if (obj == null) - return; - if (obj.fieldName != null) { - this.fieldName = new String(obj.fieldName); - } - if (obj.type != null) { - this.type = new String(obj.type); - } - if (obj.indexed != null) { - this.indexed = new Boolean(obj.indexed); - } - if (obj.stored != null) { - this.stored = new Boolean(obj.stored); - } - if (obj.multiValued != null) { - this.multiValued = new Boolean(obj.multiValued); - } - if (obj.required != null) { - this.required = new Boolean(obj.required); - } - if (obj.termVectors != null) { - this.termVectors = new Boolean(obj.termVectors); - } - if (obj.omitNorms != null) { - this.omitNorms = new Boolean(obj.omitNorms); - } - if (obj.omitPositions != null) { - this.omitPositions = new Boolean(obj.omitPositions); - } - if (obj.omitTermFreqAndPositions != null) { - this.omitTermFreqAndPositions = new Boolean(obj.omitTermFreqAndPositions); - } - if (obj.skipdelete != null) { - this.skipdelete = new Boolean(obj.skipdelete); - } - if (obj.dynamicField != null) { - this.dynamicField = new Boolean(obj.dynamicField); - } - if (obj.searchDefault != null) { - this.searchDefault = new Boolean(obj.searchDefault); - } - if (obj.autocomplete != null) { - this.autocomplete = new Boolean(obj.autocomplete); - } - if (obj.spellcheck != null) { - this.spellcheck = new Boolean(obj.spellcheck); - } - if (obj.copyFields != null) { - this.copyFields = new ArrayList(obj.getCopyFields().size()); - for (String listObj : obj.getCopyFields()) { - this.copyFields.add(new String(listObj)); - } - } - if (obj.idxCopyField != null) { - this.idxCopyField = new Boolean(obj.idxCopyField); - } - } +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class CVSchemaField { + + private String name; + private String type; + private boolean uniqueKey; + @JsonIgnore + private boolean indexed; + @JsonIgnore + private boolean stored; + @JsonIgnore + private boolean multiValued; + @JsonIgnore + private boolean termVectors; + @JsonIgnore + private boolean omitPositions; + @JsonIgnore + private boolean skipdelete; + @JsonIgnore + private boolean omitNorms; + @JsonIgnore + private boolean required; + @JsonIgnore + private boolean omitTermFreqAndPositions; public CVSchemaField() { - } - - public String getFieldName(boolean init) { - if (init) { - return getFieldName(); - } else { - return this.fieldName; - } - } - - public Boolean getIdxCopyField() { - return idxCopyField; - } - - public Boolean getIdxCopyField(boolean init) { - if (init) { - return getIdxCopyField(); - } else { - - return this.idxCopyField; - } - } - - public String getFieldName() { - if (fieldName == null) { - this.fieldName = ""; - } - return this.fieldName; - } - - public void setFieldName(String name) { - this.fieldName = name; - - } - - public String getType(boolean init) { - if (init) { - return getType(); - } else { - return this.type; - } + indexed = stored = false; } public String getType() { - if (type == null) { - this.type = ""; - } - return this.type; + return type; } public void setType(String type) { this.type = type; - } - public Boolean getIndexed(boolean init) { - if (init) { - return getIndexed(); - } else { - return this.indexed; - } + public boolean isUniqueKey() { + return uniqueKey; } - public Boolean getIndexed() { - if (indexed == null) { - this.indexed = Boolean.valueOf(false); - } - return this.indexed; + public void setUniqueKey(boolean uniqueKey) { + this.uniqueKey = uniqueKey; } - public void setIndexed(Boolean indexed) { - this.indexed = indexed; - + public boolean isIndexed() { + return indexed; } - public Boolean getStored(boolean init) { - if (init) { - return getStored(); - } else { - return this.stored; - } + public void setIndexed(boolean indexed) { + this.indexed = indexed; } - public Boolean getStored() { - if (stored == null) { - this.stored = Boolean.valueOf(false); - } - return this.stored; + public boolean isStored() { + return stored; } - public void setStored(Boolean stored) { + public void setStored(boolean stored) { this.stored = stored; - } - public Boolean getMultiValued(boolean init) { - if (init) { - return getMultiValued(); - } else { - return this.multiValued; - } + public String getName() { + return name; } - public Boolean getMultiValued() { - if (multiValued == null) { - this.multiValued = Boolean.valueOf(false); - } - return this.multiValued; + public void setName(String name) { + this.name = name; } - public void setMultiValued(Boolean multiValued) { - this.multiValued = multiValued; - - } - - public Boolean getRequired(boolean init) { - if (init) { - return getRequired(); - } else { - return this.required; - } - } - - public Boolean getRequired() { - if (required == null) { - this.required = Boolean.valueOf(false); - } - return this.required; - } - - public void setRequired(Boolean required) { - this.required = required; - + public boolean isMultiValued() { + return multiValued; } - public Boolean getTermVectors(boolean init) { - if (init) { - return getTermVectors(); - } else { - return this.termVectors; - } + public void setMultiValued(boolean multiValued) { + this.multiValued = multiValued; } - public Boolean getTermVectors() { - if (termVectors == null) { - this.termVectors = Boolean.valueOf(false); - } - return this.termVectors; + public boolean isTermVectors() { + return termVectors; } - public void setTermVectors(Boolean termVectors) { + public void setTermVectors(boolean termVectors) { this.termVectors = termVectors; - - } - - public Boolean getOmitNorms(boolean init) { - if (init) { - return getOmitNorms(); - } else { - return this.omitNorms; - } - } - - public Boolean getOmitNorms() { - if (omitNorms == null) { - this.omitNorms = Boolean.valueOf(false); - } - return this.omitNorms; - } - - public void setOmitNorms(Boolean omitNorms) { - this.omitNorms = omitNorms; - } - public Boolean getOmitPositions(boolean init) { - if (init) { - return getOmitPositions(); - } else { - return this.omitPositions; - } + public boolean isOmitPositions() { + return omitPositions; } - public Boolean getOmitPositions() { - if (omitPositions == null) { - this.omitPositions = Boolean.valueOf(false); - } - return this.omitPositions; - } - - public void setOmitPositions(Boolean omitPositions) { + public void setOmitPositions(boolean omitPositions) { this.omitPositions = omitPositions; - - } - - public Boolean getOmitTermFreqAndPositions(boolean init) { - if (init) { - return getOmitTermFreqAndPositions(); - } else { - return this.omitTermFreqAndPositions; - } - } - - public Boolean getOmitTermFreqAndPositions() { - if (omitTermFreqAndPositions == null) { - this.omitTermFreqAndPositions = Boolean.valueOf(false); - } - return this.omitTermFreqAndPositions; - } - - public void setOmitTermFreqAndPositions(Boolean omitTermFreqAndPositions) { - this.omitTermFreqAndPositions = omitTermFreqAndPositions; - - } - - public Boolean getSkipdelete(boolean init) { - if (init) { - return getSkipdelete(); - } else { - return this.skipdelete; - } } - public Boolean getSkipdelete() { - if (skipdelete == null) { - this.skipdelete = Boolean.valueOf(false); - } - return this.skipdelete; + public boolean isSkipdelete() { + return skipdelete; } - public void setSkipdelete(Boolean skipdelete) { + public void setSkipdelete(boolean skipdelete) { this.skipdelete = skipdelete; - - } - - public Boolean getDynamicField(boolean init) { - if (init) { - return getDynamicField(); - } else { - return this.dynamicField; - } } - public Boolean getDynamicField() { - if (dynamicField == null) { - this.dynamicField = Boolean.valueOf(false); - } - return this.dynamicField; - } - - public void setDynamicField(Boolean dynamicField) { - this.dynamicField = dynamicField; - + public boolean isOmitNorms() { + return omitNorms; } - public Boolean getSearchDefault(boolean init) { - if (init) { - return getSearchDefault(); - } else { - return this.searchDefault; - } - } - - public Boolean getSearchDefault() { - if (searchDefault == null) { - this.searchDefault = Boolean.valueOf(false); - } - return this.searchDefault; - } - - public void setSearchDefault(Boolean searchDefault) { - this.searchDefault = searchDefault; - - } - - public Boolean getAutocomplete(boolean init) { - if (init) { - return getAutocomplete(); - } else { - return this.autocomplete; - } - } - - public Boolean getAutocomplete() { - if (autocomplete == null) { - this.autocomplete = Boolean.valueOf(false); - } - return this.autocomplete; - } - - public void setAutocomplete(Boolean autocomplete) { - this.autocomplete = autocomplete; - - } - - public Boolean getSpellcheck(boolean init) { - if (init) { - return getSpellcheck(); - } else { - return this.spellcheck; - } - } - - public Boolean getSpellcheck() { - if (spellcheck == null) { - this.spellcheck = Boolean.valueOf(false); - } - return this.spellcheck; - } - - public void setSpellcheck(Boolean spellcheck) { - this.spellcheck = spellcheck; - - } - - public List getCopyFields(boolean init) { - if (init) { - return getCopyFields(); - } else { - return this.copyFields; - } + public void setOmitNorms(boolean omitNorms) { + this.omitNorms = omitNorms; } - public List getCopyFields() { - if (copyFields == null) { - this.copyFields = new ArrayList(); - } - return this.copyFields; + public boolean isRequired() { + return required; } - public void setCopyFields(List copyFields) { - this.copyFields = copyFields; - + public void setRequired(boolean required) { + this.required = required; } - public static List getAllFields() { - ArrayList list = new ArrayList(); - list.add(Fields.FIELD_NAME); - list.add(Fields.TYPE); - list.add(Fields.INDEXED); - list.add(Fields.STORED); - list.add(Fields.MULTI_VALUED); - list.add(Fields.REQUIRED); - list.add(Fields.TERM_VECTORS); - list.add(Fields.OMIT_NORMS); - list.add(Fields.OMIT_POSITIONS); - list.add(Fields.OMIT_TERM_FREQ_AND_POSITIONS); - list.add(Fields.SKIPDELETE); - list.add(Fields.DYNAMIC_FIELD); - list.add(Fields.SEARCH_DEFAULT); - list.add(Fields.AUTOCOMPLETE); - list.add(Fields.SPELLCHECK); - list.add(Fields.COPY_FIELDS); - list.add(Fields.IDX_COPY_FIELD); - return list; + public boolean isOmitTermFreqAndPositions() { + return omitTermFreqAndPositions; } - public static interface Fields { - String FIELD_NAME = "name"; - String TYPE = "type"; - String INDEXED = "indexed"; - String STORED = "stored"; - String MULTI_VALUED = "multiValued"; - String REQUIRED = "required"; - String TERM_VECTORS = "termVectors"; - String OMIT_NORMS = "omitNorms"; - String OMIT_POSITIONS = "omitPositions"; - String OMIT_TERM_FREQ_AND_POSITIONS = "omitTermFreqAndPositions"; - String SKIPDELETE = "skipdelete"; - String DYNAMIC_FIELD = "dynamicField"; - String SEARCH_DEFAULT = "searchDefault"; - String AUTOCOMPLETE = "autocomplete"; - String SPELLCHECK = "spellcheck"; - String COPY_FIELDS = "copyFields"; - String IDX_COPY_FIELD = "idxCopyField"; + public void setOmitTermFreqAndPositions(boolean omitTermFreqAndPositions) { + this.omitTermFreqAndPositions = omitTermFreqAndPositions; } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java index 82b8ce0bcbc..96180548908 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchema.java @@ -17,28 +17,19 @@ */ package org.apache.drill.exec.store.solr.schema; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutionException; - -import javax.inject.Inject; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.drill.exec.planner.logical.DrillTable; import org.apache.drill.exec.planner.logical.DynamicDrillTable; import org.apache.drill.exec.store.AbstractSchema; -import org.apache.drill.exec.store.solr.SolrClientAPIExec; import org.apache.drill.exec.store.solr.SolrScanSpec; import org.apache.drill.exec.store.solr.SolrStoragePlugin; import org.apache.drill.exec.store.solr.SolrStoragePluginConfig; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; public class SolrSchema extends AbstractSchema { @@ -49,11 +40,16 @@ public class SolrSchema extends AbstractSchema { private String currentSchema = "root"; private final Map drillTables = Maps.newHashMap(); private final SolrStoragePlugin solrStoragePlugin; + private final List schemaPath; - public SolrSchema(List schemaPath,String currentSchema, SolrStoragePlugin solrStoragePlugin) { + public SolrSchema(List schemaPath, String currentSchema, + SolrStoragePlugin solrStoragePlugin) { super(schemaPath, currentSchema); + this.schemaPath = schemaPath; + this.currentSchema = currentSchema; this.solrStoragePlugin = solrStoragePlugin; - availableSolrCores = solrStoragePlugin.getSolrClientApiExec().getSolrCoreList(); + availableSolrCores = solrStoragePlugin.getSolrClientApiExec() + .getSolrCoreList(); } @Override @@ -85,7 +81,8 @@ public Table getTable(String coreName) { DrillTable getDrillTable(String dbName, String collectionName) { logger.info("SolrSchema :: getDrillTable"); SolrScanSpec solrScanSpec = new SolrScanSpec(collectionName); - return new DynamicDrillTable(solrStoragePlugin, SolrStoragePluginConfig.NAME, solrScanSpec); + return new DynamicDrillTable(solrStoragePlugin, + SolrStoragePluginConfig.NAME, solrScanSpec); } @Override @@ -94,4 +91,8 @@ public Set getTableNames() { return availableSolrCores; } + @Override + public boolean showInInformationSchema() { + return true; + } } diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java index 891fae3d0ee..3e3e86f92a3 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/schema/SolrSchemaFactory.java @@ -19,24 +19,15 @@ import java.io.IOException; import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import javax.inject.Inject; import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.exec.store.SchemaConfig; import org.apache.drill.exec.store.SchemaFactory; -import org.apache.drill.exec.store.solr.SolrClientAPIExec; import org.apache.drill.exec.store.solr.SolrStoragePlugin; import org.apache.drill.exec.store.solr.SolrStoragePluginConfig; -import org.apache.solr.client.solrj.SolrClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import com.google.common.collect.Lists; public class SolrSchemaFactory implements SchemaFactory { @@ -47,10 +38,9 @@ public class SolrSchemaFactory implements SchemaFactory { public SolrSchemaFactory(SolrStoragePlugin solrStorage, String storageName) { this.solrStorage = solrStorage; - this.storageName = storageName; + this.storageName = storageName; } - /* * (non-Javadoc) * @@ -63,10 +53,10 @@ public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { logger.info("registering schema...."); List schemaPath = Lists.newArrayList(); - //schemaPath.add(SolrStoragePluginConfig.NAME); - SolrSchema schema = new SolrSchema(schemaPath,"root", solrStorage); + schemaPath.add(SolrStoragePluginConfig.NAME); + SolrSchema schema = new SolrSchema(schemaPath, "root", solrStorage); SchemaPlus hPlus = parent.add(this.storageName, schema); - + } } diff --git a/contrib/storage-solr/src/test/java/org/apache/drill/exec/store/solr/TestSolr.java b/contrib/storage-solr/src/test/java/org/apache/drill/exec/store/solr/TestSolr.java index 1992f4b80c6..2cffda6b35f 100644 --- a/contrib/storage-solr/src/test/java/org/apache/drill/exec/store/solr/TestSolr.java +++ b/contrib/storage-solr/src/test/java/org/apache/drill/exec/store/solr/TestSolr.java @@ -24,13 +24,14 @@ public class TestSolr extends BaseTestQuery { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory .getLogger(TestSolr.class); - private static final String solrServer = "http://sgdev1:20000/solr/"; + private static final String solrServer = "http://localhost:20000/solr/"; private static final String solrCoreName = "bootstrap_5"; private SolrScanSpec solrScanSpec; + @Before - public void setUp(){ - solrScanSpec=new SolrScanSpec(solrCoreName); - + public void setUp() { + solrScanSpec = new SolrScanSpec(solrCoreName); + } @Test @@ -38,4 +39,12 @@ public void solrBasicQuery() throws Exception { testBuilder().sqlQuery("select * from solr.`bootstrap_5`").unOrdered() .build().run(); } + + @Test + public void solrFilterQuery() throws Exception { + testBuilder() + .sqlQuery( + "select * from solr.`bikemini_3` where start_station_id='514'") + .unOrdered().build().run(); + } } From d59b873b1326a6f099e52bbbbd6becdc9cdfc0fc Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Thu, 27 Aug 2015 16:46:20 +0530 Subject: [PATCH 14/15] Solr Storage Plugin initial code checkin --- .../exec/store/solr/SolrQueryFilterRule.java | 104 ++++++++++++------ 1 file changed, 68 insertions(+), 36 deletions(-) diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java index 7a71448a963..9e8283adbfc 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrQueryFilterRule.java @@ -17,24 +17,21 @@ */ package org.apache.drill.exec.store.solr; -import java.io.IOException; - import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rex.RexNode; -import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillAggregateRel; import org.apache.drill.exec.planner.logical.DrillFilterRel; import org.apache.drill.exec.planner.logical.DrillOptiq; import org.apache.drill.exec.planner.logical.DrillParseContext; import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillRel; import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.DrillScreenRel; import org.apache.drill.exec.planner.logical.RelOptHelper; -import org.apache.drill.exec.planner.physical.FilterPrel; import org.apache.drill.exec.planner.physical.PrelUtil; -import org.apache.drill.exec.planner.physical.ProjectPrel; -import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,23 +41,23 @@ public abstract class SolrQueryFilterRule extends StoragePluginOptimizerRule { static final Logger logger = LoggerFactory .getLogger(SolrQueryFilterRule.class); - + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new SolrQueryFilterRule( - RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), - "SolrQueryFilterRule:Filter_On_Scan"){ + RelOptHelper.some(DrillFilterRel.class, + RelOptHelper.any(DrillScanRel.class)), + "SolrQueryFilterRule:Filter_On_Scan") { @Override public void onMatch(RelOptRuleCall call) { - logger.debug("SolrQueryFilterRule :: onMatch"); - final FilterPrel filter = (FilterPrel) call.rel(0); - final ScanPrel scan = (ScanPrel) call.rel(1); - - doOnMatch(call, filter, null, scan); + logger.debug("SolrQueryFilterRule :: onMatch :: Filter_On_Scan"); + final DrillFilterRel filterRel = (DrillFilterRel) call.rel(0); + final DrillScanRel scan = (DrillScanRel) call.rel(1); + doOnMatch(call, filterRel, null, scan); } @Override public boolean matches(RelOptRuleCall call) { - final ScanPrel scan = (ScanPrel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(1); if (scan.getGroupScan() instanceof SolrGroupScan) { return super.matches(call); } @@ -70,24 +67,51 @@ public boolean matches(RelOptRuleCall call) { public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new SolrQueryFilterRule( RelOptHelper.some( - ProjectPrel.class, - RelOptHelper.some(ProjectPrel.class, - RelOptHelper.any(ScanPrel.class))), - "StoragePluginOptimizerRule:Filter_On_Project"){ + DrillScreenRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "StoragePluginOptimizerRule:Filter_On_Project") { @Override public void onMatch(RelOptRuleCall call) { - logger.debug("SolrQueryFilterRule :: onMatch"); - final FilterPrel filterRel = (FilterPrel) call.rel(0); - final ProjectPrel projectRel = (ProjectPrel) call.rel(1); - final ScanPrel scanRel = (ScanPrel) call.rel(2); + logger.debug("SolrQueryFilterRule :: onMatch:: Filter_On_Project"); + final DrillScreenRel screenRel = (DrillScreenRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + + DrillRel inputRel = projectRel != null ? projectRel : scanRel; + SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); + // TODO: optimize the solr query to return the exact field lists (fl) + } - doOnMatch(call, filterRel, projectRel, scanRel); + @Override + public boolean matches(RelOptRuleCall call) { + final DrillScanRel scan = (DrillScanRel) call.rel(2); + if (scan.getGroupScan() instanceof SolrGroupScan) { + return super.matches(call); + } + return false; + } + }; + public static final StoragePluginOptimizerRule AGG_PUSH_DOWN = new SolrQueryFilterRule( + RelOptHelper.some( + DrillAggregateRel.class, + RelOptHelper.some(DrillProjectRel.class, + RelOptHelper.any(DrillScanRel.class))), + "StoragePluginOptimizerRule:AGG_PUSH_DOWN") { + + @Override + public void onMatch(RelOptRuleCall call) { + logger.debug("SolrQueryFilterRule :: onMatch :: Agg_Push_Down"); + final DrillAggregateRel aggrRel = (DrillAggregateRel) call.rel(0); + final DrillProjectRel projectRel = (DrillProjectRel) call.rel(1); + final DrillScanRel scanRel = (DrillScanRel) call.rel(2); + // optimize the solr query for different funcType } @Override public boolean matches(RelOptRuleCall call) { - final ScanPrel scan = (ScanPrel) call.rel(1); + final DrillScanRel scan = (DrillScanRel) call.rel(2); if (scan.getGroupScan() instanceof SolrGroupScan) { return super.matches(call); } @@ -95,24 +119,21 @@ public boolean matches(RelOptRuleCall call) { } }; - - public SolrQueryFilterRule(RelOptRuleOperand operand, String description) { super(operand, description); logger.info("SolrQueryFilterRule :: contructor"); } - + protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, + DrillProjectRel projectRel, DrillScanRel scanRel) { - protected void doOnMatch(RelOptRuleCall call, FilterPrel filterRel, - ProjectPrel projectRel, ScanPrel scanRel) { + DrillRel inputRel = projectRel != null ? projectRel : scanRel; SolrGroupScan solrGroupScan = (SolrGroupScan) scanRel.getGroupScan(); final RexNode condition = filterRel.getCondition(); + LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext( PrelUtil.getPlannerSettings(call.getPlanner())), scanRel, condition); - logger.debug("conditionExp " + conditionExp); - SolrQueryBuilder sQueryBuilder = new SolrQueryBuilder(solrGroupScan, conditionExp); SolrScanSpec newScanSpec = sQueryBuilder.parseTree(); @@ -120,15 +141,26 @@ protected void doOnMatch(RelOptRuleCall call, FilterPrel filterRel, return; SolrGroupScan newGroupScan = new SolrGroupScan(solrGroupScan.getUserName(), solrGroupScan.getSolrPlugin(), newScanSpec, solrGroupScan.getColumns()); - final ScanPrel newScanPrel = ScanPrel.create(scanRel, - filterRel.getTraitSet(), newGroupScan, scanRel.getRowType()); + + DrillScanRel newScanRel = new DrillScanRel(scanRel.getCluster(), scanRel + .getTraitSet().plus(DrillRel.DRILL_LOGICAL), scanRel.getTable(), + newGroupScan, scanRel.getRowType(), scanRel.getColumns()); + if (projectRel != null) { + DrillProjectRel newProjectRel = DrillProjectRel.create( + projectRel.getCluster(), projectRel.getTraitSet(), newScanRel, + projectRel.getProjects(), projectRel.getRowType()); + inputRel = newProjectRel; + } else { + inputRel = newScanRel; + } if (sQueryBuilder.isAllExpressionsConverted()) { logger.info("all expressions converted.. "); - call.transformTo(newScanPrel); + call.transformTo(inputRel); } else { call.transformTo(filterRel.copy(filterRel.getTraitSet(), - ImmutableList.of((RelNode) newScanPrel))); + ImmutableList.of((RelNode) inputRel))); } } + } From 30435a1de03066f1db8bb54ec302433031707c31 Mon Sep 17 00:00:00 2001 From: sudipmukherjee Date: Thu, 27 Aug 2015 16:58:21 +0530 Subject: [PATCH 15/15] Solr Storage Plugin initial code checkin --- .../34/60768288ae4c001512449b35bb2b24ab | 181 ++++++++++++++++++ .../51/50104889ae4c001512449b35bb2b24ab | 179 +++++++++++++++++ .../59/b05051a8ae4c001512449b35bb2b24ab | 177 +++++++++++++++++ .../e1/f02848a8ae4c001512449b35bb2b24ab | 179 +++++++++++++++++ .../exec/store/solr/SolrClientAPIExec.java | 6 +- 5 files changed, 717 insertions(+), 5 deletions(-) create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/34/60768288ae4c001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/51/50104889ae4c001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/59/b05051a8ae4c001512449b35bb2b24ab create mode 100644 .metadata/.plugins/org.eclipse.core.resources/.history/e1/f02848a8ae4c001512449b35bb2b24ab diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/34/60768288ae4c001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/34/60768288ae4c001512449b35bb2b24ab new file mode 100644 index 00000000000..ced9f531b7b --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/34/60768288ae4c001512449b35bb2b24ab @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.drill.exec.store.solr.schema.CVSchema; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.SolrStream; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; + +import com.beust.jcommander.internal.Lists; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; + +public class SolrClientAPIExec { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrClientAPIExec.class); + private SolrClient solrClient; + + public SolrClient getSolrClient() { + return solrClient; + } + + public void setSolrClient(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec() { + + } + + public Set getSolrCoreList() { + // Request core list + logger.debug("getting cores from solr.."); + CoreAdminRequest request = new CoreAdminRequest(); + request.setAction(CoreAdminAction.STATUS); + Set coreList = null; + try { + CoreAdminResponse cores = request.process(solrClient); + coreList = new HashSet(cores.getCoreStatus().size()); + for (int i = 0; i < cores.getCoreStatus().size(); i++) { + coreList.add(cores.getCoreStatus().getName(i)); + } + } catch (SolrServerException | IOException e) { + logger.info("error getting core info from solr server..."); + } + return coreList; + } + + public CVSchema getSchemaForCore(String coreName, String solrServerUrl) { + String schemaUrl = "{0}{1}/schema/fields"; + schemaUrl = MessageFormat.format(schemaUrl, solrServerUrl, coreName); + HttpClient client = new DefaultHttpClient(); + HttpGet request = new HttpGet(schemaUrl); + CVSchema oCVSchema = null; + request.setHeader("Content-Type", "application/json"); + try { + logger.info("getting schema details for core... " + coreName + + " schemaUrl :" + schemaUrl); + HttpResponse response = client.execute(request); + BufferedReader rd = new BufferedReader(new InputStreamReader(response + .getEntity().getContent())); + StringBuffer result = new StringBuffer(); + String line = ""; + while ((line = rd.readLine()) != null) { + result.append(line); + } + ObjectMapper mapper = new ObjectMapper(); + mapper + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + oCVSchema = mapper.readValue(result.toString(), CVSchema.class); + } catch (Exception e) { + logger.info("exception occured while fetching schema details..." + + e.getMessage()); + } + return oCVSchema; + } + + public SolrDocumentList getSolrDocs(String solrServer, String solrCoreName, + List fields, StringBuilder filters) { + logger.debug("getSolrDocs :: " + solrCoreName); + SolrClient solrClient = new HttpSolrClient(solrServer + solrCoreName); + SolrDocumentList sList = null; + SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive") + .setQuery("*:*").setRows(Integer.MAX_VALUE); + + if (fields != null) { + logger.debug("solr fields are " + fields); + String fieldStr = Joiner.on(",").join(fields); + solrQuery.setParam("fl", fieldStr); + } + if (filters.length() > 0) { + logger.info("adding filter query :: " + filters.toString()); + solrQuery.setParam("fq", filters.toString()); + } + try { + logger.debug("setting solrquery.."); + QueryResponse rsp = solrClient.query(solrQuery); + logger.debug("response recieved from " + solrServer + " core " + + solrCoreName); + sList = rsp.getResults(); + } catch (SolrServerException | IOException e) { + logger.debug("error occured while fetching results from solr server " + + e.getMessage()); + } + return sList; + } + + public List getSolrResponse(String solrServer, SolrClient solrClient, + String solrCoreName, Map solrParams) { + logger.info("sending request to solr server " + solrServer + " on core " + + solrCoreName); + SolrStream solrStream = new SolrStream(solrServer, solrParams); + List resultTuple = Lists.newArrayList(); + try { + solrStream.open(); + + Tuple tuple = null; + while (true) { + tuple = solrStream.read(); + if (tuple.EOF) { + break; + } + resultTuple.add(tuple); + } + } catch (Exception e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } finally { + try { + solrStream.close(); + } catch (IOException e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } + + } + return resultTuple; + } + +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/51/50104889ae4c001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/51/50104889ae4c001512449b35bb2b24ab new file mode 100644 index 00000000000..98689fd4224 --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/51/50104889ae4c001512449b35bb2b24ab @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.drill.exec.store.solr.schema.CVSchema; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.SolrStream; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; + +import com.beust.jcommander.internal.Lists; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; + +public class SolrClientAPIExec { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrClientAPIExec.class); + private SolrClient solrClient; + + public SolrClient getSolrClient() { + return solrClient; + } + + public void setSolrClient(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec() { + + } + + public Set getSolrCoreList() { + // Request core list + logger.debug("getting cores from solr.."); + CoreAdminRequest request = new CoreAdminRequest(); + request.setAction(CoreAdminAction.STATUS); + Set coreList = null; + try { + CoreAdminResponse cores = request.process(solrClient); + coreList = new HashSet(cores.getCoreStatus().size()); + for (int i = 0; i < cores.getCoreStatus().size(); i++) { + coreList.add(cores.getCoreStatus().getName(i)); + } + } catch (SolrServerException | IOException e) { + logger.info("error getting core info from solr server..."); + } + return coreList; + } + + public CVSchema getSchemaForCore(String coreName, String solrServerUrl) { + String schemaUrl = "{0}{1}/schema/fields"; + schemaUrl = MessageFormat.format(schemaUrl, solrServerUrl, coreName); + HttpClient client = new DefaultHttpClient(); + HttpGet request = new HttpGet(schemaUrl); + CVSchema oCVSchema = null; + request.setHeader("Content-Type", "application/json"); + try { + HttpResponse response = client.execute(request); + BufferedReader rd = new BufferedReader(new InputStreamReader(response + .getEntity().getContent())); + StringBuffer result = new StringBuffer(); + String line = ""; + while ((line = rd.readLine()) != null) { + result.append(line); + } + ObjectMapper mapper = new ObjectMapper(); + mapper + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + oCVSchema = mapper.readValue(result.toString(), CVSchema.class); + } catch (Exception e) { + logger.info("exception occured while fetching schema details..." + + e.getMessage()); + } + return oCVSchema; + } + + public SolrDocumentList getSolrDocs(String solrServer, String solrCoreName, + List fields, StringBuilder filters) { + logger.debug("getSolrDocs :: " + solrCoreName); + SolrClient solrClient = new HttpSolrClient(solrServer + solrCoreName); + SolrDocumentList sList = null; + SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive") + .setQuery("*:*").setRows(Integer.MAX_VALUE); + + if (fields != null) { + logger.debug("solr fields are " + fields); + String fieldStr = Joiner.on(",").join(fields); + solrQuery.setParam("fl", fieldStr); + } + if (filters.length() > 0) { + logger.info("adding filter query :: " + filters.toString()); + solrQuery.setParam("fq", filters.toString()); + } + try { + logger.debug("setting solrquery.."); + QueryResponse rsp = solrClient.query(solrQuery); + logger.debug("response recieved from " + solrServer + " core " + + solrCoreName); + sList = rsp.getResults(); + } catch (SolrServerException | IOException e) { + logger.debug("error occured while fetching results from solr server " + + e.getMessage()); + } + return sList; + } + + public List getSolrResponse(String solrServer, SolrClient solrClient, + String solrCoreName, Map solrParams) { + logger.info("sending request to solr server " + solrServer + " on core " + + solrCoreName); + SolrStream solrStream = new SolrStream(solrServer, solrParams); + List resultTuple = Lists.newArrayList(); + try { + solrStream.open(); + + Tuple tuple = null; + while (true) { + tuple = solrStream.read(); + if (tuple.EOF) { + break; + } + resultTuple.add(tuple); + } + } catch (Exception e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } finally { + try { + solrStream.close(); + } catch (IOException e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } + + } + return resultTuple; + } + +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/59/b05051a8ae4c001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/59/b05051a8ae4c001512449b35bb2b24ab new file mode 100644 index 00000000000..0862bb4ed2c --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/59/b05051a8ae4c001512449b35bb2b24ab @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.drill.exec.store.solr.schema.CVSchema; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.SolrStream; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; + +import com.beust.jcommander.internal.Lists; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; + +public class SolrClientAPIExec { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrClientAPIExec.class); + private SolrClient solrClient; + + public SolrClient getSolrClient() { + return solrClient; + } + + public void setSolrClient(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec() { + + } + + public Set getSolrCoreList() { + // Request core list + logger.debug("getting cores from solr.."); + CoreAdminRequest request = new CoreAdminRequest(); + request.setAction(CoreAdminAction.STATUS); + Set coreList = null; + try { + CoreAdminResponse cores = request.process(solrClient); + coreList = new HashSet(cores.getCoreStatus().size()); + for (int i = 0; i < cores.getCoreStatus().size(); i++) { + coreList.add(cores.getCoreStatus().getName(i)); + } + } catch (SolrServerException | IOException e) { + logger.info("error getting core info from solr server..."); + } + return coreList; + } + + public CVSchema getSchemaForCore(String coreName, String solrServerUrl) { + String schemaUrl = "{0}{1}/schema/fields"; + schemaUrl = MessageFormat.format(schemaUrl, solrServerUrl, coreName); + HttpClient client = new DefaultHttpClient(); + HttpGet request = new HttpGet(schemaUrl); + CVSchema oCVSchema = null; + request.setHeader("Content-Type", "application/json"); + try { + HttpResponse response = client.execute(request); + BufferedReader rd = new BufferedReader(new InputStreamReader(response + .getEntity().getContent())); + StringBuffer result = new StringBuffer(); + String line = ""; + while ((line = rd.readLine()) != null) { + result.append(line); + } + ObjectMapper mapper = new ObjectMapper(); + mapper + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + oCVSchema = mapper.readValue(result.toString(), CVSchema.class); + } catch (Exception e) { + logger.info("exception occured while fetching schema details..." + + e.getMessage()); + } + return oCVSchema; + } + + public SolrDocumentList getSolrDocs(String solrServer, String solrCoreName, + List fields, StringBuilder filters) { + SolrClient solrClient = new HttpSolrClient(solrServer + solrCoreName); + SolrDocumentList sList = null; + SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive") + .setQuery("*:*").setRows(Integer.MAX_VALUE); + + if (fields != null) { + + String fieldStr = Joiner.on(",").join(fields); + solrQuery.setParam("fl", fieldStr); + } + if (filters.length() > 0) { + solrQuery.setParam("fq", filters.toString()); + } + try { + logger.debug("setting solrquery.."); + QueryResponse rsp = solrClient.query(solrQuery); + logger.debug("response recieved from " + solrServer + " core " + + solrCoreName); + sList = rsp.getResults(); + } catch (SolrServerException | IOException e) { + logger.debug("error occured while fetching results from solr server " + + e.getMessage()); + } + return sList; + } + + public List getSolrResponse(String solrServer, SolrClient solrClient, + String solrCoreName, Map solrParams) { + logger.info("sending request to solr server " + solrServer + " on core " + + solrCoreName); + SolrStream solrStream = new SolrStream(solrServer, solrParams); + List resultTuple = Lists.newArrayList(); + try { + solrStream.open(); + + Tuple tuple = null; + while (true) { + tuple = solrStream.read(); + if (tuple.EOF) { + break; + } + resultTuple.add(tuple); + } + } catch (Exception e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } finally { + try { + solrStream.close(); + } catch (IOException e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } + + } + return resultTuple; + } + +} diff --git a/.metadata/.plugins/org.eclipse.core.resources/.history/e1/f02848a8ae4c001512449b35bb2b24ab b/.metadata/.plugins/org.eclipse.core.resources/.history/e1/f02848a8ae4c001512449b35bb2b24ab new file mode 100644 index 00000000000..b82480d111f --- /dev/null +++ b/.metadata/.plugins/org.eclipse.core.resources/.history/e1/f02848a8ae4c001512449b35bb2b24ab @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.solr; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.drill.exec.store.solr.schema.CVSchema; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.SolrStream; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; + +import com.beust.jcommander.internal.Lists; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; + +public class SolrClientAPIExec { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory + .getLogger(SolrClientAPIExec.class); + private SolrClient solrClient; + + public SolrClient getSolrClient() { + return solrClient; + } + + public void setSolrClient(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec(SolrClient solrClient) { + this.solrClient = solrClient; + } + + public SolrClientAPIExec() { + + } + + public Set getSolrCoreList() { + // Request core list + logger.debug("getting cores from solr.."); + CoreAdminRequest request = new CoreAdminRequest(); + request.setAction(CoreAdminAction.STATUS); + Set coreList = null; + try { + CoreAdminResponse cores = request.process(solrClient); + coreList = new HashSet(cores.getCoreStatus().size()); + for (int i = 0; i < cores.getCoreStatus().size(); i++) { + coreList.add(cores.getCoreStatus().getName(i)); + } + } catch (SolrServerException | IOException e) { + logger.info("error getting core info from solr server..."); + } + return coreList; + } + + public CVSchema getSchemaForCore(String coreName, String solrServerUrl) { + String schemaUrl = "{0}{1}/schema/fields"; + schemaUrl = MessageFormat.format(schemaUrl, solrServerUrl, coreName); + HttpClient client = new DefaultHttpClient(); + HttpGet request = new HttpGet(schemaUrl); + CVSchema oCVSchema = null; + request.setHeader("Content-Type", "application/json"); + try { + HttpResponse response = client.execute(request); + BufferedReader rd = new BufferedReader(new InputStreamReader(response + .getEntity().getContent())); + StringBuffer result = new StringBuffer(); + String line = ""; + while ((line = rd.readLine()) != null) { + result.append(line); + } + ObjectMapper mapper = new ObjectMapper(); + mapper + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + oCVSchema = mapper.readValue(result.toString(), CVSchema.class); + } catch (Exception e) { + logger.info("exception occured while fetching schema details..." + + e.getMessage()); + } + return oCVSchema; + } + + public SolrDocumentList getSolrDocs(String solrServer, String solrCoreName, + List fields, StringBuilder filters) { + logger.debug("getSolrDocs :: " + solrCoreName); + SolrClient solrClient = new HttpSolrClient(solrServer + solrCoreName); + SolrDocumentList sList = null; + SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive") + .setQuery("*:*").setRows(Integer.MAX_VALUE); + + if (fields != null) { + logger.debug("solr fields are " + fields); + String fieldStr = Joiner.on(",").join(fields); + solrQuery.setParam("fl", fieldStr); + } + if (filters.length() > 0) { + logger.info("adding filter query :: " + filters.toString()); + solrQuery.setParam("fq", filters.toString()); + } + try { + logger.debug("setting solrquery.."); + QueryResponse rsp = solrClient.query(solrQuery); + logger.debug("response recieved from " + solrServer + " core " + + solrCoreName); + sList = rsp.getResults(); + } catch (SolrServerException | IOException e) { + logger.debug("error occured while fetching results from solr server " + + e.getMessage()); + } + return sList; + } + + public List getSolrResponse(String solrServer, SolrClient solrClient, + String solrCoreName, Map solrParams) { + logger.info("sending request to solr server " + solrServer + " on core " + + solrCoreName); + SolrStream solrStream = new SolrStream(solrServer, solrParams); + List resultTuple = Lists.newArrayList(); + try { + solrStream.open(); + + Tuple tuple = null; + while (true) { + tuple = solrStream.read(); + if (tuple.EOF) { + break; + } + resultTuple.add(tuple); + } + } catch (Exception e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } finally { + try { + solrStream.close(); + } catch (IOException e) { + logger.info("error occured while fetching results from solr server " + + e.getMessage()); + } + + } + return resultTuple; + } + +} diff --git a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java index ced9f531b7b..d3a67a62c31 100644 --- a/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java +++ b/contrib/storage-solr/src/main/java/org/apache/drill/exec/store/solr/SolrClientAPIExec.java @@ -95,8 +95,6 @@ public CVSchema getSchemaForCore(String coreName, String solrServerUrl) { CVSchema oCVSchema = null; request.setHeader("Content-Type", "application/json"); try { - logger.info("getting schema details for core... " + coreName - + " schemaUrl :" + schemaUrl); HttpResponse response = client.execute(request); BufferedReader rd = new BufferedReader(new InputStreamReader(response .getEntity().getContent())); @@ -118,19 +116,17 @@ public CVSchema getSchemaForCore(String coreName, String solrServerUrl) { public SolrDocumentList getSolrDocs(String solrServer, String solrCoreName, List fields, StringBuilder filters) { - logger.debug("getSolrDocs :: " + solrCoreName); SolrClient solrClient = new HttpSolrClient(solrServer + solrCoreName); SolrDocumentList sList = null; SolrQuery solrQuery = new SolrQuery().setTermsRegexFlag("case_insensitive") .setQuery("*:*").setRows(Integer.MAX_VALUE); if (fields != null) { - logger.debug("solr fields are " + fields); + String fieldStr = Joiner.on(",").join(fields); solrQuery.setParam("fl", fieldStr); } if (filters.length() > 0) { - logger.info("adding filter query :: " + filters.toString()); solrQuery.setParam("fq", filters.toString()); } try {