From 7b3e4a557c014d94597f72ef2f2420063f249822 Mon Sep 17 00:00:00 2001 From: andrew Date: Tue, 25 Aug 2015 15:41:05 -0700 Subject: [PATCH 1/2] Initial commit for memory storage plugin --- contrib/pom.xml | 1 + contrib/storage-memory/pom.xml | 59 +++++++ .../store/memory/MemorySchemaFactory.java | 142 ++++++++++++++++ .../store/memory/MemorySchemaProvider.java | 100 +++++++++++ .../store/memory/MemoryStoragePlugin.java | 72 ++++++++ .../memory/MemoryStoragePluginConfig.java | 55 ++++++ .../drill/exec/store/memory/MemoryTable.java | 157 ++++++++++++++++++ .../resources/bootstrap-storage-plugins.json | 8 + .../src/main/resources/drill-module.conf | 25 +++ .../store/memory/TestMemoryStorePlugin.java | 76 +++++++++ .../resources/bootstrap-storage-plugins.json | 8 + .../src/test/resources/drill-module.conf | 25 +++ .../src/test/resources/logback-test.xml | 42 +++++ 13 files changed, 770 insertions(+) create mode 100644 contrib/storage-memory/pom.xml create mode 100644 contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaFactory.java create mode 100644 contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaProvider.java create mode 100644 contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryStoragePlugin.java create mode 100644 contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryStoragePluginConfig.java create mode 100644 contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryTable.java create mode 100644 contrib/storage-memory/src/main/resources/bootstrap-storage-plugins.json create mode 100644 contrib/storage-memory/src/main/resources/drill-module.conf create mode 100644 contrib/storage-memory/src/test/java/org/apache/drill/exec/store/memory/TestMemoryStorePlugin.java create mode 100644 contrib/storage-memory/src/test/resources/bootstrap-storage-plugins.json create mode 100644 contrib/storage-memory/src/test/resources/drill-module.conf create mode 100644 contrib/storage-memory/src/test/resources/logback-test.xml diff --git a/contrib/pom.xml b/contrib/pom.xml index 8c00e762d48..a85e51d12e1 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -35,6 +35,7 @@ storage-hbase storage-hive storage-mongo + storage-memory sqlline data diff --git a/contrib/storage-memory/pom.xml b/contrib/storage-memory/pom.xml new file mode 100644 index 00000000000..1ac3d503d52 --- /dev/null +++ b/contrib/storage-memory/pom.xml @@ -0,0 +1,59 @@ + + + + 4.0.0 + + drill-contrib-parent + org.apache.drill.contrib + 1.2.0-SNAPSHOT + + + drill-storage-memory + + contrib/memory-storage-plugin + + + + + + + org.apache.drill.exec + drill-java-exec + ${project.version} + + + + + org.apache.drill.exec + drill-java-exec + tests + ${project.version} + test + + + org.apache.drill + drill-common + tests + ${project.version} + test + + + + + diff --git a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaFactory.java b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaFactory.java new file mode 100644 index 00000000000..6d699b3b03b --- /dev/null +++ b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaFactory.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.memory; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; + +import org.apache.drill.exec.planner.logical.CreateTableEntry; +import org.apache.drill.exec.store.AbstractSchema; +import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.SchemaFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +/** + * Schema factory for memory storage plugin. + */ +public class MemorySchemaFactory implements SchemaFactory { + + private final String schema; + private final MemoryStoragePlugin plugin; + private final MemorySchemaProvider provider; + + public MemorySchemaFactory(MemoryStoragePlugin plugin, String schema) { + this.plugin = plugin; + this.schema = schema; + this.provider = new MemorySchemaProvider(); + logger.info("Factory created for memory storage plugin: [{}]", schema); + } + + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { + MemorySchema m = new MemorySchema(schema); + SchemaPlus plus = parent.add(schema, m); + m.setHolder(plus); + logger.info("Schema registered for memory storage plugin: [{}]", schema); + } + + // XXX - Remove this and make a proper writer interface. + public MemorySchemaProvider getSchemaProvider() { + return provider; + } + + public class MemorySchema extends AbstractSchema { + + public MemorySchema(String name) { + super(ImmutableList.of(), name); + } + + public MemorySchema(List parentSchemaPath, String name) { + super(parentSchemaPath, name); + } + + @Override + public String getTypeName() { + return MemoryStoragePluginConfig.NAME; + } + + public void setHolder(SchemaPlus plusOfThis) { + for (String s : getSubSchemaNames()) { + plusOfThis.add(s, getSubSchema(s)); + } + } + + @Override + public AbstractSchema getSubSchema(String name) { + if (!provider.exists(name)) { + logger.info("Schema: [{}] does not exist in memory", name); + return null; + } + Set tables = provider.tables(name); + return new MemoryObjectSchema(this, name, tables); + } + + @Override + public Set getSubSchemaNames() { + return Sets.newHashSet(provider.schemas()); + } + + @Override + public Set getTableNames() { + return provider.tables(schema); + } + + @Override + public CreateTableEntry createNewTable(String tableName, List partitionColumns) { + return super.createNewTable(tableName, partitionColumns); + } + } + + public class MemoryObjectSchema extends MemorySchema { + + private final Set tables; + + public MemoryObjectSchema(MemorySchema memorySchema, String name, Set tables) { + super(memorySchema.getSchemaPath(), name); + this.tables = tables; + } + + @Override + public String getTypeName() { + return MemoryStoragePluginConfig.NAME; + } + + @Override + public Table getTable(final String table) { + + MemoryTable.MemoryTableDefinition definition = provider.table(schema, table); + + Object spec = null; + + MemoryTable mt = new MemoryTable(definition, schema, plugin, spec); + + return mt; + } + + @Override + public Set getTableNames() { + return tables; + } + } +} diff --git a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaProvider.java b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaProvider.java new file mode 100644 index 00000000000..ca411f9beec --- /dev/null +++ b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaProvider.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.memory; + +import com.google.common.base.Preconditions; + +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.store.memory.MemoryTable.MemoryTableDefinition; +import org.apache.drill.exec.store.memory.MemoryTable.MemoryColumnDefinition; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * + */ +public class MemorySchemaProvider { + + private static final Logger logger = LoggerFactory.getLogger(MemorySchemaProvider.class); + + public boolean exists(String schema) { + return false; + } + + public List schemas() { + return new ArrayList<>(); + } + + public Set tables(String schema) { + return null; + } + + public MemoryTableDefinition table(String schema, String table) { + return null; + } + + public void createSchema(String schema) { + ; + } + + public MemoryTableDefinition createTable(String schema, String table, Object prototype) { + return createTable(schema, table, prototype.getClass()); + } + + public MemoryTableDefinition createTable(String schema, String table, Class c) { + + Preconditions.checkNotNull(schema, "Schema name may not be null"); + Preconditions.checkNotNull(table, "Table name may not be null"); + Preconditions.checkNotNull(c, "Class may not be null"); + + List columns = new ArrayList<>(); + + for (Field field : c.getDeclaredFields()) { + + Class type = field.getType(); + + if (type.isPrimitive()) { + columns.add(new MemoryColumnDefinition( + field.getName(), + SchemaPath.getSimplePath(field.getName()), + true)); + } + else { + MemoryColumnDefinition column = createColumn(type); + } + } + + MemoryTableDefinition definition = new MemoryTableDefinition(schema, table, columns); + logger.info("Created new memory table: [{}]", definition); + + return definition; + } + + private MemoryColumnDefinition createColumn(Class c) { + + List columns = new ArrayList<>(); + + return null; + } +} diff --git a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryStoragePlugin.java b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryStoragePlugin.java new file mode 100644 index 00000000000..42aa0049233 --- /dev/null +++ b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryStoragePlugin.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.memory; + +import org.apache.calcite.schema.SchemaPlus; + +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.SchemaConfig; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * Storage plugin for in-memory data. + */ +public class MemoryStoragePlugin extends AbstractStoragePlugin { + + private static final Logger logger = LoggerFactory.getLogger(MemoryStoragePlugin.class); + + private final DrillbitContext context; + private final MemoryStoragePluginConfig config; + private final MemorySchemaFactory factory; + + public MemoryStoragePlugin(MemoryStoragePluginConfig config, DrillbitContext context, String name) { + this.context = context; + this.config = config; + this.factory = new MemorySchemaFactory(this, name); + } + + @Override + public StoragePluginConfig getConfig() { + return config; + } + + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { + factory.registerSchemas(schemaConfig, parent); + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public boolean supportsWrite() { + return false; + } + + public MemorySchemaFactory getFactory() { + return factory; + } +} diff --git a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryStoragePluginConfig.java b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryStoragePluginConfig.java new file mode 100644 index 00000000000..702de9c0c60 --- /dev/null +++ b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryStoragePluginConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.memory; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +import org.apache.drill.common.logical.StoragePluginConfig; + +import java.util.Map; + +/** + * Memory storage plugin config. + */ +@JsonTypeName(MemoryStoragePluginConfig.NAME) +public class MemoryStoragePluginConfig extends StoragePluginConfig { + + public static final String NAME = "memory"; + + private final Map config; + + @JsonCreator + public MemoryStoragePluginConfig(@JsonProperty("config") Map config) { + this.config = config; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + MemoryStoragePluginConfig that = (MemoryStoragePluginConfig) o; + return config.equals(that.config); + } +} diff --git a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryTable.java b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryTable.java new file mode 100644 index 00000000000..866bf56ade0 --- /dev/null +++ b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryTable.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.memory; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; + +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.store.StoragePlugin; + +import java.util.ArrayList; +import java.util.List; + +/** + * A table backed by memory. + */ +public class MemoryTable extends DrillTable { + + private final MemoryTableDefinition definition; + + public MemoryTable(MemoryTableDefinition definition, String storageEngineName, StoragePlugin plugin, + Object selection) { + super(storageEngineName, plugin, selection); + this.definition = definition; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return null; + } + + /** + * Schema in which this table resides. + */ + public String schema() { + return definition.schema; + } + + /** + * Name of the table. + */ + public String name() { + return definition.name; + } + + /** + * A list of this table's columns. + */ + public List columns() { + return definition.columns; + } + + @Override + public String toString() { + return "MemoryTable{" + + "definition=" + definition + + '}'; + } + + public static class MemoryTableDefinition { + + private final String schema; + private final String name; + private final List columns; + + public MemoryTableDefinition(String schema, String name, List columns) { + this.schema = schema; + this.name = name; + this.columns = columns; + } + + public String schema() { + return schema; + } + + public String name() { + return name; + } + + public List columns() { + return columns; + } + + @Override + public String toString() { + return "MemoryTableDefinition{" + + "schema='" + schema + '\'' + + ", name='" + name + '\'' + + ", columns=" + columns + + '}'; + } + } + + public static class MemoryColumnDefinition { + + private final String name; + private final SchemaPath path; + private final boolean nullable; + private final List children = new ArrayList<>(); + + public MemoryColumnDefinition(String name, SchemaPath path, boolean nullable) { + this.name = name; + this.path = path; + this.nullable = nullable; + } + + public String name() { + return name; + } + + public SchemaPath path() { + return path; + } + + public boolean nullable() { + return nullable; + } + + public boolean hasChildren() { + return children.size() > 0; + } + + public List children() { + return children; + } + + public void addChild(MemoryColumnDefinition memoryColumnDefinition) { + children.add(memoryColumnDefinition); + } + + @Override + public String toString() { + return "MemoryColumnDefinition{" + + "name='" + name + '\'' + + ", path=" + path + + ", nullable=" + nullable + + ", children=" + children + + '}'; + } + } +} diff --git a/contrib/storage-memory/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-memory/src/main/resources/bootstrap-storage-plugins.json new file mode 100644 index 00000000000..a73513f4e73 --- /dev/null +++ b/contrib/storage-memory/src/main/resources/bootstrap-storage-plugins.json @@ -0,0 +1,8 @@ +{ + "storage":{ + memory : { + type:"memory", + enabled: true + } + } +} diff --git a/contrib/storage-memory/src/main/resources/drill-module.conf b/contrib/storage-memory/src/main/resources/drill-module.conf new file mode 100644 index 00000000000..6c4d2b8e7d1 --- /dev/null +++ b/contrib/storage-memory/src/main/resources/drill-module.conf @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file tells Drill to consider this module when class path scanning. +// This file can also include any supplementary configuration information. +// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. + +drill.exec: { + sys.store.provider: { + memory : { + } + } +} diff --git a/contrib/storage-memory/src/test/java/org/apache/drill/exec/store/memory/TestMemoryStorePlugin.java b/contrib/storage-memory/src/test/java/org/apache/drill/exec/store/memory/TestMemoryStorePlugin.java new file mode 100644 index 00000000000..ae73777138c --- /dev/null +++ b/contrib/storage-memory/src/test/java/org/apache/drill/exec/store/memory/TestMemoryStorePlugin.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.memory; + +import org.apache.drill.BaseTestQuery; +import org.apache.drill.exec.store.StoragePluginRegistry; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; + +/** + * Integration tests for memory store plugin. + */ +public class TestMemoryStorePlugin extends BaseTestQuery { + + private static final Logger logger = LoggerFactory.getLogger(TestMemoryStorePlugin.class); + + private static final String TEST_SCHEMA = "test_schema"; + private static final String TEST_TABLE = "test_table"; + + @BeforeClass + public static void beforeClass() throws Exception { + MemoryStoragePluginConfig config = new MemoryStoragePluginConfig(new HashMap()); + StoragePluginRegistry registry = getDrillbitContext().getStorage(); + registry.createOrUpdate(MemoryStoragePluginConfig.NAME, config, true); + } + + @Test + public void testMemoryStorePlugin() throws Exception { + + MemoryStoragePlugin plugin = (MemoryStoragePlugin) getDrillbitContext().getStorage().getPlugin("memory"); + MemorySchemaFactory factory = plugin.getFactory(); + MemorySchemaProvider provider = factory.getSchemaProvider(); + + provider.createTable(TEST_SCHEMA, TEST_TABLE, SimpleClass.class); + provider.createTable(TEST_SCHEMA, TEST_TABLE, ComplexClass.class); + + /* + String sql = "select * from memory.`" + TEST_SCHEMA + "`." + TEST_TABLE; + testSqlWithResults(sql); + */ + } + + public static class SimpleClass { + int x; + double y; + String z; + } + + public static class ComplexClass { + int xxx; + double yyy; + String zzz; + SimpleClass a; + } +} diff --git a/contrib/storage-memory/src/test/resources/bootstrap-storage-plugins.json b/contrib/storage-memory/src/test/resources/bootstrap-storage-plugins.json new file mode 100644 index 00000000000..a73513f4e73 --- /dev/null +++ b/contrib/storage-memory/src/test/resources/bootstrap-storage-plugins.json @@ -0,0 +1,8 @@ +{ + "storage":{ + memory : { + type:"memory", + enabled: true + } + } +} diff --git a/contrib/storage-memory/src/test/resources/drill-module.conf b/contrib/storage-memory/src/test/resources/drill-module.conf new file mode 100644 index 00000000000..6c4d2b8e7d1 --- /dev/null +++ b/contrib/storage-memory/src/test/resources/drill-module.conf @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file tells Drill to consider this module when class path scanning. +// This file can also include any supplementary configuration information. +// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. + +drill.exec: { + sys.store.provider: { + memory : { + } + } +} diff --git a/contrib/storage-memory/src/test/resources/logback-test.xml b/contrib/storage-memory/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..38520b84095 --- /dev/null +++ b/contrib/storage-memory/src/test/resources/logback-test.xml @@ -0,0 +1,42 @@ + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 70194b2bbd1fc0089c1fb7d30ddd38546f7b3cfc Mon Sep 17 00:00:00 2001 From: andrew Date: Wed, 26 Aug 2015 13:06:23 -0700 Subject: [PATCH 2/2] Better type detection --- .../store/memory/MemorySchemaProvider.java | 46 ++++++++++++++----- .../drill/exec/store/memory/MemoryTable.java | 14 +++--- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaProvider.java b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaProvider.java index ca411f9beec..5ed021784db 100644 --- a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaProvider.java +++ b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemorySchemaProvider.java @@ -19,6 +19,10 @@ import com.google.common.base.Preconditions; +import org.apache.calcite.sql.type.SqlTypeName; + +import org.apache.commons.lang3.ClassUtils; + import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.store.memory.MemoryTable.MemoryTableDefinition; import org.apache.drill.exec.store.memory.MemoryTable.MemoryColumnDefinition; @@ -27,9 +31,12 @@ import org.slf4j.LoggerFactory; import java.lang.reflect.Field; +import java.sql.Date; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Set; +import java.sql.Timestamp; /** * @@ -75,13 +82,34 @@ public MemoryTableDefinition createTable(String schema, String table, Class c) { Class type = field.getType(); if (type.isPrimitive()) { - columns.add(new MemoryColumnDefinition( - field.getName(), - SchemaPath.getSimplePath(field.getName()), - true)); + + Class wrapper = ClassUtils.primitiveToWrapper(type); + SqlTypeName sqlType = SqlTypeName.get(wrapper.getSimpleName().toUpperCase(Locale.ENGLISH)); + + if (sqlType != null) { + columns.add(new MemoryColumnDefinition( + field.getName(), SchemaPath.getSimplePath(field.getName()), sqlType)); + } + else { + logger.warn("Unsupported data type: [{}]", type); + } } else { - MemoryColumnDefinition column = createColumn(type); + if (type == String.class) { + columns.add(new MemoryColumnDefinition( + field.getName(), SchemaPath.getSimplePath(field.getName()), SqlTypeName.VARCHAR)); + } + else if (type == Timestamp.class) { + columns.add(new MemoryColumnDefinition( + field.getName(), SchemaPath.getSimplePath(field.getName()), SqlTypeName.TIMESTAMP)); + } + else if (type == Date.class) { + columns.add(new MemoryColumnDefinition( + field.getName(), SchemaPath.getSimplePath(field.getName()), SqlTypeName.DATE)); + } + else { + logger.warn("Unsupported data type: [{}]", type); + } } } @@ -90,11 +118,5 @@ public MemoryTableDefinition createTable(String schema, String table, Class c) { return definition; } - - private MemoryColumnDefinition createColumn(Class c) { - - List columns = new ArrayList<>(); - - return null; - } + } diff --git a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryTable.java b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryTable.java index 866bf56ade0..5770ff85baf 100644 --- a/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryTable.java +++ b/contrib/storage-memory/src/main/java/org/apache/drill/exec/store/memory/MemoryTable.java @@ -17,9 +17,11 @@ */ package org.apache.drill.exec.store.memory; +import com.google.common.collect.Lists; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.planner.logical.DrillTable; import org.apache.drill.exec.store.StoragePlugin; @@ -111,13 +113,13 @@ public static class MemoryColumnDefinition { private final String name; private final SchemaPath path; - private final boolean nullable; + private final SqlTypeName type; private final List children = new ArrayList<>(); - public MemoryColumnDefinition(String name, SchemaPath path, boolean nullable) { + public MemoryColumnDefinition(String name, SchemaPath path, SqlTypeName type) { this.name = name; this.path = path; - this.nullable = nullable; + this.type = type; } public String name() { @@ -128,8 +130,8 @@ public SchemaPath path() { return path; } - public boolean nullable() { - return nullable; + public SqlTypeName type() { + return type; } public boolean hasChildren() { @@ -149,7 +151,7 @@ public String toString() { return "MemoryColumnDefinition{" + "name='" + name + '\'' + ", path=" + path + - ", nullable=" + nullable + + ", type=" + type + ", children=" + children + '}'; }