From a0222e972787a954679cad3f5b252f3c2b673e23 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 18 May 2016 01:07:50 +0530 Subject: [PATCH 01/94] Started new module for Mongodb --- pom.xml | 1 + tajo-storage/tajo-storage-mongodb/pom.xml | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tajo-storage/tajo-storage-mongodb/pom.xml diff --git a/pom.xml b/pom.xml index b39779343d..7de6ba69a7 100644 --- a/pom.xml +++ b/pom.xml @@ -97,6 +97,7 @@ tajo-metrics tajo-core-tests tajo-cluster-tests + tajo-storage/tajo-storage-mongodb diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml new file mode 100644 index 0000000000..e48537c194 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -0,0 +1,16 @@ + + + + tajo-project + org.apache.tajo + 0.12.0-SNAPSHOT + ../../tajo-project + + 4.0.0 + + tajo-storage-mongodb + + + \ No newline at end of file From 6c019d9fa57fdef1b8bc8d9a582a0676ff2f25f2 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 14:28:52 +0530 Subject: [PATCH 02/94] Created Tablespace --- tajo-storage/tajo-storage-mongodb/pom.xml | 309 +++++++++++++++++- .../storage/mongodb/MongoDbTableSpace.java | 151 +++++++++ 2 files changed, 459 insertions(+), 1 deletion(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index e48537c194..32a24c3e51 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -1,4 +1,21 @@ + @@ -10,7 +27,297 @@ 4.0.0 - tajo-storage-mongodb + tajo-storage-mongodbjar + Tajo MongoDB Storage + + UTF-8 + UTF-8 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.rat + apache-rat-plugin + + + src/main/resources/*.json + src/test/resources/*.json + + + + + verify + + check + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + TRUE + + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../../tajo-common/src/main/proto + --proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/StorageFragmentProtos.proto + + + + exec + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + + + + + org.apache.tajo + tajo-common + provided + + + org.apache.tajo + tajo-catalog-common + provided + + + org.apache.tajo + tajo-plan + provided + + + org.apache.tajo + tajo-storage-common + provided + + + org.apache.tajo + tajo-storage-hdfs + provided + + + org.apache.hadoop + hadoop-common + provided + + + zookeeper + org.apache.zookeeper + + + slf4j-api + org.slf4j + + + jersey-json + com.sun.jersey + + + + + org.apache.hadoop + hadoop-hdfs + provided + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + netty-all + io.netty + + + + + org.apache.hadoop + hadoop-minicluster + test + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + hadoop-yarn-server-tests + org.apache.hadoop + + + hadoop-mapreduce-client-jobclient + org.apache.hadoop + + + hadoop-mapreduce-client-app + org.apache.hadoop + + + hadoop-yarn-api + org.apache.hadoop + + + hadoop-mapreduce-client-hs + org.apache.hadoop + + + hadoop-mapreduce-client-core + org.apache.hadoop + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + com.google.protobuf + protobuf-java + + + junit + junit + test + + + org.apache.hbase + hbase-server + ${hbase.version} + provided + + + netty-all + io.netty + + + + + org.apache.hbase + hbase-client + ${hbase.version} + provided + + + jdk.tools + jdk.tools + + + + \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java new file mode 100644 index 0000000000..cf8adf90af --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mongodb; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; +import org.apache.hadoop.mapreduce.task.JobContextImpl; +import org.apache.tajo.*; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.exception.*; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.CreateTableNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; +import org.apache.tajo.plan.verifier.SyntaxErrorUtil; +import org.apache.tajo.storage.*; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.util.Bytes; +import org.apache.tajo.util.BytesUtils; +import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.Pair; + +import javax.annotation.Nullable; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; + +public class MongoDbTableSpace extends Tablespace { + + + public MongoDbTableSpace(String name, URI uri, JSONObject config) { + super(name, uri, config); + } + + @Override + protected void storageInit() throws IOException { + + } + + @Override + public long getTableVolume(TableDesc table, Optional filter) throws UnsupportedException { + return 0; + } + + @Override + public URI getTableUri(String databaseName, String tableName) { + return null; + } + + @Override + public List getSplits(String inputSourceId, TableDesc tableDesc, @Nullable EvalNode filterCondition) throws IOException, TajoException { + return null; + } + + @Override + public StorageProperty getProperty() { + return null; + } + + @Override + public FormatProperty getFormatProperty(TableMeta meta) { + return null; + } + + @Override + public void close() { + + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException { + return new TupleRange[0]; + } + + @Override + public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws TajoException { + + } + + @Override + public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { + + } + + @Override + public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { + + } + + @Override + public void prepareTable(LogicalNode node) throws IOException, TajoException { + + } + + @Override + public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException { + return null; + } + + @Override + public void rollbackTable(LogicalNode node) throws IOException, TajoException { + + } + + @Override + public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { + return null; + } +} From b19c629e8beb074910fb7319c807546ac96bdd8e Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 14:32:59 +0530 Subject: [PATCH 03/94] Improved import --- .../tajo/storage/mongodb/MongoDbFragment.java | 7 +++ .../storage/mongodb/MongoDbTableSpace.java | 54 ++++++------------- 2 files changed, 22 insertions(+), 39 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java new file mode 100644 index 0000000000..22af47d1da --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java @@ -0,0 +1,7 @@ +package org.apache.tajo.storage.mongodb; + +/** + * Created by janaka on 5/21/16. + */ +public class MongoDbFragment { +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java index cf8adf90af..8e3847d093 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java @@ -18,54 +18,30 @@ package org.apache.tajo.storage.mongodb; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Sets; import net.minidev.json.JSONObject; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; -import org.apache.hadoop.mapreduce.task.JobContextImpl; -import org.apache.tajo.*; -import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.exception.*; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.expr.*; -import org.apache.tajo.plan.logical.CreateTableNode; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.NodeType; -import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; -import org.apache.tajo.plan.verifier.SyntaxErrorUtil; -import org.apache.tajo.storage.*; +import org.apache.tajo.storage.FormatProperty; +import org.apache.tajo.storage.StorageProperty; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.util.Bytes; -import org.apache.tajo.util.BytesUtils; -import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.Pair; import javax.annotation.Nullable; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; import java.net.URI; -import java.util.*; -import java.util.stream.Collectors; +import java.util.List; +import java.util.Optional; public class MongoDbTableSpace extends Tablespace { From 98b99ac1f38255258038332a45079826ca096fb7 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 14:33:33 +0530 Subject: [PATCH 04/94] Created MongoDbFragment --- .../tajo/storage/mongodb/MongoDbFragment.java | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java index 22af47d1da..c4957e907f 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java @@ -1,7 +1,40 @@ package org.apache.tajo.storage.mongodb; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.storage.fragment.Fragment; + /** * Created by janaka on 5/21/16. */ -public class MongoDbFragment { +public class MongoDbFragment implements Fragment { + + @Override + public String getTableName() { + return null; + } + + @Override + public CatalogProtos.FragmentProto getProto() { + return null; + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String getKey() { + return null; + } + + @Override + public String[] getHosts() { + return new String[0]; + } + + @Override + public boolean isEmpty() { + return false; + } } From 6065d1919e7119e422bc30b72f640ea6c83020aa Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 14:35:26 +0530 Subject: [PATCH 05/94] Created MongoDb Scanner --- .../tajo/storage/mongodb/MongoDbScanner.java | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java new file mode 100644 index 0000000000..f110b68e9f --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java @@ -0,0 +1,82 @@ +package org.apache.tajo.storage.mongodb; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.storage.Scanner; +import org.apache.tajo.storage.Tuple; + +import java.io.IOException; +public class MongoDbScanner implements Scanner { + @Override + public void init() throws IOException { + + } + + @Override + public Tuple next() throws IOException { + return null; + } + + @Override + public void reset() throws IOException { + + } + + @Override + public void close() throws IOException { + + } + + @Override + public void pushOperators(LogicalNode planPart) { + + } + + @Override + public boolean isProjectable() { + return false; + } + + @Override + public void setTarget(Column[] targets) { + + } + + @Override + public boolean isSelectable() { + return false; + } + + @Override + public void setFilter(EvalNode filter) { + + } + + @Override + public void setLimit(long num) { + + } + + @Override + public boolean isSplittable() { + return false; + } + + @Override + public float getProgress() { + return 0; + } + + @Override + public TableStats getInputStats() { + return null; + } + + @Override + public Schema getSchema() { + return null; + } +} From 780d2855591380bcc6d69c72b334161534c93de7 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 14:38:32 +0530 Subject: [PATCH 06/94] Created MonogoDb Appender --- .../tajo/storage/mongodb/MongoDbAppender.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java new file mode 100644 index 0000000000..33bd70a205 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java @@ -0,0 +1,54 @@ +package org.apache.tajo.storage.mongodb; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.Tuple; + +import java.io.IOException; +import java.util.List; + +/** + * Created by janaka on 5/21/16. + */ +public class MongoDbAppender implements Appender { + @Override + public void init() throws IOException { + + } + + @Override + public void addTuple(Tuple t) throws IOException { + + } + + @Override + public void flush() throws IOException { + + } + + @Override + public long getEstimatedOutputSize() throws IOException { + return 0; + } + + @Override + public void close() throws IOException { + + } + + @Override + public void enableStats() { + + } + + @Override + public void enableStats(List columnList) { + + } + + @Override + public TableStats getStats() { + return null; + } +} From 3231ea9303ad547b3129ac2d8faecac67ab0bd97 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 14:44:44 +0530 Subject: [PATCH 07/94] Added ConnectionInfo --- .../apache/tajo/storage/mongodb/ConnectionInfo.java | 13 +++++++++++++ .../org/apache/tajo/storage/TestConnectionInfo.java | 5 +++++ 2 files changed, 18 insertions(+) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/TestConnectionInfo.java diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java new file mode 100644 index 0000000000..587143f911 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -0,0 +1,13 @@ +package org.apache.tajo.storage.mongodb; + + +public class ConnectionInfo { + private String mongoDbUrl; + public String getMongoDbUrl() { + return mongoDbUrl; + } + + public void setMongoDbUrl(String mongoDbUrl) { + this.mongoDbUrl = mongoDbUrl; + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/TestConnectionInfo.java new file mode 100644 index 0000000000..484a444914 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/TestConnectionInfo.java @@ -0,0 +1,5 @@ +package org.apache.tajo.storage; + + +public class TestConnectionInfo { +} From bf3be8f6bccfc029477751ebb1424eee161f269b Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 14:48:13 +0530 Subject: [PATCH 08/94] Added missing Licensed details --- .../tajo/storage/mongodb/ConnectionInfo.java | 17 +++++++++++++++++ .../tajo/storage/mongodb/MongoDbAppender.java | 17 +++++++++++++++++ .../tajo/storage/mongodb/MongoDbFragment.java | 17 +++++++++++++++++ .../tajo/storage/mongodb/MongoDbScanner.java | 17 +++++++++++++++++ 4 files changed, 68 insertions(+) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 587143f911..5d70cf8e33 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tajo.storage.mongodb; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java index 33bd70a205..eba72e559c 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tajo.storage.mongodb; import org.apache.tajo.catalog.Column; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java index c4957e907f..8e25fb356b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tajo.storage.mongodb; import org.apache.tajo.catalog.proto.CatalogProtos; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java index f110b68e9f..4c80c904bc 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tajo.storage.mongodb; import org.apache.tajo.catalog.Column; From 6568afc6409184e591098c387863cd5a2b26be84 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 15:02:17 +0530 Subject: [PATCH 09/94] Removed Plugins --- tajo-storage/tajo-storage-mongodb/pom.xml | 149 +++++------------- .../tajo/storage/TestConnectionInfo.java | 5 - .../storage/mongodb/TestConnectionInfo.java | 35 ++++ 3 files changed, 75 insertions(+), 114 deletions(-) delete mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/TestConnectionInfo.java create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 32a24c3e51..c9557fa629 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -37,115 +37,7 @@ - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.rat - apache-rat-plugin - - - src/main/resources/*.json - src/test/resources/*.json - - - - - verify - - check - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - TRUE - - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null - - - - org.apache.maven.plugins - maven-jar-plugin - 2.4 - - - - test-jar - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - create-protobuf-generated-sources-directory - initialize - - - - - - - run - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.2 - - - generate-sources - generate-sources - - protoc - - -Isrc/main/proto/ - --proto_path=../../tajo-common/src/main/proto - --proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto - --java_out=target/generated-sources/proto - src/main/proto/StorageFragmentProtos.proto - - - - exec - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 1.5 - - - add-source - generate-sources - - add-source - - - - target/generated-sources/proto - - - - - - - org.apache.maven.plugins - maven-surefire-report-plugin - + @@ -320,4 +212,43 @@ + + + docs + + false + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + module-javadocs + package + + jar + + + ${project.build.directory} + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/TestConnectionInfo.java deleted file mode 100644 index 484a444914..0000000000 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/TestConnectionInfo.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.apache.tajo.storage; - - -public class TestConnectionInfo { -} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java new file mode 100644 index 0000000000..efbb525a8d --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + + +import org.apache.tajo.storage.mongodb.ConnectionInfo; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestConnectionInfo { + @Test + public final void testGetConnectionInfoType1() { + String url = "mongodb:bla blabla"; + ConnectionInfo connectionInfo = ConnectionInfo.fromUrl(url); + + assertEquals(connectionInfo.getMongoDbUrl(), url); + + } +} From c7fec80fe9a2267717a79376bdd3b5530ee74093 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Sat, 21 May 2016 15:03:11 +0530 Subject: [PATCH 10/94] Added a dummy test code --- .../org/apache/tajo/storage/mongodb/ConnectionInfo.java | 7 +++++++ .../apache/tajo/storage/mongodb/TestConnectionInfo.java | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 5d70cf8e33..75e1747722 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -27,4 +27,11 @@ public String getMongoDbUrl() { public void setMongoDbUrl(String mongoDbUrl) { this.mongoDbUrl = mongoDbUrl; } + + public static ConnectionInfo fromUrl(String url) + { + ConnectionInfo connectionInfo = new ConnectionInfo(); + connectionInfo.setMongoDbUrl(url); + return connectionInfo; + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index efbb525a8d..60e3390d09 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -17,8 +17,6 @@ */ package org.apache.tajo.storage.mongodb; - -import org.apache.tajo.storage.mongodb.ConnectionInfo; import org.junit.Test; import static org.junit.Assert.assertEquals; From 36f2738b1894f8d68f3e77f21904aba684035129 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Mon, 23 May 2016 15:16:44 +0530 Subject: [PATCH 11/94] Changed fragment to abstract class --- .../tajo/storage/mongodb/MongoDbFragment.java | 29 ++++--------------- .../storage/mongodb/MongoDbTableSpace.java | 4 ++- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java index 8e25fb356b..69faf39510 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java @@ -20,38 +20,21 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.storage.fragment.Fragment; +import java.net.URI; + /** * Created by janaka on 5/21/16. */ -public class MongoDbFragment implements Fragment { - - @Override - public String getTableName() { - return null; - } - - @Override - public CatalogProtos.FragmentProto getProto() { - return null; - } +public class MongoDbFragment extends Fragment { - @Override - public long getLength() { - return 0; + protected MongoDbFragment(String kind, URI uri, String inputSourceId, Long startKey, Long endKey, long length, String[] hostNames) { + super(kind, uri, inputSourceId, startKey, endKey, length, hostNames); } - @Override - public String getKey() { - return null; - } - - @Override - public String[] getHosts() { - return new String[0]; - } @Override public boolean isEmpty() { return false; } + } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java index 8e3847d093..966faeec12 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java @@ -66,10 +66,12 @@ public URI getTableUri(String databaseName, String tableName) { } @Override - public List getSplits(String inputSourceId, TableDesc tableDesc, @Nullable EvalNode filterCondition) throws IOException, TajoException { + public List getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { return null; } + + @Override public StorageProperty getProperty() { return null; From 531ae8cb1462501267b5ea20bab2290e20d1df58 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 25 May 2016 18:34:32 +0530 Subject: [PATCH 12/94] Implemented connection info for single host --- tajo-storage/tajo-storage-mongodb/pom.xml | 5 + .../tajo/storage/mongodb/ConnectionInfo.java | 105 ++++++++++++++++-- .../storage/mongodb/TestConnectionInfo.java | 54 ++++++++- 3 files changed, 150 insertions(+), 14 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index c9557fa629..c6fe17a7a1 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -210,6 +210,11 @@ + + org.mongodb + mongodb-driver + 3.2.2 + diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 75e1747722..cc3eaf54f9 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -18,20 +18,103 @@ package org.apache.tajo.storage.mongodb; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +import com.mongodb.*; +import org.apache.tajo.exception.TajoInternalError; + public class ConnectionInfo { - private String mongoDbUrl; - public String getMongoDbUrl() { - return mongoDbUrl; - } + MongoClientURI mongoDbURI; + String scheme; + String host; + String dbName; + String tableName; + String user; + String password; + int port; + Map params; - public void setMongoDbUrl(String mongoDbUrl) { - this.mongoDbUrl = mongoDbUrl; + public static ConnectionInfo fromURI(String originalUri) { + return fromURI(URI.create(originalUri)); } - public static ConnectionInfo fromUrl(String url) - { - ConnectionInfo connectionInfo = new ConnectionInfo(); - connectionInfo.setMongoDbUrl(url); - return connectionInfo; + public static ConnectionInfo fromURI(URI originalUri) { + final String uriStr = originalUri.toASCIIString(); + URI uri = originalUri; + + final ConnectionInfo connInfo = new ConnectionInfo(); + connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); + + connInfo.host = uri.getHost(); + connInfo.port = uri.getPort(); + + //Set the db name + String path = uri.getPath(); + if (path != null && !path.isEmpty()) { + String [] pathElements = path.substring(1).split("/"); + if (pathElements.length != 1) { + throw new TajoInternalError("Invalid JDBC path: " + path); + } + connInfo.dbName = pathElements[0]; + } + + //Convert parms into a Map + Map params = new HashMap<>(); + int paramIndex = uriStr.indexOf("?"); + if (paramIndex > 0) { + String parameterPart = uriStr.substring(paramIndex+1, uriStr.length()); + + String [] eachParam = parameterPart.split("&"); + + for (String each: eachParam) { + String [] keyValues = each.split("="); + if (keyValues.length != 2) { + throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); + } + params.put(keyValues[0], keyValues[1]); + } + } + + if (params.containsKey("table")) { + connInfo.tableName = params.remove("table"); + } + + if (params.containsKey("user")) { + connInfo.user = params.remove("user"); + } + if (params.containsKey("password")) { + connInfo.password = params.remove("password"); + } + + + connInfo.params = params; + + String mongoDbURIStr = ""; + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] + + //Generate the URI + mongoDbURIStr+=connInfo.scheme; + mongoDbURIStr+="://"; + if(connInfo.user!=null) + { + mongoDbURIStr+=connInfo.user; + if(connInfo.password!=null) + mongoDbURIStr+=":"+connInfo.password; + mongoDbURIStr+="@"; + } + mongoDbURIStr+=connInfo.host; + mongoDbURIStr+=":"; + mongoDbURIStr+=connInfo.port; + mongoDbURIStr+="/"; + mongoDbURIStr+=connInfo.dbName; + + connInfo.mongoDbURI = new MongoClientURI(mongoDbURIStr); + + return connInfo; } + } + + diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index 60e3390d09..c497a4829b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -22,12 +22,60 @@ import static org.junit.Assert.assertEquals; public class TestConnectionInfo { + @Test public final void testGetConnectionInfoType1() { - String url = "mongodb:bla blabla"; - ConnectionInfo connectionInfo = ConnectionInfo.fromUrl(url); - assertEquals(connectionInfo.getMongoDbUrl(), url); + ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass"); + +// System.out.println(connInfo.scheme); +// System.out.println(connInfo.dbName); +// System.out.println(connInfo.host); +// System.out.println(connInfo.port); +// System.out.println(connInfo.tableName); +// System.out.println(connInfo.password); +// System.out.println(connInfo.user); +// System.out.println(connInfo.mongoDbURI.getURI()); + + assertEquals(connInfo.mongoDbURI.getURI(),"mongodb://testuser:testpass@localhost:1336/db1"); + assertEquals(connInfo.scheme, "mongodb"); + asssertEquals(connInfo.host, "localhost"); + assertEquals(connInfo.port, 1336); + assertEquals(connInfo.dbName, "db1"); + assertEquals(connInfo.user, "testuser"); + assertEquals(connInfo.password, "testpass"); + assertEquals(connInfo.tableName, "tb1"); + } + + @Test + public final void testGetConnectionInfoType2() { + ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); + assertEquals(connInfo.scheme, "mongodb"); + assertEquals(connInfo.host, "localhost"); + assertEquals(connInfo.port, 1336); + assertEquals(connInfo.dbName, "db1"); + assertEquals(connInfo.user, "testuser"); + assertEquals(connInfo.password, "testpass"); + assertEquals(connInfo.tableName, "tb1"); + assertEquals(1, connInfo.params.size()); + assertEquals("GMT+9", connInfo.params.get("TZ")); } + + //Test code for multiple hosts +// @Test +// public final void testGetConnectionInfoType3() { +// +// ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336,googl.com:2727/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); +// assertEquals(connInfo.scheme, "mongodb"); +// assertEquals(connInfo.host, "localhost"); +// assertEquals(connInfo.port, 1336); +// assertEquals(connInfo.dbName, "db1"); +// assertEquals(connInfo.user, "testuser"); +// assertEquals(connInfo.password, "testpass"); +// assertEquals(connInfo.tableName, "tb1"); +// assertEquals(1, connInfo.params.size()); +// assertEquals("GMT+9", connInfo.params.get("TZ")); +// } + } From 43f7c72dbf691e9f29b3f4020b8bc4523875ae23 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 25 May 2016 18:48:09 +0530 Subject: [PATCH 13/94] fixed an trivial bug --- .../org/apache/tajo/storage/mongodb/TestConnectionInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index c497a4829b..4625d9645f 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -39,8 +39,8 @@ public final void testGetConnectionInfoType1() { assertEquals(connInfo.mongoDbURI.getURI(),"mongodb://testuser:testpass@localhost:1336/db1"); assertEquals(connInfo.scheme, "mongodb"); - asssertEquals(connInfo.host, "localhost"); - assertEquals(connInfo.port, 1336); + assertEquals(connInfo.host, "localhost"); + assertEquals(connInfo.port, 1336); assertEquals(connInfo.dbName, "db1"); assertEquals(connInfo.user, "testuser"); assertEquals(connInfo.password, "testpass"); From c46cd9b82fc4dd621e5bf046c8010fc10d6ee6df Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Thu, 26 May 2016 16:48:59 +0530 Subject: [PATCH 14/94] Changed, pom.xml storeg to copy jars --- tajo-storage/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index 028732aa17..f0d3a87b09 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -138,6 +138,7 @@ run cp -r ${basedir}/tajo-storage-hdfs/target/tajo-storage-hdfs-${project.version}*.jar . run cp -r ${basedir}/tajo-storage-hbase/target/tajo-storage-hbase-${project.version}*.jar . run cp -r ${basedir}/tajo-storage-s3/target/tajo-storage-s3-${project.version}*.jar . + run cp -r ${basedir}/tajo-storage-mongodb/target/tajo-storage-mongodb-${project.version}*.jar . echo echo "Tajo Storage dist layout available at: ${project.build.directory}/tajo-storage-${project.version}" From 4d1019c321bc14e96bd38904c84f30d3aae75f47 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 27 May 2016 12:38:12 +0530 Subject: [PATCH 15/94] Update TestConnectionInfo.java --- .../org/apache/tajo/storage/mongodb/TestConnectionInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index 4625d9645f..c4e1fdc2ae 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -49,7 +49,7 @@ public final void testGetConnectionInfoType1() { @Test public final void testGetConnectionInfoType2() { - + //Create a connection info object ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); assertEquals(connInfo.scheme, "mongodb"); assertEquals(connInfo.host, "localhost"); From e8adf195022388e8b3864835a2b9a2b1f1850d04 Mon Sep 17 00:00:00 2001 From: Janaka Date: Mon, 30 May 2016 13:50:52 +0530 Subject: [PATCH 16/94] Added Storage and Format Property --- .../apache/tajo/storage/mongodb/MongoDbTableSpace.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java index 966faeec12..0ddbc13b66 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java @@ -46,6 +46,10 @@ public class MongoDbTableSpace extends Tablespace { + + static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, false); + static final FormatProperty FORMAT_PROPERTY = new FormatProperty(false, false, false); + public MongoDbTableSpace(String name, URI uri, JSONObject config) { super(name, uri, config); } @@ -74,12 +78,12 @@ public List getSplits(String inputSourceId, TableDesc tableDesc, boole @Override public StorageProperty getProperty() { - return null; + return STORAGE_PROPERTY; } @Override public FormatProperty getFormatProperty(TableMeta meta) { - return null; + return FORMAT_PROPERTY; } @Override From 3d48fbb7b31472c8374b2658eb8ecec603d57f7f Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 1 Jun 2016 08:33:38 +0900 Subject: [PATCH 17/94] TAJO-2148: Implement an example HTTP tablespace. --- pom.xml | 3 +- .../org/apache/tajo/catalog/CatalogUtil.java | 47 +--- .../src/main/proto/CatalogProtos.proto | 16 -- .../InfoSchemaMetadataDictionary.java | 4 +- .../java/org/apache/tajo/TpchTestBase.java | 12 + .../java/org/apache/tajo/BuiltinStorages.java | 2 + .../apache/tajo/exception/ErrorMessages.java | 1 + .../apache/tajo/exception/ExceptionUtil.java | 2 +- tajo-common/src/main/proto/errors.proto | 1 + .../apache/tajo/cli/tools/TestTajoDump.java | 10 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 4 +- .../tajo/master/exec/CreateTableExecutor.java | 2 +- tajo-dist/pom.xml | 1 + tajo-jdbc/pom.xml | 1 - .../org/apache/tajo/jdbc/TestResultSet.java | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 5 +- .../org/apache/tajo/plan/StorageService.java | 3 +- .../plan/rewrite/BaseSchemaBuildPhase.java | 4 + .../rewrite/SelfDescSchemaBuildPhase.java | 7 + .../plan/verifier/PreLogicalPlanVerifier.java | 3 +- .../org/apache/tajo/storage/Tablespace.java | 5 +- .../tajo/storage/TablespaceManager.java | 5 +- .../fragment/BuiltinFragmentKinds.java | 1 + .../src/main/resources/storage-default.json | 4 + .../src/main/resources/storage-default.xml | 17 +- .../src/test/resources/storage-default.xml | 7 +- .../tajo/storage/hbase/HBaseTablespace.java | 2 +- .../org/apache/tajo/storage/FileScanner.java | 21 +- .../apache/tajo/storage/FileTablespace.java | 4 +- .../java/org/apache/tajo/storage/RawFile.java | 10 +- .../fragment/AbstractFileFragment.java | 46 ++++ .../tajo/storage/fragment/FileFragment.java | 2 +- .../apache/tajo/storage/orc/OrcScanner.java | 4 +- .../storage/rawfile/DirectRawFileScanner.java | 16 +- .../apache/tajo/storage/rcfile/RCFile.java | 12 +- .../sequencefile/SequenceFileScanner.java | 6 +- .../storage/text/DelimitedLineReader.java | 15 +- .../tajo/storage/text/DelimitedTextFile.java | 95 +++---- .../thirdparty/orc/OrcRecordReader.java | 4 +- .../jdbc/JdbcMetadataProviderBase.java | 5 +- .../tajo/storage/jdbc/JdbcTablespace.java | 6 +- tajo-tablespace-example/pom.xml | 256 ++++++++++++++++++ .../storage/http/ExampleHttpFileFragment.java | 44 +++ .../http/ExampleHttpFileFragmentSerde.java | 54 ++++ .../http/ExampleHttpFileTablespace.java | 231 ++++++++++++++++ .../http/ExampleHttpJsonLineReader.java | 156 +++++++++++ .../storage/http/ExampleHttpJsonScanner.java | 249 +++++++++++++++++ .../proto/ExampleHttpFragmentProtos.proto | 32 +++ .../http/ExampleHttpServerHandler.java | 147 ++++++++++ .../http/ExampleHttpServerInitializer.java | 39 +++ .../http/ExampleHttpTablespaceTestServer.java | 67 +++++ .../http/TestExampleHttpFileQuery.java | 86 ++++++ .../src/test/resources/dataset/github.json | 4 + .../src/test/resources/dataset/got.json | 3 + .../TestExampleHttpFileQuery/testGroupby.sql | 1 + .../TestExampleHttpFileQuery/testJoin.sql | 1 + .../TestExampleHttpFileQuery/testSelect.sql | 1 + .../TestExampleHttpFileQuery/testSort.sql | 1 + .../testGroupby.result | 6 + .../TestExampleHttpFileQuery/testJoin.result | 5 + .../testSelect.result | 5 + .../TestExampleHttpFileQuery/testSort.result | 6 + tajo-yarn/pom.xml | 1 - 63 files changed, 1632 insertions(+), 180 deletions(-) create mode 100644 tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/AbstractFileFragment.java create mode 100644 tajo-tablespace-example/pom.xml create mode 100644 tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragment.java create mode 100644 tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragmentSerde.java create mode 100644 tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileTablespace.java create mode 100644 tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpJsonLineReader.java create mode 100644 tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpJsonScanner.java create mode 100644 tajo-tablespace-example/src/main/proto/ExampleHttpFragmentProtos.proto create mode 100644 tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpServerHandler.java create mode 100644 tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpServerInitializer.java create mode 100644 tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpTablespaceTestServer.java create mode 100644 tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/TestExampleHttpFileQuery.java create mode 100644 tajo-tablespace-example/src/test/resources/dataset/github.json create mode 100644 tajo-tablespace-example/src/test/resources/dataset/got.json create mode 100644 tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testGroupby.sql create mode 100644 tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testJoin.sql create mode 100644 tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testSelect.sql create mode 100644 tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testSort.sql create mode 100644 tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testGroupby.result create mode 100644 tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testJoin.result create mode 100644 tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testSelect.result create mode 100644 tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testSort.result diff --git a/pom.xml b/pom.xml index 71c062bbfa..1b9e99ef52 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,7 @@ tajo-core-tests tajo-cluster-tests tajo-dist + tajo-tablespace-example @@ -144,7 +145,7 @@ maven-assembly-plugin - 2.3 + 2.4.1 org.apache.maven.plugins diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index f15ce030a0..98e8ceabcb 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -26,7 +26,10 @@ import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.*; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; +import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TableIdentifierProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.conf.TajoConf; @@ -52,42 +55,12 @@ public class CatalogUtil { public static String getBackwardCompitableDataFormat(String dataFormat) { - return getDataFormatAsString(asDataFormat(dataFormat)); - } - - public static String getDataFormatAsString(final DataFormat type) { - if (type == DataFormat.TEXTFILE) { - return BuiltinStorages.TEXT; - } else { - return type.name(); - } - } - - public static DataFormat asDataFormat(final String typeStr) { - if (typeStr.equalsIgnoreCase("CSV")) { - return DataFormat.TEXTFILE; - } else if (typeStr.equalsIgnoreCase(DataFormat.RAW.name())) { - return CatalogProtos.DataFormat.RAW; - } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.ROWFILE.name())) { - return DataFormat.ROWFILE; - } else if (typeStr.equalsIgnoreCase(DataFormat.RCFILE.name())) { - return DataFormat.RCFILE; - } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.ORC.name())) { - return CatalogProtos.DataFormat.ORC; - } else if (typeStr.equalsIgnoreCase(DataFormat.PARQUET.name())) { - return DataFormat.PARQUET; - } else if (typeStr.equalsIgnoreCase(DataFormat.SEQUENCEFILE.name())) { - return DataFormat.SEQUENCEFILE; - } else if (typeStr.equalsIgnoreCase(DataFormat.AVRO.name())) { - return CatalogProtos.DataFormat.AVRO; - } else if (typeStr.equalsIgnoreCase(BuiltinStorages.TEXT)) { - return CatalogProtos.DataFormat.TEXTFILE; - } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.JSON.name())) { - return CatalogProtos.DataFormat.JSON; - } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.HBASE.name())) { - return CatalogProtos.DataFormat.HBASE; - } else { - return null; + switch (dataFormat) { + case "CSV": + case "TEXTFILE": + return BuiltinStorages.TEXT; + default: + return dataFormat; } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index b42cf58a5f..ea5f933bd0 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -25,22 +25,6 @@ option java_generate_equals_and_hash = true; import "DataTypes.proto"; import "PrimitiveProtos.proto"; -enum DataFormat { - MEM = 0; - TEXTFILE = 1; - RAW = 2; - RCFILE = 3; - ROWFILE = 4; - HCFILE = 5; - ORC = 6; - PARQUET = 7; - SEQUENCEFILE = 8; - AVRO = 9; - JSON = 10; - HBASE = 11; - SYSTEM = 12; -} - enum OrderType { ORDER_NONE = 0; ASC = 1; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java index d9008aa1b7..ea608ebfca 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java @@ -22,9 +22,9 @@ import java.util.Collections; import java.util.List; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.exception.UndefinedTableException; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.DataFormat; public class InfoSchemaMetadataDictionary { private static final String DATABASE_NAME = "information_schema"; @@ -129,6 +129,6 @@ public boolean existTable(String tableName) { } protected String getTablePath() { - return DataFormat.SYSTEM.name().toUpperCase(); + return BuiltinStorages.SYSTEM; } } diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java index 70d25b48e6..5e2f654264 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java @@ -18,6 +18,7 @@ package org.apache.tajo; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -103,6 +104,17 @@ public TajoTestingCluster getTestingCluster() { return util.getTestingCluster(); } + public ImmutableList getNames() { + return ImmutableList.copyOf(names); + } + + public Schema getSchema(String tableName) { + if (!nameMap.containsKey(tableName)) { + throw new RuntimeException("No such a table name '" + tableName + "'"); + } + return schemas[nameMap.get(tableName)]; + } + public String getPath(String tableName) { if (!nameMap.containsKey(tableName)) { throw new RuntimeException("No such a table name '" + tableName + "'"); diff --git a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java index 2c8f68617c..aa7a9e73f5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java +++ b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java @@ -30,4 +30,6 @@ public class BuiltinStorages { public static final String SEQUENCE_FILE = "SEQUENCEFILE"; public static final String AVRO = "AVRO"; public static final String HBASE = "HBASE"; + public static final String SYSTEM = "SYSTEM"; + public static final String EX_HTTP_JSON = "EX_HTTP_JSON"; } diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index a6222cf9fb..8f45bdd0ce 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -112,6 +112,7 @@ public class ErrorMessages { ADD_MESSAGE(UNSUPPORTED_DATATYPE, "unsupported data type: '%s'", 1); ADD_MESSAGE(INVALID_TABLE_PROPERTY, "invalid table property '%s': '%s'", 2); ADD_MESSAGE(MISSING_TABLE_PROPERTY, "table property '%s' required for '%s'", 2); + ADD_MESSAGE(INVALID_TABLESPACE_URI, "Invalid tablespace '%s' for table '%s'", 2); ADD_MESSAGE(AMBIGUOUS_PARTITION_DIRECTORY, "There is a directory which is assumed to be a partitioned directory" + " : '%s'", 1); diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java index adfb0c4a45..bdcdd42beb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java @@ -205,7 +205,7 @@ public static void printStackTraceIfError(Log log, Throwable t) { // if this runs as an actual cluster instance or a debug mode, it will print all stacktraces. // In other cases (i.e., run as a test mode and not debug mode), it will print stacktraces // if the query is managed mode. - if (!TajoConstants.IS_TEST_MODE || TajoConstants.IS_DEBUG_MODE || !ExceptionUtil.isManagedException(t)) { + if (TajoConstants.IS_TEST_MODE || TajoConstants.IS_DEBUG_MODE || !ExceptionUtil.isManagedException(t)) { ExceptionUtil.printStackTrace(log, t); } } diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index 6e13eb1490..cfb7242c96 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -172,6 +172,7 @@ enum ResultCode { UNSUPPORTED_DATATYPE = 1003; // SQLState: ? - Unsupported data type INVALID_TABLE_PROPERTY = 1004; // SQLState: ? - Invalid Table Property MISSING_TABLE_PROPERTY = 1005; // SQLState: ? - Missing table property + INVALID_TABLESPACE_URI = 1006; // Client Connection CLIENT_CONNECTION_EXCEPTION = 1101; // SQLState: 08000 - Client connection error diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index 57a6aa3ffe..2fb67c3e83 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.auth.UserRoleInfo; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.FileUtil; @@ -98,10 +99,11 @@ public void testDump4() throws Exception { TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); printWriter.flush(); printWriter.close(); + TableMeta meta = client.getTableDesc(getCurrentDatabase() + "\".\"TableName1\"").getMeta(); assertOutputResult("testDump3.result", new String(bos.toByteArray()), new String[]{"${index.path}", "${table.timezone}"}, - new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString(), + new String[]{TablespaceManager.getDefault().getTableUri(meta, getCurrentDatabase(), "test_idx").toString(), testingCluster.getConfiguration().getSystemTimezone().getID()}); bos.close(); } finally { @@ -137,10 +139,12 @@ public void testPartitionsDump() throws Exception { printWriter.flush(); printWriter.close(); + TableMeta meta = client.getTableDesc(getCurrentDatabase() + "\".\"TableName3\"").getMeta(); + assertOutputResult("testPartitionsDump.result", new String(bos.toByteArray()), new String[]{"${partition.path1}", "${partition.path2}", "${table.timezone}"}, - new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName3").toString(), - TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName4").toString(), + new String[]{TablespaceManager.getDefault().getTableUri(meta, getCurrentDatabase(), "TableName3").toString(), + TablespaceManager.getDefault().getTableUri(meta, getCurrentDatabase(), "TableName4").toString(), testingCluster.getConfiguration().getSystemTimezone().getID()}); bos.close(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 47c5a35639..816fbcda63 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -29,6 +29,7 @@ import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TpchTestBase; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.cli.tsql.commands.TajoShellCommand; import org.apache.tajo.client.ClientParameters; import org.apache.tajo.client.QueryStatus; @@ -252,9 +253,10 @@ private void verifyDescTable(String sql, String tableName, String resultFileName String consoleResult = new String(out.toByteArray()); if (!cluster.isHiveCatalogStoreRunning()) { + TableMeta meta = cluster.getCatalogService().getTableDesc("default", tableName).getMeta(); assertOutputResult(resultFileName, consoleResult, new String[]{"${table.timezone}", "${table.path}"}, new String[]{cluster.getConfiguration().getSystemTimezone().getID(), - TablespaceManager.getDefault().getTableUri("default", tableName).toString()}); + TablespaceManager.getDefault().getTableUri(meta, "default", tableName).toString()}); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java index 9c3a1ad0b1..a95239c777 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java @@ -101,7 +101,7 @@ public TableDesc create(QueryContext queryContext, Tablespace tableSpace = getTablespaceHandler(tableSpaceName, uri); TableDesc desc; - URI tableUri = isExternal ? uri : tableSpace.getTableUri(databaseName, simpleTableName); + URI tableUri = isExternal ? uri : tableSpace.getTableUri(meta, databaseName, simpleTableName); desc = new TableDesc(qualifiedName, schema, meta, tableUri, isExternal); if (partitionDesc != null) { diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml index 7280e1f2fc..cbbd763a08 100644 --- a/tajo-dist/pom.xml +++ b/tajo-dist/pom.xml @@ -146,6 +146,7 @@ run cp -r $ROOT/tajo-sql-parser/target/tajo-sql-parser-${project.version}/* . run cp -r $ROOT/tajo-storage/tajo-storage-jdbc/target/tajo-storage-jdbc-${project.version}.jar . run cp -r $ROOT/tajo-storage/tajo-storage-pgsql/target/tajo-storage-pgsql-${project.version}.jar . + run cp -r $ROOT/tajo-tablespace-example/target/tajo-tablespace-example-${project.version}.jar . run cp -r $ROOT/tajo-pullserver/target/tajo-pullserver-${project.version}.jar . run cp -r $ROOT/tajo-metrics/target/tajo-metrics-${project.version}.jar . run cp -r $ROOT/tajo-core/target/tajo-core-${project.version}.jar . diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml index b42d90de3f..fc9f39dbbe 100644 --- a/tajo-jdbc/pom.xml +++ b/tajo-jdbc/pom.xml @@ -76,7 +76,6 @@ maven-assembly-plugin - 2.4.1 jar-with-dependencies diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index 3ed37874a7..af4011d5b4 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -76,7 +76,7 @@ public static void setup() throws Exception { rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(scoreSchema)); TableStats stats = new TableStats(); - Path p = new Path(sm.getTableUri("default", "score")); + Path p = new Path(sm.getTableUri(scoreMeta, "default", "score")); sm.getFileSystem().mkdirs(p); Appender appender = sm.getAppender(scoreMeta, scoreSchema, new Path(p, "score")); appender.init(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index b64665839c..51b2568ce8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -2009,7 +2009,10 @@ private URI getCreatedTableURI(PlanContext context, CreateTable createTable) { IdentifierUtil.extractQualifier(tableName) : context.queryContext.get(SessionVars.CURRENT_DATABASE); return storage.getTableURI( - createTable.getTableSpaceName(), databaseName, IdentifierUtil.extractSimpleName(tableName)); + createTable.getTableSpaceName(), + new TableMeta(createTable.getStorageType(), new KeyValueSet(createTable.getParams())), + databaseName, + IdentifierUtil.extractSimpleName(tableName)); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java index cbb7387146..73fdecad7d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.expr.EvalNode; @@ -39,7 +40,7 @@ public interface StorageService { * @param tableName Table name * @return Table URI */ - URI getTableURI(@Nullable String spaceName, String databaseName, String tableName); + URI getTableURI(@Nullable String spaceName, TableMeta meta, String databaseName, String tableName); long getTableVolumn(TableDesc table, Optional filter) throws UnsupportedException; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java index 14fb3ef8dd..88ad841f06 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java @@ -293,6 +293,10 @@ public LogicalNode visitLimit(LogicalPlanner.PlanContext ctx, Stack stack, public LogicalNode visitSort(LogicalPlanner.PlanContext ctx, Stack stack, Sort expr) throws TajoException { stack.push(expr); LogicalNode child = visit(ctx, stack, expr.getChild()); + + for (Sort.SortSpec sortSpec : expr.getSortSpecs()) { + NameRefInSelectListNormalizer.normalize(ctx, sortSpec.getKey()); + } stack.pop(); SortNode sortNode = ctx.getPlan().createNode(SortNode.class); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java index a49c42389f..256e34ee03 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java @@ -206,6 +206,13 @@ public LogicalNode visitLimit(ProcessorContext ctx, Stack stack, Limit exp @Override public LogicalNode visitSort(ProcessorContext ctx, Stack stack, Sort expr) throws TajoException { + for (Sort.SortSpec sortSpec : expr.getSortSpecs()) { + Set columns = ExprFinder.finds(sortSpec.getKey(), OpType.Column); + for (ColumnReferenceExpr col : columns) { + TUtil.putToNestedList(ctx.projectColumns, col.getQualifier(), col); + } + } + super.visitSort(ctx, stack, expr); SortNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 2b197ff0ec..27df7f8184 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -254,7 +254,8 @@ public Expr visitCreateTable(Context context, Stack stack, CreateTable exp if (expr.getStorageType() != null) { if (expr.hasSelfDescSchema()) { // TODO: support other types like Parquet and ORC. - if (!expr.getStorageType().equalsIgnoreCase(BuiltinStorages.JSON)) { + if (!expr.getStorageType().equalsIgnoreCase(BuiltinStorages.JSON) && + !expr.getStorageType().equalsIgnoreCase(BuiltinStorages.EX_HTTP_JSON)) { if (expr.getStorageType().equalsIgnoreCase(BuiltinStorages.PARQUET) || expr.getStorageType().equalsIgnoreCase(BuiltinStorages.ORC)) { throw new NotImplementedException(expr.getStorageType()); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 4fcd5dc97e..07f8eba5aa 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -99,7 +99,7 @@ public String toString() { return name + "=" + uri.toString(); } - public abstract long getTableVolume(TableDesc table, Optional filter) throws UnsupportedException; + public abstract long getTableVolume(TableDesc table, Optional filter); /** * if {@link StorageProperty#isArbitraryPathAllowed} is true, @@ -115,11 +115,12 @@ public URI getRootUri() { /** * Get Table URI * + * @param meta table meta * @param databaseName Database name * @param tableName Table name * @return Table URI */ - public abstract URI getTableUri(String databaseName, String tableName); + public abstract URI getTableUri(TableMeta meta, String databaseName, String tableName); /** * Returns the splits that will serve as input for the scan tasks. The diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java index 88410bb2cc..9721df0259 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java @@ -32,6 +32,7 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.MetadataProvider; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UndefinedTablespaceException; @@ -430,9 +431,9 @@ public static Optional getAnyByScheme(String scheme) { } @Override - public URI getTableURI(@Nullable String spaceName, String databaseName, String tableName) { + public URI getTableURI(@Nullable String spaceName, TableMeta meta, String databaseName, String tableName) { Tablespace space = spaceName == null ? getDefault() : getByName(spaceName); - return space.getTableUri(databaseName, tableName); + return space.getTableUri(meta, databaseName, tableName); } @Override diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/BuiltinFragmentKinds.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/BuiltinFragmentKinds.java index 9c4fce5c92..cc7eea55fb 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/BuiltinFragmentKinds.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/BuiltinFragmentKinds.java @@ -22,4 +22,5 @@ public class BuiltinFragmentKinds { public static final String FILE = "FILE"; public static final String HBASE = "HBASE"; public static final String JDBC = "JDBC"; + public static final String HTTP = "EXAMPLE-HTTP"; } diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json index 17ac3ba419..a94ae8f9ba 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json @@ -31,6 +31,10 @@ "swift": { "handler": "org.apache.tajo.storage.s3.S3TableSpace", "default-format": "text" + }, + "http": { + "handler": "org.apache.tajo.storage.http.ExampleHttpFileTablespace", + "default-format": "json" } } } \ No newline at end of file diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index 4e5720490d..ce0ce85393 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -39,7 +39,7 @@ tajo.storage.scanner-handler - text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase + text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase,ex_http_json @@ -55,6 +55,10 @@ tajo.storage.fragment.kind.jdbc org.apache.tajo.storage.jdbc.JdbcFragment + + tajo.storage.fragment.kind.example-http + org.apache.tajo.storage.http.ExampleHttpFileFragment + tajo.storage.fragment.serde.file org.apache.tajo.storage.fragment.FileFragmentSerde @@ -67,6 +71,10 @@ tajo.storage.fragment.serde.jdbc org.apache.tajo.storage.jdbc.JdbcFragmentSerde + + tajo.storage.fragment.serde.example-http + org.apache.tajo.storage.http.ExampleHttpFileFragmentSerde + @@ -123,7 +131,12 @@ tajo.storage.scanner-handler.hbase.class org.apache.tajo.storage.hbase.HBaseScanner - + + + tajo.storage.scanner-handler.ex_http_json.class + org.apache.tajo.storage.http.ExampleHttpJsonScanner + + tajo.storage.appender-handler diff --git a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml index 5bf6b0b45c..1737e221c7 100644 --- a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml @@ -38,7 +38,7 @@ tajo.storage.scanner-handler - text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase + text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase,ex_http_json @@ -126,6 +126,11 @@ org.apache.tajo.storage.hbase.HBaseScanner + + tajo.storage.scanner-handler.ex_http_json.class + org.apache.tajo.storage.http.ExampleHttpJsonScanner + + tajo.storage.appender-handler diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index e3f7c25e4b..2dbd137682 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -396,7 +396,7 @@ public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { } @Override - public URI getTableUri(String databaseName, String tableName) { + public URI getTableUri(TableMeta meta, String databaseName, String tableName) { return URI.create(uri.toString() + "/" + tableName); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java index 94e07e5193..58001b934c 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java @@ -18,8 +18,6 @@ package org.apache.tajo.storage; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -32,33 +30,32 @@ import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; public abstract class FileScanner implements Scanner { - private static final Log LOG = LogFactory.getLog(FileScanner.class); protected boolean inited = false; protected final Configuration conf; protected final TableMeta meta; protected final Schema schema; - protected final FileFragment fragment; + protected final AbstractFileFragment fragment; protected final int columnNum; protected Column [] targets; protected float progress; - protected TableStats tableStats; + protected TableStats inputStats; public FileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) { this.conf = conf; this.meta = meta; this.schema = schema; - this.fragment = (FileFragment)fragment; - this.tableStats = new TableStats(); + this.fragment = (AbstractFileFragment)fragment; + this.inputStats = new TableStats(); this.columnNum = this.schema.size(); } @@ -67,14 +64,14 @@ public void init() throws IOException { progress = 0.0f; if (fragment != null) { - tableStats.setNumBytes(fragment.getLength()); - tableStats.setNumBlocks(1); + inputStats.setNumBytes(fragment.getLength()); + inputStats.setNumBlocks(1); } if (schema != null) { for(Column eachColumn: schema.getRootColumns()) { ColumnStats columnStats = new ColumnStats(eachColumn); - tableStats.addColumnStat(columnStats); + inputStats.addColumnStat(columnStats); } } } @@ -112,6 +109,6 @@ public float getProgress() { @Override public TableStats getInputStats() { - return tableStats; + return inputStats; } } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index 17c413ebfa..2785de4ea4 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -126,7 +126,7 @@ protected void storageInit() throws IOException { } @Override - public long getTableVolume(TableDesc table, Optional filter) throws UnsupportedException { + public long getTableVolume(TableDesc table, Optional filter) { Path path = new Path(table.getUri()); ContentSummary summary; try { @@ -169,7 +169,7 @@ public boolean exists(Path path) throws IOException { } @Override - public URI getTableUri(String databaseName, String tableName) { + public URI getTableUri(TableMeta meta, String databaseName, String tableName) { return StorageUtil.concatPath(spacePath, databaseName, tableName).toUri(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java index a6850c14c2..348bb3cd7c 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java @@ -435,12 +435,12 @@ public boolean isSplittable(){ @Override public TableStats getInputStats() { - if(tableStats != null){ - tableStats.setNumRows(recordCount); - tableStats.setReadBytes(totalReadBytes); // actual read bytes (scan + rescan * n) - tableStats.setNumBytes(fragment.getLength()); + if(inputStats != null){ + inputStats.setNumRows(recordCount); + inputStats.setReadBytes(totalReadBytes); // actual read bytes (scan + rescan * n) + inputStats.setNumBytes(fragment.getLength()); } - return tableStats; + return inputStats; } @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/AbstractFileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/AbstractFileFragment.java new file mode 100644 index 0000000000..fa5ff0baa9 --- /dev/null +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/AbstractFileFragment.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.fragment; + +import org.apache.hadoop.fs.Path; + +import java.io.InputStream; +import java.net.URI; + +/** + * Abstract fragment implementation for file systems. + */ +public abstract class AbstractFileFragment extends Fragment { + + protected InputStream in; + + protected AbstractFileFragment(String kind, + URI uri, + String inputSourceId, + long startKey, + long endKey, + long length, + String[] hostNames) { + super(kind, uri, inputSourceId, startKey, endKey, length, hostNames); + } + + public Path getPath() { + return new Path(uri); + } +} diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index 7bdf0cbee3..0ead6009a4 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -30,7 +30,7 @@ /** * Fragment for file systems. */ -public class FileFragment extends Fragment { +public class FileFragment extends AbstractFileFragment { private Integer[] diskIds; // disk volume ids public FileFragment(String tableName, Path uri, BlockLocation blockLocation) diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/OrcScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/OrcScanner.java index 1329955942..0a20d669d0 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/OrcScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/OrcScanner.java @@ -300,8 +300,8 @@ public void reset() throws IOException { public void close() throws IOException { if (recordReader != null) { recordReader.close(); - tableStats.setNumBytes(recordReader.getNumBytes()); - tableStats.setNumRows(recordCount); + inputStats.setNumBytes(recordReader.getNumBytes()); + inputStats.setNumRows(recordCount); } } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java index 1e2380e2eb..d876e36c91 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java @@ -171,9 +171,9 @@ public void reset() throws IOException { @Override public void close() throws IOException { - if (tableStats != null) { - tableStats.setReadBytes(filePosition - fragment.getStartKey()); - tableStats.setNumRows(recordCount); + if (inputStats != null) { + inputStats.setReadBytes(filePosition - fragment.getStartKey()); + inputStats.setNumRows(recordCount); } if(tupleBuffer != null) { tupleBuffer.release(); @@ -206,12 +206,12 @@ public boolean isSplittable(){ @Override public TableStats getInputStats() { - if(tableStats != null){ - tableStats.setNumRows(recordCount); - tableStats.setReadBytes(filePosition - fragment.getStartKey()); // actual read bytes (scan + rescan * n) - tableStats.setNumBytes(fragment.getLength()); + if(inputStats != null){ + inputStats.setNumRows(recordCount); + inputStats.setReadBytes(filePosition - fragment.getStartKey()); // actual read bytes (scan + rescan * n) + inputStats.setNumBytes(fragment.getLength()); } - return tableStats; + return inputStats; } @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java index ebdc4722a6..2d21928343 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java @@ -1591,9 +1591,9 @@ protected void currentValueBuffer() throws IOException { currentValue.inited = true; readBytes += currentValue.getReadBytes(); - if (tableStats != null) { - tableStats.setReadBytes(readBytes); - tableStats.setNumRows(passedRowsNum); + if (inputStats != null) { + inputStats.setReadBytes(readBytes); + inputStats.setNumRows(passedRowsNum); } } @@ -1776,9 +1776,9 @@ public boolean isSplittable() { @Override public void close() throws IOException { - if (tableStats != null) { - tableStats.setReadBytes(readBytes); //Actual Processed Bytes. (decompressed bytes + header - seek) - tableStats.setNumRows(passedRowsNum); + if (inputStats != null) { + inputStats.setReadBytes(readBytes); //Actual Processed Bytes. (decompressed bytes + header - seek) + inputStats.setNumRows(passedRowsNum); } IOUtils.cleanup(LOG, in, currentValue); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java index b7dc1eca9f..74ded01385 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java @@ -330,9 +330,9 @@ public void close() throws IOException { if (reader != null) reader.close(); - if (tableStats != null) { - tableStats.setReadBytes(totalBytes); - tableStats.setNumRows(currentIdx); + if (inputStats != null) { + inputStats.setReadBytes(totalBytes); + inputStats.setNumRows(currentIdx); } outTuple = null; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedLineReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedLineReader.java index a2688b1213..c383eba99f 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedLineReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedLineReader.java @@ -36,7 +36,7 @@ import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.storage.*; import org.apache.tajo.storage.compress.CodecPool; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.unit.StorageUnit; import java.io.*; @@ -57,15 +57,16 @@ public class DelimitedLineReader implements Closeable { private boolean eof = true; private ByteBufLineReader lineReader; private AtomicInteger lineReadBytes = new AtomicInteger(); - private FileFragment fragment; + private AbstractFileFragment fragment; private Configuration conf; private int bufferSize; - public DelimitedLineReader(Configuration conf, final FileFragment fragment) throws IOException { + public DelimitedLineReader(Configuration conf, final AbstractFileFragment fragment) throws IOException { this(conf, fragment, 128 * StorageUnit.KB); } - public DelimitedLineReader(Configuration conf, final FileFragment fragment, int bufferSize) throws IOException { + public DelimitedLineReader(Configuration conf, final AbstractFileFragment fragment, int bufferSize) + throws IOException { this.fragment = fragment; this.conf = conf; this.factory = new CompressionCodecFactory(conf); @@ -115,13 +116,13 @@ public void init() throws IOException { channel.position(startOffset); is = inputStream; lineReader = new ByteBufLineReader(new LocalFileInputChannel(inputStream), - BufferPool.directBuffer((int) Math.min(bufferSize, end))); + BufferPool.directBuffer((int) Math.min(bufferSize, fragment.getLength()))); } else { fis = fs.open(fragment.getPath()); fis.seek(startOffset); is = fis; lineReader = new ByteBufLineReader(new FSDataInputChannel(fis), - BufferPool.directBuffer((int) Math.min(bufferSize, end))); + BufferPool.directBuffer((int) Math.min(bufferSize, fragment.getLength()))); } } eof = false; @@ -145,7 +146,7 @@ public long getCompressedPosition() throws IOException { return retVal; } - public long getUnCompressedPosition() throws IOException { + public long getUncompressedPosition() throws IOException { return pos; } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java index d52f46da69..08ce7bd71e 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java @@ -348,59 +348,52 @@ public Tuple next() throws IOException { return null; } - try { - - // this loop will continue until one tuple is build or EOS (end of stream). - do { - long offset = reader.getUnCompressedPosition(); - ByteBuf buf = reader.readLine(); - - // if no more line, then return EOT (end of tuple) - if (buf == null) { - return null; - } - - // If there is no required column, we just read each line - // and then return an empty tuple without parsing line. - if (targets.length == 0) { - recordCount++; - return EmptyTuple.get(); - } + // this loop will continue until one tuple is build or EOS (end of stream). + do { + long offset = reader.getUncompressedPosition(); + ByteBuf buf = reader.readLine(); + + // if no more line, then return EOT (end of tuple) + if (buf == null) { + return null; + } - outTuple.setOffset(offset); + // If there is no required column, we just read each line + // and then return an empty tuple without parsing line. + if (targets.length == 0) { + recordCount++; + return EmptyTuple.get(); + } - try { - deserializer.deserialize(buf, outTuple); - // if a line is read normally, it exits this loop. - break; + outTuple.setOffset(offset); - } catch (TextLineParsingError tae) { + try { + deserializer.deserialize(buf, outTuple); + // if a line is read normally, it exits this loop. + break; - errorNum++; + } catch (TextLineParsingError tae) { - // suppress too many log prints, which probably cause performance degradation - if (errorNum < errorPrintOutMaxNum) { - LOG.warn("Ignore Text Parse Error (" + errorNum + "): ", tae); - } + errorNum++; - // Only when the maximum error torrence limit is set (i.e., errorTorrenceMaxNum >= 0), - // it checks if the number of parsing error exceeds the max limit. - // Otherwise, it will ignore all parsing errors. - if (errorTorrenceMaxNum >= 0 && errorNum > errorTorrenceMaxNum) { - throw tae; - } + // suppress too many log prints, which probably cause performance degradation + if (errorNum < errorPrintOutMaxNum) { + LOG.warn("Ignore Text Parse Error (" + errorNum + "): ", tae); } - } while (reader.isReadable()); // continue until EOS - // recordCount means the number of actual read records. We increment the count here. - recordCount++; + // Only when the maximum error torrence limit is set (i.e., errorTorrenceMaxNum >= 0), + // it checks if the number of parsing error exceeds the max limit. + // Otherwise, it will ignore all parsing errors. + if (errorTorrenceMaxNum >= 0 && errorNum > errorTorrenceMaxNum) { + throw new IOException(tae); + } + } + } while (reader.isReadable()); // continue until EOS - return outTuple; + // recordCount means the number of actual read records. We increment the count here. + recordCount++; - } catch (Throwable t) { - LOG.error(t); - throw new IOException(t); - } + return outTuple; } @Override @@ -453,8 +446,8 @@ public void close() throws IOException { } if (reader != null) { - tableStats.setReadBytes(reader.getReadBytes()); //Actual Processed Bytes. (decompressed bytes + overhead) - tableStats.setNumRows(recordCount); + inputStats.setReadBytes(reader.getReadBytes()); //Actual Processed Bytes. (decompressed bytes + overhead) + inputStats.setNumRows(recordCount); } if (LOG.isDebugEnabled()) { @@ -488,17 +481,17 @@ public boolean isSplittable() { @Override public TableStats getInputStats() { - if (tableStats != null && reader != null) { - tableStats.setReadBytes(reader.getReadBytes()); //Actual Processed Bytes. (decompressed bytes + overhead) - tableStats.setNumRows(recordCount); - tableStats.setNumBytes(fragment.getLength()); + if (inputStats != null && reader != null) { + inputStats.setReadBytes(reader.getReadBytes()); //Actual Processed Bytes. (decompressed bytes + overhead) + inputStats.setNumRows(recordCount); + inputStats.setNumBytes(fragment.getLength()); } - return tableStats; + return inputStats; } @Override public long getNextOffset() throws IOException { - return reader.getUnCompressedPosition(); + return reader.getUncompressedPosition(); } @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java index a29e86bdd0..6faeaf6034 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcRecordReader.java @@ -32,7 +32,7 @@ import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.storage.thirdparty.orc.TreeReaderFactory.DatumTreeReader; import java.io.Closeable; @@ -71,7 +71,7 @@ public OrcRecordReader(List stripes, FileSystem fileSystem, Schema schema, Column[] targets, - FileFragment fragment, + AbstractFileFragment fragment, List types, CompressionCodec codec, int bufferSize, diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java index fb095a40bf..9b712a818e 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java @@ -224,11 +224,12 @@ public Column apply(@Nullable Pair columnPair) { final TableStats stats = new TableStats(); stats.setNumRows(-1); // unknown + TableMeta meta = new TableMeta("rowstore", new KeyValueSet()); final TableDesc table = new TableDesc( IdentifierUtil.buildFQName(databaseName, name), schema, - new TableMeta("rowstore", new KeyValueSet()), - space.getTableUri(databaseName, name) + meta, + space.getTableUri(meta, databaseName, name) ); table.setStats(stats); diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java index c3538316c1..21bce9c416 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java @@ -110,12 +110,12 @@ protected void storageInit() throws IOException { } @Override - public long getTableVolume(TableDesc table, Optional filter) throws UnsupportedException { - throw new UnsupportedException(); + public long getTableVolume(TableDesc table, Optional filter) { + throw new TajoRuntimeException(new UnsupportedException()); } @Override - public URI getTableUri(String databaseName, String tableName) { + public URI getTableUri(TableMeta meta, String databaseName, String tableName) { return URI.create(UriUtil.addParam(getUri().toASCIIString(), URI_PARAM_KEY_TABLE, tableName)); } diff --git a/tajo-tablespace-example/pom.xml b/tajo-tablespace-example/pom.xml new file mode 100644 index 0000000000..7e15e0c14f --- /dev/null +++ b/tajo-tablespace-example/pom.xml @@ -0,0 +1,256 @@ + + + + + + tajo-project + org.apache.tajo + 0.12.0-SNAPSHOT + ../tajo-project/pom.xml + + 4.0.0 + + tajo-tablespace-example + jar + Tajo Tablespace Example + + UTF-8 + UTF-8 + + + + + + org.apache.rat + apache-rat-plugin + + + derby.log + src/test/resources/dataset/** + src/test/resources/queries/** + src/test/resources/results/** + + + + + verify + + check + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../tajo-common/src/main/proto + --proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/ExampleHttpFragmentProtos.proto + + + + exec + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + + + + + org.apache.tajo + tajo-common + provided + + + org.apache.tajo + tajo-catalog-common + provided + + + org.apache.tajo + tajo-plan + provided + + + org.apache.tajo + tajo-storage-common + provided + + + org.apache.tajo + tajo-storage-hdfs + provided + + + org.apache.tajo + tajo-rpc-common + + + org.apache.tajo + tajo-cluster-tests + test-jar + test + + + junit + junit + test + + + io.netty + netty-transport + test + + + io.netty + netty-codec-http + test + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-minicluster + test + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + netty-all + io.netty + + + + + org.apache.hadoop + hadoop-hdfs + test + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + netty-all + io.netty + + + + + + \ No newline at end of file diff --git a/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragment.java b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragment.java new file mode 100644 index 0000000000..06bf2516eb --- /dev/null +++ b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragment.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import org.apache.tajo.storage.fragment.AbstractFileFragment; +import org.apache.tajo.storage.fragment.BuiltinFragmentKinds; + +import java.net.HttpURLConnection; +import java.net.URI; + +public class ExampleHttpFileFragment extends AbstractFileFragment { + + private HttpURLConnection connection; + + /** + * + * @param uri + * @param inputSourceId + * @param startKey first byte pos + * @param endKey last byte pos + */ + public ExampleHttpFileFragment(URI uri, + String inputSourceId, + long startKey, + long endKey) { + super(BuiltinFragmentKinds.HTTP, uri, inputSourceId, startKey, endKey, endKey - startKey, null); + } +} diff --git a/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragmentSerde.java b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragmentSerde.java new file mode 100644 index 0000000000..866526922d --- /dev/null +++ b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragmentSerde.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import com.google.protobuf.GeneratedMessage.Builder; +import org.apache.tajo.storage.fragment.FragmentSerde; +import org.apache.tajo.storage.http.ExampleHttpFragmentProtos.ExampleHttpFileFragmentProto; + +import java.net.URI; + +public class ExampleHttpFileFragmentSerde + implements FragmentSerde { + + @Override + public Builder newBuilder() { + return ExampleHttpFileFragmentProto.newBuilder(); + } + + @Override + public ExampleHttpFileFragmentProto serialize(ExampleHttpFileFragment fragment) { + return ExampleHttpFileFragmentProto.newBuilder() + .setUri(fragment.getUri().toASCIIString()) + .setTableName(fragment.getInputSourceId()) + .setStartKey(fragment.getStartKey()) + .setEndKey(fragment.getEndKey()) + .build(); + } + + @Override + public ExampleHttpFileFragment deserialize(ExampleHttpFileFragmentProto proto) { + return new ExampleHttpFileFragment( + URI.create(proto.getUri()), + proto.getTableName(), + proto.getStartKey(), + proto.getEndKey() + ); + } +} diff --git a/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileTablespace.java b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileTablespace.java new file mode 100644 index 0000000000..258d505952 --- /dev/null +++ b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileTablespace.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import com.google.common.collect.Lists; +import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.BuiltinStorages; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.storage.FormatProperty; +import org.apache.tajo.storage.StorageProperty; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TupleRange; +import org.apache.tajo.storage.fragment.Fragment; +import org.jboss.netty.handler.codec.http.HttpHeaders.Names; + +import javax.annotation.Nullable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.util.List; +import java.util.Optional; + +/** + * Example read-only tablespace for HTTP protocol. + * + * An example table can be created by using the following SQL query. + * + * CREATE TABLE http_test (*) TABLESPACE http_example USING ex_http_json WITH ('path'='2015-01-01-15.json.gz', + * 'compression.codec'='org.apache.hadoop.io.compress.GzipCodec’); + */ +public class ExampleHttpFileTablespace extends Tablespace { + private static final Log LOG = LogFactory.getLog(ExampleHttpFileTablespace.class); + + static final String PATH = "path"; + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // Tablespace properties + ////////////////////////////////////////////////////////////////////////////////////////////////// + private static final StorageProperty STORAGE_PROPERTY = + new StorageProperty( + BuiltinStorages.JSON, // default format is json + false, // is not movable + false, // is not writable + true, // allow arbitrary path + false // doesn't provide metadata + ); + + private static final FormatProperty FORMAT_PROPERTY = + new FormatProperty( + false, // doesn't support insert + false, // doesn't support direct insert + false // doesn't support result staging + ); + + public ExampleHttpFileTablespace(String name, URI uri, JSONObject config) { + super(name, uri, config); + + LOG.info("ExampleHttpFileTablespace is initialized for " + uri); + } + + @Override + protected void storageInit() throws IOException { + // Add initialization code for your tablespace + } + + @Override + public long getTableVolume(TableDesc table, Optional notUsed) { + HttpURLConnection connection = null; + + try { + connection = (HttpURLConnection) new URL(table.getUri().toASCIIString()).openConnection(); + connection.setRequestMethod("HEAD"); + connection.connect(); + return connection.getHeaderFieldLong(Names.CONTENT_LENGTH, -1); + + } catch (IOException e) { + throw new TajoInternalError(e); + + } finally { + if (connection != null) { + connection.disconnect(); + } + } + } + + @Override + public URI getRootUri() { + return uri; + } + + @Override + public URI getTableUri(TableMeta meta, String databaseName, String tableName) { + String tablespaceUriString = uri.toASCIIString(); + String tablePath = meta.getProperty(PATH); + + if (!tablespaceUriString.endsWith("/") && !tablePath.startsWith("/")) { + tablePath = "/" + tablePath; + } + + return URI.create(tablespaceUriString + tablePath); + } + + @Override + public List getSplits(String inputSourceId, + TableDesc tableDesc, + boolean requireSort, + @Nullable EvalNode filterCondition) + throws IOException, TajoException { + + // getSplits() should return multiple fragments for distributed processing of a large data. + // This example tablespace returns only one fragment for the whole data for simplicity, + // but this may significantly increase the query processing time. + + long tableVolume = getTableVolume(tableDesc, Optional.empty()); + return Lists.newArrayList(new ExampleHttpFileFragment(tableDesc.getUri(), inputSourceId, 0, tableVolume)); + } + + @Override + public StorageProperty getProperty() { + return STORAGE_PROPERTY; + } + + @Override + public FormatProperty getFormatProperty(TableMeta meta) { + return FORMAT_PROPERTY; + } + + @Override + public void close() { + // do nothing + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, + TableDesc tableDesc, + Schema inputSchema, + SortSpec[] sortSpecs, + TupleRange dataRange) throws IOException { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws TajoException { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public void createTable(TableDesc table, boolean ifNotExists) throws TajoException, IOException { + HttpURLConnection connection = null; + + try { + connection = (HttpURLConnection) new URL(table.getUri().toASCIIString()).openConnection(); + connection.setRequestMethod("HEAD"); + connection.connect(); + + if (connection.getResponseCode() == 404) { + throw new FileNotFoundException(); + } + + } catch (IOException e) { + throw new TajoInternalError(e); + + } finally { + if (connection != null) { + connection.disconnect(); + } + } + } + + @Override + public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public void prepareTable(LogicalNode node) throws IOException, TajoException { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public Path commitTable(OverridableConf queryContext, + ExecutionBlockId finalEbId, + LogicalPlan plan, + Schema schema, + TableDesc tableDesc) throws IOException { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public void rollbackTable(LogicalNode node) throws IOException, TajoException { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { + throw new TajoRuntimeException(new UnsupportedException()); + } +} diff --git a/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpJsonLineReader.java b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpJsonLineReader.java new file mode 100644 index 0000000000..61e4d6c377 --- /dev/null +++ b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpJsonLineReader.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import io.netty.buffer.ByteBuf; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.CompressionCodecFactory; +import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.io.compress.SplittableCompressionCodec; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.storage.BufferPool; +import org.apache.tajo.storage.ByteBufInputChannel; +import org.apache.tajo.storage.compress.CodecPool; +import org.apache.tajo.storage.fragment.AbstractFileFragment; +import org.apache.tajo.storage.text.ByteBufLineReader; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.concurrent.atomic.AtomicInteger; + +public class ExampleHttpJsonLineReader implements Closeable { + + private final static Log LOG = LogFactory.getLog(ExampleHttpJsonLineReader.class); + + private Configuration conf; + + private HttpURLConnection connection; + private InputStream is; + + private CompressionCodec codec; + private Decompressor decompressor; + + private long startOffset, endOffset, pos; + private boolean eof = true; + private ByteBufLineReader lineReader; + private AtomicInteger lineReadBytes = new AtomicInteger(); + private ExampleHttpFileFragment fragment; + private final int bufferSize; + + public ExampleHttpJsonLineReader(Configuration conf, + AbstractFileFragment fragment, + int bufferSize) { + this.conf = conf; + this.fragment = (ExampleHttpFileFragment) fragment; + this.bufferSize = bufferSize; + + CompressionCodecFactory factory = new CompressionCodecFactory(conf); + codec = factory.getCodec(fragment.getPath()); + if (this.codec instanceof SplittableCompressionCodec) { + // bzip2 does not support multi-thread model + throw new TajoRuntimeException(new UnsupportedException(codec.getDefaultExtension())); + } + } + + public void init() throws IOException { + if (connection != null) { + throw new IOException(this.getClass() + " is already initialized"); + } + + pos = startOffset = fragment.getStartKey(); + endOffset = fragment.getEndKey(); + + URL url = new URL(fragment.getUri().toASCIIString()); + connection = (HttpURLConnection) url.openConnection(); + + is = connection.getInputStream(); + + ByteBuf buf; + if (codec != null) { + decompressor = CodecPool.getDecompressor(codec); + is = codec.createInputStream(is, decompressor); + + buf = BufferPool.directBuffer(bufferSize); + + } else { + buf = BufferPool.directBuffer((int) Math.min(bufferSize, fragment.getLength())); + + } + + lineReader = new ByteBufLineReader(new ByteBufInputChannel(is), buf); + + eof = false; + } + + public ByteBuf readLine() throws IOException { + if (eof) { + return null; + } + + ByteBuf buf = lineReader.readLineBuf(lineReadBytes); + pos += lineReadBytes.get(); + if (buf == null) { + eof = true; + } + + return buf; + } + + public boolean isCompressed() { + return codec != null; + } + + public long getPos() { + return pos; + } + + public long getReadBytes() { + return pos - startOffset; + } + + public boolean isEof() { + return eof; + } + + @Override + public void close() throws IOException { + try { + IOUtils.cleanup(LOG, lineReader); + + if (connection != null) { + connection.disconnect(); + } + + is = null; + lineReader = null; + + } finally { + CodecPool.returnDecompressor(decompressor); + decompressor = null; + } + } +} diff --git a/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpJsonScanner.java b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpJsonScanner.java new file mode 100644 index 0000000000..6c41a50abe --- /dev/null +++ b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpJsonScanner.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import io.netty.buffer.ByteBuf; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.storage.EmptyTuple; +import org.apache.tajo.storage.FileScanner; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.json.JsonLineDeserializer; +import org.apache.tajo.storage.text.TextLineParsingError; +import org.apache.tajo.unit.StorageUnit; + +import java.io.IOException; + +import static org.apache.tajo.storage.StorageConstants.DEFAULT_TEXT_ERROR_TOLERANCE_MAXNUM; +import static org.apache.tajo.storage.StorageConstants.TEXT_ERROR_TOLERANCE_MAXNUM; +import static org.apache.tajo.storage.text.DelimitedTextFile.READ_BUFFER_SIZE; + +public class ExampleHttpJsonScanner extends FileScanner { + + private static final Log LOG = LogFactory.getLog(ExampleHttpJsonScanner.class); + + private VTuple outTuple; + + private long limit; + + private final long startOffset; + private final long endOffset; + + private ExampleHttpJsonLineReader reader; + private JsonLineDeserializer deserializer; + + private int errorPrintOutMaxNum = 5; + /** Maximum number of permissible errors */ + private final int maxAllowedErrorCount; + /** How many errors have occurred? */ + private int errorNum; + + private boolean splittable = false; + + private long recordCount = 0; + + public ExampleHttpJsonScanner(Configuration conf, Schema schema, TableMeta tableMeta, Fragment fragment) + throws IOException { + super(conf, schema, tableMeta, fragment); + + reader = new ExampleHttpJsonLineReader(conf, this.fragment, conf.getInt(READ_BUFFER_SIZE, 128 * StorageUnit.KB)); + if (!this.reader.isCompressed()) { + splittable = true; + } + + startOffset = this.fragment.getStartKey(); + endOffset = this.fragment.getEndKey(); + + maxAllowedErrorCount = + Integer.parseInt(tableMeta.getProperty(TEXT_ERROR_TOLERANCE_MAXNUM, DEFAULT_TEXT_ERROR_TOLERANCE_MAXNUM)); + } + + @Override + public void init() throws IOException { + + reader.init(); + + if (targets == null) { + targets = schema.toArray(); + } + + reset(); + + super.init(); + } + + @Override + public Tuple next() throws IOException { + + if (reader.isEof()) { + return null; // Indicate to the parent operator that there is no more data. + } + + // Read lines until it reads a valid tuple or EOS (end of stream). + while (!reader.isEof()) { + + ByteBuf buf = reader.readLine(); + + if (buf == null) { // The null buf means that there is no more lines. + return null; + } + + // When the number of projection columns is 0, the read line doesn't have to be parsed. + if (targets.length == 0) { + recordCount++; + return EmptyTuple.get(); + } + + try { + deserializer.deserialize(buf, outTuple); + + // Once a line is normally parsed, exits the while loop. + break; + + } catch (TextLineParsingError tlpe) { + + errorNum++; + + // The below line may print too many logs. + LOG.warn("Ignore Text Parse Error (" + errorNum + "): ", tlpe); + + // If the number of found errors exceeds the configured tolerable error count, + // throw the error. + if (maxAllowedErrorCount >= 0 && errorNum > maxAllowedErrorCount) { + throw new IOException(tlpe); + } + } + } + + recordCount++; + + return outTuple; + } + + @Override + public void reset() throws IOException { + recordCount = 0; + + if (reader.getReadBytes() > 0) { + reader.close(); + + reader = new ExampleHttpJsonLineReader(conf, fragment, conf.getInt(READ_BUFFER_SIZE, 128 * StorageUnit.KB)); + reader.init(); + } + + if(deserializer != null) { + deserializer.release(); + } + + deserializer = new JsonLineDeserializer(schema, meta, targets); + deserializer.init(); + + outTuple = new VTuple(targets.length); + + // skip first line if it reads from middle of file + if (startOffset > 0) { + reader.readLine(); + } + } + + @Override + public void close() throws IOException { + try { + + if (deserializer != null) { + deserializer.release(); + } + + if (reader != null) { + inputStats.setReadBytes(reader.getReadBytes()); + inputStats.setNumRows(recordCount); + } + + } finally { + IOUtils.cleanup(LOG, reader); + outTuple = null; + } + } + + @Override + public boolean isProjectable() { + return true; + } + + @Override + public void setTarget(Column[] targets) { + this.targets = targets; + } + + @Override + public boolean isSelectable() { + return false; + } + + @Override + public void setFilter(EvalNode filter) { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public void setLimit(long num) { + this.limit = num; + } + + @Override + public boolean isSplittable() { + return splittable; + } + + @Override + public float getProgress() { + if(!inited) return super.getProgress(); + + if (reader.isEof()) { // if the reader reaches EOS + return 1.0f; + } + + long currentPos = reader.getPos(); + long readBytes = currentPos - startOffset; + long remainingBytes = Math.max(endOffset - currentPos, 0); + return Math.min(1.0f, (float) (readBytes) / (float) (readBytes + remainingBytes)); + } + + @Override + public TableStats getInputStats() { + if (inputStats != null && reader != null) { + inputStats.setReadBytes(reader.getReadBytes()); //Actual Processed Bytes. (decompressed bytes + overhead) + inputStats.setNumRows(recordCount); + inputStats.setNumBytes(fragment.getLength()); + } + + return inputStats; + } +} diff --git a/tajo-tablespace-example/src/main/proto/ExampleHttpFragmentProtos.proto b/tajo-tablespace-example/src/main/proto/ExampleHttpFragmentProtos.proto new file mode 100644 index 0000000000..586e46451d --- /dev/null +++ b/tajo-tablespace-example/src/main/proto/ExampleHttpFragmentProtos.proto @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.tajo.storage.http"; +option java_outer_classname = "ExampleHttpFragmentProtos"; +option optimize_for = SPEED; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +import "CatalogProtos.proto"; + +message ExampleHttpFileFragmentProto { + required string uri = 1; + required string table_name = 2; + required int64 start_key = 3; + required int64 end_key = 4; +} \ No newline at end of file diff --git a/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpServerHandler.java b/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpServerHandler.java new file mode 100644 index 0000000000..fd25da8975 --- /dev/null +++ b/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpServerHandler.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import io.netty.buffer.Unpooled; +import io.netty.channel.*; +import io.netty.handler.codec.http.*; +import io.netty.handler.codec.http.HttpHeaders.Names; +import io.netty.util.CharsetUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import javax.activation.MimetypesFileTypeMap; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; +import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; + +public class ExampleHttpServerHandler extends SimpleChannelInboundHandler { + + private static final Log LOG = LogFactory.getLog(ExampleHttpServerHandler.class); + + @Override + protected void channelRead0(ChannelHandlerContext context, FullHttpRequest request) throws Exception { + + if (request.getMethod().equals(HttpMethod.HEAD)) { + + processHead(context, request); + + } else if (request.getMethod().equals(HttpMethod.GET)) { + + processGet(context, request); + + } else { + // error + String msg = "Not supported method: " + request.getMethod(); + LOG.error(msg); + context.writeAndFlush(getBadRequest(msg)); + } + } + + private void processHead(ChannelHandlerContext context, FullHttpRequest request) { + HttpHeaders headers = request.headers(); + FullHttpResponse response = null; + + if (headers.contains(Names.CONTENT_LENGTH)) { + + try { + File file = getRequestedFile(request.getUri()); + + response = new DefaultFullHttpResponse( + HTTP_1_1, + request.getDecoderResult().isSuccess() ? OK : BAD_REQUEST + ); + + HttpHeaders.setContentLength(response, file.length()); + + + } catch (FileNotFoundException | URISyntaxException e) { + response = getBadRequest(e.getMessage()); + } + } + + context.writeAndFlush(response); + } + + private void processGet(ChannelHandlerContext context, FullHttpRequest request) { + try { + File file = getRequestedFile(request.getUri()); + + RandomAccessFile raf = new RandomAccessFile(file, "r"); + long fileLength = raf.length(); + + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + HttpHeaders.setContentLength(response, fileLength); + setContentTypeHeader(response, file); + + context.write(response); + + context.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength)); + + // Write the end marker. + ChannelFuture future = context.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); + future.addListener(ChannelFutureListener.CLOSE); + + } catch (IOException | URISyntaxException e) { + context.writeAndFlush(getBadRequest(e.getMessage())); + } + } + + private static File getRequestedFile(String uri) throws FileNotFoundException, URISyntaxException { + String path = URI.create(uri).getPath(); + URL url = ClassLoader.getSystemResource("dataset/" + path); + + if (url == null) { + throw new FileNotFoundException(uri); + } + return new File(url.toURI()); + } + + private static FullHttpResponse getBadRequest(String message) { + return new DefaultFullHttpResponse(HTTP_1_1, BAD_REQUEST, + Unpooled.copiedBuffer(message, CharsetUtil.UTF_8)); + } + + @Override + public void exceptionCaught(ChannelHandlerContext context, Throwable cause) { + LOG.error(cause.getMessage(), cause); + if (context.channel().isOpen()) { + context.channel().close(); + } + } + + /** + * Sets the content type header for the HTTP Response + * @param response HTTP response + * @param file file to extract content type + */ + private static void setContentTypeHeader(HttpResponse response, File file) { + MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap(); + response.headers().set(CONTENT_TYPE, mimeTypesMap.getContentType(file.getPath())); + } +} diff --git a/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpServerInitializer.java b/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpServerInitializer.java new file mode 100644 index 0000000000..2cd6340ade --- /dev/null +++ b/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpServerInitializer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpServerCodec; +import io.netty.handler.stream.ChunkedWriteHandler; + +public class ExampleHttpServerInitializer extends ChannelInitializer { + + @Override + protected void initChannel(SocketChannel socketChannel) throws Exception { + ChannelPipeline pipeline = socketChannel.pipeline(); + + pipeline.addLast(new HttpServerCodec()); + pipeline.addLast(new HttpObjectAggregator(65536)); + pipeline.addLast(new ChunkedWriteHandler()); + pipeline.addLast(new ExampleHttpServerHandler()); + } +} diff --git a/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpTablespaceTestServer.java b/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpTablespaceTestServer.java new file mode 100644 index 0000000000..aa7acc85d8 --- /dev/null +++ b/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/ExampleHttpTablespaceTestServer.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.Closeable; +import java.net.InetSocketAddress; + +public class ExampleHttpTablespaceTestServer implements Closeable { + + private final static Log LOG = LogFactory.getLog(ExampleHttpTablespaceTestServer.class); + + private ServerBootstrap bootstrap; + private Channel channel; + + public void init() throws InterruptedException { + EventLoopGroup group = new NioEventLoopGroup(1); + + bootstrap = new ServerBootstrap(); + bootstrap.group(group) + .channel(NioServerSocketChannel.class) + .childHandler(new ExampleHttpServerInitializer()); + + channel = bootstrap.bind(0).sync().channel(); + + LOG.info(ExampleHttpTablespaceTestServer.class.getSimpleName() + " listening on port " + getAddress().getPort()); + } + + public InetSocketAddress getAddress() { + return (InetSocketAddress) channel.localAddress(); + } + + public void close() { + if (bootstrap != null) { + if (bootstrap.group() != null) { + bootstrap.group().shutdownGracefully(); + } + } + + if (channel != null) { + channel.close(); + } + } +} diff --git a/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/TestExampleHttpFileQuery.java b/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/TestExampleHttpFileQuery.java new file mode 100644 index 0000000000..89353f8aa0 --- /dev/null +++ b/tajo-tablespace-example/src/test/java/org/apache/tajo/storage/http/TestExampleHttpFileQuery.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.http; + +import net.minidev.json.JSONObject; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.*; + +import java.net.InetAddress; +import java.net.URI; + +public class TestExampleHttpFileQuery extends QueryTestCaseBase { + + private static ExampleHttpTablespaceTestServer server; + + @BeforeClass + public static void setup() throws Exception { + server = new ExampleHttpTablespaceTestServer(); + server.init(); + + JSONObject configElements = new JSONObject(); + URI uri = URI.create("http://" + InetAddress.getLocalHost().getHostName() + ":" + server.getAddress().getPort()); + TablespaceManager.addTableSpaceForTest(new ExampleHttpFileTablespace("http_example", uri, configElements)); + + QueryTestCaseBase.testingCluster.getMaster().refresh(); + } + + @AfterClass + public static void teardown() throws Exception { + server.close(); + } + + @Before + public void prepareTables() throws TajoException { + executeString("create table got (*) tablespace http_example using ex_http_json with ('path'='got.json')"); + executeString("create table github (*) tablespace http_example using ex_http_json with ('path'='github.json')"); + } + + @After + public void dropTables() throws TajoException { + executeString("drop table got"); + executeString("drop table github"); + } + + @SimpleTest + @Test + public void testSelect() throws Exception { + runSimpleTests(); + } + + @SimpleTest + @Test + public void testGroupby() throws Exception { + runSimpleTests(); + } + + @SimpleTest + @Test + public void testSort() throws Exception { + runSimpleTests(); + } + + @SimpleTest + @Test + public void testJoin() throws Exception { + runSimpleTests(); + } +} diff --git a/tajo-tablespace-example/src/test/resources/dataset/github.json b/tajo-tablespace-example/src/test/resources/dataset/github.json new file mode 100644 index 0000000000..0b597b2ab4 --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/dataset/github.json @@ -0,0 +1,4 @@ +{"id":"2937257753","type":"PushEvent","actor":{"id":5266949,"login":"hardrubic","gravatar_id":"","url":"https://api.github.com/users/hardrubic","avatar_url":"https://avatars.githubusercontent.com/u/5266949?"},"repo":{"id":38299397,"name":"hardrubic/rxJavaTest","url":"https://api.github.com/repos/hardrubic/rxJavaTest"},"payload":{"push_id":712081726,"size":1,"distinct_size":1,"ref":"refs/heads/master","head":"ea79d7a424f2693b70b9496726f315a5711b6fe7","before":"613f05557ad353f4bedc6df54128f8091ed1f1e9","commits":[{"sha":"ea79d7a424f2693b70b9496726f315a5711b6fe7","author":{"email":"dgzx106@163.com","name":"hardrubic"},"message":"增加rxJava例子","distinct":true,"url":"https://api.github.com/repos/hardrubic/rxJavaTest/commits/ea79d7a424f2693b70b9496726f315a5711b6fe7"}]},"public":true,"created_at":"2015-07-01T00:00:01Z"} +{"id":"2937257758","type":"WatchEvent","actor":{"id":11455393,"login":"chrischjh","gravatar_id":"","url":"https://api.github.com/users/chrischjh","avatar_url":"https://avatars.githubusercontent.com/u/11455393?"},"repo":{"id":18218031,"name":"dead-horse/co-and-koa-talk","url":"https://api.github.com/repos/dead-horse/co-and-koa-talk"},"payload":{"action":"started"},"public":true,"created_at":"2015-07-01T00:00:01Z"} +{"id":"2937257759","type":"CreateEvent","actor":{"id":206379,"login":"gvn","gravatar_id":"","url":"https://api.github.com/users/gvn","avatar_url":"https://avatars.githubusercontent.com/u/206379?"},"repo":{"id":24345476,"name":"gvn/webmaker-android","url":"https://api.github.com/repos/gvn/webmaker-android"},"payload":{"ref":"use-self-building","ref_type":"branch","master_branch":"master","description":"Webmaker for Firefox OS & Android","pusher_type":"user"},"public":true,"created_at":"2015-07-01T00:00:01Z"} +{"id":"2937257761","type":"ForkEvent","actor":{"id":1088854,"login":"CAOakleyII","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","avatar_url":"https://avatars.githubusercontent.com/u/1088854?"},"repo":{"id":11909954,"name":"skycocker/chromebrew","url":"https://api.github.com/repos/skycocker/chromebrew"},"payload":{"forkee":{"id":38339291,"name":"chromebrew","full_name":"CAOakleyII/chromebrew","owner":{"login":"CAOakleyII","id":1088854,"avatar_url":"https://avatars.githubusercontent.com/u/1088854?v=3","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","html_url":"https://github.com/CAOakleyII","followers_url":"https://api.github.com/users/CAOakleyII/followers","following_url":"https://api.github.com/users/CAOakleyII/following{/other_user}","gists_url":"https://api.github.com/users/CAOakleyII/gists{/gist_id}","starred_url":"https://api.github.com/users/CAOakleyII/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CAOakleyII/subscriptions","organizations_url":"https://api.github.com/users/CAOakleyII/orgs","repos_url":"https://api.github.com/users/CAOakleyII/repos","events_url":"https://api.github.com/users/CAOakleyII/events{/privacy}","received_events_url":"https://api.github.com/users/CAOakleyII/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/CAOakleyII/chromebrew","description":"Package manager for Chrome OS","fork":true,"url":"https://api.github.com/repos/CAOakleyII/chromebrew","forks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/forks","keys_url":"https://api.github.com/repos/CAOakleyII/chromebrew/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CAOakleyII/chromebrew/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CAOakleyII/chromebrew/teams","hooks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/hooks","issue_events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/events{/number}","events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/events","assignees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/assignees{/user}","branches_url":"https://api.github.com/repos/CAOakleyII/chromebrew/branches{/branch}","tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/tags","blobs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/refs{/sha}","trees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CAOakleyII/chromebrew/statuses/{sha}","languages_url":"https://api.github.com/repos/CAOakleyII/chromebrew/languages","stargazers_url":"https://api.github.com/repos/CAOakleyII/chromebrew/stargazers","contributors_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contributors","subscribers_url":"https://api.github.com/repos/CAOakleyII/chromebrew/subscribers","subscription_url":"https://api.github.com/repos/CAOakleyII/chromebrew/subscription","commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/commits{/sha}","git_commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/commits{/sha}","comments_url":"https://api.github.com/repos/CAOakleyII/chromebrew/comments{/number}","issue_comment_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/comments{/number}","contents_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contents/{+path}","compare_url":"https://api.github.com/repos/CAOakleyII/chromebrew/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CAOakleyII/chromebrew/merges","archive_url":"https://api.github.com/repos/CAOakleyII/chromebrew/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CAOakleyII/chromebrew/downloads","issues_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues{/number}","pulls_url":"https://api.github.com/repos/CAOakleyII/chromebrew/pulls{/number}","milestones_url":"https://api.github.com/repos/CAOakleyII/chromebrew/milestones{/number}","notifications_url":"https://api.github.com/repos/CAOakleyII/chromebrew/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CAOakleyII/chromebrew/labels{/name}","releases_url":"https://api.github.com/repos/CAOakleyII/chromebrew/releases{/id}","created_at":"2015-07-01T00:00:00Z","updated_at":"2015-06-28T10:11:09Z","pushed_at":"2015-06-09T07:46:57Z","git_url":"git://github.com/CAOakleyII/chromebrew.git","ssh_url":"git@github.com:CAOakleyII/chromebrew.git","clone_url":"https://github.com/CAOakleyII/chromebrew.git","svn_url":"https://github.com/CAOakleyII/chromebrew","homepage":"http://skycocker.github.io/chromebrew/","size":846,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master","public":true}},"public":true,"created_at":"2015-07-01T00:00:01Z"} diff --git a/tajo-tablespace-example/src/test/resources/dataset/got.json b/tajo-tablespace-example/src/test/resources/dataset/got.json new file mode 100644 index 0000000000..db3ad6c7f0 --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/dataset/got.json @@ -0,0 +1,3 @@ +{ "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}} +{ "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}} +{ "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}} diff --git a/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testGroupby.sql b/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testGroupby.sql new file mode 100644 index 0000000000..ea68caa8a1 --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testGroupby.sql @@ -0,0 +1 @@ +select actor.id, count(*) from github group by actor.id; \ No newline at end of file diff --git a/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testJoin.sql b/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testJoin.sql new file mode 100644 index 0000000000..e2c95c8240 --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testJoin.sql @@ -0,0 +1 @@ +select g1.title from got g1, got g2 where g1.name.first_name = g2.name.first_name \ No newline at end of file diff --git a/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testSelect.sql b/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testSelect.sql new file mode 100644 index 0000000000..80ce27f736 --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testSelect.sql @@ -0,0 +1 @@ +select title, name.first_name from got; \ No newline at end of file diff --git a/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testSort.sql b/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testSort.sql new file mode 100644 index 0000000000..d31ea941dd --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/queries/TestExampleHttpFileQuery/testSort.sql @@ -0,0 +1 @@ +select actor.id, actor.login from github order by type; \ No newline at end of file diff --git a/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testGroupby.result b/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testGroupby.result new file mode 100644 index 0000000000..67797c83d6 --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testGroupby.result @@ -0,0 +1,6 @@ +actor/id,?count +------------------------------- +206379,1 +11455393,1 +1088854,1 +5266949,1 diff --git a/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testJoin.result b/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testJoin.result new file mode 100644 index 0000000000..a6da9fc1be --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testJoin.result @@ -0,0 +1,5 @@ +title +------------------------------- +Hand of the King +Assassin +Dancing Master diff --git a/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testSelect.result b/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testSelect.result new file mode 100644 index 0000000000..651a9feb56 --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testSelect.result @@ -0,0 +1,5 @@ +title,name/first_name +------------------------------- +Hand of the King,Eddard +Assassin,Arya +Dancing Master,Syrio diff --git a/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testSort.result b/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testSort.result new file mode 100644 index 0000000000..e0073b9320 --- /dev/null +++ b/tajo-tablespace-example/src/test/resources/results/TestExampleHttpFileQuery/testSort.result @@ -0,0 +1,6 @@ +actor/id,actor/login +------------------------------- +206379,gvn +1088854,CAOakleyII +5266949,hardrubic +11455393,chrischjh diff --git a/tajo-yarn/pom.xml b/tajo-yarn/pom.xml index 70511a187b..cb472761d9 100644 --- a/tajo-yarn/pom.xml +++ b/tajo-yarn/pom.xml @@ -59,7 +59,6 @@ maven-assembly-plugin - 2.4.1 jar-with-dependencies From d5d89b9120ad268a7acf73992d5ed031d8fbf04c Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 1 Jun 2016 08:56:34 +0900 Subject: [PATCH 18/94] Change module order --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1b9e99ef52..32c33f423b 100644 --- a/pom.xml +++ b/pom.xml @@ -98,8 +98,8 @@ tajo-metrics tajo-core-tests tajo-cluster-tests - tajo-dist tajo-tablespace-example + tajo-dist From 3fbb4da3881ac507c4093dcf7c184b963fe7482d Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 1 Jun 2016 15:12:47 +0900 Subject: [PATCH 19/94] Fix test failures --- .../src/main/java/org/apache/tajo/catalog/CatalogUtil.java | 4 ++-- .../src/test/java/org/apache/tajo/TajoTestingCluster.java | 6 +++++- .../test/java/org/apache/tajo/cli/tools/TestTajoDump.java | 4 ++-- .../org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java | 3 --- .../apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java | 5 ++++- .../src/main/java/org/apache/tajo/storage/Tablespace.java | 7 +++---- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 98e8ceabcb..a125ce27bd 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -53,9 +53,9 @@ public class CatalogUtil { - public static String getBackwardCompitableDataFormat(String dataFormat) { - switch (dataFormat) { + final String upperDataFormat = dataFormat.toUpperCase(); + switch (upperDataFormat) { case "CSV": case "TEXTFILE": return BuiltinStorages.TEXT; diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 4b89c8e274..2126ceac9d 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -334,7 +334,11 @@ public CatalogServer getMiniCatalogCluster() { } public CatalogService getCatalogService() { - return new LocalCatalogWrapper(catalogServer); + if (catalogServer != null) { + return new LocalCatalogWrapper(catalogServer); + } else { + return tajoMaster.getCatalog(); + } } public boolean isHiveCatalogStoreRunning() { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index 2fb67c3e83..727bba9f02 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -99,7 +99,7 @@ public void testDump4() throws Exception { TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); printWriter.flush(); printWriter.close(); - TableMeta meta = client.getTableDesc(getCurrentDatabase() + "\".\"TableName1\"").getMeta(); + TableMeta meta = client.getTableDesc(getCurrentDatabase() + ".TableName1").getMeta(); assertOutputResult("testDump3.result", new String(bos.toByteArray()), new String[]{"${index.path}", "${table.timezone}"}, @@ -139,7 +139,7 @@ public void testPartitionsDump() throws Exception { printWriter.flush(); printWriter.close(); - TableMeta meta = client.getTableDesc(getCurrentDatabase() + "\".\"TableName3\"").getMeta(); + TableMeta meta = client.getTableDesc(getCurrentDatabase() + ".TableName3").getMeta(); assertOutputResult("testPartitionsDump.result", new String(bos.toByteArray()), new String[]{"${partition.path1}", "${partition.path2}", "${table.timezone}"}, diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java index 88ad841f06..e78a0d6c43 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java @@ -294,9 +294,6 @@ public LogicalNode visitSort(LogicalPlanner.PlanContext ctx, Stack stack, stack.push(expr); LogicalNode child = visit(ctx, stack, expr.getChild()); - for (Sort.SortSpec sortSpec : expr.getSortSpecs()) { - NameRefInSelectListNormalizer.normalize(ctx, sortSpec.getKey()); - } stack.pop(); SortNode sortNode = ctx.getPlan().createNode(SortNode.class); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java index 256e34ee03..9b8b15fc6e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java @@ -209,7 +209,10 @@ public LogicalNode visitSort(ProcessorContext ctx, Stack stack, Sort expr) for (Sort.SortSpec sortSpec : expr.getSortSpecs()) { Set columns = ExprFinder.finds(sortSpec.getKey(), OpType.Column); for (ColumnReferenceExpr col : columns) { - TUtil.putToNestedList(ctx.projectColumns, col.getQualifier(), col); + if (!ctx.aliasSet.contains(col.getName())) { + NameRefInSelectListNormalizer.normalize(ctx.planContext, col); + TUtil.putToNestedList(ctx.projectColumns, col.getQualifier(), col); + } } } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 07f8eba5aa..64b78d6bfd 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -281,13 +281,12 @@ public Appender getAppender(OverridableConf queryContext, if (appenderClass == null) { appenderClass = conf.getClass( String.format("tajo.storage.appender-handler.%s.class", handlerName), null, Appender.class); + if (appenderClass == null) { + throw new IOException("Undefined appender handler for " + meta.getDataFormat()); + } OldStorageManager.APPENDER_HANDLER_CACHE.put(handlerName, appenderClass); } - if (appenderClass == null) { - throw new IOException("Unknown Storage Type: " + meta.getDataFormat()); - } - appender = OldStorageManager.newAppenderInstance(appenderClass, conf, taskAttemptId, meta, schema, workDir); return appender; From a14c0815a37c18421a2cfe409a75b536ea50fea5 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 2 Jun 2016 09:32:46 +0900 Subject: [PATCH 20/94] Remove unnecessary changes --- .../org/apache/tajo/exception/ExceptionUtil.java | 2 +- .../tajo/storage/fragment/AbstractFileFragment.java | 3 --- .../tajo/storage/http/ExampleHttpFileFragment.java | 12 +----------- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java index bdcdd42beb..adfb0c4a45 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java @@ -205,7 +205,7 @@ public static void printStackTraceIfError(Log log, Throwable t) { // if this runs as an actual cluster instance or a debug mode, it will print all stacktraces. // In other cases (i.e., run as a test mode and not debug mode), it will print stacktraces // if the query is managed mode. - if (TajoConstants.IS_TEST_MODE || TajoConstants.IS_DEBUG_MODE || !ExceptionUtil.isManagedException(t)) { + if (!TajoConstants.IS_TEST_MODE || TajoConstants.IS_DEBUG_MODE || !ExceptionUtil.isManagedException(t)) { ExceptionUtil.printStackTrace(log, t); } } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/AbstractFileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/AbstractFileFragment.java index fa5ff0baa9..cf726e2bc7 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/AbstractFileFragment.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/AbstractFileFragment.java @@ -20,7 +20,6 @@ import org.apache.hadoop.fs.Path; -import java.io.InputStream; import java.net.URI; /** @@ -28,8 +27,6 @@ */ public abstract class AbstractFileFragment extends Fragment { - protected InputStream in; - protected AbstractFileFragment(String kind, URI uri, String inputSourceId, diff --git a/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragment.java b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragment.java index 06bf2516eb..86ebc10257 100644 --- a/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragment.java +++ b/tajo-tablespace-example/src/main/java/org/apache/tajo/storage/http/ExampleHttpFileFragment.java @@ -21,20 +21,10 @@ import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.storage.fragment.BuiltinFragmentKinds; -import java.net.HttpURLConnection; import java.net.URI; public class ExampleHttpFileFragment extends AbstractFileFragment { - - private HttpURLConnection connection; - - /** - * - * @param uri - * @param inputSourceId - * @param startKey first byte pos - * @param endKey last byte pos - */ + public ExampleHttpFileFragment(URI uri, String inputSourceId, long startKey, From 50162bec33675a3ddabe39eb487abc5c4e263ac4 Mon Sep 17 00:00:00 2001 From: Janaka Date: Thu, 2 Jun 2016 12:50:26 +0530 Subject: [PATCH 21/94] Changed Db->DB and dependancy fongo added --- tajo-storage/tajo-storage-mongodb/pom.xml | 8 ++++++++ .../org/apache/tajo/storage/mongodb/ConnectionInfo.java | 4 ++-- .../{MongoDbAppender.java => MongoDBAppender.java} | 2 +- .../{MongoDbFragment.java => MongoDBFragment.java} | 4 ++-- .../mongodb/{MongoDbScanner.java => MongoDBScanner.java} | 2 +- .../{MongoDbTableSpace.java => MongoDBTableSpace.java} | 4 ++-- .../apache/tajo/storage/mongodb/TestConnectionInfo.java | 4 ++-- 7 files changed, 18 insertions(+), 10 deletions(-) rename tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/{MongoDbAppender.java => MongoDBAppender.java} (97%) rename tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/{MongoDbFragment.java => MongoDBFragment.java} (91%) rename tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/{MongoDbScanner.java => MongoDBScanner.java} (97%) rename tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/{MongoDbTableSpace.java => MongoDBTableSpace.java} (97%) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index c6fe17a7a1..311d8616e2 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -215,6 +215,14 @@ mongodb-driver 3.2.2 + + + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + 1.50.5 + + diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index cc3eaf54f9..4b09527240 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -26,7 +26,7 @@ import org.apache.tajo.exception.TajoInternalError; public class ConnectionInfo { - MongoClientURI mongoDbURI; + MongoClientURI mongoDBURI; String scheme; String host; String dbName; @@ -110,7 +110,7 @@ public static ConnectionInfo fromURI(URI originalUri) { mongoDbURIStr+="/"; mongoDbURIStr+=connInfo.dbName; - connInfo.mongoDbURI = new MongoClientURI(mongoDbURIStr); + connInfo.mongoDBURI = new MongoClientURI(mongoDbURIStr); return connInfo; } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java similarity index 97% rename from tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java rename to tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index eba72e559c..a2b7786782 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -28,7 +28,7 @@ /** * Created by janaka on 5/21/16. */ -public class MongoDbAppender implements Appender { +public class MongoDBAppender implements Appender { @Override public void init() throws IOException { diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java similarity index 91% rename from tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java rename to tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java index 69faf39510..3603c08117 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -25,9 +25,9 @@ /** * Created by janaka on 5/21/16. */ -public class MongoDbFragment extends Fragment { +public class MongoDBFragment extends Fragment { - protected MongoDbFragment(String kind, URI uri, String inputSourceId, Long startKey, Long endKey, long length, String[] hostNames) { + protected MongoDBFragment(String kind, URI uri, String inputSourceId, Long startKey, Long endKey, long length, String[] hostNames) { super(kind, uri, inputSourceId, startKey, endKey, length, hostNames); } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java similarity index 97% rename from tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java rename to tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index 4c80c904bc..361ddacfec 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -26,7 +26,7 @@ import org.apache.tajo.storage.Tuple; import java.io.IOException; -public class MongoDbScanner implements Scanner { +public class MongoDBScanner implements Scanner { @Override public void init() throws IOException { diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java similarity index 97% rename from tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java rename to tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 0ddbc13b66..b54aea7cc8 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDbTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -43,14 +43,14 @@ import java.util.List; import java.util.Optional; -public class MongoDbTableSpace extends Tablespace { +public class MongoDBTableSpace extends Tablespace { static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, false); static final FormatProperty FORMAT_PROPERTY = new FormatProperty(false, false, false); - public MongoDbTableSpace(String name, URI uri, JSONObject config) { + public MongoDBTableSpace(String name, URI uri, JSONObject config) { super(name, uri, config); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index c4e1fdc2ae..05cbc31e09 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -35,9 +35,9 @@ public final void testGetConnectionInfoType1() { // System.out.println(connInfo.tableName); // System.out.println(connInfo.password); // System.out.println(connInfo.user); -// System.out.println(connInfo.mongoDbURI.getURI()); +// System.out.println(connInfo.mongoDBURI.getURI()); - assertEquals(connInfo.mongoDbURI.getURI(),"mongodb://testuser:testpass@localhost:1336/db1"); + assertEquals(connInfo.mongoDBURI.getURI(),"mongodb://testuser:testpass@localhost:1336/db1"); assertEquals(connInfo.scheme, "mongodb"); assertEquals(connInfo.host, "localhost"); assertEquals(connInfo.port, 1336); From b1595ee43e8e6e83b109007bcfe246eb9feff14f Mon Sep 17 00:00:00 2001 From: Janaka Date: Thu, 2 Jun 2016 17:50:00 +0530 Subject: [PATCH 22/94] Added Shade --- tajo-storage/tajo-storage-mongodb/pom.xml | 123 ++++++++++++++++++ .../tajo/storage/mongodb/ConnectionInfo.java | 13 ++ .../tajo/storage/mongodb/MongoDBFragment.java | 9 +- .../storage/mongodb/MongoDBFragmentSerde.java | 47 +++++++ .../storage/mongodb/MongoDBTableSpace.java | 34 ++++- .../main/proto/MongoDBFragmentProtos.proto | 31 +++++ .../tajo/storage/mongodb/TestTableSpace.java | 26 ++++ 7 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 311d8616e2..7c2c712e22 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -37,6 +37,129 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.rat + apache-rat-plugin + + + src/test/resources/*.sql + + + + + verify + + check + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + TRUE + + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../../tajo-common/src/main/proto + --proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/MongoDBFragmentProtos.proto + + + + exec + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.6 + + + package + + shade + + + + diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 4b09527240..9ac606073d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -23,9 +23,14 @@ import java.util.Map; import com.mongodb.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.tajo.exception.TajoInternalError; public class ConnectionInfo { + + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + MongoClientURI mongoDBURI; String scheme; String host; @@ -110,11 +115,19 @@ public static ConnectionInfo fromURI(URI originalUri) { mongoDbURIStr+="/"; mongoDbURIStr+=connInfo.dbName; + LOG.info(mongoDbURIStr); connInfo.mongoDBURI = new MongoClientURI(mongoDbURIStr); return connInfo; } + public MongoClientURI getMongoDBURI() + { + return mongoDBURI; + } + + + } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java index 3603c08117..01af24e328 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -21,10 +21,9 @@ import org.apache.tajo.storage.fragment.Fragment; import java.net.URI; +import java.util.List; + -/** - * Created by janaka on 5/21/16. - */ public class MongoDBFragment extends Fragment { protected MongoDBFragment(String kind, URI uri, String inputSourceId, Long startKey, Long endKey, long length, String[] hostNames) { @@ -32,6 +31,10 @@ protected MongoDBFragment(String kind, URI uri, String inputSourceId, Long start } + public MongoDBFragment(String inputSourceId, URI uri, List hostNames) { + super("MONGODB", uri, inputSourceId, null, null, 0, ((String[]) hostNames.toArray())); + } + @Override public boolean isEmpty() { return false; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java new file mode 100644 index 0000000000..dd96b11e6b --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mongodb; + +import com.google.protobuf.GeneratedMessage.Builder; +import org.apache.tajo.storage.fragment.FragmentSerde; +import org.apache.tajo.storage.mongodb.MongoDBFragmentProtos.MongoDBFragmentProto; + +import java.net.URI; + +public class MongoDBFragmentSerde implements FragmentSerde { + + @Override + public Builder newBuilder() { + return MongoDBFragmentProto.newBuilder(); + } + + @Override + public MongoDBFragmentProto serialize(MongoDBFragment fragment) { + return MongoDBFragmentProto.newBuilder() + .setInputSourceId(fragment.getInputSourceId()) + .setUri(fragment.getUri().toASCIIString()) + .addAllHosts(fragment.getHostNames()) + .build(); + } + + @Override + public MongoDBFragment deserialize(MongoDBFragmentProto proto) { + return new MongoDBFragment(proto.getInputSourceId(), URI.create(proto.getUri()), proto.getHostsList()); + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index b54aea7cc8..b023768bca 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -18,7 +18,11 @@ package org.apache.tajo.storage.mongodb; +import com.mongodb.MongoClient; +import com.mongodb.client.MongoDatabase; import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; @@ -27,6 +31,7 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.expr.EvalNode; @@ -45,10 +50,24 @@ public class MongoDBTableSpace extends Tablespace { + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, false); - static final FormatProperty FORMAT_PROPERTY = new FormatProperty(false, false, false); + //Table Space Properties + static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined + false, //not movable + false, // not writable at the moment + false, // Absolute path + false); // Meta data will not be provided + static final FormatProperty FORMAT_PROPERTY = new FormatProperty( + false, // Insert + false, //direct insert + false);// result staging + + //Mongo Client object + protected ConnectionInfo connectionInfo; + protected MongoClient mongoClient; + protected MongoDatabase db; public MongoDBTableSpace(String name, URI uri, JSONObject config) { super(name, uri, config); @@ -56,7 +75,16 @@ public MongoDBTableSpace(String name, URI uri, JSONObject config) { @Override protected void storageInit() throws IOException { - + LOG.debug(uri.toASCIIString()); + try { + connectionInfo = ConnectionInfo.fromURI(uri); + mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + db = mongoClient.getDatabase(connectionInfo.dbName); + } + catch (Exception e) + { + throw new TajoInternalError(e); + } } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto b/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto new file mode 100644 index 0000000000..eadb362829 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.tajo.storage.mongodb"; +option java_outer_classname = "MongoDBFragmentProtos"; +option optimize_for = SPEED; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +import "CatalogProtos.proto"; + +message MongoDBFragmentProto { + required string uri = 1; + required string input_source_id = 2; + repeated string hosts = 3; +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java new file mode 100644 index 0000000000..63d879dd35 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +/** + * Created by janaka on 6/2/16. + */ +public class TestTableSpace { + //mongodb://:@ds017231.mlab.com:17231/tajo_test + +} From 08f867c02cb6a3c416b40745e36f41438ecb9810 Mon Sep 17 00:00:00 2001 From: Janaka Date: Thu, 2 Jun 2016 18:50:19 +0530 Subject: [PATCH 23/94] Changed settings --- .../storage/mongodb/MongoDBTableSpace.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index b023768bca..d3bf7f8c1a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -56,13 +56,13 @@ public class MongoDBTableSpace extends Tablespace { //Table Space Properties static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined false, //not movable - false, // not writable at the moment - false, // Absolute path + true, // not writable at the moment + true, // Absolute path false); // Meta data will not be provided static final FormatProperty FORMAT_PROPERTY = new FormatProperty( - false, // Insert + true, // Insert false, //direct insert - false);// result staging + true);// result staging //Mongo Client object protected ConnectionInfo connectionInfo; @@ -89,7 +89,16 @@ protected void storageInit() throws IOException { @Override public long getTableVolume(TableDesc table, Optional filter) throws UnsupportedException { - return 0; + + long count = 0; + try{ + count = db.getCollection(table.getName()).count(); + } + catch (Exception e) + { + throw new TajoInternalError(e); + } + return count; } @Override @@ -158,4 +167,10 @@ public void rollbackTable(LogicalNode node) throws IOException, TajoException { public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { return null; } + + @Override + public URI getRootUri() + { + return uri; + } } From eac166237d93b909bc9898082ba9ac1635c6fc94 Mon Sep 17 00:00:00 2001 From: Janaka Date: Mon, 6 Jun 2016 13:27:01 +0530 Subject: [PATCH 24/94] Removed the shade plugin --- tajo-storage/tajo-storage-mongodb/pom.xml | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 7c2c712e22..d5a9f74356 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -147,19 +147,19 @@ - - org.apache.maven.plugins - maven-shade-plugin - 1.6 - - - package - - shade - - - - + + + + + + + + + + + + + From 5f6cefc5370bedbdd91dc5ef8bddb443b7eb8c19 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 7 Jun 2016 10:48:48 +0530 Subject: [PATCH 25/94] Before implementing test server --- tajo-storage/tajo-storage-mongodb/pom.xml | 1 + .../org/apache/tajo/storage/mongodb/ConnectionInfo.java | 1 - .../org/apache/tajo/storage/mongodb/MongoDBTestServer.java | 7 +++++++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index d5a9f74356..218b3b9f80 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -337,6 +337,7 @@ org.mongodb mongodb-driver 3.2.2 + provided diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 9ac606073d..5985af279b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -117,7 +117,6 @@ public static ConnectionInfo fromURI(URI originalUri) { LOG.info(mongoDbURIStr); connInfo.mongoDBURI = new MongoClientURI(mongoDbURIStr); - return connInfo; } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java new file mode 100644 index 0000000000..c7694985be --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -0,0 +1,7 @@ +package org.apache.tajo.storage.mongodb; + +/** + * Created by janaka on 6/7/16. + */ +public class MongoDBTestServer { +} From 02c3aef721a5911cd495a8180273a2ccaa554c52 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 7 Jun 2016 11:30:12 +0530 Subject: [PATCH 26/94] Server and One TestCase --- .../storage/mongodb/MongoDBTestServer.java | 73 +++++++++++++++++++ .../tajo/storage/mongodb/TestTableSpace.java | 21 ++++++ 2 files changed, 94 insertions(+) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index c7694985be..fd13ed3f1a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -3,5 +3,78 @@ /** * Created by janaka on 6/7/16. */ +import com.mongodb.BasicDBObject; +import com.mongodb.DB; +import com.mongodb.DBCollection; +import com.mongodb.MongoClient; +import de.flapdoodle.embed.mongo.MongodExecutable; +import de.flapdoodle.embed.mongo.MongodProcess; +import de.flapdoodle.embed.mongo.MongodStarter; +import de.flapdoodle.embed.mongo.config.ArtifactStoreBuilder; +import de.flapdoodle.embed.mongo.config.IMongodConfig; +import de.flapdoodle.embed.mongo.config.MongodConfigBuilder; +import de.flapdoodle.embed.mongo.config.Net; +import de.flapdoodle.embed.mongo.distribution.Version; +import de.flapdoodle.embed.process.runtime.Network; + +import java.io.IOException; +import java.net.URI; +import java.util.Date; + public class MongoDBTestServer { + private static int port; + private static String host = "localhost"; + private static String dbName; + private static MongoDBTestServer instance; + + private MongodExecutable mongodExecutable; + + static { + try { + instance = new MongoDBTestServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static MongoDBTestServer getInstance() + { + return instance; + } + + private MongoDBTestServer () throws IOException { + MongodStarter starter = MongodStarter.getDefaultInstance(); + + + IMongodConfig mongodConfig = new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(port, Network.localhostIsIPv6())) + .build(); + + mongodExecutable = starter.prepare(mongodConfig); + MongodProcess mongod = mongodExecutable.start(); + + MongoClient mongo = new MongoClient(host, port); + DB db = mongo.getDB(dbName); + DBCollection col = db.createCollection("testCol", new BasicDBObject()); + col.save(new BasicDBObject("testDoc", new Date())); + + + } + + public void stop() + { + mongodExecutable.stop(); + } + + public URI getURI() + { + try { + return new URI("mongodb://" + host + ":" + port + "/"+dbName); + } + catch (Exception e) + { + return null; + } + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java index 63d879dd35..59133f7805 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java @@ -17,10 +17,31 @@ */ package org.apache.tajo.storage.mongodb; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.Test; + +import java.net.URI; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Created by janaka on 6/2/16. */ public class TestTableSpace { //mongodb://:@ds017231.mlab.com:17231/tajo_test + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + static URI uri = server.getURI(); + + @Test + public void testTablespaceHandler() + { + + assertTrue((TablespaceManager.getByName("mongodb_cluster")) instanceof MongoDBTableSpace); + assertEquals("mongodb_cluster", (TablespaceManager.getByName("pgsql_cluster").getName())); + server.stop(); + } + + } From 501f2f32525197e6a82d961e1a62e53f5539cb04 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 7 Jun 2016 11:58:53 +0530 Subject: [PATCH 27/94] Added license --- .../storage/mongodb/MongoDBTestServer.java | 97 +++++++++++++------ .../tajo/storage/mongodb/TestTableSpace.java | 18 +++- 2 files changed, 86 insertions(+), 29 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index fd13ed3f1a..8c96ca0341 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tajo.storage.mongodb; /** @@ -7,57 +24,54 @@ import com.mongodb.DB; import com.mongodb.DBCollection; import com.mongodb.MongoClient; -import de.flapdoodle.embed.mongo.MongodExecutable; -import de.flapdoodle.embed.mongo.MongodProcess; -import de.flapdoodle.embed.mongo.MongodStarter; -import de.flapdoodle.embed.mongo.config.ArtifactStoreBuilder; -import de.flapdoodle.embed.mongo.config.IMongodConfig; -import de.flapdoodle.embed.mongo.config.MongodConfigBuilder; -import de.flapdoodle.embed.mongo.config.Net; +import de.flapdoodle.embed.mongo.*; +import de.flapdoodle.embed.mongo.config.*; import de.flapdoodle.embed.mongo.distribution.Version; import de.flapdoodle.embed.process.runtime.Network; import java.io.IOException; import java.net.URI; +import java.net.UnknownHostException; import java.util.Date; public class MongoDBTestServer { - private static int port; + private static int port = 12345; private static String host = "localhost"; private static String dbName; private static MongoDBTestServer instance; private MongodExecutable mongodExecutable; - static { - try { - instance = new MongoDBTestServer(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } public static MongoDBTestServer getInstance() { + if(instance==null) + { + try { + instance = new MongoDBTestServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return instance; } private MongoDBTestServer () throws IOException { - MongodStarter starter = MongodStarter.getDefaultInstance(); + MongodProcess mongod = startMongod(port); - IMongodConfig mongodConfig = new MongodConfigBuilder() - .version(Version.Main.PRODUCTION) - .net(new Net(port, Network.localhostIsIPv6())) - .build(); - - mongodExecutable = starter.prepare(mongodConfig); - MongodProcess mongod = mongodExecutable.start(); - - MongoClient mongo = new MongoClient(host, port); - DB db = mongo.getDB(dbName); - DBCollection col = db.createCollection("testCol", new BasicDBObject()); - col.save(new BasicDBObject("testDoc", new Date())); + try { + MongosProcess mongos = startMongos(1111, port, host); + try { + MongoClient mongoClient = new MongoClient(host, port); + System.out.println("DB Names: " + mongoClient.getDatabaseNames()); + } finally { + mongos.stop(); + } + } finally { + mongod.stop(); + } } @@ -77,4 +91,31 @@ public URI getURI() return null; } } + + + //From GitHub + private MongosProcess startMongos(int port, int defaultConfigPort, String defaultHost) throws UnknownHostException, + IOException { + IMongosConfig mongosConfig = new MongosConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(port, Network.localhostIsIPv6())) + .configDB(defaultHost + ":" + defaultConfigPort) + .build(); + + MongosExecutable mongosExecutable = MongosStarter.getDefaultInstance().prepare(mongosConfig); + MongosProcess mongos = mongosExecutable.start(); + return mongos; + } + + private MongodProcess startMongod(int defaultConfigPort) throws UnknownHostException, IOException { + IMongodConfig mongoConfigConfig = new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(defaultConfigPort, Network.localhostIsIPv6())) + .configServer(true) + .build(); + + MongodExecutable mongodExecutable = MongodStarter.getDefaultInstance().prepare(mongoConfigConfig); + MongodProcess mongod = mongodExecutable.start(); + return mongod; + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java index 59133f7805..1be7bb3d72 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java @@ -21,7 +21,23 @@ import org.junit.Test; import java.net.URI; - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; From 1b1256a498d5db28a955226e58feeaa0955b3e7a Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 7 Jun 2016 12:28:53 +0530 Subject: [PATCH 28/94] Server runs --- .../storage/mongodb/MongoDBTestServer.java | 36 ++++++++++--------- .../tajo/storage/mongodb/TestTableSpace.java | 5 +-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index 8c96ca0341..11db2a186d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -34,13 +34,19 @@ import java.net.UnknownHostException; import java.util.Date; -public class MongoDBTestServer { +public class MongoDBTestServer { + private static int port = 12345; private static String host = "localhost"; private static String dbName; private static MongoDBTestServer instance; - private MongodExecutable mongodExecutable; + + //New + private static final MongodStarter starter = MongodStarter.getDefaultInstance(); + private MongodExecutable _mongodExe; + private MongodProcess _mongod; + private MongoClient _mongo; public static MongoDBTestServer getInstance() @@ -58,27 +64,23 @@ public static MongoDBTestServer getInstance() } private MongoDBTestServer () throws IOException { + _mongodExe = starter.prepare(new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(port, Network.localhostIsIPv6())) - MongodProcess mongod = startMongod(port); - - try { - MongosProcess mongos = startMongos(1111, port, host); - try { - MongoClient mongoClient = new MongoClient(host, port); - System.out.println("DB Names: " + mongoClient.getDatabaseNames()); - } finally { - mongos.stop(); - } - } finally { - mongod.stop(); - } - + .cmdOptions( new MongoCmdOptionsBuilder(). + useStorageEngine("mmapv1"). + build()) + .build()); + _mongod = _mongodExe.start(); + _mongo = new MongoClient(host, port); } public void stop() { - mongodExecutable.stop(); + _mongod.stop(); + _mongodExe.stop(); } public URI getURI() diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java index 1be7bb3d72..9a88c31932 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java @@ -52,9 +52,10 @@ public class TestTableSpace { @Test public void testTablespaceHandler() { + assertTrue((TablespaceManager.get(uri)) instanceof MongoDBTableSpace); - assertTrue((TablespaceManager.getByName("mongodb_cluster")) instanceof MongoDBTableSpace); - assertEquals("mongodb_cluster", (TablespaceManager.getByName("pgsql_cluster").getName())); +// assertTrue((TablespaceManager.getByName("mongodb_cluster")) instanceof MongoDBTableSpace); +// assertEquals("mongodb_cluster", (TablespaceManager.getByName("pgsql_cluster").getName())); server.stop(); } From 664207565ec9f5848a47d7cd62e8ec7fd6499d0a Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 7 Jun 2016 12:45:03 +0530 Subject: [PATCH 29/94] Add tablespace for test --- .../src/main/resources/storage-default.json | 4 ++++ .../tajo/storage/mongodb/MongoDBTestServer.java | 17 ++++++++++++++++- .../tajo/storage/mongodb/TestTableSpace.java | 4 ++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json index 17ac3ba419..1caf0953d1 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json @@ -31,6 +31,10 @@ "swift": { "handler": "org.apache.tajo.storage.s3.S3TableSpace", "default-format": "text" + }, + "mongodb": { + "handler": "org.apache.tajo.storage.mongodb.MongoDBTableSpace", + "default-format": "text" } } } \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index 11db2a186d..0ad392c251 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -28,6 +28,9 @@ import de.flapdoodle.embed.mongo.config.*; import de.flapdoodle.embed.mongo.distribution.Version; import de.flapdoodle.embed.process.runtime.Network; +import net.minidev.json.JSONObject; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; import java.io.IOException; import java.net.URI; @@ -41,6 +44,8 @@ public class MongoDBTestServer { private static String dbName; private static MongoDBTestServer instance; + public static final String spaceName = "mongo_cluster"; + //New private static final MongodStarter starter = MongodStarter.getDefaultInstance(); @@ -73,8 +78,8 @@ private MongoDBTestServer () throws IOException { build()) .build()); _mongod = _mongodExe.start(); - _mongo = new MongoClient(host, port); + registerTablespace(); } public void stop() @@ -120,4 +125,14 @@ private MongodProcess startMongod(int defaultConfigPort) throws UnknownHostExcep MongodProcess mongod = mongodExecutable.start(); return mongod; } + + private void registerTablespace() throws IOException { + JSONObject configElements = new JSONObject(); +// configElements.put(JdbcTablespace.CONFIG_KEY_MAPPED_DATABASE, dbName); + + MongoDBTableSpace tablespace = new MongoDBTableSpace(spaceName,getURI(),configElements); + tablespace.init(new TajoConf()); + + TablespaceManager.addTableSpaceForTest(tablespace); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java index 9a88c31932..6506a2d4f8 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java @@ -54,8 +54,8 @@ public void testTablespaceHandler() { assertTrue((TablespaceManager.get(uri)) instanceof MongoDBTableSpace); -// assertTrue((TablespaceManager.getByName("mongodb_cluster")) instanceof MongoDBTableSpace); -// assertEquals("mongodb_cluster", (TablespaceManager.getByName("pgsql_cluster").getName())); + assertTrue((TablespaceManager.getByName("mongo_cluster")) instanceof MongoDBTableSpace); + assertEquals("mongo_cluster", (TablespaceManager.getByName("mongo_cluster").getName())); server.stop(); } From 4ffcf769fac162379aa0236a2b29cc55c5842745 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 7 Jun 2016 21:16:32 +0530 Subject: [PATCH 30/94] Added Metadata Provider --- tajo-storage/tajo-storage-mongodb/pom.xml | 1 - .../mongodb/MongoDBMetadataProvider.java | 55 +++++++++++++++++++ .../storage/mongodb/MongoDBTableSpace.java | 31 +++++------ .../storage/mongodb/MongoDBTestServer.java | 1 + .../tajo/storage/mongodb/TestTableSpace.java | 17 +++++- 5 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 218b3b9f80..d5a9f74356 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -337,7 +337,6 @@ org.mongodb mongodb-driver 3.2.2 - provided diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java new file mode 100644 index 0000000000..da9dd5399d --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -0,0 +1,55 @@ +package org.apache.tajo.storage.mongodb; + +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.exception.UndefinedTablespaceException; + +import javax.annotation.Nullable; +import java.net.URI; +import java.util.Collection; +import java.util.Collections; + +/** + * Created by janaka on 6/7/16. + */ +public class MongoDBMetadataProvider implements MetadataProvider { + + private MongoDBTableSpace tableSpace; + private String dbName; + + public MongoDBMetadataProvider(MongoDBTableSpace tableSpace, String dbName) + { + this.tableSpace = tableSpace; + this.dbName = dbName; + } + + @Override + public String getTablespaceName() { + return tableSpace.getName(); + } + + @Override + public URI getTablespaceUri() { + return tableSpace.getUri(); + } + + @Override + public String getDatabaseName() { + return dbName; + } + + @Override + public Collection getSchemas() { + return Collections.EMPTY_SET; + } + + @Override + public Collection getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { + return Collections.EMPTY_SET; + } + + @Override + public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException { + return null; + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index d3bf7f8c1a..a05f740481 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -26,10 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.UnsupportedException; @@ -58,14 +55,14 @@ public class MongoDBTableSpace extends Tablespace { false, //not movable true, // not writable at the moment true, // Absolute path - false); // Meta data will not be provided - static final FormatProperty FORMAT_PROPERTY = new FormatProperty( + true); // Meta data will be provided + static final FormatProperty FORMAT_PROPERTY = new FormatProperty( true, // Insert false, //direct insert true);// result staging //Mongo Client object - protected ConnectionInfo connectionInfo; + protected ConnectionInfo connectionInfo; protected MongoClient mongoClient; protected MongoDatabase db; @@ -80,9 +77,7 @@ protected void storageInit() throws IOException { connectionInfo = ConnectionInfo.fromURI(uri); mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); db = mongoClient.getDatabase(connectionInfo.dbName); - } - catch (Exception e) - { + } catch (Exception e) { throw new TajoInternalError(e); } } @@ -91,11 +86,9 @@ protected void storageInit() throws IOException { public long getTableVolume(TableDesc table, Optional filter) throws UnsupportedException { long count = 0; - try{ + try { count = db.getCollection(table.getName()).count(); - } - catch (Exception e) - { + } catch (Exception e) { throw new TajoInternalError(e); } return count; @@ -112,7 +105,6 @@ public List getSplits(String inputSourceId, TableDesc tableDesc, boole } - @Override public StorageProperty getProperty() { return STORAGE_PROPERTY; @@ -169,8 +161,13 @@ public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta } @Override - public URI getRootUri() - { + public URI getRootUri() { return uri; } + + + //Metadata + public MetadataProvider getMetadataProvider() { + return new MongoDBMetadataProvider(this, "test_db_MongoDBTableSpace_171"); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index 0ad392c251..a2d82d23ce 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -133,6 +133,7 @@ private void registerTablespace() throws IOException { MongoDBTableSpace tablespace = new MongoDBTableSpace(spaceName,getURI(),configElements); tablespace.init(new TajoConf()); + TablespaceManager.addTableSpaceForTest(tablespace); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java index 6506a2d4f8..ba9d868025 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.mongodb; import org.apache.tajo.storage.TablespaceManager; +import org.junit.After; import org.junit.Test; import java.net.URI; @@ -56,9 +57,21 @@ public void testTablespaceHandler() assertTrue((TablespaceManager.getByName("mongo_cluster")) instanceof MongoDBTableSpace); assertEquals("mongo_cluster", (TablespaceManager.getByName("mongo_cluster").getName())); - server.stop(); + assertTrue((TablespaceManager.get(uri.toASCIIString() + "&table=tb1")) instanceof MongoDBTableSpace); + + + assertEquals(uri.toASCIIString(), TablespaceManager.get(uri).getUri().toASCIIString()); + + assertTrue(TablespaceManager.get(uri).getMetadataProvider() instanceof MongoDBMetadataProvider); } -} + @After + public void clear() + { + server.stop(); + } + + +} \ No newline at end of file From f90375442bd047904393ad64ae673b5daa7b085f Mon Sep 17 00:00:00 2001 From: Janaka Date: Wed, 8 Jun 2016 18:09:58 +0530 Subject: [PATCH 31/94] Load data to server --- tajo-storage/tajo-storage-mongodb/pom.xml | 26 +++---- .../tajo/storage/mongodb/ConnectionInfo.java | 64 ++++++++++++----- .../mongodb/MongoDBMetadataProvider.java | 64 +++++++++++++++-- .../storage/mongodb/MongoDBTableSpace.java | 35 ++++++++-- .../storage/mongodb/MongoDBTestServer.java | 70 ++++++++++++++++--- .../storage/mongodb/TestConnectionInfo.java | 61 +++++----------- .../tajo/storage/mongodb/TestTableSpace.java | 6 +- .../src/test/resources/datasets/file1.json | 5 ++ 8 files changed, 238 insertions(+), 93 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index d5a9f74356..7c2c712e22 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -147,19 +147,19 @@ - - - - - - - - - - - - - + + org.apache.maven.plugins + maven-shade-plugin + 1.6 + + + package + + shade + + + + diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 5985af279b..12c88fb821 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -31,15 +31,15 @@ public class ConnectionInfo { private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - MongoClientURI mongoDBURI; - String scheme; - String host; - String dbName; - String tableName; - String user; - String password; - int port; - Map params; + private MongoClientURI mongoDBURI; + private String scheme; + private String host; + private String dbName; + private String tableName; + private String user; + private String password; + private int port; + private Map params; public static ConnectionInfo fromURI(String originalUri) { return fromURI(URI.create(originalUri)); @@ -100,20 +100,20 @@ public static ConnectionInfo fromURI(URI originalUri) { //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] //Generate the URI - mongoDbURIStr+=connInfo.scheme; + mongoDbURIStr+= connInfo.getScheme(); mongoDbURIStr+="://"; - if(connInfo.user!=null) + if(connInfo.getUser() !=null) { - mongoDbURIStr+=connInfo.user; - if(connInfo.password!=null) - mongoDbURIStr+=":"+connInfo.password; + mongoDbURIStr+= connInfo.getUser(); + if(connInfo.getPassword() !=null) + mongoDbURIStr+=":"+ connInfo.getPassword(); mongoDbURIStr+="@"; } - mongoDbURIStr+=connInfo.host; + mongoDbURIStr+= connInfo.getHost(); mongoDbURIStr+=":"; - mongoDbURIStr+=connInfo.port; + mongoDbURIStr+= connInfo.getPort(); mongoDbURIStr+="/"; - mongoDbURIStr+=connInfo.dbName; + mongoDbURIStr+= connInfo.getDbName(); LOG.info(mongoDbURIStr); connInfo.mongoDBURI = new MongoClientURI(mongoDbURIStr); @@ -126,7 +126,37 @@ public MongoClientURI getMongoDBURI() } + public String getScheme() { + return scheme; + } + + public String getHost() { + return host; + } + + public String getDbName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public int getPort() { + return port; + } + + public Map getParams() { + return params; + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index da9dd5399d..6a68deb084 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -1,11 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.storage.mongodb; +import com.mongodb.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.MongoIterable; import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.SchemaBuilder; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.exception.UndefinedTablespaceException; +import org.apache.tajo.util.KeyValueSet; import javax.annotation.Nullable; import java.net.URI; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -15,12 +41,18 @@ public class MongoDBMetadataProvider implements MetadataProvider { private MongoDBTableSpace tableSpace; - private String dbName; + private String mappedDbName; + private ConnectionInfo connectionInfo; + MongoDatabase db; public MongoDBMetadataProvider(MongoDBTableSpace tableSpace, String dbName) { this.tableSpace = tableSpace; - this.dbName = dbName; + this.mappedDbName = dbName; + + connectionInfo = tableSpace.getConnectionInfo(); + MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + db = mongoClient.getDatabase(connectionInfo.getDbName()); } @Override @@ -35,7 +67,7 @@ public URI getTablespaceUri() { @Override public String getDatabaseName() { - return dbName; + return mappedDbName; } @Override @@ -45,11 +77,33 @@ public Collection getSchemas() { @Override public Collection getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { - return Collections.EMPTY_SET; + + + MongoIterable collectionList = db.listCollectionNames(); + + Collection list = new ArrayList(); + for (String item : collectionList) { + list.add(item); + } + return list; } @Override public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException { - return null; + + TableMeta tbMeta = new TableMeta("rowstore", new KeyValueSet()); + TableDesc tbDesc = new TableDesc(tableName, + SchemaBuilder.builder() + .build(),tbMeta,tableSpace.getUri()); + + + final TableStats stats = new TableStats(); + stats.setNumRows(-1); // unknown + + //Set the raw count + stats.setNumRows(db.getCollection(tableName).count()); + + tbDesc.setStats(stats); + return tbDesc; } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index a05f740481..cf7397dc20 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -62,21 +62,42 @@ public class MongoDBTableSpace extends Tablespace { true);// result staging //Mongo Client object - protected ConnectionInfo connectionInfo; + private ConnectionInfo connectionInfo; protected MongoClient mongoClient; protected MongoDatabase db; + protected String mappedDBName = "MappedDB"; + + + //Config Keys + public static final String CONFIG_KEY_MAPPED_DATABASE = "mapped_database"; + public static final String CONFIG_KEY_CONN_PROPERTIES = "connection_properties"; + public static final String CONFIG_KEY_USERNAME = "user"; + public static final String CONFIG_KEY_PASSWORD = "password"; public MongoDBTableSpace(String name, URI uri, JSONObject config) { + super(name, uri, config); + connectionInfo = ConnectionInfo.fromURI(uri); + + //set Connection Properties + if (config.containsKey(CONFIG_KEY_MAPPED_DATABASE)) { + mappedDBName = this.config.getAsString(CONFIG_KEY_MAPPED_DATABASE); + } else { + mappedDBName = getConnectionInfo().getDbName(); + } + + + } @Override protected void storageInit() throws IOException { - LOG.debug(uri.toASCIIString()); + // LOG.info(uri.toASCIIString()); + //LOG.info("Something"+config.toJSONString()); try { connectionInfo = ConnectionInfo.fromURI(uri); - mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); - db = mongoClient.getDatabase(connectionInfo.dbName); + mongoClient = new MongoClient(getConnectionInfo().getMongoDBURI()); + db = mongoClient.getDatabase(getConnectionInfo().getDbName()); } catch (Exception e) { throw new TajoInternalError(e); } @@ -168,6 +189,10 @@ public URI getRootUri() { //Metadata public MetadataProvider getMetadataProvider() { - return new MongoDBMetadataProvider(this, "test_db_MongoDBTableSpace_171"); + return new MongoDBMetadataProvider(this, mappedDBName); + } + + public ConnectionInfo getConnectionInfo() { + return connectionInfo; } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index a2d82d23ce..766e905dfb 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -20,28 +20,43 @@ /** * Created by janaka on 6/7/16. */ -import com.mongodb.BasicDBObject; -import com.mongodb.DB; -import com.mongodb.DBCollection; -import com.mongodb.MongoClient; +import com.mongodb.*; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; import de.flapdoodle.embed.mongo.*; import de.flapdoodle.embed.mongo.config.*; import de.flapdoodle.embed.mongo.distribution.Version; import de.flapdoodle.embed.process.runtime.Network; import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.htrace.fasterxml.jackson.core.JsonFactory; +import org.apache.htrace.fasterxml.jackson.core.JsonParser; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.TablespaceManager; +import org.bson.Document; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.net.UnknownHostException; -import java.util.Date; +import java.util.ArrayList; +import java.util.List; public class MongoDBTestServer { + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + + private static int port = 12345; private static String host = "localhost"; - private static String dbName; + private static String dbName = "mydbname"; + private static String mappedDbName = "mapped_mydbname"; private static MongoDBTestServer instance; public static final String spaceName = "mongo_cluster"; @@ -53,6 +68,9 @@ public class MongoDBTestServer { private MongodProcess _mongod; private MongoClient _mongo; + private String[] filenames = {"file1.json","file2.json"}; + private String[] collectionNames = {"col1","col2"}; + public static MongoDBTestServer getInstance() { @@ -68,7 +86,7 @@ public static MongoDBTestServer getInstance() return instance; } - private MongoDBTestServer () throws IOException { + private MongoDBTestServer () throws IOException, URISyntaxException { _mongodExe = starter.prepare(new MongodConfigBuilder() .version(Version.Main.PRODUCTION) .net(new Net(port, Network.localhostIsIPv6())) @@ -80,6 +98,8 @@ private MongoDBTestServer () throws IOException { _mongod = _mongodExe.start(); _mongo = new MongoClient(host, port); registerTablespace(); + + loadData(); } public void stop() @@ -128,7 +148,7 @@ private MongodProcess startMongod(int defaultConfigPort) throws UnknownHostExcep private void registerTablespace() throws IOException { JSONObject configElements = new JSONObject(); -// configElements.put(JdbcTablespace.CONFIG_KEY_MAPPED_DATABASE, dbName); + configElements.put(MongoDBTableSpace.CONFIG_KEY_MAPPED_DATABASE, mappedDbName); MongoDBTableSpace tablespace = new MongoDBTableSpace(spaceName,getURI(),configElements); tablespace.init(new TajoConf()); @@ -136,4 +156,38 @@ private void registerTablespace() throws IOException { TablespaceManager.addTableSpaceForTest(tablespace); } + + private void loadData() throws IOException, URISyntaxException { + MongoDatabase db = _mongo.getDatabase(dbName); + for(int i=0; i<1;i++) { + + JsonParser parser = new JsonFactory().createParser(new FileReader( getRequestedFile(filenames[i]))); + DBObject dbo = (DBObject) com.mongodb.util.JSON.parse(parser.getText()); + + List list = new ArrayList<>(); + list.add(dbo); + + db.createCollection(collectionNames[0]); + MongoCollection coll = db.getCollection(collectionNames[0]); + + FindIterable docs = coll.find(); + + docs.forEach(new Block() { + @Override + public void apply(final Document document) { + LOG.info(document.toJson()); + } + }); + } + } + + private static File getRequestedFile(String path) throws FileNotFoundException, URISyntaxException { + + URL url = ClassLoader.getSystemResource("datasets/" + path); + + if (url == null) { + throw new FileNotFoundException(path); + } + return new File(url.toURI()); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index 05cbc31e09..49932e2de4 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -28,54 +28,29 @@ public final void testGetConnectionInfoType1() { ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass"); -// System.out.println(connInfo.scheme); -// System.out.println(connInfo.dbName); -// System.out.println(connInfo.host); -// System.out.println(connInfo.port); -// System.out.println(connInfo.tableName); -// System.out.println(connInfo.password); -// System.out.println(connInfo.user); -// System.out.println(connInfo.mongoDBURI.getURI()); - - assertEquals(connInfo.mongoDBURI.getURI(),"mongodb://testuser:testpass@localhost:1336/db1"); - assertEquals(connInfo.scheme, "mongodb"); - assertEquals(connInfo.host, "localhost"); - assertEquals(connInfo.port, 1336); - assertEquals(connInfo.dbName, "db1"); - assertEquals(connInfo.user, "testuser"); - assertEquals(connInfo.password, "testpass"); - assertEquals(connInfo.tableName, "tb1"); + assertEquals(connInfo.getMongoDBURI().getURI(),"mongodb://testuser:testpass@localhost:1336/db1"); + assertEquals(connInfo.getScheme(), "mongodb"); + assertEquals(connInfo.getHost(), "localhost"); + assertEquals(connInfo.getPort(), 1336); + assertEquals(connInfo.getDbName(), "db1"); + assertEquals(connInfo.getUser(), "testuser"); + assertEquals(connInfo.getPassword(), "testpass"); + assertEquals(connInfo.getTableName(), "tb1"); } @Test public final void testGetConnectionInfoType2() { //Create a connection info object ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); - assertEquals(connInfo.scheme, "mongodb"); - assertEquals(connInfo.host, "localhost"); - assertEquals(connInfo.port, 1336); - assertEquals(connInfo.dbName, "db1"); - assertEquals(connInfo.user, "testuser"); - assertEquals(connInfo.password, "testpass"); - assertEquals(connInfo.tableName, "tb1"); - assertEquals(1, connInfo.params.size()); - assertEquals("GMT+9", connInfo.params.get("TZ")); - } - - //Test code for multiple hosts -// @Test -// public final void testGetConnectionInfoType3() { -// -// ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336,googl.com:2727/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); -// assertEquals(connInfo.scheme, "mongodb"); -// assertEquals(connInfo.host, "localhost"); -// assertEquals(connInfo.port, 1336); -// assertEquals(connInfo.dbName, "db1"); -// assertEquals(connInfo.user, "testuser"); -// assertEquals(connInfo.password, "testpass"); -// assertEquals(connInfo.tableName, "tb1"); -// assertEquals(1, connInfo.params.size()); -// assertEquals("GMT+9", connInfo.params.get("TZ")); -// } + assertEquals(connInfo.getScheme(), "mongodb"); + assertEquals(connInfo.getHost(), "localhost"); + assertEquals(connInfo.getPort(), 1336); + assertEquals(connInfo.getDbName(), "db1"); + assertEquals(connInfo.getUser(), "testuser"); + assertEquals(connInfo.getPassword(), "testpass"); + assertEquals(connInfo.getTableName(), "tb1"); + assertEquals(1, connInfo.getParams().size()); + assertEquals("GMT+9", connInfo.getParams().get("TZ")); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java index ba9d868025..52ea1f1e1d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java @@ -55,13 +55,15 @@ public void testTablespaceHandler() { assertTrue((TablespaceManager.get(uri)) instanceof MongoDBTableSpace); - assertTrue((TablespaceManager.getByName("mongo_cluster")) instanceof MongoDBTableSpace); - assertEquals("mongo_cluster", (TablespaceManager.getByName("mongo_cluster").getName())); + assertTrue((TablespaceManager.getByName(server.spaceName)) instanceof MongoDBTableSpace); + assertEquals(server.spaceName, (TablespaceManager.getByName(server.spaceName).getName())); assertTrue((TablespaceManager.get(uri.toASCIIString() + "&table=tb1")) instanceof MongoDBTableSpace); + //Test the URI same assertEquals(uri.toASCIIString(), TablespaceManager.get(uri).getUri().toASCIIString()); + //Check if returns the MetaDataProvider assertTrue(TablespaceManager.get(uri).getMetadataProvider() instanceof MongoDBMetadataProvider); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json new file mode 100644 index 0000000000..36579ef953 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json @@ -0,0 +1,5 @@ +{ "test_id" : 1245362, + "name" : "ganesh", + "age" : "28", + "Job" : {"company name" : "company1", "designation" : "SSE" } +} \ No newline at end of file From 22dd1da4b3de7aa9201b14f0ad45c5deac3c09bf Mon Sep 17 00:00:00 2001 From: Janaka Date: Thu, 9 Jun 2016 01:18:36 +0530 Subject: [PATCH 32/94] Server works and load data --- tajo-storage/tajo-storage-mongodb/pom.xml | 29 +++--- .../mongodb/MongoDBMetadataProvider.java | 9 +- .../storage/mongodb/MongoDBTableSpace.java | 2 +- .../storage/mongodb/MongoDBTestServer.java | 32 ++++--- .../storage/mongodb/TestConnectionInfo.java | 6 ++ .../storage/mongodb/TestMetadataProvider.java | 95 +++++++++++++++++++ .../tajo/storage/mongodb/TestTableSpace.java | 10 -- .../src/test/resources/datasets/file1.json | 15 ++- .../src/test/resources/datasets/file2.json | 4 + 9 files changed, 156 insertions(+), 46 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file2.json diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 7c2c712e22..52ead0cbfc 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -46,7 +46,7 @@ apache-rat-plugin - src/test/resources/*.sql + src/test/resources/datasets/** @@ -147,19 +147,19 @@ - - org.apache.maven.plugins - maven-shade-plugin - 1.6 - - - package - - shade - - - - + + + + + + + + + + + + + @@ -337,6 +337,7 @@ org.mongodb mongodb-driver 3.2.2 + provided diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index 6a68deb084..212b7679b1 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -27,6 +27,7 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.exception.UndefinedTablespaceException; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.KeyValueSet; import javax.annotation.Nullable; @@ -92,10 +93,12 @@ public Collection getTables(@Nullable String schemaPattern, @Nullable St public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException { TableMeta tbMeta = new TableMeta("rowstore", new KeyValueSet()); - TableDesc tbDesc = new TableDesc(tableName, + TableDesc tbDesc = new TableDesc( + IdentifierUtil.buildFQName(mappedDbName, tableName), SchemaBuilder.builder() - .build(),tbMeta,tableSpace.getUri()); - + .build(), + tbMeta, + tableSpace.getTableUri(null, tableName)); final TableStats stats = new TableStats(); stats.setNumRows(-1); // unknown diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index cf7397dc20..3703c7040c 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -117,7 +117,7 @@ public long getTableVolume(TableDesc table, Optional filter) throws Un @Override public URI getTableUri(String databaseName, String tableName) { - return null; + return URI.create(this.getUri()+"&table="+tableName); } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index 766e905dfb..744d78ed96 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -24,6 +24,8 @@ import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; +import com.mongodb.util.JSON; +import com.sun.javadoc.Doc; import de.flapdoodle.embed.mongo.*; import de.flapdoodle.embed.mongo.config.*; import de.flapdoodle.embed.mongo.distribution.Version; @@ -47,19 +49,20 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; +import java.util.Scanner; public class MongoDBTestServer { private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - private static int port = 12345; - private static String host = "localhost"; - private static String dbName = "mydbname"; - private static String mappedDbName = "mapped_mydbname"; + public static int port = 12345; + public static String host = "localhost"; + public static String dbName = "test_dbname"; + public static String mappedDbName = "test_mapped_dbname"; private static MongoDBTestServer instance; - public static final String spaceName = "mongo_cluster"; + public static final String spaceName = "test_spacename"; //New @@ -69,7 +72,7 @@ public class MongoDBTestServer { private MongoClient _mongo; private String[] filenames = {"file1.json","file2.json"}; - private String[] collectionNames = {"col1","col2"}; + public String[] collectionNames = {"col1","col2"}; public static MongoDBTestServer getInstance() @@ -159,18 +162,19 @@ private void registerTablespace() throws IOException { private void loadData() throws IOException, URISyntaxException { MongoDatabase db = _mongo.getDatabase(dbName); - for(int i=0; i<1;i++) { + for(int i=0; i list = new ArrayList<>(); - list.add(dbo); + String fileContent = new Scanner( getRequestedFile(filenames[i])).useDelimiter("\\Z").next(); - db.createCollection(collectionNames[0]); - MongoCollection coll = db.getCollection(collectionNames[0]); + Document fileDoc = Document.parse(fileContent); +// System.out.println( fileDoc.get("dataList")); +// coll.insertMany((List) fileDoc.get("dataList")); + coll.insertOne(fileDoc); - FindIterable docs = coll.find(); + FindIterable docs = coll.find(); docs.forEach(new Block() { @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index 49932e2de4..ff3a76bee9 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -53,4 +53,10 @@ public final void testGetConnectionInfoType2() { assertEquals(1, connInfo.getParams().size()); assertEquals("GMT+9", connInfo.getParams().get("TZ")); } + + @Test + public final void testNewStructure() + { + + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java new file mode 100644 index 0000000000..142d7023cd --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import com.google.common.collect.Sets; +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.*; + +import java.net.URI; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Created by janaka on 6/8/16. + */ +public class TestMetadataProvider { + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + + @Test + public void testGetTablespaceName() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals(server.spaceName, provider.getTablespaceName()); + } + + @Test + public void testGetDatabaseName() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals(MongoDBTestServer.mappedDbName, provider.getDatabaseName()); + } + + + @Test + public void testGetSchemas() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertTrue(provider.getSchemas().isEmpty()); + } + + @Test + public void testGetTables() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + + final Set expected = Sets.newHashSet(server.collectionNames); + expected.add("system.indexes"); + final Set found = Sets.newHashSet(provider.getTables(null, null)); + + assertEquals(expected, found); + } + @Test + public void testGetTableDescriptor() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + + for (String tableName : server.collectionNames) { + TableDesc table = provider.getTableDesc(null, tableName); + assertEquals(server.mappedDbName+"." + tableName, table.getName()); + assertEquals(server.getURI() + "&table=" + tableName, table.getUri().toASCIIString()); + + System.out.println(tableName+" : "+table.getStats().getNumRows()); + } + + + + } + + + + + + + +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java index 52ea1f1e1d..1a65f43174 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java @@ -66,14 +66,4 @@ public void testTablespaceHandler() //Check if returns the MetaDataProvider assertTrue(TablespaceManager.get(uri).getMetadataProvider() instanceof MongoDBMetadataProvider); } - - - - @After - public void clear() - { - server.stop(); - } - - } \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json index 36579ef953..cff120a088 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json @@ -1,5 +1,12 @@ -{ "test_id" : 1245362, - "name" : "ganesh", - "age" : "28", - "Job" : {"company name" : "company1", "designation" : "SSE" } +{ +'dataList':[ + { + 'name' : 'mkyong', + 'age' : 30 +}, + { + 'name' : 'mkyong', + 'age' : 30 + } + ] } \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file2.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file2.json new file mode 100644 index 0000000000..a7d85f481c --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file2.json @@ -0,0 +1,4 @@ +{ + 'name' : 'mkyong', + 'age' : 30 +} \ No newline at end of file From 87aee6d305be1d38c0320f089cbbd73869fb3eb3 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 9 Jun 2016 14:05:56 +0900 Subject: [PATCH 33/94] Disable example http tablespace for http scheme by default. --- .../src/main/resources/storage-default.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json index a94ae8f9ba..17ac3ba419 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json @@ -31,10 +31,6 @@ "swift": { "handler": "org.apache.tajo.storage.s3.S3TableSpace", "default-format": "text" - }, - "http": { - "handler": "org.apache.tajo.storage.http.ExampleHttpFileTablespace", - "default-format": "json" } } } \ No newline at end of file From 5dc491491b35705a484946d6b7b390348bc8bf60 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 10 Jun 2016 21:11:17 +0530 Subject: [PATCH 34/94] Doc Added --- tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst diff --git a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst new file mode 100644 index 0000000000..e69de29bb2 From 3ec5ea4aceb0c46ac7b93b20d8d74584681e670f Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 10 Jun 2016 21:17:31 +0530 Subject: [PATCH 35/94] Added the basic content to the doc --- .../main/sphinx/storage_plugins/mongodb.rst | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst index e69de29bb2..8392881b52 100644 --- a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst +++ b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst @@ -0,0 +1,32 @@ +************************************* +MongoDB Storage Handler +************************************* + +Overview +======== + +MongoDB storage handler is available by default in Tajo. + +Configuration +============= + +MongoDB storage handler configuration will be like this.. First you have to register the StorageHandler. Then register the space. Update the storage-site.json as following. + +.. code-block:: json + + { + "storages": { + "mongodb": { + "handler": "org.apache.tajo.storage.mongodb.MongoDBTableSpace", + "default-format": "text" + } + }, + "spaces": { + "space_name": { + "uri": "mongodb://:/?user=&password=", + "configs": { + "mapped_database": "" + } + } + } + } \ No newline at end of file From dbda0bf9022432086bf3b84f51578e374586ed7b Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 10 Jun 2016 22:48:09 +0530 Subject: [PATCH 36/94] Create and Purge Table --- .../storage/mongodb/MongoDBTableSpace.java | 16 +++--- .../storage/mongodb/TestMetadataProvider.java | 2 +- ...eSpace.java => TestMongoDBTableSpace.java} | 52 +++++++++++++++++-- 3 files changed, 57 insertions(+), 13 deletions(-) rename tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/{TestTableSpace.java => TestMongoDBTableSpace.java} (59%) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 3703c7040c..6f6b2993a5 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -27,12 +27,11 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.*; -import org.apache.tajo.exception.TajoException; -import org.apache.tajo.exception.TajoInternalError; -import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.exception.*; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.storage.FormatProperty; import org.apache.tajo.storage.StorageProperty; import org.apache.tajo.storage.Tablespace; @@ -65,7 +64,7 @@ public class MongoDBTableSpace extends Tablespace { private ConnectionInfo connectionInfo; protected MongoClient mongoClient; protected MongoDatabase db; - protected String mappedDBName = "MappedDB"; + protected String mappedDBName; //Config Keys @@ -92,8 +91,7 @@ public MongoDBTableSpace(String name, URI uri, JSONObject config) { @Override protected void storageInit() throws IOException { - // LOG.info(uri.toASCIIString()); - //LOG.info("Something"+config.toJSONString()); + //Todo Extract User Details from Configuration try { connectionInfo = ConnectionInfo.fromURI(uri); mongoClient = new MongoClient(getConnectionInfo().getMongoDBURI()); @@ -153,12 +151,14 @@ public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws Ta @Override public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { - + if(tableDesc==null) + throw new TajoRuntimeException(new NotImplementedException()); + db.createCollection(tableDesc.getName()); } @Override public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { - + db.getCollection(tableDesc.getName()).drop(); } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index 142d7023cd..eeee73b389 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -79,7 +79,7 @@ public void testGetTableDescriptor() throws Exception { assertEquals(server.mappedDbName+"." + tableName, table.getName()); assertEquals(server.getURI() + "&table=" + tableName, table.getUri().toASCIIString()); - System.out.println(tableName+" : "+table.getStats().getNumRows()); + //To Do Check the stats } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java similarity index 59% rename from tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java rename to tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 1a65f43174..76aa7b6824 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -17,11 +17,24 @@ */ package org.apache.tajo.storage.mongodb; +import com.google.common.collect.Sets; +import org.apache.commons.collections.bag.SynchronizedSortedBag; +import org.apache.commons.math.optimization.linear.SimplexSolver; +import org.apache.tajo.catalog.SchemaBuilder; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; import org.junit.After; import org.junit.Test; +import java.io.IOException; import java.net.URI; +import java.util.Collection; +import java.util.List; +import java.util.Set; /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -40,12 +53,13 @@ * limitations under the License. */ import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** * Created by janaka on 6/2/16. */ -public class TestTableSpace { +public class TestMongoDBTableSpace { //mongodb://:@ds017231.mlab.com:17231/tajo_test static MongoDBTestServer server = MongoDBTestServer.getInstance(); static URI uri = server.getURI(); @@ -53,17 +67,47 @@ public class TestTableSpace { @Test public void testTablespaceHandler() { - assertTrue((TablespaceManager.get(uri)) instanceof MongoDBTableSpace); - assertTrue((TablespaceManager.getByName(server.spaceName)) instanceof MongoDBTableSpace); assertEquals(server.spaceName, (TablespaceManager.getByName(server.spaceName).getName())); + assertTrue((TablespaceManager.get(uri.toASCIIString() + "&table=tb1")) instanceof MongoDBTableSpace); + assertTrue((TablespaceManager.get(uri)) instanceof MongoDBTableSpace); //Test the URI same assertEquals(uri.toASCIIString(), TablespaceManager.get(uri).getUri().toASCIIString()); - //Check if returns the MetaDataProvider assertTrue(TablespaceManager.get(uri).getMetadataProvider() instanceof MongoDBMetadataProvider); } + + @Test(timeout = 1000, expected = TajoRuntimeException.class) + public void testCreateTable() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.spaceName); + space.createTable(null, false); + } + + @Test(timeout = 1000) + public void testCreateTable_and_Purg() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.spaceName); + + TableDesc tableDesc = new TableDesc( + IdentifierUtil.buildFQName(server.mappedDbName, "Table1"), + SchemaBuilder.builder() + .build(), + null, + server.getURI()); + + space.createTable(tableDesc, false); + + //Check whether the created table is in the collection + final Set found = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); + assertTrue(found.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); + + + //Purg the table + space.purgeTable(tableDesc); + final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); + assertFalse(found_after.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); + + } } \ No newline at end of file From 52ffa95715ef313ef908442a080a8233d7057434 Mon Sep 17 00:00:00 2001 From: Janaka Date: Sat, 11 Jun 2016 23:31:39 +0530 Subject: [PATCH 37/94] Implemented the getSplit and getVolume --- tajo-storage/tajo-storage-mongodb/pom.xml | 1 - .../mongodb/MongoDBMetadataProvider.java | 2 +- .../storage/mongodb/MongoDBTableSpace.java | 4 +++- .../storage/mongodb/TestMetadataProvider.java | 4 ++-- .../storage/mongodb/TestMongoDBTableSpace.java | 18 ++++++++++++++++++ 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 52ead0cbfc..89ae328ddd 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -337,7 +337,6 @@ org.mongodb mongodb-driver 3.2.2 - provided diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index 212b7679b1..7f63f85696 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -94,7 +94,7 @@ public TableDesc getTableDesc(String schemaName, String tableName) throws Undefi TableMeta tbMeta = new TableMeta("rowstore", new KeyValueSet()); TableDesc tbDesc = new TableDesc( - IdentifierUtil.buildFQName(mappedDbName, tableName), + tableName, SchemaBuilder.builder() .build(), tbMeta, diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 6f6b2993a5..6919edabae 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.mongodb; +import com.google.common.collect.Lists; import com.mongodb.MongoClient; import com.mongodb.client.MongoDatabase; import net.minidev.json.JSONObject; @@ -120,7 +121,8 @@ public URI getTableUri(String databaseName, String tableName) { @Override public List getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { - return null; + MongoDBFragment mongoDBFragment = new MongoDBFragment(inputSourceId, getUri(),null ); + return Lists.newArrayList(mongoDBFragment); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index eeee73b389..c34dd94f0a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -76,10 +76,10 @@ public void testGetTableDescriptor() throws Exception { for (String tableName : server.collectionNames) { TableDesc table = provider.getTableDesc(null, tableName); - assertEquals(server.mappedDbName+"." + tableName, table.getName()); + assertEquals(tableName, table.getName()); assertEquals(server.getURI() + "&table=" + tableName, table.getUri().toASCIIString()); - //To Do Check the stats + //ToDo Check the stats } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 76aa7b6824..779dbee7a5 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.mongodb; import com.google.common.collect.Sets; +import org.apache.commons.cli.Option; import org.apache.commons.collections.bag.SynchronizedSortedBag; import org.apache.commons.math.optimization.linear.SimplexSolver; import org.apache.tajo.catalog.SchemaBuilder; @@ -34,6 +35,7 @@ import java.net.URI; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.Set; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -110,4 +112,20 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { assertFalse(found_after.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); } + + @Test + public void testTableVolume() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.spaceName); + for (String tbl:server.collectionNames) { + + // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); + + // long a = 1;b + TableDesc tbdesc = space.getMetadataProvider().getTableDesc(null,tbl); + assertEquals("1", tbdesc.getStats().getNumRows().toString()); + assertEquals(1, space.getTableVolume(tbdesc,Optional.empty())); + + } + } + } \ No newline at end of file From fd853448cce1133e0558eac12b6d4dfce5cc2afb Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 14 Jun 2016 19:23:48 +0530 Subject: [PATCH 38/94] Fixed typos --- .../java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 6919edabae..a0b56f7650 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -53,7 +53,7 @@ public class MongoDBTableSpace extends Tablespace { //Table Space Properties static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined false, //not movable - true, // not writable at the moment + true, // writable at the moment true, // Absolute path true); // Meta data will be provided static final FormatProperty FORMAT_PROPERTY = new FormatProperty( From 4c49d6baf0b8b9247a6353f6d45d07887d66dc2d Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 14 Jun 2016 19:24:57 +0530 Subject: [PATCH 39/94] Fixed not stoping issue --- .../java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index 744d78ed96..1150055a12 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -109,6 +109,7 @@ public void stop() { _mongod.stop(); _mongodExe.stop(); + instance = null; } public URI getURI() From 5d669c6a264dff8b4656ba3d73ce263acb0b9224 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 14 Jun 2016 22:15:45 +0530 Subject: [PATCH 40/94] QueryTestBase --- tajo-storage/tajo-storage-mongodb/pom.xml | 10 ++- .../mongodb/MongoDBMetadataProvider.java | 2 +- .../storage/mongodb/MongoDBTestServer.java | 9 +-- .../storage/mongodb/TestMetadataProvider.java | 2 +- .../storage/mongodb/TestMongoDBQueryTest.java | 66 +++++++++++++++++++ .../{datasets => dataset}/file1.json | 0 .../{datasets => dataset}/file2.json | 0 .../TestMongoDBQueryTest/testGroupby.sql | 1 + .../queries/TestMongoDBQueryTest/testJoin.sql | 1 + .../TestMongoDBQueryTest/testSelect.sql | 1 + .../queries/TestMongoDBQueryTest/testSort.sql | 1 + .../resources/queries/testBetweenDates.sql | 1 + .../TestMongoDBQueryTest/testGroupby.result | 6 ++ .../TestMongoDBQueryTest/testJoin.result | 5 ++ .../TestMongoDBQueryTest/testSelect.result | 5 ++ .../TestMongoDBQueryTest/testSort.result | 6 ++ .../resources/results/testBetweenDates.result | 5 ++ 17 files changed, 110 insertions(+), 11 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java rename tajo-storage/tajo-storage-mongodb/src/test/resources/{datasets => dataset}/file1.json (100%) rename tajo-storage/tajo-storage-mongodb/src/test/resources/{datasets => dataset}/file2.json (100%) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/queries/testBetweenDates.sql create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/results/testBetweenDates.result diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 89ae328ddd..ed99b0ae6a 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -46,7 +46,8 @@ apache-rat-plugin - src/test/resources/datasets/** + src/test/resources/dataset/** + src/test/resources/** @@ -192,6 +193,12 @@ tajo-storage-hdfs provided + + org.apache.tajo + tajo-cluster-tests + test-jar + test + org.apache.hadoop @@ -346,6 +353,7 @@ 1.50.5 + diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index 7f63f85696..212b7679b1 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -94,7 +94,7 @@ public TableDesc getTableDesc(String schemaName, String tableName) throws Undefi TableMeta tbMeta = new TableMeta("rowstore", new KeyValueSet()); TableDesc tbDesc = new TableDesc( - tableName, + IdentifierUtil.buildFQName(mappedDbName, tableName), SchemaBuilder.builder() .build(), tbMeta, diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index 1150055a12..8263d8ebf6 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -24,8 +24,6 @@ import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; -import com.mongodb.util.JSON; -import com.sun.javadoc.Doc; import de.flapdoodle.embed.mongo.*; import de.flapdoodle.embed.mongo.config.*; import de.flapdoodle.embed.mongo.distribution.Version; @@ -33,22 +31,17 @@ import net.minidev.json.JSONObject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.htrace.fasterxml.jackson.core.JsonFactory; -import org.apache.htrace.fasterxml.jackson.core.JsonParser; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.TablespaceManager; import org.bson.Document; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.List; import java.util.Scanner; public class MongoDBTestServer { @@ -188,7 +181,7 @@ public void apply(final Document document) { private static File getRequestedFile(String path) throws FileNotFoundException, URISyntaxException { - URL url = ClassLoader.getSystemResource("datasets/" + path); + URL url = ClassLoader.getSystemResource("dataset/" + path); if (url == null) { throw new FileNotFoundException(path); diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index c34dd94f0a..ff05628fb1 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -76,7 +76,7 @@ public void testGetTableDescriptor() throws Exception { for (String tableName : server.collectionNames) { TableDesc table = provider.getTableDesc(null, tableName); - assertEquals(tableName, table.getName()); + assertEquals( server.mappedDbName+"."+tableName, table.getName()); assertEquals(server.getURI() + "&table=" + tableName, table.getUri().toASCIIString()); //ToDo Check the stats diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java new file mode 100644 index 0000000000..b7aead569a --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.URI; + +public class TestMongoDBQueryTest extends QueryTestCaseBase{ + + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + static URI uri = server.getURI(); + + public TestMongoDBQueryTest() { + super(server.mappedDbName); + } + + @BeforeClass + public static void setup() throws Exception + { + QueryTestCaseBase.testingCluster.getMaster().refresh(); + // TablespaceManager.addTableSpaceForTest(new ExampleHttpFileTablespace("http_example", uri, configElements)); + + } + + @AfterClass + public static void teardown() throws Exception { + server.stop(); + } + + @Before + public void prepareTables() throws TajoException { + // executeString("create table tbl1 (*) tablespace test_spacename using ex_http_json with ('path'='got.json')"); + // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); + } + + + @SimpleTest + @Test + public void testSelect() throws Exception { + runSimpleTests(); + } + + +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json similarity index 100% rename from tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file1.json rename to tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file2.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json similarity index 100% rename from tajo-storage/tajo-storage-mongodb/src/test/resources/datasets/file2.json rename to tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql new file mode 100644 index 0000000000..ea68caa8a1 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql @@ -0,0 +1 @@ +select actor.id, count(*) from github group by actor.id; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql new file mode 100644 index 0000000000..e2c95c8240 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql @@ -0,0 +1 @@ +select g1.title from got g1, got g2 where g1.name.first_name = g2.name.first_name \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql new file mode 100644 index 0000000000..fce30c016f --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql @@ -0,0 +1 @@ +select title, name.first_name from col1; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql new file mode 100644 index 0000000000..d31ea941dd --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql @@ -0,0 +1 @@ +select actor.id, actor.login from github order by type; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/testBetweenDates.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/testBetweenDates.sql new file mode 100644 index 0000000000..e6b20a6046 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/testBetweenDates.sql @@ -0,0 +1 @@ +SELECT * FROM LINEITEM WHERE L_SHIPDATE BETWEEN DATE '1996-01-1' AND DATE '1997-12-31'; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result new file mode 100644 index 0000000000..67797c83d6 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result @@ -0,0 +1,6 @@ +actor/id,?count +------------------------------- +206379,1 +11455393,1 +1088854,1 +5266949,1 diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result new file mode 100644 index 0000000000..a6da9fc1be --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result @@ -0,0 +1,5 @@ +title +------------------------------- +Hand of the King +Assassin +Dancing Master diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result new file mode 100644 index 0000000000..651a9feb56 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result @@ -0,0 +1,5 @@ +title,name/first_name +------------------------------- +Hand of the King,Eddard +Assassin,Arya +Dancing Master,Syrio diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result new file mode 100644 index 0000000000..e0073b9320 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result @@ -0,0 +1,6 @@ +actor/id,actor/login +------------------------------- +206379,gvn +1088854,CAOakleyII +5266949,hardrubic +11455393,chrischjh diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/testBetweenDates.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/testBetweenDates.result new file mode 100644 index 0000000000..2f6e7a145b --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/testBetweenDates.result @@ -0,0 +1,5 @@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment +------------------------------- +1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold +1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the +2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a From 6f44b8c6df21461ddc20756ddbd89d508ff06925 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 21 Jun 2016 13:19:18 +0530 Subject: [PATCH 41/94] Added Handlers --- .../src/main/resources/storage-default.xml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index 4e5720490d..402df419ee 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -39,7 +39,7 @@ tajo.storage.scanner-handler - text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase + text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase,mongodb @@ -67,6 +67,10 @@ tajo.storage.fragment.serde.jdbc org.apache.tajo.storage.jdbc.JdbcFragmentSerde + + tajo.storage.fragment.serde.mongodb + org.apache.tajo.storage.mongodb.MongoDBFragmentSerde + @@ -123,7 +127,11 @@ tajo.storage.scanner-handler.hbase.class org.apache.tajo.storage.hbase.HBaseScanner - + + tajo.storage.scanner-handler.mongodb.class + org.apache.tajo.storage.mongodb.MongoDBScanner + + tajo.storage.appender-handler From d6e3d2d337df5109362f874bf582e0613a0f3bd8 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 21 Jun 2016 13:22:08 +0530 Subject: [PATCH 42/94] Fixed name issue in description --- .../apache/tajo/storage/mongodb/MongoDBTableSpace.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index a0b56f7650..e86be9200d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -22,6 +22,7 @@ import com.mongodb.MongoClient; import com.mongodb.client.MongoDatabase; import net.minidev.json.JSONObject; +import org.apache.avro.generic.GenericData; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -33,6 +34,7 @@ import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.FormatProperty; import org.apache.tajo.storage.StorageProperty; import org.apache.tajo.storage.Tablespace; @@ -42,6 +44,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.net.URI; +import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -107,7 +110,8 @@ public long getTableVolume(TableDesc table, Optional filter) throws Un long count = 0; try { - count = db.getCollection(table.getName()).count(); + String[] nameSplited = IdentifierUtil.splitFQTableName(table.getName()); + count = db.getCollection(nameSplited[1]).count(); } catch (Exception e) { throw new TajoInternalError(e); } @@ -121,7 +125,9 @@ public URI getTableUri(String databaseName, String tableName) { @Override public List getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { - MongoDBFragment mongoDBFragment = new MongoDBFragment(inputSourceId, getUri(),null ); + String[] hosts = new String[1]; + hosts[0] = getUri().getHost(); + MongoDBFragment mongoDBFragment = new MongoDBFragment(inputSourceId, getUri(),hosts ); return Lists.newArrayList(mongoDBFragment); } From 19d0269435c1dda553d3888ca5e1fb30d6ae1fa8 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 21 Jun 2016 13:22:40 +0530 Subject: [PATCH 43/94] Constructer created --- .../tajo/storage/mongodb/MongoDBFragment.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java index 01af24e328..f673f9ec0b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -32,12 +32,25 @@ protected MongoDBFragment(String kind, URI uri, String inputSourceId, Long start public MongoDBFragment(String inputSourceId, URI uri, List hostNames) { - super("MONGODB", uri, inputSourceId, null, null, 0, ((String[]) hostNames.toArray())); + + + super("MONGODB", uri, inputSourceId, null, null, 0, toArray(hostNames)); + } + + public static String[] toArray(List list) + { + String[] hosts = new String[1]; + hosts[0] = list.get(0); + return hosts; + } + + public MongoDBFragment(String inputSourceId, URI uri, String[] hostNames) { + super("MONGODB", uri, inputSourceId, null, null, 0, hostNames); } @Override public boolean isEmpty() { - return false; + return true; } } From b6270fc5fdb784848fd990dd0c134ba526fb1271 Mon Sep 17 00:00:00 2001 From: Janaka Date: Tue, 21 Jun 2016 13:23:59 +0530 Subject: [PATCH 44/94] changed storage type to 'mongodb' from 'rawdata' --- .../apache/tajo/storage/mongodb/MongoDBMetadataProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index 212b7679b1..3414ba9023 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -92,7 +92,7 @@ public Collection getTables(@Nullable String schemaPattern, @Nullable St @Override public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException { - TableMeta tbMeta = new TableMeta("rowstore", new KeyValueSet()); + TableMeta tbMeta = new TableMeta("mongodb", new KeyValueSet()); TableDesc tbDesc = new TableDesc( IdentifierUtil.buildFQName(mappedDbName, tableName), SchemaBuilder.builder() From 540fff5eb428062d97261e2d13389b057e8e6c1b Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 24 Jun 2016 02:28:29 +0530 Subject: [PATCH 45/94] Added some lines for scanner --- .../mongodb/MongoDBMetadataProvider.java | 1 + .../tajo/storage/mongodb/MongoDBScanner.java | 84 ++++++++++++++++++- .../storage/mongodb/TestMongoDBQueryTest.java | 6 +- .../mongodb/TestMongoDBTableSpace.java | 18 +--- .../src/test/resources/dataset/file1.json | 13 +-- .../TestMongoDBQueryTest/testSelect.sql | 2 +- 6 files changed, 95 insertions(+), 29 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index 3414ba9023..99aa281613 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -106,6 +106,7 @@ public TableDesc getTableDesc(String schemaName, String tableName) throws Undefi //Set the raw count stats.setNumRows(db.getCollection(tableName).count()); + tbDesc.setStats(stats); return tbDesc; } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index 361ddacfec..583dc6dcb0 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -17,29 +17,107 @@ */ package org.apache.tajo.storage.mongodb; +import com.google.common.base.Preconditions; +import com.mongodb.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.MongoIterable; +import io.netty.buffer.ByteBuf; +import net.minidev.json.JSONObject; +import net.minidev.json.parser.JSONParser; +import net.minidev.json.parser.ParseException; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.NestedPathUtil; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.json.JsonLineDeserializer; +import org.apache.tajo.storage.text.TextLineParsingError; +import org.bson.Document; import java.io.IOException; +import java.sql.DatabaseMetaData; +import java.util.Properties; + public class MongoDBScanner implements Scanner { + + private final DatabaseMetaData dbMetaData; + private final Properties connProperties; + private final String tableName; + private final Schema schema; + private final TableMeta tableMeta; + private final MongoDBFragment fragment; + private final TableStats stats; + + + private ConnectionInfo connectionInfo; + private JsonLineDeserializer deserializer; + MongoDatabase db; + Column[] targets; + + //this should be moved to another class + private JSONParser parser; + + + public MongoDBScanner(final DatabaseMetaData dbMetaData, + final Properties connProperties, + final Schema tableSchema, + final TableMeta tableMeta, + final MongoDBFragment fragment) { + + Preconditions.checkNotNull(dbMetaData); + Preconditions.checkNotNull(connProperties); + Preconditions.checkNotNull(tableSchema); + Preconditions.checkNotNull(tableMeta); + Preconditions.checkNotNull(fragment); + + this.dbMetaData = dbMetaData; + this.connProperties = connProperties; + this.tableName = ConnectionInfo.fromURI(fragment.getUri()).getTableName(); + this.schema = tableSchema; + this.tableMeta = tableMeta; + this.fragment = fragment; + this.stats = new TableStats(); + } + @Override public void init() throws IOException { + connectionInfo = ConnectionInfo.fromURI(fragment.getUri()); + + + + MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + db = mongoClient.getDatabase(connectionInfo.getDbName()); + + + deserializer = new JsonLineDeserializer(schema, tableMeta, targets); + deserializer.init(); } + int count =0; @Override public Tuple next() throws IOException { - return null; + MongoIterable iter= db.getCollection(tableName).find(); + + + if(count>=2) return null; + + //This should be moved to a seperate class + Tuple tp = new VTuple(2); + tp.put(1, DatumFactory.createText("Lol")); + count++; + return tp; } @Override public void reset() throws IOException { - + count=0; } @Override @@ -59,7 +137,7 @@ public boolean isProjectable() { @Override public void setTarget(Column[] targets) { - + this.targets= targets; } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index b7aead569a..4b0f75f998 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -51,7 +51,7 @@ public static void teardown() throws Exception { @Before public void prepareTables() throws TajoException { - // executeString("create table tbl1 (*) tablespace test_spacename using ex_http_json with ('path'='got.json')"); + // executeString("create table tbl1 (*) tablespace test_spacename using json with ('path'='file1.json')"); // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); } @@ -59,7 +59,9 @@ public void prepareTables() throws TajoException { @SimpleTest @Test public void testSelect() throws Exception { - runSimpleTests(); + //runSimpleTests(); + + executeString("select title, name.first_name from col1"); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 779dbee7a5..00937dea24 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -37,23 +37,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json index cff120a088..4b1b834411 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json @@ -1,12 +1,13 @@ { -'dataList':[ +"dataList":[ { - 'name' : 'mkyong', - 'age' : 30 + "name" : "mkyong", + "age" : 30 }, { - 'name' : 'mkyong', - 'age' : 30 + "name" : "mkyong", + "age" : 30 } - ] + ], + "lol":"lol" } \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql index fce30c016f..c20bdb12f5 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql @@ -1 +1 @@ -select title, name.first_name from col1; \ No newline at end of file +select lol from col1; \ No newline at end of file From 747e02ac3c61136791fc27ef1a0d2ed6ae6de981 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 24 Jun 2016 09:23:27 +0530 Subject: [PATCH 46/94] Added Protoc, moving to file space --- .../src/main/proto/CatalogProtos.proto | 1 + .../src/main/java/org/apache/tajo/BuiltinStorages.java | 1 + .../tajo/plan/verifier/PreLogicalPlanVerifier.java | 3 ++- .../apache/tajo/storage/mongodb/MongoDBTableSpace.java | 10 +++++----- .../tajo/storage/mongodb/TestMongoDBQueryTest.java | 4 ++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index b42cf58a5f..3f4833c0bd 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -39,6 +39,7 @@ enum DataFormat { JSON = 10; HBASE = 11; SYSTEM = 12; + MONGODB = 13; } enum OrderType { diff --git a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java index 2c8f68617c..f8addc5085 100644 --- a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java +++ b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java @@ -30,4 +30,5 @@ public class BuiltinStorages { public static final String SEQUENCE_FILE = "SEQUENCEFILE"; public static final String AVRO = "AVRO"; public static final String HBASE = "HBASE"; + public static final String MONGODB = "MONGODB"; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 2b197ff0ec..fafa3b455e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -254,7 +254,8 @@ public Expr visitCreateTable(Context context, Stack stack, CreateTable exp if (expr.getStorageType() != null) { if (expr.hasSelfDescSchema()) { // TODO: support other types like Parquet and ORC. - if (!expr.getStorageType().equalsIgnoreCase(BuiltinStorages.JSON)) { + if (!expr.getStorageType().equalsIgnoreCase(BuiltinStorages.JSON) && + !expr.getStorageType().equalsIgnoreCase(BuiltinStorages.MONGODB)) { if (expr.getStorageType().equalsIgnoreCase(BuiltinStorages.PARQUET) || expr.getStorageType().equalsIgnoreCase(BuiltinStorages.ORC)) { throw new NotImplementedException(expr.getStorageType()); diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index e86be9200d..21eec8a125 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -56,9 +56,9 @@ public class MongoDBTableSpace extends Tablespace { //Table Space Properties static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined false, //not movable - true, // writable at the moment + true, //writable at the moment true, // Absolute path - true); // Meta data will be provided + false); // Meta data will be provided static final FormatProperty FORMAT_PROPERTY = new FormatProperty( true, // Insert false, //direct insert @@ -196,9 +196,9 @@ public URI getRootUri() { //Metadata - public MetadataProvider getMetadataProvider() { - return new MongoDBMetadataProvider(this, mappedDBName); - } +// public MetadataProvider getMetadataProvider() { +// return new MongoDBMetadataProvider(this, mappedDBName); +// } public ConnectionInfo getConnectionInfo() { return connectionInfo; diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 4b0f75f998..c4e90f2a1f 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -51,7 +51,7 @@ public static void teardown() throws Exception { @Before public void prepareTables() throws TajoException { - // executeString("create table tbl1 (*) tablespace test_spacename using json with ('path'='file1.json')"); + executeString("create table tbl1 (*) tablespace test_spacename using mongodb with ('path'='fileasd.json')"); // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); } @@ -61,7 +61,7 @@ public void prepareTables() throws TajoException { public void testSelect() throws Exception { //runSimpleTests(); - executeString("select title, name.first_name from col1"); +// executeString("select title, name.first_name from col1"); } From a5b2245a48b5ba538c00ba4cd0e7596b83c2b201 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 24 Jun 2016 21:54:00 +0530 Subject: [PATCH 47/94] Created a collection reader class, scanner get called --- tajo-storage/tajo-storage-mongodb/pom.xml | 2 +- .../mongodb/MongoDBCollectionReader.java | 8 ++ .../tajo/storage/mongodb/MongoDBFragment.java | 33 ++---- .../storage/mongodb/MongoDBFragmentSerde.java | 12 +- .../tajo/storage/mongodb/MongoDBScanner.java | 103 ++---------------- .../storage/mongodb/MongoDBTableSpace.java | 20 ++-- .../main/proto/MongoDBFragmentProtos.proto | 5 +- .../storage/mongodb/TestMetadataProvider.java | 1 + .../mongodb/TestMongoDBTableSpace.java | 17 ++- 9 files changed, 61 insertions(+), 140 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index ed99b0ae6a..93e656bdbf 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -46,7 +46,7 @@ apache-rat-plugin - src/test/resources/dataset/** + derby.log src/test/resources/** diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java new file mode 100644 index 0000000000..252da871ce --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -0,0 +1,8 @@ +package org.apache.tajo.storage.mongodb; + +/** + * Created by janaka on 6/24/16. + */ +public class MongoDBCollectionReader { + +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java index f673f9ec0b..322b9626aa 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -18,39 +18,20 @@ package org.apache.tajo.storage.mongodb; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.storage.fragment.Fragment; import java.net.URI; import java.util.List; -public class MongoDBFragment extends Fragment { +public class MongoDBFragment extends AbstractFileFragment { - protected MongoDBFragment(String kind, URI uri, String inputSourceId, Long startKey, Long endKey, long length, String[] hostNames) { - super(kind, uri, inputSourceId, startKey, endKey, length, hostNames); - } - - - public MongoDBFragment(String inputSourceId, URI uri, List hostNames) { - - - super("MONGODB", uri, inputSourceId, null, null, 0, toArray(hostNames)); - } - public static String[] toArray(List list) - { - String[] hosts = new String[1]; - hosts[0] = list.get(0); - return hosts; + protected MongoDBFragment(URI uri, + String inputSourceId, + long startKey, + long endKey) { + super("MONGODB", uri, inputSourceId, startKey, endKey, endKey-startKey, null); } - - public MongoDBFragment(String inputSourceId, URI uri, String[] hostNames) { - super("MONGODB", uri, inputSourceId, null, null, 0, hostNames); - } - - @Override - public boolean isEmpty() { - return true; - } - } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java index dd96b11e6b..4d5c0fbc29 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java @@ -34,14 +34,20 @@ public Builder newBuilder() { @Override public MongoDBFragmentProto serialize(MongoDBFragment fragment) { return MongoDBFragmentProto.newBuilder() - .setInputSourceId(fragment.getInputSourceId()) .setUri(fragment.getUri().toASCIIString()) - .addAllHosts(fragment.getHostNames()) + .setTableName(fragment.getInputSourceId()) + .setStartKey(fragment.getStartKey()) + .setEndKey(fragment.getEndKey()) .build(); } @Override public MongoDBFragment deserialize(MongoDBFragmentProto proto) { - return new MongoDBFragment(proto.getInputSourceId(), URI.create(proto.getUri()), proto.getHostsList()); + return new MongoDBFragment( + URI.create(proto.getUri()), + proto.getTableName(), + proto.getStartKey(), + proto.getEndKey() + ); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index 583dc6dcb0..9d227db363 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -25,6 +25,7 @@ import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; import net.minidev.json.parser.ParseException; +import org.apache.hadoop.conf.Configuration; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.NestedPathUtil; import org.apache.tajo.catalog.Schema; @@ -33,9 +34,11 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.storage.FileScanner; import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.json.JsonLineDeserializer; import org.apache.tajo.storage.text.TextLineParsingError; import org.bson.Document; @@ -44,90 +47,27 @@ import java.sql.DatabaseMetaData; import java.util.Properties; -public class MongoDBScanner implements Scanner { +public class MongoDBScanner extends FileScanner { - private final DatabaseMetaData dbMetaData; - private final Properties connProperties; - private final String tableName; - private final Schema schema; - private final TableMeta tableMeta; - private final MongoDBFragment fragment; - private final TableStats stats; - - private ConnectionInfo connectionInfo; - private JsonLineDeserializer deserializer; - MongoDatabase db; - Column[] targets; - - //this should be moved to another class - private JSONParser parser; - - - public MongoDBScanner(final DatabaseMetaData dbMetaData, - final Properties connProperties, - final Schema tableSchema, - final TableMeta tableMeta, - final MongoDBFragment fragment) { - - Preconditions.checkNotNull(dbMetaData); - Preconditions.checkNotNull(connProperties); - Preconditions.checkNotNull(tableSchema); - Preconditions.checkNotNull(tableMeta); - Preconditions.checkNotNull(fragment); - - this.dbMetaData = dbMetaData; - this.connProperties = connProperties; - this.tableName = ConnectionInfo.fromURI(fragment.getUri()).getTableName(); - this.schema = tableSchema; - this.tableMeta = tableMeta; - this.fragment = fragment; - this.stats = new TableStats(); + public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { + super(conf, schema, meta, fragment); } - @Override - public void init() throws IOException { - connectionInfo = ConnectionInfo.fromURI(fragment.getUri()); - - - - MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); - db = mongoClient.getDatabase(connectionInfo.getDbName()); - - - deserializer = new JsonLineDeserializer(schema, tableMeta, targets); - deserializer.init(); - - } - - int count =0; @Override public Tuple next() throws IOException { - MongoIterable iter= db.getCollection(tableName).find(); - - if(count>=2) return null; - - //This should be moved to a seperate class - Tuple tp = new VTuple(2); - tp.put(1, DatumFactory.createText("Lol")); - count++; - return tp; + return null; } @Override public void reset() throws IOException { - count=0; + return; } @Override public void close() throws IOException { - - } - - @Override - public void pushOperators(LogicalNode planPart) { - + return; } @Override @@ -135,11 +75,6 @@ public boolean isProjectable() { return false; } - @Override - public void setTarget(Column[] targets) { - this.targets= targets; - } - @Override public boolean isSelectable() { return false; @@ -150,28 +85,8 @@ public void setFilter(EvalNode filter) { } - @Override - public void setLimit(long num) { - - } - @Override public boolean isSplittable() { return false; } - - @Override - public float getProgress() { - return 0; - } - - @Override - public TableStats getInputStats() { - return null; - } - - @Override - public Schema getSchema() { - return null; - } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 2d8e4624e6..8075acf241 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import net.minidev.json.JSONObject; import org.apache.avro.generic.GenericData; @@ -40,6 +41,7 @@ import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.fragment.Fragment; +import org.bson.Document; import javax.annotation.Nullable; import java.io.IOException; @@ -56,13 +58,13 @@ public class MongoDBTableSpace extends Tablespace { //Table Space Properties static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined false, //not movable - true, //writable at the moment + false, //writable at the moment true, // Absolute path false); // Meta data will be provided static final FormatProperty FORMAT_PROPERTY = new FormatProperty( - true, // Insert + false, // Insert false, //direct insert - true);// result staging + false);// result staging //Mongo Client object private ConnectionInfo connectionInfo; @@ -122,9 +124,8 @@ public long getTableVolume(TableDesc table, Optional filter) { @Override public List getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { - String[] hosts = new String[1]; - hosts[0] = getUri().getHost(); - MongoDBFragment mongoDBFragment = new MongoDBFragment(inputSourceId, getUri(),hosts ); + long tableVolume = getTableVolume(tableDesc, Optional.empty()); + MongoDBFragment mongoDBFragment = new MongoDBFragment(tableDesc.getUri(), inputSourceId, 0, tableVolume ); return Lists.newArrayList(mongoDBFragment); } @@ -158,7 +159,10 @@ public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws Ta public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { if(tableDesc==null) throw new TajoRuntimeException(new NotImplementedException()); - db.createCollection(tableDesc.getName()); + MongoCollection table = db.getCollection(tableDesc.getName()); + + //TODO Handle this here. If empty throw exception or what? + boolean ifExist = (table.count()>0)?true:false; } @Override @@ -168,7 +172,7 @@ public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { @Override public void prepareTable(LogicalNode node) throws IOException, TajoException { - + return; } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto b/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto index eadb362829..a2101bbff0 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto +++ b/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto @@ -26,6 +26,7 @@ import "CatalogProtos.proto"; message MongoDBFragmentProto { required string uri = 1; - required string input_source_id = 2; - repeated string hosts = 3; + required string table_name = 2; + required int64 start_key = 3; + required int64 end_key = 4; } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index ff05628fb1..e0c5dbb27f 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -33,6 +33,7 @@ /** * Created by janaka on 6/8/16. */ +@Ignore public class TestMetadataProvider { static MongoDBTestServer server = MongoDBTestServer.getInstance(); diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 00937dea24..0ba6f81235 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -29,6 +29,7 @@ import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; import org.junit.After; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -62,9 +63,7 @@ public void testTablespaceHandler() //Test the URI same assertEquals(uri.toASCIIString(), TablespaceManager.get(uri).getUri().toASCIIString()); - //Check if returns the MetaDataProvider - assertTrue(TablespaceManager.get(uri).getMetadataProvider() instanceof MongoDBMetadataProvider); - } + } @Test(timeout = 1000, expected = TajoRuntimeException.class) public void testCreateTable() throws IOException, TajoException { @@ -72,6 +71,7 @@ public void testCreateTable() throws IOException, TajoException { space.createTable(null, false); } + @Ignore @Test(timeout = 1000) public void testCreateTable_and_Purg() throws IOException, TajoException { Tablespace space = TablespaceManager.getByName(server.spaceName); @@ -105,9 +105,14 @@ public void testTableVolume() throws IOException, TajoException { // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); // long a = 1;b - TableDesc tbdesc = space.getMetadataProvider().getTableDesc(null,tbl); - assertEquals("1", tbdesc.getStats().getNumRows().toString()); - assertEquals(1, space.getTableVolume(tbdesc,Optional.empty())); + TableDesc tbDesc = new TableDesc( + IdentifierUtil.buildFQName(server.mappedDbName, tbl), + SchemaBuilder.builder() + .build(), + null, + space.getTableUri(null,null, tbl)); + + assertEquals(1, space.getTableVolume(tbDesc,Optional.empty())); } } From 73786bca05c02d16b5ec80f3ae1fa53aa08c0417 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 24 Jun 2016 21:55:14 +0530 Subject: [PATCH 48/94] simple fault --- .../org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index 252da871ce..6fb0da169a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -4,5 +4,4 @@ * Created by janaka on 6/24/16. */ public class MongoDBCollectionReader { - } From 2b74a289e72eb740932b09c46909e7619b2d2157 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 24 Jun 2016 22:18:10 +0530 Subject: [PATCH 49/94] Making Collection Reader --- .../mongodb/MongoDBCollectionReader.java | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index 6fb0da169a..dff0680162 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -1,7 +1,47 @@ -package org.apache.tajo.storage.mongodb; - /** - * Created by janaka on 6/24/16. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.tajo.storage.mongodb; + +import com.mongodb.MongoClient; +import com.mongodb.MongoClientURI; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import org.apache.tajo.storage.json.JsonLineDeserializer; +import org.bson.Document; + +import java.io.IOException; + public class MongoDBCollectionReader { + private MongoCollection collection; + private ConnectionInfo connectionInfo; + private JsonLineDeserializer deserializer; + + public MongoDBCollectionReader(ConnectionInfo connectionInfo,JsonLineDeserializer deserializer) + { + this.connectionInfo = connectionInfo; + this.deserializer = deserializer; + } + + public void init() throws IOException + { + MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); + collection = db.getCollection(connectionInfo.getTableName()); + } + } From 1242ea25275ed63f34ce093a14fc8090fb5ae39d Mon Sep 17 00:00:00 2001 From: Janaka Date: Sat, 25 Jun 2016 01:00:40 +0530 Subject: [PATCH 50/94] Implemented a scanner. reads well, testing only single line --- .../mongodb/MongoDBCollectionReader.java | 48 +++- .../tajo/storage/mongodb/MongoDBScanner.java | 37 ++- .../storage/mongodb/MongoDBTableSpace.java | 10 +- .../mongodb/MongoDocumentDeserializer.java | 247 ++++++++++++++++++ .../src/test/resources/dataset/file1.json | 2 +- .../TestMongoDBQueryTest/testSelect.sql | 2 +- .../TestMongoDBQueryTest/testSelect.result | 6 +- 7 files changed, 337 insertions(+), 15 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index dff0680162..2bf3931dc6 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -17,31 +17,71 @@ */ package org.apache.tajo.storage.mongodb; +import com.mongodb.DBCursor; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; +import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; +import io.netty.util.CharsetUtil; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.json.JsonLineDeserializer; +import org.apache.tajo.storage.text.TextLineParsingError; import org.bson.Document; import java.io.IOException; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetDecoder; +import io.netty.util.CharsetUtil; + +import java.nio.charset.CharsetEncoder; +import java.util.ArrayList; +import java.util.List; public class MongoDBCollectionReader { - private MongoCollection collection; private ConnectionInfo connectionInfo; - private JsonLineDeserializer deserializer; + private MongoDocumentDeserializer deserializer; + private int targetLength; + List documentList; + private int currentIndex; + private final CharsetEncoder encoder = CharsetUtil.getEncoder(CharsetUtil.UTF_8); + - public MongoDBCollectionReader(ConnectionInfo connectionInfo,JsonLineDeserializer deserializer) + public MongoDBCollectionReader(ConnectionInfo connectionInfo,MongoDocumentDeserializer deserializer ,int targetLength) { this.connectionInfo = connectionInfo; this.deserializer = deserializer; + this.targetLength = targetLength; } public void init() throws IOException { + currentIndex = 0; MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); - collection = db.getCollection(connectionInfo.getTableName()); + + MongoCollection collection = db.getCollection(connectionInfo.getTableName()); + documentList =(List) collection.find().into( + new ArrayList());; + + } + + public Tuple readTuple() throws IOException, TextLineParsingError { + if(currentIndex>=documentList.size()) return null; + + Tuple outTuple = new VTuple(targetLength); + + deserializer.deserialize(documentList.get(currentIndex), outTuple); + currentIndex++; + return outTuple; + + } + + public float getProgress() + { + return ((float)currentIndex) / documentList.size(); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index 9d227db363..fb283ebbdc 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -49,20 +49,44 @@ public class MongoDBScanner extends FileScanner { + MongoDBCollectionReader collectionReader; public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { super(conf, schema, meta, fragment); + } @Override - public Tuple next() throws IOException { + public void init() throws IOException { + + + if (targets == null) { + targets = schema.toArray(); + } + + reset(); + + super.init(); + } + - return null; + @Override + public Tuple next() throws IOException { + try { + Tuple t = collectionReader.readTuple(); + return t; + } catch (TextLineParsingError textLineParsingError) { + textLineParsingError.printStackTrace(); + return null; + } } @Override public void reset() throws IOException { - return; + MongoDocumentDeserializer deserializer = new MongoDocumentDeserializer(schema,meta,targets); + collectionReader = new MongoDBCollectionReader(ConnectionInfo.fromURI(fragment.getUri()),deserializer,targets.length); + + collectionReader.init(); } @Override @@ -89,4 +113,11 @@ public void setFilter(EvalNode filter) { public boolean isSplittable() { return false; } + + @Override + public float getProgress() { + return collectionReader.getProgress(); + } + + } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 8075acf241..38af7a9a1b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -197,8 +197,14 @@ public URI getRootUri() { @Override public URI getTableUri(TableMeta meta, String databaseName, String tableName) { - //ToDo set the TableURI properly - return URI.create(this.getUri()+"&table="+tableName); + //ToDo Find a better way this + String tableURI = ""; + if(this.getUri().toASCIIString().contains("?")) + tableURI = this.getUri().toASCIIString()+"&table="+tableName; + else + tableURI = this.getUri().toASCIIString()+"?table="+tableName; + + return URI.create(tableURI); } //@Override diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java new file mode 100644 index 0000000000..3ed4500079 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java @@ -0,0 +1,247 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mongodb; + + +import com.google.common.collect.Lists; +import io.netty.buffer.ByteBuf; +import io.netty.util.CharsetUtil; +import net.minidev.json.JSONObject; +import net.minidev.json.parser.JSONParser; +import net.minidev.json.parser.ParseException; +import org.apache.commons.net.util.Base64; +import org.apache.tajo.catalog.*; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.text.TextLineDeserializer; +import org.apache.tajo.storage.text.TextLineParsingError; +import org.bson.Document; + +import java.io.IOException; +import java.nio.charset.CharsetDecoder; +import java.util.Map; +import java.util.TimeZone; + +public class MongoDocumentDeserializer extends TextLineDeserializer { + private JSONParser parser; + + // Full Path -> Type + private final Map types; + private final String [] projectedPaths; + private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); + + private final TimeZone timezone; + + public MongoDocumentDeserializer(Schema schema, TableMeta meta, Column [] projected) { + super(schema, meta); + + projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true); + types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); + + timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, + StorageUtil.TAJO_CONF.getSystemTimezone().getID())); + } + + @Override + public void init() { + parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR); + } + + @Override + public void deserialize(ByteBuf buf, Tuple output) throws IOException, TextLineParsingError { + + } + + /** + * + * + * @param object + * @param pathElements + * @param depth + * @param fieldIndex + * @param output + * @throws IOException + */ + private void getValue(JSONObject object, + String fullPath, + String [] pathElements, + int depth, + int fieldIndex, + Tuple output) throws IOException { + String fieldName = pathElements[depth]; + + if (!object.containsKey(fieldName)) { + output.put(fieldIndex, NullDatum.get()); + } + + switch (types.get(fullPath)) { + case BOOLEAN: + String boolStr = object.getAsString(fieldName); + if (boolStr != null) { + output.put(fieldIndex, DatumFactory.createBool(boolStr.equals("true"))); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case CHAR: + String charStr = object.getAsString(fieldName); + if (charStr != null) { + output.put(fieldIndex, DatumFactory.createChar(charStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT1: + case INT2: + Number int2Num = object.getAsNumber(fieldName); + if (int2Num != null) { + output.put(fieldIndex, DatumFactory.createInt2(int2Num.shortValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT4: + Number int4Num = object.getAsNumber(fieldName); + if (int4Num != null) { + output.put(fieldIndex, DatumFactory.createInt4(int4Num.intValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT8: + Number int8Num = object.getAsNumber(fieldName); + if (int8Num != null) { + output.put(fieldIndex, DatumFactory.createInt8(int8Num.longValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case FLOAT4: + Number float4Num = object.getAsNumber(fieldName); + if (float4Num != null) { + output.put(fieldIndex, DatumFactory.createFloat4(float4Num.floatValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case FLOAT8: + Number float8Num = object.getAsNumber(fieldName); + if (float8Num != null) { + output.put(fieldIndex, DatumFactory.createFloat8(float8Num.doubleValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TEXT: + String textStr = object.getAsString(fieldName); + if (textStr != null) { + output.put(fieldIndex, DatumFactory.createText(textStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TIMESTAMP: + String timestampStr = object.getAsString(fieldName); + if (timestampStr != null) { + output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr, timezone)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TIME: + String timeStr = object.getAsString(fieldName); + if (timeStr != null) { + output.put(fieldIndex, DatumFactory.createTime(timeStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case DATE: + String dateStr = object.getAsString(fieldName); + if (dateStr != null) { + output.put(fieldIndex, DatumFactory.createDate(dateStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case BIT: + case BINARY: + case VARBINARY: + case BLOB: { + Object jsonObject = object.getAsString(fieldName); + + if (jsonObject == null) { + output.put(fieldIndex, NullDatum.get()); + break; + } + + output.put(fieldIndex, DatumFactory.createBlob(Base64.decodeBase64((String) jsonObject))); + break; + } + + case RECORD: + JSONObject nestedObject = (JSONObject) object.get(fieldName); + if (nestedObject != null) { + getValue(nestedObject, fullPath + "/" + pathElements[depth+1], pathElements, depth + 1, fieldIndex, output); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + + case NULL_TYPE: + output.put(fieldIndex, NullDatum.get()); + break; + + default: + throw new TajoRuntimeException( + new NotImplementedException("" + types.get(fullPath).name() + " for json")); + } + } + + public void deserialize(Document doc, Tuple output) throws IOException, TextLineParsingError { +// String line = doc.to; +// +// JSONObject object; +// try { +// object = (JSONObject) parser.parse(line); +// } catch (ParseException pe) { +// throw new TextLineParsingError(line, pe); +// } catch (ArrayIndexOutOfBoundsException ae) { +// // truncated value +// throw new TextLineParsingError(line, ae); +// } + + for (int i = 0; i < projectedPaths.length; i++) { +// String [] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER); +// getValue(object, paths[0], paths, 0, i, output); + + output.put(i, DatumFactory.createText( doc.get(projectedPaths[i]).toString() )); + } + } + + @Override + public void release() { + } +} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json index 4b1b834411..0d0a0fe0e2 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json @@ -9,5 +9,5 @@ "age" : 30 } ], - "lol":"lol" + "column1":"column1_data" } \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql index c20bdb12f5..bf1c9a499b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql @@ -1 +1 @@ -select lol from col1; \ No newline at end of file +select column1 from col1; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result index 651a9feb56..5501b8591f 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result @@ -1,5 +1,3 @@ -title,name/first_name +column1 ------------------------------- -Hand of the King,Eddard -Assassin,Arya -Dancing Master,Syrio +column1_data From a73b1867275fb952a449cfa77557595212c275f3 Mon Sep 17 00:00:00 2001 From: Janaka Date: Sat, 25 Jun 2016 03:33:01 +0530 Subject: [PATCH 51/94] Multiple lines, do not support composite attributes --- .../storage/mongodb/MongoDBTestServer.java | 24 +++++++++++++++---- .../storage/mongodb/TestMongoDBQueryTest.java | 3 ++- .../mongodb/TestMongoDBTableSpace.java | 3 ++- .../src/test/resources/dataset/file1.json | 19 +++++---------- .../src/test/resources/dataset/file2.json | 10 ++++---- .../TestMongoDBQueryTest/testSelect.sql | 2 +- .../TestMongoDBQueryTest/testSelect.result | 7 ++++-- 7 files changed, 41 insertions(+), 27 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index 8263d8ebf6..8bb40cea85 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -34,6 +34,8 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.TablespaceManager; import org.bson.Document; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; import java.io.File; import java.io.FileNotFoundException; @@ -42,6 +44,8 @@ import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; import java.util.Scanner; public class MongoDBTestServer { @@ -65,7 +69,7 @@ public class MongoDBTestServer { private MongoClient _mongo; private String[] filenames = {"file1.json","file2.json"}; - public String[] collectionNames = {"col1","col2"}; + public String[] collectionNames = {"github","got"}; public static MongoDBTestServer getInstance() @@ -163,10 +167,20 @@ private void loadData() throws IOException, URISyntaxException { String fileContent = new Scanner( getRequestedFile(filenames[i])).useDelimiter("\\Z").next(); - Document fileDoc = Document.parse(fileContent); -// System.out.println( fileDoc.get("dataList")); -// coll.insertMany((List) fileDoc.get("dataList")); - coll.insertOne(fileDoc); + //Document list + List documentList = new ArrayList(); + try { + JSONArray jsonarray = new JSONArray(fileContent); + for (int j = 0; j < jsonarray.length(); j++) { + String jsonStr = jsonarray.getJSONObject(j).toString(); + documentList.add(Document.parse(jsonStr)); + } + + } catch (JSONException e) { + e.printStackTrace(); + } + + coll.insertMany(documentList); FindIterable docs = coll.find(); diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index f21490430d..d535aead89 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -51,7 +51,8 @@ public static void teardown() throws Exception { @Before public void prepareTables() throws TajoException { - executeString("create table col1 (*) tablespace test_spacename using mongodb with ('path'='fileasd.json')"); + executeString("create table got (*) tablespace test_spacename using mongodb"); + executeString("create table github (*) tablespace test_spacename using mongodb"); // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 0ba6f81235..1c9e85d9cd 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -100,6 +100,7 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { @Test public void testTableVolume() throws IOException, TajoException { Tablespace space = TablespaceManager.getByName(server.spaceName); + int[] tableSizes = new int[]{4,3}; for (String tbl:server.collectionNames) { // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); @@ -112,7 +113,7 @@ public void testTableVolume() throws IOException, TajoException { null, space.getTableUri(null,null, tbl)); - assertEquals(1, space.getTableVolume(tbDesc,Optional.empty())); + assertEquals(4, space.getTableVolume(tbDesc,Optional.empty())); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json index 0d0a0fe0e2..b3c74e8123 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json @@ -1,13 +1,6 @@ -{ -"dataList":[ - { - "name" : "mkyong", - "age" : 30 -}, - { - "name" : "mkyong", - "age" : 30 - } - ], - "column1":"column1_data" -} \ No newline at end of file +[ + {"id":"2937257753","type":"PushEvent","actor":{"id":5266949,"login":"hardrubic","gravatar_id":"","url":"https://api.github.com/users/hardrubic","avatar_url":"https://avatars.githubusercontent.com/u/5266949?"},"repo":{"id":38299397,"name":"hardrubic/rxJavaTest","url":"https://api.github.com/repos/hardrubic/rxJavaTest"},"payload":{"push_id":712081726,"size":1,"distinct_size":1,"ref":"refs/heads/master","head":"ea79d7a424f2693b70b9496726f315a5711b6fe7","before":"613f05557ad353f4bedc6df54128f8091ed1f1e9","commits":[{"sha":"ea79d7a424f2693b70b9496726f315a5711b6fe7","author":{"email":"dgzx106@163.com","name":"hardrubic"},"message":"增加rxJava例子","distinct":true,"url":"https://api.github.com/repos/hardrubic/rxJavaTest/commits/ea79d7a424f2693b70b9496726f315a5711b6fe7"}]},"public":true,"created_at":"2015-07-01T00:00:01Z"}, + {"id":"2937257758","type":"WatchEvent","actor":{"id":11455393,"login":"chrischjh","gravatar_id":"","url":"https://api.github.com/users/chrischjh","avatar_url":"https://avatars.githubusercontent.com/u/11455393?"},"repo":{"id":18218031,"name":"dead-horse/co-and-koa-talk","url":"https://api.github.com/repos/dead-horse/co-and-koa-talk"},"payload":{"action":"started"},"public":true,"created_at":"2015-07-01T00:00:01Z"}, + {"id":"2937257759","type":"CreateEvent","actor":{"id":206379,"login":"gvn","gravatar_id":"","url":"https://api.github.com/users/gvn","avatar_url":"https://avatars.githubusercontent.com/u/206379?"},"repo":{"id":24345476,"name":"gvn/webmaker-android","url":"https://api.github.com/repos/gvn/webmaker-android"},"payload":{"ref":"use-self-building","ref_type":"branch","master_branch":"master","description":"Webmaker for Firefox OS & Android","pusher_type":"user"},"public":true,"created_at":"2015-07-01T00:00:01Z"}, + {"id":"2937257761","type":"ForkEvent","actor":{"id":1088854,"login":"CAOakleyII","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","avatar_url":"https://avatars.githubusercontent.com/u/1088854?"},"repo":{"id":11909954,"name":"skycocker/chromebrew","url":"https://api.github.com/repos/skycocker/chromebrew"},"payload":{"forkee":{"id":38339291,"name":"chromebrew","full_name":"CAOakleyII/chromebrew","owner":{"login":"CAOakleyII","id":1088854,"avatar_url":"https://avatars.githubusercontent.com/u/1088854?v=3","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","html_url":"https://github.com/CAOakleyII","followers_url":"https://api.github.com/users/CAOakleyII/followers","following_url":"https://api.github.com/users/CAOakleyII/following{/other_user}","gists_url":"https://api.github.com/users/CAOakleyII/gists{/gist_id}","starred_url":"https://api.github.com/users/CAOakleyII/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CAOakleyII/subscriptions","organizations_url":"https://api.github.com/users/CAOakleyII/orgs","repos_url":"https://api.github.com/users/CAOakleyII/repos","events_url":"https://api.github.com/users/CAOakleyII/events{/privacy}","received_events_url":"https://api.github.com/users/CAOakleyII/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/CAOakleyII/chromebrew","description":"Package manager for Chrome OS","fork":true,"url":"https://api.github.com/repos/CAOakleyII/chromebrew","forks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/forks","keys_url":"https://api.github.com/repos/CAOakleyII/chromebrew/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CAOakleyII/chromebrew/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CAOakleyII/chromebrew/teams","hooks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/hooks","issue_events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/events{/number}","events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/events","assignees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/assignees{/user}","branches_url":"https://api.github.com/repos/CAOakleyII/chromebrew/branches{/branch}","tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/tags","blobs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/refs{/sha}","trees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CAOakleyII/chromebrew/statuses/{sha}","languages_url":"https://api.github.com/repos/CAOakleyII/chromebrew/languages","stargazers_url":"https://api.github.com/repos/CAOakleyII/chromebrew/stargazers","contributors_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contributors","subscribers_url":"https://api.github.com/repos/CAOakleyII/chromebrew/subscribers","subscription_url":"https://api.github.com/repos/CAOakleyII/chromebrew/subscription","commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/commits{/sha}","git_commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/commits{/sha}","comments_url":"https://api.github.com/repos/CAOakleyII/chromebrew/comments{/number}","issue_comment_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/comments{/number}","contents_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contents/{+path}","compare_url":"https://api.github.com/repos/CAOakleyII/chromebrew/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CAOakleyII/chromebrew/merges","archive_url":"https://api.github.com/repos/CAOakleyII/chromebrew/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CAOakleyII/chromebrew/downloads","issues_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues{/number}","pulls_url":"https://api.github.com/repos/CAOakleyII/chromebrew/pulls{/number}","milestones_url":"https://api.github.com/repos/CAOakleyII/chromebrew/milestones{/number}","notifications_url":"https://api.github.com/repos/CAOakleyII/chromebrew/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CAOakleyII/chromebrew/labels{/name}","releases_url":"https://api.github.com/repos/CAOakleyII/chromebrew/releases{/id}","created_at":"2015-07-01T00:00:00Z","updated_at":"2015-06-28T10:11:09Z","pushed_at":"2015-06-09T07:46:57Z","git_url":"git://github.com/CAOakleyII/chromebrew.git","ssh_url":"git@github.com:CAOakleyII/chromebrew.git","clone_url":"https://github.com/CAOakleyII/chromebrew.git","svn_url":"https://github.com/CAOakleyII/chromebrew","homepage":"http://skycocker.github.io/chromebrew/","size":846,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master","public":true}},"public":true,"created_at":"2015-07-01T00:00:01Z"} +] \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json index a7d85f481c..6f405bdf89 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json @@ -1,4 +1,6 @@ -{ - 'name' : 'mkyong', - 'age' : 30 -} \ No newline at end of file +[ + { "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}}, + { "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}}, + { "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}}, + { "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}} +] \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql index bf1c9a499b..9cfc3a1aa5 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSelect.sql @@ -1 +1 @@ -select column1 from col1; \ No newline at end of file +select title from got; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result index 5501b8591f..55ab36eeed 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result @@ -1,3 +1,6 @@ -column1 +title ------------------------------- -column1_data +Hand of the King +Assassin +Dancing Master +Dancing Master From 8d0e0745eccf4dcc44321d8fe28477e9a6d8c4ea Mon Sep 17 00:00:00 2001 From: Janaka Date: Sun, 26 Jun 2016 01:42:02 +0530 Subject: [PATCH 52/94] Configured to support both file and not file --- .../mongodb/MongoDBCollectionReader.java | 1 + .../storage/mongodb/MongoDBTableSpace.java | 13 ++++++--- .../mongodb/MongoDocumentDeserializer.java | 28 +++++++++---------- .../storage/mongodb/TestMetadataProvider.java | 6 ++-- .../storage/mongodb/TestMongoDBQueryTest.java | 21 +++++++++----- .../mongodb/TestMongoDBTableSpace.java | 22 +++++++++------ 6 files changed, 54 insertions(+), 37 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index 2bf3931dc6..5c1360430a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -66,6 +66,7 @@ public void init() throws IOException documentList =(List) collection.find().into( new ArrayList());; + deserializer.init(); } public Tuple readTuple() throws IOException, TextLineParsingError { diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 38af7a9a1b..7406cf338e 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -163,10 +163,15 @@ public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoExc //TODO Handle this here. If empty throw exception or what? boolean ifExist = (table.count()>0)?true:false; + + //If meta data provides. Create a table + if(STORAGE_PROPERTY.isMetadataProvided()) + db.createCollection(tableDesc.getName()); } @Override public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { + if(STORAGE_PROPERTY.isMetadataProvided()) db.getCollection(tableDesc.getName()).drop(); } @@ -220,10 +225,10 @@ public URI getTableUri( String databaseName, String tableName) { // } - //Metadata -// public MetadataProvider getMetadataProvider() { -// return new MongoDBMetadataProvider(this, mappedDBName); -// } + // Metadata + public MetadataProvider getMetadataProvider() { + return new MongoDBMetadataProvider(this, mappedDBName); + } public ConnectionInfo getConnectionInfo() { return connectionInfo; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java index 3ed4500079..65ce0de5aa 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java @@ -221,23 +221,23 @@ private void getValue(JSONObject object, } public void deserialize(Document doc, Tuple output) throws IOException, TextLineParsingError { -// String line = doc.to; -// -// JSONObject object; -// try { -// object = (JSONObject) parser.parse(line); -// } catch (ParseException pe) { -// throw new TextLineParsingError(line, pe); -// } catch (ArrayIndexOutOfBoundsException ae) { -// // truncated value -// throw new TextLineParsingError(line, ae); -// } + + String line = doc.toJson(); + JSONObject object; + try { + object = (JSONObject) parser.parse(line); + } catch (ParseException pe) { + throw new TextLineParsingError(line, pe); + } catch (ArrayIndexOutOfBoundsException ae) { + // truncated value + throw new TextLineParsingError(line, ae); + } for (int i = 0; i < projectedPaths.length; i++) { -// String [] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER); -// getValue(object, paths[0], paths, 0, i, output); + String [] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER); + getValue(object, paths[0], paths, 0, i, output); - output.put(i, DatumFactory.createText( doc.get(projectedPaths[i]).toString() )); + //output.put(i, DatumFactory.createText( doc.get(projectedPaths[i]).toString() )); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index e0c5dbb27f..03d063128a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -33,7 +33,7 @@ /** * Created by janaka on 6/8/16. */ -@Ignore + public class TestMetadataProvider { static MongoDBTestServer server = MongoDBTestServer.getInstance(); @@ -71,14 +71,14 @@ public void testGetTables() throws Exception { assertEquals(expected, found); } @Test - public void testGetTableDescriptor() throws Exception { + public void testGetTableDescription() throws Exception { Tablespace tablespace = TablespaceManager.get(server.getURI()); MetadataProvider provider = tablespace.getMetadataProvider(); for (String tableName : server.collectionNames) { TableDesc table = provider.getTableDesc(null, tableName); assertEquals( server.mappedDbName+"."+tableName, table.getName()); - assertEquals(server.getURI() + "&table=" + tableName, table.getUri().toASCIIString()); + assertEquals(server.getURI() + "?table=" + tableName, table.getUri().toASCIIString()); //ToDo Check the stats } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index d535aead89..119a86b2c4 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -20,10 +20,7 @@ import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.exception.TajoException; import org.apache.tajo.storage.TablespaceManager; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; import java.net.URI; @@ -51,9 +48,19 @@ public static void teardown() throws Exception { @Before public void prepareTables() throws TajoException { - executeString("create table got (*) tablespace test_spacename using mongodb"); - executeString("create table github (*) tablespace test_spacename using mongodb"); - // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); + if(!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + executeString("create table got (*) tablespace test_spacename using mongodb"); + executeString("create table github (*) tablespace test_spacename using mongodb"); + // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); + } + } + + @After + public void dropTables() throws TajoException { + if(!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + executeString("drop table got"); + executeString("drop table github"); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 1c9e85d9cd..d44cd5e134 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -71,7 +71,8 @@ public void testCreateTable() throws IOException, TajoException { space.createTable(null, false); } - @Ignore + + //Todo delete only metadat is provide @Test(timeout = 1000) public void testCreateTable_and_Purg() throws IOException, TajoException { Tablespace space = TablespaceManager.getByName(server.spaceName); @@ -83,18 +84,21 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { null, server.getURI()); - space.createTable(tableDesc, false); + //Test create and delete if meta data provided + if(MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + space.createTable(tableDesc, false); - //Check whether the created table is in the collection - final Set found = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); - assertTrue(found.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); + //Check whether the created table is in the collection + final Set found = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); + assertTrue(found.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); - //Purg the table - space.purgeTable(tableDesc); - final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); - assertFalse(found_after.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); + //Purg the table + space.purgeTable(tableDesc); + final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); + assertFalse(found_after.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); + } } @Test From 0fa19bf6c0cd825eb69b4a97f96b14358590cbff Mon Sep 17 00:00:00 2001 From: Janaka Date: Wed, 27 Jul 2016 23:30:39 +0530 Subject: [PATCH 53/94] Added test cases to test Join, Groupby, Sort --- .../storage/mongodb/TestMongoDBQueryTest.java | 17 ++++++++++++++++- .../storage/mongodb/TestMongoDBTableSpace.java | 11 +++++------ .../src/test/resources/dataset/file2.json | 9 +++++---- .../TestMongoDBQueryTest/testGroupby.sql | 2 +- .../queries/TestMongoDBQueryTest/testJoin.sql | 2 +- .../queries/TestMongoDBQueryTest/testSort.sql | 2 +- .../TestMongoDBQueryTest/testGroupby.result | 8 +++----- .../TestMongoDBQueryTest/testJoin.result | 2 ++ .../TestMongoDBQueryTest/testSelect.result | 3 ++- .../TestMongoDBQueryTest/testSort.result | 11 ++++++----- 10 files changed, 42 insertions(+), 25 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 119a86b2c4..1917e18288 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -68,9 +68,24 @@ public void dropTables() throws TajoException { @Test public void testSelect() throws Exception { runSimpleTests(); + } + -// executeString("select title, name.first_name from col1"); + @SimpleTest + @Test + public void testSort() throws Exception { + runSimpleTests(); } + @SimpleTest + @Test + public void testGroupby() throws Exception { + runSimpleTests(); + } + @SimpleTest + @Test + public void testJoin() throws Exception { + runSimpleTests(); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index d44cd5e134..ce8f86198d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -34,10 +34,7 @@ import java.io.IOException; import java.net.URI; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.Set; +import java.util.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -104,7 +101,9 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { @Test public void testTableVolume() throws IOException, TajoException { Tablespace space = TablespaceManager.getByName(server.spaceName); - int[] tableSizes = new int[]{4,3}; + Map tableSizes = new HashMap(); + tableSizes.put("github",4); + tableSizes.put("got",5); for (String tbl:server.collectionNames) { // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); @@ -117,7 +116,7 @@ public void testTableVolume() throws IOException, TajoException { null, space.getTableUri(null,null, tbl)); - assertEquals(4, space.getTableVolume(tbDesc,Optional.empty())); + assertEquals((int)tableSizes.get(tbl), space.getTableVolume(tbDesc,Optional.empty())); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json index 6f405bdf89..ad2fc3f159 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json @@ -1,6 +1,7 @@ [ - { "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}}, - { "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}}, - { "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}}, - { "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}} + { "title" : "Hand of the King", "first_name": "Eddard", "last_name": "Stark"}, + { "title" : "Assassin", "first_name": "Arya", "last_name": "Stark"}, + { "title" : "Lady", "first_name": "Sansa", "last_name": "Stark"}, + { "title" : "Dancing Master", "first_name": "Syrio", "last_name": "Forel"}, + { "title" : "Dancing Master's Father", "first_name": "Nyrio", "last_name": "Forel"} ] \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql index ea68caa8a1..26fd5b5358 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testGroupby.sql @@ -1 +1 @@ -select actor.id, count(*) from github group by actor.id; \ No newline at end of file +select last_name, count(*) from got group by last_name; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql index e2c95c8240..b0915358c0 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testJoin.sql @@ -1 +1 @@ -select g1.title from got g1, got g2 where g1.name.first_name = g2.name.first_name \ No newline at end of file +select g1.title from got g1, got g2 where g1.first_name = g2.first_name \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql index d31ea941dd..38d00e83bc 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testSort.sql @@ -1 +1 @@ -select actor.id, actor.login from github order by type; \ No newline at end of file +select title,first_name from got order by first_name; \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result index 67797c83d6..f14620d207 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testGroupby.result @@ -1,6 +1,4 @@ -actor/id,?count +last_name,?count ------------------------------- -206379,1 -11455393,1 -1088854,1 -5266949,1 +Stark,3 +Forel,2 diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result index a6da9fc1be..0888ed5321 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testJoin.result @@ -2,4 +2,6 @@ title ------------------------------- Hand of the King Assassin +Lady Dancing Master +Dancing Master's Father diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result index 55ab36eeed..0888ed5321 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSelect.result @@ -2,5 +2,6 @@ title ------------------------------- Hand of the King Assassin +Lady Dancing Master -Dancing Master +Dancing Master's Father diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result index e0073b9320..d1794ef189 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testSort.result @@ -1,6 +1,7 @@ -actor/id,actor/login +title,first_name ------------------------------- -206379,gvn -1088854,CAOakleyII -5266949,hardrubic -11455393,chrischjh +Assassin,Arya +Hand of the King,Eddard +Dancing Master's Father,Nyrio +Lady,Sansa +Dancing Master,Syrio From b59d553080ab87d9d2e4132fa808dfc9b5a333e8 Mon Sep 17 00:00:00 2001 From: Janaka Date: Thu, 28 Jul 2016 20:55:14 +0530 Subject: [PATCH 54/94] Inheritance removed because there is no use in parent class. --- .../storage/mongodb/MongoDocumentDeserializer.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java index 65ce0de5aa..29fb3e1f0d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java @@ -44,7 +44,7 @@ import java.util.Map; import java.util.TimeZone; -public class MongoDocumentDeserializer extends TextLineDeserializer { +public class MongoDocumentDeserializer { private JSONParser parser; // Full Path -> Type @@ -53,9 +53,12 @@ public class MongoDocumentDeserializer extends TextLineDeserializer { private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); private final TimeZone timezone; + protected final Schema schema; + protected final TableMeta meta; public MongoDocumentDeserializer(Schema schema, TableMeta meta, Column [] projected) { - super(schema, meta); + this.schema = schema; + this.meta = meta; projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true); types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); @@ -64,12 +67,10 @@ public MongoDocumentDeserializer(Schema schema, TableMeta meta, Column [] projec StorageUtil.TAJO_CONF.getSystemTimezone().getID())); } - @Override public void init() { parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR); } - @Override public void deserialize(ByteBuf buf, Tuple output) throws IOException, TextLineParsingError { } @@ -241,7 +242,6 @@ public void deserialize(Document doc, Tuple output) throws IOException, TextLine } } - @Override public void release() { } } From aed6b35c814bfd50e4297f0e5f4ed32a21727af0 Mon Sep 17 00:00:00 2001 From: Janaka Date: Thu, 28 Jul 2016 21:11:39 +0530 Subject: [PATCH 55/94] Commeted the previous code --- .../apache/tajo/storage/mongodb/ConnectionInfo.java | 11 ++++++++--- .../tajo/storage/mongodb/MongoDBCollectionReader.java | 4 ++++ .../apache/tajo/storage/mongodb/MongoDBFragment.java | 5 +++++ .../tajo/storage/mongodb/MongoDBFragmentSerde.java | 4 ++++ .../tajo/storage/mongodb/MongoDBMetadataProvider.java | 11 +++++++---- .../apache/tajo/storage/mongodb/MongoDBScanner.java | 7 ++++++- .../tajo/storage/mongodb/MongoDBTableSpace.java | 4 ++++ .../storage/mongodb/MongoDocumentDeserializer.java | 5 +++++ 8 files changed, 43 insertions(+), 8 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 12c88fb821..c113f651c9 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -27,6 +27,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.tajo.exception.TajoInternalError; +/* +ConnectionInfo keeps the details about the mongodb server connection. +MongoURI, db credentials and URI specific details such as schema, host, port + */ + public class ConnectionInfo { private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); @@ -41,10 +46,12 @@ public class ConnectionInfo { private int port; private Map params; + //To create an instance using provided string of a URI public static ConnectionInfo fromURI(String originalUri) { return fromURI(URI.create(originalUri)); } + //creates a instance using provided URI public static ConnectionInfo fromURI(URI originalUri) { final String uriStr = originalUri.toASCIIString(); URI uri = originalUri; @@ -97,9 +104,8 @@ public static ConnectionInfo fromURI(URI originalUri) { connInfo.params = params; String mongoDbURIStr = ""; - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] - //Generate the URI + //Generate the MongoURI mongoDbURIStr+= connInfo.getScheme(); mongoDbURIStr+="://"; if(connInfo.getUser() !=null) @@ -125,7 +131,6 @@ public MongoClientURI getMongoDBURI() return mongoDBURI; } - public String getScheme() { return scheme; } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index 5c1360430a..fa29f9f734 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -40,6 +40,10 @@ import java.util.ArrayList; import java.util.List; +/* + Reads data from a mongodb collection, line by line. + Used within the MongoScanner to read tuples. + */ public class MongoDBCollectionReader { private ConnectionInfo connectionInfo; private MongoDocumentDeserializer deserializer; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java index 322b9626aa..e24936d258 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -24,6 +24,11 @@ import java.net.URI; import java.util.List; +/* + Fragment is similar to splits in Map Reduce. + Which contains enough details to access, a relevant section from the storage. + This will be serialized and passed to the cluster. + */ public class MongoDBFragment extends AbstractFileFragment { diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java index 4d5c0fbc29..9099d8e38c 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java @@ -24,6 +24,10 @@ import java.net.URI; +/** + * Fragment Serde for MongoFragment. Which is used to serialize and deserialize + * MongoDBFragment objects using ProtocolBuffer. + */ public class MongoDBFragmentSerde implements FragmentSerde { @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index 88f2a467c1..57d50f1bcf 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -36,9 +36,10 @@ import java.util.Collection; import java.util.Collections; -/** - * Created by janaka on 6/7/16. - */ +/* +* Used to provide MetaData about a particular mongodb tablespace. +* Provides table details, list of tables and statistics of tables +* * */ public class MongoDBMetadataProvider implements MetadataProvider { private MongoDBTableSpace tableSpace; @@ -79,9 +80,10 @@ public Collection getSchemas() { @Override public Collection getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { - + //Get a list of table(=collection) names MongoIterable collectionList = db.listCollectionNames(); + //Map to a string list and return Collection list = new ArrayList(); for (String item : collectionList) { list.add(item); @@ -92,6 +94,7 @@ public Collection getTables(@Nullable String schemaPattern, @Nullable St @Override public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException { + //Create table description and meta fro a specific table TableMeta tbMeta = new TableMeta("mongodb", new KeyValueSet()); TableDesc tbDesc = new TableDesc( IdentifierUtil.buildFQName(mappedDbName, tableName), diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index fb283ebbdc..37858163ef 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -47,6 +47,12 @@ import java.sql.DatabaseMetaData; import java.util.Properties; +/* +* Reads data from a MongoDB table +* Uses MongoDBCollectionReader to read tuples +* */ + +//Todo Remove FileScanner public class MongoDBScanner extends FileScanner { MongoDBCollectionReader collectionReader; @@ -69,7 +75,6 @@ public void init() throws IOException { super.init(); } - @Override public Tuple next() throws IOException { try { diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 7406cf338e..0d3bf1c281 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -50,6 +50,10 @@ import java.util.List; import java.util.Optional; +/* +* TableSpace for MongoDB +* */ + public class MongoDBTableSpace extends Tablespace { private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java index 29fb3e1f0d..0d39ee3eac 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java @@ -44,6 +44,11 @@ import java.util.Map; import java.util.TimeZone; +/* +* Used inside the Collection Reader to map mongo documents to tuples. +* Use org.apache.tajo.storage.json.JsonLineDeserializer to understand the structure +* */ + public class MongoDocumentDeserializer { private JSONParser parser; From 005af166fd25cd5809a1e09e630f5b48511e24c7 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 3 Aug 2016 11:16:05 +0530 Subject: [PATCH 56/94] New test case to insert --- .../apache/tajo/storage/mongodb/TestMongoDBQueryTest.java | 7 +++++++ .../resources/queries/TestMongoDBQueryTest/testInsert.sql | 1 + .../results/TestMongoDBQueryTest/testInsert.result | 7 +++++++ 3 files changed, 15 insertions(+) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testInsert.result diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 1917e18288..08c4e628b8 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -88,4 +88,11 @@ public void testGroupby() throws Exception { public void testJoin() throws Exception { runSimpleTests(); } + + + @SimpleTest + @Test + public void testInsert() throws Exception { + runSimpleTests(); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql new file mode 100644 index 0000000000..7835a3af18 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql @@ -0,0 +1 @@ +INSERT INTO got VALUES (value1,value2,value3,...); \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testInsert.result b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testInsert.result new file mode 100644 index 0000000000..d1794ef189 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/results/TestMongoDBQueryTest/testInsert.result @@ -0,0 +1,7 @@ +title,first_name +------------------------------- +Assassin,Arya +Hand of the King,Eddard +Dancing Master's Father,Nyrio +Lady,Sansa +Dancing Master,Syrio From 68d17745447cec26b1cb5eca70675a1575c2fd72 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 3 Aug 2016 11:33:37 +0530 Subject: [PATCH 57/94] Commented the insert test which in not yet setup --- .../org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 08c4e628b8..720a9d20b5 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -93,6 +93,6 @@ public void testJoin() throws Exception { @SimpleTest @Test public void testInsert() throws Exception { - runSimpleTests(); + // runSimpleTests(); } } From 6eb9d261cf0765e0066612c2a27a87e0e93a2fc8 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Thu, 4 Aug 2016 12:33:46 +0530 Subject: [PATCH 58/94] commented the insert test --- .../org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java | 2 +- .../test/resources/queries/TestMongoDBQueryTest/testInsert.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 720a9d20b5..6c2d51921c 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -93,6 +93,6 @@ public void testJoin() throws Exception { @SimpleTest @Test public void testInsert() throws Exception { - // runSimpleTests(); + // runSimpleTests(); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql index 7835a3af18..3714e92b82 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql @@ -1 +1 @@ -INSERT INTO got VALUES (value1,value2,value3,...); \ No newline at end of file +INSERT INTO got (title,first_name) values (1,2); \ No newline at end of file From 192bc6990d6fa1e98108cec5ecfa5ea569e58f65 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Thu, 4 Aug 2016 14:39:27 +0530 Subject: [PATCH 59/94] commented insert --- .../org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java | 2 +- .../test/resources/queries/TestMongoDBQueryTest/testInsert.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 08c4e628b8..f4e4fcf83f 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -93,6 +93,6 @@ public void testJoin() throws Exception { @SimpleTest @Test public void testInsert() throws Exception { - runSimpleTests(); +// runSimpleTests(); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql index a5333b45cc..31d40fe86b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql @@ -1 +1 @@ -INSERT INTO got select (title,first_name) got values (1,2); \ No newline at end of file +INSERT OVERWRITE INTO got SELECT * FROM got values 'a','b','c'; \ No newline at end of file From 87375c0b5db0d528cd149e50678b4a83638d31fe Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 5 Aug 2016 11:50:52 +0530 Subject: [PATCH 60/94] Added appender, CollectionWriter, DocumentSerializer --- .../tajo/storage/mongodb/MongoDBAppender.java | 44 ++++++++++++++- .../mongodb/MongoDBCollectionReader.java | 12 +---- .../mongodb/MongoDBCollectionWriter.java | 54 +++++++++++++++++++ ....java => MongoDBDocumentDeserializer.java} | 15 +++--- .../mongodb/MongoDBDocumentSerializer.java | 25 +++++++++ .../tajo/storage/mongodb/MongoDBScanner.java | 21 +------- .../storage/mongodb/MongoDBTableSpace.java | 6 +-- .../storage/mongodb/MongoDBTestServer.java | 48 ++++++++++++----- .../storage/mongodb/TestMetadataProvider.java | 9 ++-- .../storage/mongodb/TestMongoDBQueryTest.java | 5 +- .../mongodb/TestMongoDBTableSpace.java | 41 +++++++------- .../TestMongoDBQueryTest/testInsert.sql | 2 +- 12 files changed, 198 insertions(+), 84 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java rename tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/{MongoDocumentDeserializer.java => MongoDBDocumentDeserializer.java} (97%) create mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index a2b7786782..a71d353e98 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -17,21 +17,55 @@ */ package org.apache.tajo.storage.mongodb; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.TaskAttemptId; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.Tuple; import java.io.IOException; +import java.net.URI; import java.util.List; /** * Created by janaka on 5/21/16. */ public class MongoDBAppender implements Appender { + + //Given at intialization + private final Configuration conf; + private final Schema schema; + private final TableMeta meta; + private final Path stagingDir; + private final TaskAttemptId taskAttemptId; + private final URI uri; + + + protected boolean inited = false; + private boolean[] columnStatsEnabled; + private boolean tableStatsEnabled; + + public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path stagingDir, URI uri) { + this.conf = conf; + this.schema = schema; + this.meta = meta; + this.stagingDir = stagingDir; + this.taskAttemptId = taskAttemptId; + this.uri = uri; + } + @Override public void init() throws IOException { - + if (inited) { + throw new IllegalStateException("FileAppender is already initialized."); + } + inited = true; } @Override @@ -46,7 +80,7 @@ public void flush() throws IOException { @Override public long getEstimatedOutputSize() throws IOException { - return 0; + throw new IOException(new NotImplementedException()); } @Override @@ -56,6 +90,12 @@ public void close() throws IOException { @Override public void enableStats() { + if (inited) { + throw new IllegalStateException("Should enable this option before init()"); + } + + this.tableStatsEnabled = true; + this.columnStatsEnabled = new boolean[schema.size()]; } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index fa29f9f734..2b3b3f6428 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -17,24 +17,16 @@ */ package org.apache.tajo.storage.mongodb; -import com.mongodb.DBCursor; import com.mongodb.MongoClient; -import com.mongodb.MongoClientURI; -import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import io.netty.util.CharsetUtil; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; -import org.apache.tajo.storage.json.JsonLineDeserializer; import org.apache.tajo.storage.text.TextLineParsingError; import org.bson.Document; import java.io.IOException; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.CharsetDecoder; -import io.netty.util.CharsetUtil; import java.nio.charset.CharsetEncoder; import java.util.ArrayList; @@ -46,14 +38,14 @@ */ public class MongoDBCollectionReader { private ConnectionInfo connectionInfo; - private MongoDocumentDeserializer deserializer; + private MongoDBDocumentDeserializer deserializer; private int targetLength; List documentList; private int currentIndex; private final CharsetEncoder encoder = CharsetUtil.getEncoder(CharsetUtil.UTF_8); - public MongoDBCollectionReader(ConnectionInfo connectionInfo,MongoDocumentDeserializer deserializer ,int targetLength) + public MongoDBCollectionReader(ConnectionInfo connectionInfo, MongoDBDocumentDeserializer deserializer , int targetLength) { this.connectionInfo = connectionInfo; this.deserializer = deserializer; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java new file mode 100644 index 0000000000..c7d86b1979 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import org.apache.tajo.storage.Tuple; +import org.bson.Document; + +/** + * Writes tuples into a mongodb collection + */ +public class MongoDBCollectionWriter { + private ConnectionInfo connectionInfo; + MongoCollection collection; + + public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentDeserializer deserializer , int targetLength) + { + this.connectionInfo = connectionInfo; +// this.deserializer = deserializer; +// this.targetLength = targetLength; + } + + public void init() + { + + MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); + + collection = db.getCollection(connectionInfo.getTableName()); + } + + public void writeTuple(Tuple tuple) + { + + } + +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java similarity index 97% rename from tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java rename to tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java index 0d39ee3eac..e8554e6a94 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDocumentDeserializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java @@ -49,7 +49,7 @@ * Use org.apache.tajo.storage.json.JsonLineDeserializer to understand the structure * */ -public class MongoDocumentDeserializer { +public class MongoDBDocumentDeserializer { private JSONParser parser; // Full Path -> Type @@ -61,7 +61,7 @@ public class MongoDocumentDeserializer { protected final Schema schema; protected final TableMeta meta; - public MongoDocumentDeserializer(Schema schema, TableMeta meta, Column [] projected) { + public MongoDBDocumentDeserializer(Schema schema, TableMeta meta, Column [] projected) { this.schema = schema; this.meta = meta; @@ -72,14 +72,12 @@ public MongoDocumentDeserializer(Schema schema, TableMeta meta, Column [] projec StorageUtil.TAJO_CONF.getSystemTimezone().getID())); } + + //Initialize and setup parser public void init() { parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR); } - public void deserialize(ByteBuf buf, Tuple output) throws IOException, TextLineParsingError { - - } - /** * * @@ -226,6 +224,8 @@ private void getValue(JSONObject object, } } + + //Map the given documents into the given tuple and fill with calues public void deserialize(Document doc, Tuple output) throws IOException, TextLineParsingError { String line = doc.toJson(); @@ -246,7 +246,4 @@ public void deserialize(Document doc, Tuple output) throws IOException, TextLine //output.put(i, DatumFactory.createText( doc.get(projectedPaths[i]).toString() )); } } - - public void release() { - } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java new file mode 100644 index 0000000000..6f6df16d7e --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +/** + * Serialize tuples into Mongo Documents + */ +public class MongoDBDocumentSerializer { + +} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index 37858163ef..7383cbc681 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -17,35 +17,16 @@ */ package org.apache.tajo.storage.mongodb; -import com.google.common.base.Preconditions; -import com.mongodb.MongoClient; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.MongoIterable; -import io.netty.buffer.ByteBuf; -import net.minidev.json.JSONObject; -import net.minidev.json.parser.JSONParser; -import net.minidev.json.parser.ParseException; import org.apache.hadoop.conf.Configuration; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.NestedPathUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.storage.FileScanner; -import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.json.JsonLineDeserializer; import org.apache.tajo.storage.text.TextLineParsingError; -import org.bson.Document; import java.io.IOException; -import java.sql.DatabaseMetaData; -import java.util.Properties; /* * Reads data from a MongoDB table @@ -88,7 +69,7 @@ public Tuple next() throws IOException { @Override public void reset() throws IOException { - MongoDocumentDeserializer deserializer = new MongoDocumentDeserializer(schema,meta,targets); + MongoDBDocumentDeserializer deserializer = new MongoDBDocumentDeserializer(schema,meta,targets); collectionReader = new MongoDBCollectionReader(ConnectionInfo.fromURI(fragment.getUri()),deserializer,targets.length); collectionReader.init(); diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 0d3bf1c281..df45f3351e 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -62,9 +62,9 @@ public class MongoDBTableSpace extends Tablespace { //Table Space Properties static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined false, //not movable - false, //writable at the moment + true, //writable at the moment true, // Absolute path - false); // Meta data will be provided + true); // Meta data will be provided static final FormatProperty FORMAT_PROPERTY = new FormatProperty( false, // Insert false, //direct insert @@ -170,7 +170,7 @@ public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoExc //If meta data provides. Create a table if(STORAGE_PROPERTY.isMetadataProvided()) - db.createCollection(tableDesc.getName()); + db.createCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())); } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index 8bb40cea85..fee9441697 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -53,25 +53,32 @@ public class MongoDBTestServer { private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - public static int port = 12345; - public static String host = "localhost"; - public static String dbName = "test_dbname"; - public static String mappedDbName = "test_mapped_dbname"; - private static MongoDBTestServer instance; + //mongo server details + public static final int PORT = 12345; + public static final String HOST = "localhost"; + public static final String DBNAME = "test_dbname"; + public static final String MAPPEDDBNAME = "test_mapped_dbname"; + + //tajo tableSpace name + public static final String SPACE_NAME = "test_spacename"; - public static final String spaceName = "test_spacename"; + // instance for singleton server + private static MongoDBTestServer instance; - //New + //Mongo Server componenets private static final MongodStarter starter = MongodStarter.getDefaultInstance(); private MongodExecutable _mongodExe; private MongodProcess _mongod; private MongoClient _mongo; + //File names to load data private String[] filenames = {"file1.json","file2.json"}; + //Collection names (table names) to be created inside mongodb public String[] collectionNames = {"github","got"}; + //server object can be created using this method public static MongoDBTestServer getInstance() { if(instance==null) @@ -86,22 +93,24 @@ public static MongoDBTestServer getInstance() return instance; } + // private constructor private MongoDBTestServer () throws IOException, URISyntaxException { _mongodExe = starter.prepare(new MongodConfigBuilder() .version(Version.Main.PRODUCTION) - .net(new Net(port, Network.localhostIsIPv6())) + .net(new Net(PORT, Network.localhostIsIPv6())) .cmdOptions( new MongoCmdOptionsBuilder(). useStorageEngine("mmapv1"). build()) .build()); _mongod = _mongodExe.start(); - _mongo = new MongoClient(host, port); + _mongo = new MongoClient(HOST, PORT); registerTablespace(); loadData(); } + //To stop the server public void stop() { _mongod.stop(); @@ -109,10 +118,11 @@ public void stop() instance = null; } + //Returns the url which can be used to connet to the server instance public URI getURI() { try { - return new URI("mongodb://" + host + ":" + port + "/"+dbName); + return new URI("mongodb://" + HOST + ":" + PORT + "/"+ DBNAME); } catch (Exception e) { @@ -121,7 +131,7 @@ public URI getURI() } - //From GitHub + //Start mongo process private MongosProcess startMongos(int port, int defaultConfigPort, String defaultHost) throws UnknownHostException, IOException { IMongosConfig mongosConfig = new MongosConfigBuilder() @@ -147,19 +157,22 @@ private MongodProcess startMongod(int defaultConfigPort) throws UnknownHostExcep return mongod; } + + //Register the new table space for testing private void registerTablespace() throws IOException { JSONObject configElements = new JSONObject(); - configElements.put(MongoDBTableSpace.CONFIG_KEY_MAPPED_DATABASE, mappedDbName); + configElements.put(MongoDBTableSpace.CONFIG_KEY_MAPPED_DATABASE, MAPPEDDBNAME); - MongoDBTableSpace tablespace = new MongoDBTableSpace(spaceName,getURI(),configElements); + MongoDBTableSpace tablespace = new MongoDBTableSpace(SPACE_NAME,getURI(),configElements); tablespace.init(new TajoConf()); TablespaceManager.addTableSpaceForTest(tablespace); } + //Create tables and Load data into the mongo server instance private void loadData() throws IOException, URISyntaxException { - MongoDatabase db = _mongo.getDatabase(dbName); + MongoDatabase db = _mongo.getDatabase(DBNAME); for(int i=0; i found = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); - assertTrue(found.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); - + assertTrue(found.contains("Table1")); + + //Check whether the created table is in the mongo database + MongoClient mongoClient = server.getMongoClient(); + Boolean foundInMongoDB = false; + MongoIterable collectionNames = mongoClient.getDatabase(server.DBNAME).listCollectionNames(); + for (final String name : collectionNames) { + if (name.equalsIgnoreCase("Table1")) { + foundInMongoDB = true; + } + } + assertTrue(foundInMongoDB); //Purg the table space.purgeTable(tableDesc); final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); - assertFalse(found_after.contains(IdentifierUtil.buildFQName(server.mappedDbName, "Table1"))); + assertFalse(found_after.contains(IdentifierUtil.buildFQName(server.MAPPEDDBNAME, "Table1"))); } } @Test public void testTableVolume() throws IOException, TajoException { - Tablespace space = TablespaceManager.getByName(server.spaceName); + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); Map tableSizes = new HashMap(); tableSizes.put("github",4); tableSizes.put("got",5); @@ -110,7 +115,7 @@ public void testTableVolume() throws IOException, TajoException { // long a = 1;b TableDesc tbDesc = new TableDesc( - IdentifierUtil.buildFQName(server.mappedDbName, tbl), + IdentifierUtil.buildFQName(server.MAPPEDDBNAME, tbl), SchemaBuilder.builder() .build(), null, diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql index 31d40fe86b..6bca9fd267 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql @@ -1 +1 @@ -INSERT OVERWRITE INTO got SELECT * FROM got values 'a','b','c'; \ No newline at end of file +INSERT INTO got SELECT 'a','b','c'; \ No newline at end of file From ca18ca82855e1ad2e47c1535f1a2974475f13507 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 5 Aug 2016 16:19:29 +0530 Subject: [PATCH 61/94] Implemented the text type for appender --- .../tajo/storage/mongodb/MongoDBAppender.java | 11 +++- .../mongodb/MongoDBCollectionWriter.java | 12 +++- .../mongodb/MongoDBDocumentSerializer.java | 25 ++++++++ .../TestMongoDBDocumentSerializer.java | 62 +++++++++++++++++++ 4 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index a71d353e98..93f9d3b938 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -37,7 +37,7 @@ */ public class MongoDBAppender implements Appender { - //Given at intialization + //Given at constructor private final Configuration conf; private final Schema schema; private final TableMeta meta; @@ -50,6 +50,8 @@ public class MongoDBAppender implements Appender { private boolean[] columnStatsEnabled; private boolean tableStatsEnabled; + private MongoDBCollectionWriter mongoDBCollectionWriter; + public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schema, TableMeta meta, Path stagingDir, URI uri) { this.conf = conf; @@ -66,11 +68,14 @@ public void init() throws IOException { throw new IllegalStateException("FileAppender is already initialized."); } inited = true; + MongoDBDocumentSerializer md = new MongoDBDocumentSerializer(schema,meta); + mongoDBCollectionWriter = new MongoDBCollectionWriter(ConnectionInfo.fromURI(uri),md); + mongoDBCollectionWriter.init(); } @Override public void addTuple(Tuple t) throws IOException { - + mongoDBCollectionWriter.writeTuple(t); } @Override @@ -85,7 +90,7 @@ public long getEstimatedOutputSize() throws IOException { @Override public void close() throws IOException { - + mongoDBCollectionWriter.close(); } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java index c7d86b1979..76efeea08e 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java @@ -28,9 +28,11 @@ */ public class MongoDBCollectionWriter { private ConnectionInfo connectionInfo; + MongoClient mongoClient; MongoCollection collection; - public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentDeserializer deserializer , int targetLength) + + public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer deserializer) { this.connectionInfo = connectionInfo; // this.deserializer = deserializer; @@ -39,8 +41,7 @@ public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentDes public void init() { - - MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); collection = db.getCollection(connectionInfo.getTableName()); @@ -51,4 +52,9 @@ public void writeTuple(Tuple tuple) } + public void close() + { + mongoClient.close(); + } + } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java index 6f6df16d7e..8b5c920651 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java @@ -17,9 +17,34 @@ */ package org.apache.tajo.storage.mongodb; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.storage.Tuple; +import org.bson.Document; + +import java.io.IOException; + /** * Serialize tuples into Mongo Documents */ public class MongoDBDocumentSerializer { + private final Schema schema; + private final TableMeta meta; + + public MongoDBDocumentSerializer(Schema schema, TableMeta meta) { + this.schema = schema; + this.meta = meta; + } + + + public void deserialize(Tuple inputTuple,Document outputDoc) throws IOException + { + for(int i=0; i< inputTuple.size();i++) + { + outputDoc.put(schema.getColumn(i).getSimpleName(),inputTuple.getText(i)); + } + } + } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java new file mode 100644 index 0000000000..371d1d51ae --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import com.sun.org.apache.xml.internal.security.encryption.DocumentSerializer; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SchemaBuilder; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; +import org.bson.Document; +import org.junit.Test; + +import java.io.IOException; + +public class TestMongoDBDocumentSerializer { + + + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + + @Test + public void testSeriaalizeTextType() throws IOException { + Schema schem = SchemaBuilder.builder(). + add(new Column("title", TajoDataTypes.Type.TEXT)) + .add(new Column("first_name", TajoDataTypes.Type.TEXT)) + .add(new Column("last_name", TajoDataTypes.Type.TEXT)) + .build(); + + MongoDBDocumentSerializer dc = new MongoDBDocumentSerializer(schem,null); + + Tuple tuple = new VTuple(3); + tuple.put(0,DatumFactory.createText("Good_Man")); + tuple.put(1,DatumFactory.createText("Janaka")); + tuple.put(2,DatumFactory.createText("Chathuranga")); + Document md = new Document(); + + dc.deserialize(tuple,md); + + LOG.info(md.toJson()); + + } + +} From adaa3a8c8bce2341bef1e98952b41dc898d8bae6 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 5 Aug 2016 16:26:07 +0530 Subject: [PATCH 62/94] Set metadataProvided=false --- .../java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index df45f3351e..76ebf78658 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -64,7 +64,7 @@ public class MongoDBTableSpace extends Tablespace { false, //not movable true, //writable at the moment true, // Absolute path - true); // Meta data will be provided + false); // Meta data will be provided static final FormatProperty FORMAT_PROPERTY = new FormatProperty( false, // Insert false, //direct insert From 41f27d43b968cfbb563c48637714e92b6341c608 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 5 Aug 2016 16:34:25 +0530 Subject: [PATCH 63/94] Refactor Deserialize to Serialize --- .../tajo/storage/mongodb/MongoDBCollectionWriter.java | 9 +++++++-- .../tajo/storage/mongodb/MongoDBDocumentSerializer.java | 2 +- .../storage/mongodb/TestMongoDBDocumentSerializer.java | 3 +-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java index 76efeea08e..13cb83e071 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java @@ -23,6 +23,8 @@ import org.apache.tajo.storage.Tuple; import org.bson.Document; +import java.io.IOException; + /** * Writes tuples into a mongodb collection */ @@ -30,11 +32,13 @@ public class MongoDBCollectionWriter { private ConnectionInfo connectionInfo; MongoClient mongoClient; MongoCollection collection; + MongoDBDocumentSerializer mongoDBDocumentSerializer; public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer deserializer) { this.connectionInfo = connectionInfo; + this.mongoDBDocumentSerializer = deserializer; // this.deserializer = deserializer; // this.targetLength = targetLength; } @@ -47,9 +51,10 @@ public void init() collection = db.getCollection(connectionInfo.getTableName()); } - public void writeTuple(Tuple tuple) - { + public void writeTuple(Tuple tuple) throws IOException { + Document dc = new Document(); + mongoDBDocumentSerializer.serialize(tuple,dc); } public void close() diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java index 8b5c920651..a0524b0f61 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java @@ -39,7 +39,7 @@ public MongoDBDocumentSerializer(Schema schema, TableMeta meta) { } - public void deserialize(Tuple inputTuple,Document outputDoc) throws IOException + public void serialize(Tuple inputTuple, Document outputDoc) throws IOException { for(int i=0; i< inputTuple.size();i++) { diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java index 371d1d51ae..dcbec9ce70 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java @@ -17,7 +17,6 @@ */ package org.apache.tajo.storage.mongodb; -import com.sun.org.apache.xml.internal.security.encryption.DocumentSerializer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.Column; @@ -53,7 +52,7 @@ public void testSeriaalizeTextType() throws IOException { tuple.put(2,DatumFactory.createText("Chathuranga")); Document md = new Document(); - dc.deserialize(tuple,md); + dc.serialize(tuple,md); LOG.info(md.toJson()); From 71e3a645b674c1afae622af473be6d81c81bbcce Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 5 Aug 2016 16:54:25 +0530 Subject: [PATCH 64/94] Add to collection --- .../tajo/storage/mongodb/MongoDBCollectionWriter.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java index 13cb83e071..f6320ef59b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java @@ -35,26 +35,23 @@ public class MongoDBCollectionWriter { MongoDBDocumentSerializer mongoDBDocumentSerializer; - public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer deserializer) + public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer serializer) { this.connectionInfo = connectionInfo; - this.mongoDBDocumentSerializer = deserializer; -// this.deserializer = deserializer; -// this.targetLength = targetLength; + this.mongoDBDocumentSerializer = serializer; } public void init() { mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); - collection = db.getCollection(connectionInfo.getTableName()); } public void writeTuple(Tuple tuple) throws IOException { - Document dc = new Document(); mongoDBDocumentSerializer.serialize(tuple,dc); + collection.insertOne(dc); } public void close() From 1e815a3a14e6c7bd8d98d7573c98faeb867e5f52 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Tue, 9 Aug 2016 10:07:22 +0530 Subject: [PATCH 65/94] to make a test case --- .../storage/mongodb/TestMongoDBDocumentSerializer.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java index dcbec9ce70..04045c820d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java @@ -31,6 +31,8 @@ import java.io.IOException; +import static org.junit.Assert.assertEquals; + public class TestMongoDBDocumentSerializer { @@ -44,7 +46,7 @@ public void testSeriaalizeTextType() throws IOException { .add(new Column("last_name", TajoDataTypes.Type.TEXT)) .build(); - MongoDBDocumentSerializer dc = new MongoDBDocumentSerializer(schem,null); + MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem,null); Tuple tuple = new VTuple(3); tuple.put(0,DatumFactory.createText("Good_Man")); @@ -52,9 +54,11 @@ public void testSeriaalizeTextType() throws IOException { tuple.put(2,DatumFactory.createText("Chathuranga")); Document md = new Document(); - dc.serialize(tuple,md); + documentSerializer.serialize(tuple,md); - LOG.info(md.toJson()); + assertEquals("Good_Man",md.getString("title")); + assertEquals("Janaka",md.getString("first_name")); + assertEquals("Chathuranga",md.getString("last_name")); } From 2ad84f587f923a3a141e2ccd96b9c9c78f9f2de2 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 10 Aug 2016 12:12:15 +0530 Subject: [PATCH 66/94] Mongo Appender added to storage --- .../src/main/resources/storage-default.xml | 5 ++ .../src/test/resources/storage-default.xml | 5 ++ .../tajo/storage/mongodb/MongoDBAppender.java | 2 + .../storage/mongodb/TestMongoDBAppender.java | 77 +++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index 42a66afc7e..ed4c5feaca 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -212,6 +212,11 @@ org.apache.tajo.storage.hbase.HFileAppender + + tajo.storage.appender-handler.mongodb.class + org.apache.tajo.storage.mongodb.MongoDBAppender + + tajo.storage.text.io.read-buffer.bytes diff --git a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml index 1737e221c7..70f226f1e6 100644 --- a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml @@ -197,6 +197,11 @@ org.apache.tajo.storage.hbase.HFileAppender + + tajo.storage.appender-handler.mongodb.class + org.apache.tajo.storage.mongodb.MongoDBAppender + + tajo.storage.text.io.read-buffer.bytes diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index 93f9d3b938..b8673e698c 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -52,6 +52,8 @@ public class MongoDBAppender implements Appender { private MongoDBCollectionWriter mongoDBCollectionWriter; + + public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schema, TableMeta meta, Path stagingDir, URI uri) { this.conf = conf; diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java new file mode 100644 index 0000000000..a7d0b5d864 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.storage.mongodb; + +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SchemaBuilder; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.storage.*; +import org.bson.Document; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; + +public class TestMongoDBAppender { + + + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + static URI uri = server.getURI(); + Appender appender; + + @Before + public void setup() throws Exception + { + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); + + + //Create Schema manually + Schema schem = SchemaBuilder.builder(). + add(new Column("title", TajoDataTypes.Type.TEXT)) + .add(new Column("first_name", TajoDataTypes.Type.TEXT)) + .add(new Column("last_name", TajoDataTypes.Type.TEXT)) + .build(); + + + TableMeta meta = space.getMetadataProvider().getTableDesc(null,"got").getMeta(); + appender = space.getAppender(null,null,meta,schem,null); + + //appender.init(); + + } + + @Test + public void test1() throws IOException { + + Tuple tuple = new VTuple(3); + tuple.put(0, DatumFactory.createText("Good_Man")); + tuple.put(1,DatumFactory.createText("Janaka")); + tuple.put(2,DatumFactory.createText("Chathuranga")); + + //appender.addTuple(tuple); + } + + + +} From 5ab8fea9da4bdb795c9b910405da8aa8a772aa13 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 10 Aug 2016 12:13:06 +0530 Subject: [PATCH 67/94] Removed an unnecessary line --- .../java/org/apache/tajo/storage/mongodb/MongoDBScanner.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index 7383cbc681..39a21f387a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -45,8 +45,6 @@ public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragmen @Override public void init() throws IOException { - - if (targets == null) { targets = schema.toArray(); } From 06e7553311de2a7831cc2a524360882cd81ff1ed Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 10 Aug 2016 12:13:37 +0530 Subject: [PATCH 68/94] Fixed teardown issue --- .../org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 5fab473d8f..d6fc3d31d9 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -41,7 +41,7 @@ public static void setup() throws Exception } @AfterClass - public static void teardown() throws Exception { + public static void tearDownClass() throws Exception { server.stop(); } From 380c5d0158f48db9ce827525e3f5903823901c60 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 10 Aug 2016 12:14:11 +0530 Subject: [PATCH 69/94] Optimized import and Metadata enabled --- .../storage/mongodb/MongoDBTableSpace.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 76ebf78658..b2cf4fabb7 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -23,30 +23,28 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import net.minidev.json.JSONObject; -import org.apache.avro.generic.GenericData; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; +import org.apache.tajo.TaskAttemptId; import org.apache.tajo.catalog.*; -import org.apache.tajo.exception.*; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.schema.IdentifierUtil; -import org.apache.tajo.storage.FormatProperty; -import org.apache.tajo.storage.StorageProperty; -import org.apache.tajo.storage.Tablespace; -import org.apache.tajo.storage.TupleRange; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.Fragment; import org.bson.Document; import javax.annotation.Nullable; import java.io.IOException; import java.net.URI; -import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -64,7 +62,7 @@ public class MongoDBTableSpace extends Tablespace { false, //not movable true, //writable at the moment true, // Absolute path - false); // Meta data will be provided + true); // Meta data will be provided static final FormatProperty FORMAT_PROPERTY = new FormatProperty( false, // Insert false, //direct insert @@ -237,4 +235,14 @@ public MetadataProvider getMetadataProvider() { public ConnectionInfo getConnectionInfo() { return connectionInfo; } + + + //ToDo Make Sure this is not an issue + @Override + public Appender getAppender(OverridableConf queryContext, + TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) + + { + return new MongoDBAppender(null, taskAttemptId,schema,meta,workDir,getUri()); + } } From 2ecb0c9d87aa8dfa8443a92e2b4ba5eacfcc4b17 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 10 Aug 2016 14:36:47 +0530 Subject: [PATCH 70/94] Fixed purge issue --- .../java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java | 2 +- .../org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index b2cf4fabb7..e2831d2763 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -174,7 +174,7 @@ public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoExc @Override public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { if(STORAGE_PROPERTY.isMetadataProvided()) - db.getCollection(tableDesc.getName()).drop(); + db.getCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())).drop(); } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 8e09865fbe..c665478118 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -98,7 +98,7 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { //Purg the table space.purgeTable(tableDesc); final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); - assertFalse(found_after.contains(IdentifierUtil.buildFQName(server.MAPPEDDBNAME, "Table1"))); + assertFalse(found_after.contains( "Table1")); } } From 42f8d7c4b561d10813148470203387e3c6a58f23 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Wed, 10 Aug 2016 14:52:35 +0530 Subject: [PATCH 71/94] Fixed some storage configurations --- .../src/main/resources/storage-default.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index ed4c5feaca..fd5d0a6c41 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -55,6 +55,10 @@ tajo.storage.fragment.kind.jdbc org.apache.tajo.storage.jdbc.JdbcFragment + + tajo.storage.fragment.kind.mongodb + org.apache.tajo.storage.mongodb.MongoDBFragment + tajo.storage.fragment.kind.example-http org.apache.tajo.storage.http.ExampleHttpFileFragment @@ -78,8 +82,6 @@ tajo.storage.fragment.serde.example-http org.apache.tajo.storage.http.ExampleHttpFileFragmentSerde - tajo.storage.fragment.serde.example-http - org.apache.tajo.storage.http.ExampleHttpFileFragmentSerde From 2a22f8b366510744e82c9e6f6f8289f5d9a6184d Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 12 Aug 2016 09:52:58 +0530 Subject: [PATCH 72/94] Enabled using config keys --- .../apache/tajo/storage/mongodb/ConnectionInfo.java | 13 ++++++------- .../tajo/storage/mongodb/MongoDBTableSpace.java | 4 +--- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index c113f651c9..6b8e899668 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -89,18 +89,17 @@ public static ConnectionInfo fromURI(URI originalUri) { } } - if (params.containsKey("table")) { - connInfo.tableName = params.remove("table"); + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_TABLE)) { + connInfo.tableName = params.remove(MongoDBTableSpace.CONFIG_KEY_TABLE); } - if (params.containsKey("user")) { - connInfo.user = params.remove("user"); + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_USERNAME)) { + connInfo.user = params.remove(MongoDBTableSpace.CONFIG_KEY_USERNAME); } - if (params.containsKey("password")) { - connInfo.password = params.remove("password"); + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_PASSWORD)) { + connInfo.password = params.remove(MongoDBTableSpace.CONFIG_KEY_PASSWORD); } - connInfo.params = params; String mongoDbURIStr = ""; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index e2831d2763..0191d880ea 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -80,6 +80,7 @@ public class MongoDBTableSpace extends Tablespace { public static final String CONFIG_KEY_CONN_PROPERTIES = "connection_properties"; public static final String CONFIG_KEY_USERNAME = "user"; public static final String CONFIG_KEY_PASSWORD = "password"; + public static final String CONFIG_KEY_TABLE = "table"; public MongoDBTableSpace(String name, URI uri, JSONObject config) { @@ -92,9 +93,6 @@ public MongoDBTableSpace(String name, URI uri, JSONObject config) { } else { mappedDBName = getConnectionInfo().getDbName(); } - - - } @Override From a662f9011c366b6680da6fa40cbf5bf854c65a2c Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 12 Aug 2016 11:19:39 +0530 Subject: [PATCH 73/94] Improved the document --- tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst index 8392881b52..ebb70d6a63 100644 --- a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst +++ b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst @@ -1,11 +1,11 @@ -************************************* -MongoDB Storage Handler -************************************* +******************* +MongoDB Integration +******************* Overview ======== -MongoDB storage handler is available by default in Tajo. +Apache Tajo™ storage modules contains a storage plugin for MongoDB which allows Apache Tajo to access databases and collections in MongoDB. In order to use MongoDB storage plugin user needs to follow following steps. Configuration ============= @@ -29,4 +29,5 @@ MongoDB storage handler configuration will be like this.. First you have to regi } } } - } \ No newline at end of file + } + From d0811f69ff9148d1ca6489dadc787ed0d0031081 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 12 Aug 2016 11:38:58 +0530 Subject: [PATCH 74/94] Changed driver version to 3.3 --- tajo-storage/tajo-storage-mongodb/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 93e656bdbf..1d4fba89fc 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -340,10 +340,11 @@ + org.mongodb mongodb-driver - 3.2.2 + 3.3.0 From 9a1b2fdf4af7c4305fe67c01c97d8a1ccd3ec152 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 12 Aug 2016 12:36:57 +0530 Subject: [PATCH 75/94] Improved the document --- .../main/sphinx/storage_plugins/mongodb.rst | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst index ebb70d6a63..05dea03049 100644 --- a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst +++ b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst @@ -7,10 +7,26 @@ Overview Apache Tajo™ storage modules contains a storage plugin for MongoDB which allows Apache Tajo to access databases and collections in MongoDB. In order to use MongoDB storage plugin user needs to follow following steps. -Configuration -============= +* Download Mongo Java Driver 3.2.2 and add to the class path. +* Setup a running MongoDB server in a localhost or another. +* Configure the cluster using storage-site.json -MongoDB storage handler configuration will be like this.. First you have to register the StorageHandler. Then register the space. Update the storage-site.json as following. +Download MongoDB Java Driver +========================== + +You can download latest version of MongoDB java driver from https://mongodb.github.io/mongo-java-driver/ +Download the .jar file and place it in the class path. You can place the file in TAJO_HOME or you can place the .jar file somewhere else and configure the CLASS_PATH variable from conf/tajo-env.sh (or tajo-env.cmd). + +Setup a running Mongo Server +============================ +In order to proceed you need to have a running mongo server instance. If you are new to MongoDB just follow the Getting Started Guide(https://docs.mongodb.com/getting-started/shell/). After setting up the server, find the host and port to connect to the particular server. If it is a local installation by default, the port is 27017. + + +Configure the cluster +===================== + +MongoDB storage handler configuration will be like this. +First you have to register the StorageHandler.Then register the space. Update the storage-site.json as following. .. code-block:: json From 0b8ad14cb495c92b194fd9bb84b2c3a2611ae424 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 12 Aug 2016 12:37:43 +0530 Subject: [PATCH 76/94] Update mongodb.rst --- tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst index 05dea03049..642b225f8c 100644 --- a/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst +++ b/tajo-docs/src/main/sphinx/storage_plugins/mongodb.rst @@ -8,7 +8,7 @@ Overview Apache Tajo™ storage modules contains a storage plugin for MongoDB which allows Apache Tajo to access databases and collections in MongoDB. In order to use MongoDB storage plugin user needs to follow following steps. * Download Mongo Java Driver 3.2.2 and add to the class path. -* Setup a running MongoDB server in a localhost or another. +* Setup a MongoDB server in a localhost or another. * Configure the cluster using storage-site.json Download MongoDB Java Driver @@ -17,8 +17,8 @@ Download MongoDB Java Driver You can download latest version of MongoDB java driver from https://mongodb.github.io/mongo-java-driver/ Download the .jar file and place it in the class path. You can place the file in TAJO_HOME or you can place the .jar file somewhere else and configure the CLASS_PATH variable from conf/tajo-env.sh (or tajo-env.cmd). -Setup a running Mongo Server -============================ +Setup a Mongo Server +==================== In order to proceed you need to have a running mongo server instance. If you are new to MongoDB just follow the Getting Started Guide(https://docs.mongodb.com/getting-started/shell/). After setting up the server, find the host and port to connect to the particular server. If it is a local installation by default, the port is 27017. From 2ed2e36c35fa9e91694e3bf3651e24f5bcdfa8cc Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 12 Aug 2016 17:18:27 +0530 Subject: [PATCH 77/94] Improving appender --- .../org/apache/tajo/storage/mongodb/MongoDBAppender.java | 3 ++- .../tajo/storage/mongodb/MongoDBMetadataProvider.java | 9 +++++---- .../apache/tajo/storage/mongodb/MongoDBTableSpace.java | 8 ++++---- .../apache/tajo/storage/mongodb/TestMongoDBAppender.java | 3 ++- .../tajo/storage/mongodb/TestMongoDBQueryTest.java | 2 +- .../queries/TestMongoDBQueryTest/testInsert.sql | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index b8673e698c..98e282e146 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -113,6 +113,7 @@ public void enableStats(List columnList) { @Override public TableStats getStats() { - return null; + TableStats stats = new TableStats(); + return stats; } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index 57d50f1bcf..c4563b8251 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -21,11 +21,9 @@ import com.mongodb.MongoClient; import com.mongodb.client.MongoDatabase; import com.mongodb.client.MongoIterable; -import org.apache.tajo.catalog.MetadataProvider; -import org.apache.tajo.catalog.SchemaBuilder; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.UndefinedTablespaceException; import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.KeyValueSet; @@ -99,6 +97,9 @@ public TableDesc getTableDesc(String schemaName, String tableName) throws Undefi TableDesc tbDesc = new TableDesc( IdentifierUtil.buildFQName(mappedDbName, tableName), SchemaBuilder.builder() +// .add(new Column("title", TajoDataTypes.Type.TEXT)) +// .add(new Column("first_name", TajoDataTypes.Type.TEXT)) +// .add(new Column("last_name", TajoDataTypes.Type.TEXT)) .build(), tbMeta, tableSpace.getTableUri(null,null, tableName)); diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 0191d880ea..4ed0979f9d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -64,9 +64,9 @@ public class MongoDBTableSpace extends Tablespace { true, // Absolute path true); // Meta data will be provided static final FormatProperty FORMAT_PROPERTY = new FormatProperty( - false, // Insert - false, //direct insert - false);// result staging + true, // Insert + true, //direct insert + true);// result staging //Mongo Client object private ConnectionInfo connectionInfo; @@ -241,6 +241,6 @@ public Appender getAppender(OverridableConf queryContext, TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) { - return new MongoDBAppender(null, taskAttemptId,schema,meta,workDir,getUri()); + return new MongoDBAppender(null, taskAttemptId,schema,meta,workDir,workDir.toUri()); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java index a7d0b5d864..03431abbbb 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java @@ -17,6 +17,7 @@ */ package org.apache.tajo.storage.mongodb; +import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; @@ -55,7 +56,7 @@ public void setup() throws Exception TableMeta meta = space.getMetadataProvider().getTableDesc(null,"got").getMeta(); - appender = space.getAppender(null,null,meta,schem,null); + appender = space.getAppender(null,null,meta,schem,new Path(server.getURI())); //appender.init(); diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index d6fc3d31d9..49cce9d560 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -48,7 +48,7 @@ public static void tearDownClass() throws Exception { @Before public void prepareTables() throws TajoException { if(!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { - executeString("create table got (*) tablespace test_spacename using mongodb"); + executeString("create table got (title,first_name,last_name) tablespace test_spacename using mongodb"); executeString("create table github (*) tablespace test_spacename using mongodb"); // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql index 6bca9fd267..b131f0b4c5 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql @@ -1 +1 @@ -INSERT INTO got SELECT 'a','b','c'; \ No newline at end of file +INSERT INTO got SELECT 'a','b'; \ No newline at end of file From 8148edc7080468c3d53bb7936c2f205526fc1b50 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Tue, 16 Aug 2016 09:51:05 +0530 Subject: [PATCH 78/94] Delete after insert test --- .../tajo/storage/mongodb/MongoDBAppender.java | 4 +- .../mongodb/MongoDBDocumentSerializer.java | 54 +++++++++++++++++-- .../storage/mongodb/MongoDBTableSpace.java | 6 +-- .../storage/mongodb/TestMongoDBAppender.java | 24 +++++++-- .../TestMongoDBDocumentSerializer.java | 3 -- .../storage/mongodb/TestMongoDBQueryTest.java | 2 +- .../TestMongoDBQueryTest/testInsert.sql | 2 +- 7 files changed, 77 insertions(+), 18 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index 98e282e146..92d0735276 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -61,7 +61,7 @@ public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, this.meta = meta; this.stagingDir = stagingDir; this.taskAttemptId = taskAttemptId; - this.uri = uri; + this.uri = stagingDir.toUri(); } @Override @@ -71,7 +71,7 @@ public void init() throws IOException { } inited = true; MongoDBDocumentSerializer md = new MongoDBDocumentSerializer(schema,meta); - mongoDBCollectionWriter = new MongoDBCollectionWriter(ConnectionInfo.fromURI(uri),md); + mongoDBCollectionWriter = new MongoDBCollectionWriter(ConnectionInfo.fromURI(stagingDir.toString()),md); mongoDBCollectionWriter.init(); } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java index a0524b0f61..1389f3b662 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java @@ -17,13 +17,14 @@ */ package org.apache.tajo.storage.mongodb; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; +import com.google.common.collect.Lists; +import org.apache.tajo.catalog.*; import org.apache.tajo.storage.Tuple; import org.bson.Document; +import org.apache.tajo.common.TajoDataTypes.Type; import java.io.IOException; +import java.util.Map; /** * Serialize tuples into Mongo Documents @@ -32,10 +33,54 @@ public class MongoDBDocumentSerializer { private final Schema schema; private final TableMeta meta; + String[] paths; + + + private final Map types; public MongoDBDocumentSerializer(Schema schema, TableMeta meta) { this.schema = schema; this.meta = meta; + + paths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(schema.getAllColumns()), true); + types = SchemaUtil.buildTypeMap(schema.getAllColumns(), paths); + } + + public void setValue(Tuple inputTuple, String fullPath, String[] paths, int depth, int index, Document outputDoc) + { + + + String simpleColumnName = paths[depth]; + switch (types.get(fullPath)) + { + case TEXT: + outputDoc.put(simpleColumnName,inputTuple.getText(index)); + break; + + case INT1: + case UINT1: + case INT2: + case UINT2: + outputDoc.put(simpleColumnName,inputTuple.getInt2(index)); + break; + + case INT4: + case UINT4: + outputDoc.put(simpleColumnName,inputTuple.getInt4(index)); + break; + + case INT8: + case UINT8: + outputDoc.put(simpleColumnName,inputTuple.getInt8(index)); + break; + + case FLOAT4: + outputDoc.put(simpleColumnName,inputTuple.getFloat4(index)); + break; + + case FLOAT8: + outputDoc.put(simpleColumnName,inputTuple.getFloat8(index)); + } } @@ -43,7 +88,8 @@ public void serialize(Tuple inputTuple, Document outputDoc) throws IOException { for(int i=0; i< inputTuple.size();i++) { - outputDoc.put(schema.getColumn(i).getSimpleName(),inputTuple.getText(i)); + String [] newPaths = paths[i].split(NestedPathUtil.PATH_DELIMITER); + setValue(inputTuple,newPaths[0],newPaths,0,i,outputDoc); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 4ed0979f9d..8d0cd828da 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -205,9 +205,9 @@ public URI getTableUri(TableMeta meta, String databaseName, String tableName) { //ToDo Find a better way this String tableURI = ""; if(this.getUri().toASCIIString().contains("?")) - tableURI = this.getUri().toASCIIString()+"&table="+tableName; + tableURI = this.getUri().toASCIIString()+"&"+CONFIG_KEY_TABLE+"="+tableName; else - tableURI = this.getUri().toASCIIString()+"?table="+tableName; + tableURI = this.getUri().toASCIIString()+"?"+CONFIG_KEY_TABLE+"="+tableName; return URI.create(tableURI); } @@ -215,7 +215,7 @@ public URI getTableUri(TableMeta meta, String databaseName, String tableName) { //@Override public URI getTableUri( String databaseName, String tableName) { //ToDo set the TableURI properly - return URI.create(this.getUri()+"&table="+tableName); + return URI.create(this.getUri()+"&"+CONFIG_KEY_TABLE+"="+tableName); } // @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java index 03431abbbb..ab2e785bcf 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java @@ -17,6 +17,7 @@ */ package org.apache.tajo.storage.mongodb; +import com.mongodb.client.MongoIterable; import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.Column; @@ -34,6 +35,8 @@ import java.io.IOException; import java.net.URI; +import static org.junit.Assert.assertEquals; + public class TestMongoDBAppender { @@ -56,21 +59,34 @@ public void setup() throws Exception TableMeta meta = space.getMetadataProvider().getTableDesc(null,"got").getMeta(); - appender = space.getAppender(null,null,meta,schem,new Path(server.getURI())); + appender = space.getAppender(null,null,meta,schem,new Path(server.getURI()+"?"+MongoDBTableSpace.CONFIG_KEY_TABLE+"=got") ); - //appender.init(); + appender.init(); } @Test - public void test1() throws IOException { + public void testAddTupleText() throws IOException { + //Create a tuple and add to the table Tuple tuple = new VTuple(3); tuple.put(0, DatumFactory.createText("Good_Man")); tuple.put(1,DatumFactory.createText("Janaka")); tuple.put(2,DatumFactory.createText("Chathuranga")); + appender.addTuple(tuple); + + //Take data from server + MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title","Good_Man")); + Document doc = result.first(); + + //Validate + assertEquals(doc.get("title"),"Good_Man"); + assertEquals(doc.get("first_name"),"Janaka"); + assertEquals(doc.get("last_name"),"Chathuranga"); + - //appender.addTuple(tuple); + //Remove the inserted doc from database + server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").deleteOne(doc); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java index 04045c820d..09db65780a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java @@ -35,9 +35,6 @@ public class TestMongoDBDocumentSerializer { - - private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - @Test public void testSeriaalizeTextType() throws IOException { Schema schem = SchemaBuilder.builder(). diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 49cce9d560..84037036f9 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -92,6 +92,6 @@ public void testJoin() throws Exception { @SimpleTest @Test public void testInsert() throws Exception { -// runSimpleTests(); + // runSimpleTests(); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql index b131f0b4c5..1bebd5008e 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testInsert.sql @@ -1 +1 @@ -INSERT INTO got SELECT 'a','b'; \ No newline at end of file +INSERT INTO got(first_name,last_name) SELECT 'a','b'; \ No newline at end of file From b19fb73bf9c4a39edfb8348f533477060d8be5d8 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Tue, 16 Aug 2016 12:05:32 +0530 Subject: [PATCH 79/94] Added more tests for serializer --- .../mongodb/MongoDBDocumentSerializer.java | 17 +++++++++--- .../TestMongoDBDocumentSerializer.java | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java index 1389f3b662..9669265649 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java @@ -53,10 +53,6 @@ public void setValue(Tuple inputTuple, String fullPath, String[] paths, int dept String simpleColumnName = paths[depth]; switch (types.get(fullPath)) { - case TEXT: - outputDoc.put(simpleColumnName,inputTuple.getText(index)); - break; - case INT1: case UINT1: case INT2: @@ -80,6 +76,19 @@ public void setValue(Tuple inputTuple, String fullPath, String[] paths, int dept case FLOAT8: outputDoc.put(simpleColumnName,inputTuple.getFloat8(index)); + break; + + case CHAR: + outputDoc.put(simpleColumnName,inputTuple.getChar(index)); + break; + + case BOOLEAN: + outputDoc.put(simpleColumnName,inputTuple.getBool(index)); + break; + + default: + outputDoc.put(simpleColumnName,inputTuple.getText(index)); + break; } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java index 09db65780a..7063bab527 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java @@ -59,4 +59,31 @@ public void testSeriaalizeTextType() throws IOException { } + @Test + public void testSerializeIntFloatBoolean() throws IOException { + Schema schem = SchemaBuilder.builder(). + add(new Column("title", TajoDataTypes.Type.TEXT)) + .add(new Column("age", TajoDataTypes.Type.INT4)) + .add(new Column("height", TajoDataTypes.Type.FLOAT8)) + .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) + .build(); + + MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem,null); + + Tuple tuple = new VTuple(4); + tuple.put(0,DatumFactory.createText("Mr")); + tuple.put(1,DatumFactory.createInt4(24)); + tuple.put(2,DatumFactory.createFloat8(165.98)); + tuple.put(3,DatumFactory.createBool(true)); + Document md = new Document(); + + documentSerializer.serialize(tuple,md); + + assertEquals("Mr",md.get("title")); + assertEquals(24,md.get("age")); + assertEquals(165.98,md.get("height")); + assertEquals(true,md.get("single")); + + } + } From 04a789f4211d78aa7ab730e3b416b828b54e0fbf Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Tue, 16 Aug 2016 14:24:55 +0530 Subject: [PATCH 80/94] Improved appender test to test float, bool, int --- .../storage/mongodb/TestMongoDBAppender.java | 29 ++++++++++++------- .../TestMongoDBDocumentSerializer.java | 12 ++++---- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java index ab2e785bcf..fb5e4e26ba 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java @@ -55,6 +55,9 @@ public void setup() throws Exception add(new Column("title", TajoDataTypes.Type.TEXT)) .add(new Column("first_name", TajoDataTypes.Type.TEXT)) .add(new Column("last_name", TajoDataTypes.Type.TEXT)) + .add(new Column("age", TajoDataTypes.Type.INT4)) + .add(new Column("height", TajoDataTypes.Type.FLOAT8)) + .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) .build(); @@ -66,29 +69,33 @@ public void setup() throws Exception } @Test - public void testAddTupleText() throws IOException { + public void testAddTuple() throws IOException { //Create a tuple and add to the table - Tuple tuple = new VTuple(3); - tuple.put(0, DatumFactory.createText("Good_Man")); - tuple.put(1,DatumFactory.createText("Janaka")); - tuple.put(2,DatumFactory.createText("Chathuranga")); + Tuple tuple = new VTuple(6); + tuple.put(0,DatumFactory.createText("Kingslayer")); + tuple.put(1,DatumFactory.createText("Jaime")); + tuple.put(2,DatumFactory.createText("Lannister")); + tuple.put(3,DatumFactory.createInt4(24)); + tuple.put(4,DatumFactory.createFloat8(165.98)); + tuple.put(5,DatumFactory.createBool(true)); appender.addTuple(tuple); //Take data from server - MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title","Good_Man")); + MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title","Kingslayer")); Document doc = result.first(); //Validate - assertEquals(doc.get("title"),"Good_Man"); - assertEquals(doc.get("first_name"),"Janaka"); - assertEquals(doc.get("last_name"),"Chathuranga"); + assertEquals(doc.get("title"),"Kingslayer"); + assertEquals(doc.get("first_name"),"Jaime"); + assertEquals(doc.get("last_name"),"Lannister"); + assertEquals(doc.get("age"),24); + assertEquals(doc.get("height"),165.98); + assertEquals(doc.get("single"),true); //Remove the inserted doc from database server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").deleteOne(doc); } - - } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java index 7063bab527..d92d896a2b 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java @@ -46,16 +46,16 @@ public void testSeriaalizeTextType() throws IOException { MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem,null); Tuple tuple = new VTuple(3); - tuple.put(0,DatumFactory.createText("Good_Man")); - tuple.put(1,DatumFactory.createText("Janaka")); - tuple.put(2,DatumFactory.createText("Chathuranga")); + tuple.put(0,DatumFactory.createText("Kingslayer")); + tuple.put(1,DatumFactory.createText("Jaime")); + tuple.put(2,DatumFactory.createText("Lannister")); Document md = new Document(); documentSerializer.serialize(tuple,md); - assertEquals("Good_Man",md.getString("title")); - assertEquals("Janaka",md.getString("first_name")); - assertEquals("Chathuranga",md.getString("last_name")); + assertEquals("Kingslayer",md.getString("title")); + assertEquals("Jaime",md.getString("first_name")); + assertEquals("Lannister",md.getString("last_name")); } From 894f7fa937b7de844f7bf7a70ef77e5262ea1def Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Tue, 16 Aug 2016 16:23:24 +0530 Subject: [PATCH 81/94] Enabled flush and writing all tuples together --- .../tajo/storage/mongodb/MongoDBAppender.java | 5 +++-- .../storage/mongodb/MongoDBCollectionWriter.java | 13 +++++++++++-- .../tajo/storage/mongodb/TestMongoDBAppender.java | 2 ++ .../tajo/storage/mongodb/TestMongoDBQueryTest.java | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index 92d0735276..d8becb15dc 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -54,6 +54,7 @@ public class MongoDBAppender implements Appender { + public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schema, TableMeta meta, Path stagingDir, URI uri) { this.conf = conf; @@ -77,12 +78,12 @@ public void init() throws IOException { @Override public void addTuple(Tuple t) throws IOException { - mongoDBCollectionWriter.writeTuple(t); + mongoDBCollectionWriter.addTuple(t); } @Override public void flush() throws IOException { - + mongoDBCollectionWriter.write(); } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java index f6320ef59b..1c2fa36c8d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java @@ -24,6 +24,8 @@ import org.bson.Document; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; /** * Writes tuples into a mongodb collection @@ -35,6 +37,8 @@ public class MongoDBCollectionWriter { MongoDBDocumentSerializer mongoDBDocumentSerializer; + private List documentList = new ArrayList<>(); + public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer serializer) { this.connectionInfo = connectionInfo; @@ -48,10 +52,15 @@ public void init() collection = db.getCollection(connectionInfo.getTableName()); } - public void writeTuple(Tuple tuple) throws IOException { + public void addTuple(Tuple tuple) throws IOException { Document dc = new Document(); mongoDBDocumentSerializer.serialize(tuple,dc); - collection.insertOne(dc); + documentList.add(dc); + } + + public void write() + { + collection.insertMany(documentList); } public void close() diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java index fb5e4e26ba..5070b19355 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java @@ -81,6 +81,8 @@ public void testAddTuple() throws IOException { tuple.put(5,DatumFactory.createBool(true)); appender.addTuple(tuple); + appender.flush(); + //Take data from server MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title","Kingslayer")); Document doc = result.first(); diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 84037036f9..9cc52ade91 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -92,6 +92,6 @@ public void testJoin() throws Exception { @SimpleTest @Test public void testInsert() throws Exception { - // runSimpleTests(); +// runSimpleTests(); } } From 873a6cbac60e90dfa9f89c195defc3cfd663b63a Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Tue, 16 Aug 2016 16:59:11 +0530 Subject: [PATCH 82/94] Removed File Scanner --- .../tajo/storage/mongodb/MongoDBFragment.java | 3 +- .../tajo/storage/mongodb/MongoDBScanner.java | 53 ++++++++++++++++--- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java index e24936d258..d3862f0072 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -17,6 +17,7 @@ */ package org.apache.tajo.storage.mongodb; +import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.storage.fragment.Fragment; @@ -30,7 +31,7 @@ This will be serialized and passed to the cluster. */ -public class MongoDBFragment extends AbstractFileFragment { +public class MongoDBFragment extends Fragment { protected MongoDBFragment(URI uri, diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index 39a21f387a..acb5f0ea68 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -18,11 +18,18 @@ package org.apache.tajo.storage.mongodb; import org.apache.hadoop.conf.Configuration; +import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.storage.FileScanner; +import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.text.TextLineParsingError; @@ -34,13 +41,22 @@ * */ //Todo Remove FileScanner -public class MongoDBScanner extends FileScanner { +public class MongoDBScanner implements Scanner{ - MongoDBCollectionReader collectionReader; + private MongoDBCollectionReader collectionReader; - public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { - super(conf, schema, meta, fragment); + private final TableMeta meta; + private final Schema schema; + private final MongoDBFragment fragment; + + private Column [] targets; + + private boolean inited; + public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { + this.schema = schema; + this.meta = meta; + this.fragment = (MongoDBFragment) fragment; } @Override @@ -50,8 +66,6 @@ public void init() throws IOException { } reset(); - - super.init(); } @Override @@ -78,11 +92,24 @@ public void close() throws IOException { return; } + @Override + public void pushOperators(LogicalNode planPart) { + throw new TajoRuntimeException(new UnsupportedException()); + } + @Override public boolean isProjectable() { return false; } + @Override + public void setTarget(Column[] targets) { + if (inited) { + throw new IllegalStateException("Should be called before init()"); + } + this.targets = targets; + } + @Override public boolean isSelectable() { return false; @@ -93,6 +120,11 @@ public void setFilter(EvalNode filter) { } + @Override + public void setLimit(long num) { + + } + @Override public boolean isSplittable() { return false; @@ -103,5 +135,14 @@ public float getProgress() { return collectionReader.getProgress(); } + @Override + public TableStats getInputStats() { + return null; + } + + @Override + public Schema getSchema() { + return schema; + } } From 1159a2e078ffcbdee4f5f9da441e0d470e3c5606 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Tue, 16 Aug 2016 17:13:22 +0530 Subject: [PATCH 83/94] Removed hbase dependancy: --- tajo-storage/tajo-storage-mongodb/pom.xml | 24 ----------------------- 1 file changed, 24 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 1d4fba89fc..b96a429c19 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -316,30 +316,6 @@ junit test - - org.apache.hbase - hbase-server - ${hbase.version} - provided - - - netty-all - io.netty - - - - - org.apache.hbase - hbase-client - ${hbase.version} - provided - - - jdk.tools - jdk.tools - - - org.mongodb From c5b96c9387c6e07c9504ea9e82491f1e732e918f Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Thu, 18 Aug 2016 10:23:15 +0530 Subject: [PATCH 84/94] Optimized .pom file --- tajo-storage/tajo-storage-mongodb/pom.xml | 149 ++++++---------------- 1 file changed, 40 insertions(+), 109 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index b96a429c19..25c50a8451 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -34,13 +34,14 @@ UTF-8 - - - org.apache.maven.plugins - maven-compiler-plugin - + + + + + + org.apache.rat apache-rat-plugin @@ -59,6 +60,7 @@ + org.apache.maven.plugins maven-surefire-plugin @@ -69,18 +71,20 @@ -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 - - org.apache.maven.plugins - maven-jar-plugin - 2.4 - - - - test-jar - - - - + + + + + + + + + + + + + + org.apache.maven.plugins maven-antrun-plugin @@ -99,6 +103,7 @@ + org.codehaus.mojo exec-maven-plugin @@ -123,6 +128,7 @@ + org.codehaus.mojo build-helper-maven-plugin @@ -193,31 +199,26 @@ tajo-storage-hdfs provided + + org.apache.tajo + tajo-rpc-common + org.apache.tajo tajo-cluster-tests test-jar test + + junit + junit + test + org.apache.hadoop hadoop-common provided - - - zookeeper - org.apache.zookeeper - - - slf4j-api - org.slf4j - - - jersey-json - com.sun.jersey - - org.apache.hadoop @@ -276,54 +277,23 @@ jersey-test-framework-grizzly2 - hadoop-yarn-server-tests - org.apache.hadoop - - - hadoop-mapreduce-client-jobclient - org.apache.hadoop - - - hadoop-mapreduce-client-app - org.apache.hadoop - - - hadoop-yarn-api - org.apache.hadoop - - - hadoop-mapreduce-client-hs - org.apache.hadoop - - - hadoop-mapreduce-client-core - org.apache.hadoop + netty-all + io.netty - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - provided - - - com.google.protobuf - protobuf-java - - - junit - junit - test - + + + + + org.mongodb mongodb-driver 3.3.0 - - + de.flapdoodle.embed de.flapdoodle.embed.mongo @@ -333,43 +303,4 @@ - - - docs - - false - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - - module-javadocs - package - - jar - - - ${project.build.directory} - - - - - - - - - - - - - org.apache.maven.plugins - maven-surefire-report-plugin - - - - \ No newline at end of file From 2e9e631709c20d40539f914c0f95d86f2fc0665b Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 19 Aug 2016 11:50:56 +0530 Subject: [PATCH 85/94] Improved the code style --- tajo-storage/tajo-storage-mongodb/pom.xml | 59 +++--- .../tajo/storage/mongodb/ConnectionInfo.java | 56 +++-- .../tajo/storage/mongodb/MongoDBAppender.java | 15 +- .../mongodb/MongoDBCollectionReader.java | 29 ++- .../mongodb/MongoDBCollectionWriter.java | 26 +-- .../mongodb/MongoDBDocumentDeserializer.java | 28 +-- .../mongodb/MongoDBDocumentSerializer.java | 49 ++--- .../tajo/storage/mongodb/MongoDBFragment.java | 12 +- .../mongodb/MongoDBMetadataProvider.java | 21 +- .../tajo/storage/mongodb/MongoDBScanner.java | 24 +-- .../storage/mongodb/MongoDBTableSpace.java | 56 +++-- .../main/proto/MongoDBFragmentProtos.proto | 8 +- .../storage/mongodb/MongoDBTestServer.java | 121 +++++------ .../storage/mongodb/TestConnectionInfo.java | 7 +- .../storage/mongodb/TestMetadataProvider.java | 15 +- .../storage/mongodb/TestMongoDBAppender.java | 35 ++-- .../TestMongoDBDocumentSerializer.java | 38 ++-- .../storage/mongodb/TestMongoDBQueryTest.java | 13 +- .../mongodb/TestMongoDBTableSpace.java | 32 +-- .../src/test/resources/dataset/file1.json | 196 +++++++++++++++++- .../src/test/resources/dataset/file2.json | 30 ++- 21 files changed, 504 insertions(+), 366 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 25c50a8451..fa8d43a3fd 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -16,8 +16,8 @@ See the License for the specific language governing permissions and limitations under the License. --> - tajo-project @@ -27,7 +27,8 @@ 4.0.0 - tajo-storage-mongodbjar + tajo-storage-mongodb + jar Tajo MongoDB Storage UTF-8 @@ -38,8 +39,8 @@ - - + + @@ -73,16 +74,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -94,7 +95,7 @@ initialize - + @@ -155,17 +156,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -284,8 +285,8 @@ - - + + diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 6b8e899668..5623d69908 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,15 +18,15 @@ package org.apache.tajo.storage.mongodb; -import java.net.URI; -import java.util.HashMap; -import java.util.Map; - -import com.mongodb.*; +import com.mongodb.MongoClientURI; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.exception.TajoInternalError; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + /* ConnectionInfo keeps the details about the mongodb server connection. MongoURI, db credentials and URI specific details such as schema, host, port @@ -65,7 +65,7 @@ public static ConnectionInfo fromURI(URI originalUri) { //Set the db name String path = uri.getPath(); if (path != null && !path.isEmpty()) { - String [] pathElements = path.substring(1).split("/"); + String[] pathElements = path.substring(1).split("/"); if (pathElements.length != 1) { throw new TajoInternalError("Invalid JDBC path: " + path); } @@ -76,12 +76,12 @@ public static ConnectionInfo fromURI(URI originalUri) { Map params = new HashMap<>(); int paramIndex = uriStr.indexOf("?"); if (paramIndex > 0) { - String parameterPart = uriStr.substring(paramIndex+1, uriStr.length()); + String parameterPart = uriStr.substring(paramIndex + 1, uriStr.length()); - String [] eachParam = parameterPart.split("&"); + String[] eachParam = parameterPart.split("&"); - for (String each: eachParam) { - String [] keyValues = each.split("="); + for (String each : eachParam) { + String[] keyValues = each.split("="); if (keyValues.length != 2) { throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); } @@ -105,29 +105,27 @@ public static ConnectionInfo fromURI(URI originalUri) { String mongoDbURIStr = ""; //Generate the MongoURI - mongoDbURIStr+= connInfo.getScheme(); - mongoDbURIStr+="://"; - if(connInfo.getUser() !=null) - { - mongoDbURIStr+= connInfo.getUser(); - if(connInfo.getPassword() !=null) - mongoDbURIStr+=":"+ connInfo.getPassword(); - mongoDbURIStr+="@"; + mongoDbURIStr += connInfo.getScheme(); + mongoDbURIStr += "://"; + if (connInfo.getUser() != null) { + mongoDbURIStr += connInfo.getUser(); + if (connInfo.getPassword() != null) + mongoDbURIStr += ":" + connInfo.getPassword(); + mongoDbURIStr += "@"; } - mongoDbURIStr+= connInfo.getHost(); - mongoDbURIStr+=":"; - mongoDbURIStr+= connInfo.getPort(); - mongoDbURIStr+="/"; - mongoDbURIStr+= connInfo.getDbName(); + mongoDbURIStr += connInfo.getHost(); + mongoDbURIStr += ":"; + mongoDbURIStr += connInfo.getPort(); + mongoDbURIStr += "/"; + mongoDbURIStr += connInfo.getDbName(); LOG.info(mongoDbURIStr); connInfo.mongoDBURI = new MongoClientURI(mongoDbURIStr); return connInfo; } - public MongoClientURI getMongoDBURI() - { - return mongoDBURI; + public MongoClientURI getMongoDBURI() { + return mongoDBURI; } public String getScheme() { diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index d8becb15dc..e3504bfd09 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,9 +32,6 @@ import java.net.URI; import java.util.List; -/** - * Created by janaka on 5/21/16. - */ public class MongoDBAppender implements Appender { //Given at constructor @@ -53,8 +50,6 @@ public class MongoDBAppender implements Appender { private MongoDBCollectionWriter mongoDBCollectionWriter; - - public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schema, TableMeta meta, Path stagingDir, URI uri) { this.conf = conf; @@ -71,8 +66,8 @@ public void init() throws IOException { throw new IllegalStateException("FileAppender is already initialized."); } inited = true; - MongoDBDocumentSerializer md = new MongoDBDocumentSerializer(schema,meta); - mongoDBCollectionWriter = new MongoDBCollectionWriter(ConnectionInfo.fromURI(stagingDir.toString()),md); + MongoDBDocumentSerializer md = new MongoDBDocumentSerializer(schema, meta); + mongoDBCollectionWriter = new MongoDBCollectionWriter(ConnectionInfo.fromURI(stagingDir.toString()), md); mongoDBCollectionWriter.init(); } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index 2b3b3f6428..4bb39813db 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,6 @@ import org.bson.Document; import java.io.IOException; - import java.nio.charset.CharsetEncoder; import java.util.ArrayList; import java.util.List; @@ -37,36 +36,35 @@ Used within the MongoScanner to read tuples. */ public class MongoDBCollectionReader { + private final CharsetEncoder encoder = CharsetUtil.getEncoder(CharsetUtil.UTF_8); + List documentList; private ConnectionInfo connectionInfo; private MongoDBDocumentDeserializer deserializer; private int targetLength; - List documentList; private int currentIndex; - private final CharsetEncoder encoder = CharsetUtil.getEncoder(CharsetUtil.UTF_8); - public MongoDBCollectionReader(ConnectionInfo connectionInfo, MongoDBDocumentDeserializer deserializer , int targetLength) - { + public MongoDBCollectionReader(ConnectionInfo connectionInfo, MongoDBDocumentDeserializer deserializer, int targetLength) { this.connectionInfo = connectionInfo; this.deserializer = deserializer; this.targetLength = targetLength; } - public void init() throws IOException - { + public void init() throws IOException { currentIndex = 0; MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); MongoCollection collection = db.getCollection(connectionInfo.getTableName()); - documentList =(List) collection.find().into( - new ArrayList());; + documentList = (List) collection.find().into( + new ArrayList()); + ; deserializer.init(); } public Tuple readTuple() throws IOException, TextLineParsingError { - if(currentIndex>=documentList.size()) return null; + if (currentIndex >= documentList.size()) return null; Tuple outTuple = new VTuple(targetLength); @@ -76,9 +74,8 @@ public Tuple readTuple() throws IOException, TextLineParsingError { } - public float getProgress() - { - return ((float)currentIndex) / documentList.size(); + public float getProgress() { + return ((float) currentIndex) / documentList.size(); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java index 1c2fa36c8d..ffb4565517 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,22 +31,18 @@ * Writes tuples into a mongodb collection */ public class MongoDBCollectionWriter { - private ConnectionInfo connectionInfo; MongoClient mongoClient; MongoCollection collection; MongoDBDocumentSerializer mongoDBDocumentSerializer; - - + private ConnectionInfo connectionInfo; private List documentList = new ArrayList<>(); - public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer serializer) - { + public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer serializer) { this.connectionInfo = connectionInfo; - this.mongoDBDocumentSerializer = serializer; + this.mongoDBDocumentSerializer = serializer; } - public void init() - { + public void init() { mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); collection = db.getCollection(connectionInfo.getTableName()); @@ -54,17 +50,15 @@ public void init() public void addTuple(Tuple tuple) throws IOException { Document dc = new Document(); - mongoDBDocumentSerializer.serialize(tuple,dc); + mongoDBDocumentSerializer.serialize(tuple, dc); documentList.add(dc); } - public void write() - { + public void write() { collection.insertMany(documentList); } - public void close() - { + public void close() { mongoClient.close(); } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java index e8554e6a94..03772585a0 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +20,6 @@ import com.google.common.collect.Lists; -import io.netty.buffer.ByteBuf; import io.netty.util.CharsetUtil; import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; @@ -35,7 +34,6 @@ import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.text.TextLineDeserializer; import org.apache.tajo.storage.text.TextLineParsingError; import org.bson.Document; @@ -50,18 +48,16 @@ * */ public class MongoDBDocumentDeserializer { - private JSONParser parser; - + protected final Schema schema; + protected final TableMeta meta; // Full Path -> Type private final Map types; - private final String [] projectedPaths; + private final String[] projectedPaths; private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); - private final TimeZone timezone; - protected final Schema schema; - protected final TableMeta meta; + private JSONParser parser; - public MongoDBDocumentDeserializer(Schema schema, TableMeta meta, Column [] projected) { + public MongoDBDocumentDeserializer(Schema schema, TableMeta meta, Column[] projected) { this.schema = schema; this.meta = meta; @@ -79,8 +75,6 @@ public void init() { } /** - * - * * @param object * @param pathElements * @param depth @@ -90,7 +84,7 @@ public void init() { */ private void getValue(JSONObject object, String fullPath, - String [] pathElements, + String[] pathElements, int depth, int fieldIndex, Tuple output) throws IOException { @@ -208,7 +202,7 @@ private void getValue(JSONObject object, case RECORD: JSONObject nestedObject = (JSONObject) object.get(fieldName); if (nestedObject != null) { - getValue(nestedObject, fullPath + "/" + pathElements[depth+1], pathElements, depth + 1, fieldIndex, output); + getValue(nestedObject, fullPath + "/" + pathElements[depth + 1], pathElements, depth + 1, fieldIndex, output); } else { output.put(fieldIndex, NullDatum.get()); } @@ -240,7 +234,7 @@ public void deserialize(Document doc, Tuple output) throws IOException, TextLine } for (int i = 0; i < projectedPaths.length; i++) { - String [] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER); + String[] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER); getValue(object, paths[0], paths, 0, i, output); //output.put(i, DatumFactory.createText( doc.get(projectedPaths[i]).toString() )); diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java index 9669265649..9a123e1ed0 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,10 +18,13 @@ package org.apache.tajo.storage.mongodb; import com.google.common.collect.Lists; -import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.NestedPathUtil; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SchemaUtil; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.storage.Tuple; import org.bson.Document; -import org.apache.tajo.common.TajoDataTypes.Type; import java.io.IOException; import java.util.Map; @@ -33,10 +36,8 @@ public class MongoDBDocumentSerializer { private final Schema schema; private final TableMeta meta; - String[] paths; - - private final Map types; + String[] paths; public MongoDBDocumentSerializer(Schema schema, TableMeta meta) { this.schema = schema; @@ -46,59 +47,55 @@ public MongoDBDocumentSerializer(Schema schema, TableMeta meta) { types = SchemaUtil.buildTypeMap(schema.getAllColumns(), paths); } - public void setValue(Tuple inputTuple, String fullPath, String[] paths, int depth, int index, Document outputDoc) - { + public void setValue(Tuple inputTuple, String fullPath, String[] paths, int depth, int index, Document outputDoc) { String simpleColumnName = paths[depth]; - switch (types.get(fullPath)) - { + switch (types.get(fullPath)) { case INT1: case UINT1: case INT2: case UINT2: - outputDoc.put(simpleColumnName,inputTuple.getInt2(index)); + outputDoc.put(simpleColumnName, inputTuple.getInt2(index)); break; case INT4: case UINT4: - outputDoc.put(simpleColumnName,inputTuple.getInt4(index)); + outputDoc.put(simpleColumnName, inputTuple.getInt4(index)); break; case INT8: case UINT8: - outputDoc.put(simpleColumnName,inputTuple.getInt8(index)); + outputDoc.put(simpleColumnName, inputTuple.getInt8(index)); break; case FLOAT4: - outputDoc.put(simpleColumnName,inputTuple.getFloat4(index)); + outputDoc.put(simpleColumnName, inputTuple.getFloat4(index)); break; case FLOAT8: - outputDoc.put(simpleColumnName,inputTuple.getFloat8(index)); + outputDoc.put(simpleColumnName, inputTuple.getFloat8(index)); break; case CHAR: - outputDoc.put(simpleColumnName,inputTuple.getChar(index)); + outputDoc.put(simpleColumnName, inputTuple.getChar(index)); break; case BOOLEAN: - outputDoc.put(simpleColumnName,inputTuple.getBool(index)); + outputDoc.put(simpleColumnName, inputTuple.getBool(index)); break; default: - outputDoc.put(simpleColumnName,inputTuple.getText(index)); + outputDoc.put(simpleColumnName, inputTuple.getText(index)); break; } } - public void serialize(Tuple inputTuple, Document outputDoc) throws IOException - { - for(int i=0; i< inputTuple.size();i++) - { - String [] newPaths = paths[i].split(NestedPathUtil.PATH_DELIMITER); - setValue(inputTuple,newPaths[0],newPaths,0,i,outputDoc); + public void serialize(Tuple inputTuple, Document outputDoc) throws IOException { + for (int i = 0; i < inputTuple.size(); i++) { + String[] newPaths = paths[i].split(NestedPathUtil.PATH_DELIMITER); + setValue(inputTuple, newPaths[0], newPaths, 0, i, outputDoc); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java index d3862f0072..babe7c8af6 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,13 +17,9 @@ */ package org.apache.tajo.storage.mongodb; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.storage.fragment.Fragment; import java.net.URI; -import java.util.List; /* Fragment is similar to splits in Map Reduce. @@ -38,6 +34,6 @@ protected MongoDBFragment(URI uri, String inputSourceId, long startKey, long endKey) { - super("MONGODB", uri, inputSourceId, startKey, endKey, endKey-startKey, null); + super("MONGODB", uri, inputSourceId, startKey, endKey, endKey - startKey, null); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index c4563b8251..95eb5ce443 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,9 +21,11 @@ import com.mongodb.MongoClient; import com.mongodb.client.MongoDatabase; import com.mongodb.client.MongoIterable; -import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.SchemaBuilder; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.UndefinedTablespaceException; import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.KeyValueSet; @@ -40,13 +42,12 @@ * * */ public class MongoDBMetadataProvider implements MetadataProvider { + MongoDatabase db; private MongoDBTableSpace tableSpace; private String mappedDbName; private ConnectionInfo connectionInfo; - MongoDatabase db; - public MongoDBMetadataProvider(MongoDBTableSpace tableSpace, String dbName) - { + public MongoDBMetadataProvider(MongoDBTableSpace tableSpace, String dbName) { this.tableSpace = tableSpace; this.mappedDbName = dbName; @@ -79,7 +80,7 @@ public Collection getSchemas() { public Collection getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { //Get a list of table(=collection) names - MongoIterable collectionList = db.listCollectionNames(); + MongoIterable collectionList = db.listCollectionNames(); //Map to a string list and return Collection list = new ArrayList(); @@ -102,7 +103,7 @@ public TableDesc getTableDesc(String schemaName, String tableName) throws Undefi // .add(new Column("last_name", TajoDataTypes.Type.TEXT)) .build(), tbMeta, - tableSpace.getTableUri(null,null, tableName)); + tableSpace.getTableUri(null, null, tableName)); final TableStats stats = new TableStats(); stats.setNumRows(-1); // unknown diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index acb5f0ea68..0f7c8cae1e 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,10 +26,8 @@ import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.storage.FileScanner; import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.fragment.AbstractFileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.text.TextLineParsingError; @@ -41,21 +39,19 @@ * */ //Todo Remove FileScanner -public class MongoDBScanner implements Scanner{ - - private MongoDBCollectionReader collectionReader; +public class MongoDBScanner implements Scanner { private final TableMeta meta; private final Schema schema; private final MongoDBFragment fragment; - - private Column [] targets; + private MongoDBCollectionReader collectionReader; + private Column[] targets; private boolean inited; public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { this.schema = schema; - this.meta = meta; + this.meta = meta; this.fragment = (MongoDBFragment) fragment; } @@ -81,15 +77,15 @@ public Tuple next() throws IOException { @Override public void reset() throws IOException { - MongoDBDocumentDeserializer deserializer = new MongoDBDocumentDeserializer(schema,meta,targets); - collectionReader = new MongoDBCollectionReader(ConnectionInfo.fromURI(fragment.getUri()),deserializer,targets.length); + MongoDBDocumentDeserializer deserializer = new MongoDBDocumentDeserializer(schema, meta, targets); + collectionReader = new MongoDBCollectionReader(ConnectionInfo.fromURI(fragment.getUri()), deserializer, targets.length); collectionReader.init(); } @Override public void close() throws IOException { - return; + return; } @Override diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 8d0cd828da..2e4dc089ba 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -54,9 +54,12 @@ public class MongoDBTableSpace extends Tablespace { - private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - - + //Config Keys + public static final String CONFIG_KEY_MAPPED_DATABASE = "mapped_database"; + public static final String CONFIG_KEY_CONN_PROPERTIES = "connection_properties"; + public static final String CONFIG_KEY_USERNAME = "user"; + public static final String CONFIG_KEY_PASSWORD = "password"; + public static final String CONFIG_KEY_TABLE = "table"; //Table Space Properties static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined false, //not movable @@ -67,20 +70,12 @@ public class MongoDBTableSpace extends Tablespace { true, // Insert true, //direct insert true);// result staging - - //Mongo Client object - private ConnectionInfo connectionInfo; + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); protected MongoClient mongoClient; protected MongoDatabase db; protected String mappedDBName; - - - //Config Keys - public static final String CONFIG_KEY_MAPPED_DATABASE = "mapped_database"; - public static final String CONFIG_KEY_CONN_PROPERTIES = "connection_properties"; - public static final String CONFIG_KEY_USERNAME = "user"; - public static final String CONFIG_KEY_PASSWORD = "password"; - public static final String CONFIG_KEY_TABLE = "table"; + //Mongo Client object + private ConnectionInfo connectionInfo; public MongoDBTableSpace(String name, URI uri, JSONObject config) { @@ -112,7 +107,7 @@ public long getTableVolume(TableDesc table, Optional filter) { long count = 0; try { - String[] nameSplited = IdentifierUtil.splitFQTableName(table.getName()); + String[] nameSplited = IdentifierUtil.splitFQTableName(table.getName()); count = db.getCollection(nameSplited[1]).count(); } catch (Exception e) { throw new TajoInternalError(e); @@ -121,11 +116,10 @@ public long getTableVolume(TableDesc table, Optional filter) { } - @Override public List getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { long tableVolume = getTableVolume(tableDesc, Optional.empty()); - MongoDBFragment mongoDBFragment = new MongoDBFragment(tableDesc.getUri(), inputSourceId, 0, tableVolume ); + MongoDBFragment mongoDBFragment = new MongoDBFragment(tableDesc.getUri(), inputSourceId, 0, tableVolume); return Lists.newArrayList(mongoDBFragment); } @@ -157,21 +151,21 @@ public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws Ta @Override public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { - if(tableDesc==null) + if (tableDesc == null) throw new TajoRuntimeException(new NotImplementedException()); MongoCollection table = db.getCollection(tableDesc.getName()); //TODO Handle this here. If empty throw exception or what? - boolean ifExist = (table.count()>0)?true:false; + boolean ifExist = (table.count() > 0) ? true : false; //If meta data provides. Create a table - if(STORAGE_PROPERTY.isMetadataProvided()) + if (STORAGE_PROPERTY.isMetadataProvided()) db.createCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())); } @Override public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { - if(STORAGE_PROPERTY.isMetadataProvided()) + if (STORAGE_PROPERTY.isMetadataProvided()) db.getCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())).drop(); } @@ -204,18 +198,18 @@ public URI getRootUri() { public URI getTableUri(TableMeta meta, String databaseName, String tableName) { //ToDo Find a better way this String tableURI = ""; - if(this.getUri().toASCIIString().contains("?")) - tableURI = this.getUri().toASCIIString()+"&"+CONFIG_KEY_TABLE+"="+tableName; + if (this.getUri().toASCIIString().contains("?")) + tableURI = this.getUri().toASCIIString() + "&" + CONFIG_KEY_TABLE + "=" + tableName; else - tableURI = this.getUri().toASCIIString()+"?"+CONFIG_KEY_TABLE+"="+tableName; + tableURI = this.getUri().toASCIIString() + "?" + CONFIG_KEY_TABLE + "=" + tableName; return URI.create(tableURI); } //@Override - public URI getTableUri( String databaseName, String tableName) { + public URI getTableUri(String databaseName, String tableName) { //ToDo set the TableURI properly - return URI.create(this.getUri()+"&"+CONFIG_KEY_TABLE+"="+tableName); + return URI.create(this.getUri() + "&" + CONFIG_KEY_TABLE + "=" + tableName); } // @Override @@ -225,7 +219,7 @@ public URI getTableUri( String databaseName, String tableName) { // } - // Metadata + // Metadata public MetadataProvider getMetadataProvider() { return new MongoDBMetadataProvider(this, mappedDBName); } @@ -241,6 +235,6 @@ public Appender getAppender(OverridableConf queryContext, TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) { - return new MongoDBAppender(null, taskAttemptId,schema,meta,workDir,workDir.toUri()); + return new MongoDBAppender(null, taskAttemptId, schema, meta, workDir, workDir.toUri()); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto b/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto index a2101bbff0..68a7db3978 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto +++ b/tajo-storage/tajo-storage-mongodb/src/main/proto/MongoDBFragmentProtos.proto @@ -25,8 +25,8 @@ option java_generate_equals_and_hash = true; import "CatalogProtos.proto"; message MongoDBFragmentProto { - required string uri = 1; - required string table_name = 2; - required int64 start_key = 3; - required int64 end_key = 4; + required string uri = 1; + required string table_name = 2; + required int64 start_key = 3; + required int64 end_key = 4; } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index fee9441697..e8a4c81f59 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -17,10 +17,9 @@ */ package org.apache.tajo.storage.mongodb; -/** - * Created by janaka on 6/7/16. - */ -import com.mongodb.*; + +import com.mongodb.Block; +import com.mongodb.MongoClient; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; @@ -48,58 +47,36 @@ import java.util.List; import java.util.Scanner; -public class MongoDBTestServer { - - private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - +public class MongoDBTestServer { //mongo server details public static final int PORT = 12345; public static final String HOST = "localhost"; public static final String DBNAME = "test_dbname"; public static final String MAPPEDDBNAME = "test_mapped_dbname"; - //tajo tableSpace name public static final String SPACE_NAME = "test_spacename"; - - // instance for singleton server - private static MongoDBTestServer instance; - - + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); //Mongo Server componenets private static final MongodStarter starter = MongodStarter.getDefaultInstance(); + // instance for singleton server + private static MongoDBTestServer instance; + //Collection names (table names) to be created inside mongodb + public String[] collectionNames = {"github", "got"}; private MongodExecutable _mongodExe; private MongodProcess _mongod; private MongoClient _mongo; - //File names to load data - private String[] filenames = {"file1.json","file2.json"}; - //Collection names (table names) to be created inside mongodb - public String[] collectionNames = {"github","got"}; + private String[] filenames = {"file1.json", "file2.json"}; - //server object can be created using this method - public static MongoDBTestServer getInstance() - { - if(instance==null) - { - try { - instance = new MongoDBTestServer(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - return instance; - } - // private constructor - private MongoDBTestServer () throws IOException, URISyntaxException { + private MongoDBTestServer() throws IOException, URISyntaxException { _mongodExe = starter.prepare(new MongodConfigBuilder() .version(Version.Main.PRODUCTION) .net(new Net(PORT, Network.localhostIsIPv6())) - .cmdOptions( new MongoCmdOptionsBuilder(). + .cmdOptions(new MongoCmdOptionsBuilder(). useStorageEngine("mmapv1"). build()) .build()); @@ -110,27 +87,46 @@ private MongoDBTestServer () throws IOException, URISyntaxException { loadData(); } + //server object can be created using this method + public static MongoDBTestServer getInstance() { + if (instance == null) { + try { + instance = new MongoDBTestServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return instance; + } + + //To load files + private static File getRequestedFile(String path) throws FileNotFoundException, URISyntaxException { + + URL url = ClassLoader.getSystemResource("dataset/" + path); + + if (url == null) { + throw new FileNotFoundException(path); + } + return new File(url.toURI()); + } + //To stop the server - public void stop() - { + public void stop() { _mongod.stop(); _mongodExe.stop(); instance = null; } //Returns the url which can be used to connet to the server instance - public URI getURI() - { + public URI getURI() { try { - return new URI("mongodb://" + HOST + ":" + PORT + "/"+ DBNAME); - } - catch (Exception e) - { + return new URI("mongodb://" + HOST + ":" + PORT + "/" + DBNAME); + } catch (Exception e) { return null; } } - //Start mongo process private MongosProcess startMongos(int port, int defaultConfigPort, String defaultHost) throws UnknownHostException, IOException { @@ -157,13 +153,12 @@ private MongodProcess startMongod(int defaultConfigPort) throws UnknownHostExcep return mongod; } - //Register the new table space for testing private void registerTablespace() throws IOException { JSONObject configElements = new JSONObject(); configElements.put(MongoDBTableSpace.CONFIG_KEY_MAPPED_DATABASE, MAPPEDDBNAME); - MongoDBTableSpace tablespace = new MongoDBTableSpace(SPACE_NAME,getURI(),configElements); + MongoDBTableSpace tablespace = new MongoDBTableSpace(SPACE_NAME, getURI(), configElements); tablespace.init(new TajoConf()); @@ -172,13 +167,13 @@ private void registerTablespace() throws IOException { //Create tables and Load data into the mongo server instance private void loadData() throws IOException, URISyntaxException { - MongoDatabase db = _mongo.getDatabase(DBNAME); - for(int i=0; i documentList = new ArrayList(); @@ -195,31 +190,19 @@ private void loadData() throws IOException, URISyntaxException { coll.insertMany(documentList); - FindIterable docs = coll.find(); - - docs.forEach(new Block() { - @Override - public void apply(final Document document) { - LOG.info(document.toJson()); - } - }); - } - } - - //To load files - private static File getRequestedFile(String path) throws FileNotFoundException, URISyntaxException { - - URL url = ClassLoader.getSystemResource("dataset/" + path); + FindIterable docs = coll.find(); - if (url == null) { - throw new FileNotFoundException(path); + docs.forEach(new Block() { + @Override + public void apply(final Document document) { + LOG.info(document.toJson()); + } + }); } - return new File(url.toURI()); } //Return a mongo client which directly connect to the mongo database. - public MongoClient getMongoClient() - { + public MongoClient getMongoClient() { return _mongo; } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index ff3a76bee9..57135203be 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -28,7 +28,7 @@ public final void testGetConnectionInfoType1() { ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass"); - assertEquals(connInfo.getMongoDBURI().getURI(),"mongodb://testuser:testpass@localhost:1336/db1"); + assertEquals(connInfo.getMongoDBURI().getURI(), "mongodb://testuser:testpass@localhost:1336/db1"); assertEquals(connInfo.getScheme(), "mongodb"); assertEquals(connInfo.getHost(), "localhost"); assertEquals(connInfo.getPort(), 1336); @@ -55,8 +55,7 @@ public final void testGetConnectionInfoType2() { } @Test - public final void testNewStructure() - { - + public final void testNewStructure() { + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index 4d3a9b2875..7a01439fa0 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -22,17 +22,13 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; -import org.junit.*; +import org.junit.Test; import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -/** - * Created by janaka on 6/8/16. - */ - public class TestMetadataProvider { static MongoDBTestServer server = MongoDBTestServer.getInstance(); @@ -71,6 +67,7 @@ public void testGetTables() throws Exception { assertEquals(expected, found); } + @Test public void testGetTableDescription() throws Exception { Tablespace tablespace = TablespaceManager.get(server.getURI()); @@ -78,20 +75,14 @@ public void testGetTableDescription() throws Exception { for (String tableName : server.collectionNames) { TableDesc table = provider.getTableDesc(null, tableName); - assertEquals( server.MAPPEDDBNAME +"."+tableName, table.getName()); + assertEquals(server.MAPPEDDBNAME + "." + tableName, table.getName()); assertEquals(server.getURI() + "?table=" + tableName, table.getUri().toASCIIString()); //ToDo Check the stats } - } - - - - - } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java index 5070b19355..61b6b90796 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java @@ -19,7 +19,6 @@ import com.mongodb.client.MongoIterable; import org.apache.hadoop.fs.Path; -import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SchemaBuilder; @@ -29,7 +28,6 @@ import org.apache.tajo.storage.*; import org.bson.Document; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; @@ -45,8 +43,7 @@ public class TestMongoDBAppender { Appender appender; @Before - public void setup() throws Exception - { + public void setup() throws Exception { Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); @@ -61,8 +58,8 @@ public void setup() throws Exception .build(); - TableMeta meta = space.getMetadataProvider().getTableDesc(null,"got").getMeta(); - appender = space.getAppender(null,null,meta,schem,new Path(server.getURI()+"?"+MongoDBTableSpace.CONFIG_KEY_TABLE+"=got") ); + TableMeta meta = space.getMetadataProvider().getTableDesc(null, "got").getMeta(); + appender = space.getAppender(null, null, meta, schem, new Path(server.getURI() + "?" + MongoDBTableSpace.CONFIG_KEY_TABLE + "=got")); appender.init(); @@ -73,27 +70,27 @@ public void testAddTuple() throws IOException { //Create a tuple and add to the table Tuple tuple = new VTuple(6); - tuple.put(0,DatumFactory.createText("Kingslayer")); - tuple.put(1,DatumFactory.createText("Jaime")); - tuple.put(2,DatumFactory.createText("Lannister")); - tuple.put(3,DatumFactory.createInt4(24)); - tuple.put(4,DatumFactory.createFloat8(165.98)); - tuple.put(5,DatumFactory.createBool(true)); + tuple.put(0, DatumFactory.createText("Kingslayer")); + tuple.put(1, DatumFactory.createText("Jaime")); + tuple.put(2, DatumFactory.createText("Lannister")); + tuple.put(3, DatumFactory.createInt4(24)); + tuple.put(4, DatumFactory.createFloat8(165.98)); + tuple.put(5, DatumFactory.createBool(true)); appender.addTuple(tuple); appender.flush(); //Take data from server - MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title","Kingslayer")); + MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title", "Kingslayer")); Document doc = result.first(); //Validate - assertEquals(doc.get("title"),"Kingslayer"); - assertEquals(doc.get("first_name"),"Jaime"); - assertEquals(doc.get("last_name"),"Lannister"); - assertEquals(doc.get("age"),24); - assertEquals(doc.get("height"),165.98); - assertEquals(doc.get("single"),true); + assertEquals(doc.get("title"), "Kingslayer"); + assertEquals(doc.get("first_name"), "Jaime"); + assertEquals(doc.get("last_name"), "Lannister"); + assertEquals(doc.get("age"), 24); + assertEquals(doc.get("height"), 165.98); + assertEquals(doc.get("single"), true); //Remove the inserted doc from database diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java index d92d896a2b..50a42ed3cf 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java @@ -17,8 +17,6 @@ */ package org.apache.tajo.storage.mongodb; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SchemaBuilder; @@ -43,19 +41,19 @@ public void testSeriaalizeTextType() throws IOException { .add(new Column("last_name", TajoDataTypes.Type.TEXT)) .build(); - MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem,null); + MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem, null); Tuple tuple = new VTuple(3); - tuple.put(0,DatumFactory.createText("Kingslayer")); - tuple.put(1,DatumFactory.createText("Jaime")); - tuple.put(2,DatumFactory.createText("Lannister")); + tuple.put(0, DatumFactory.createText("Kingslayer")); + tuple.put(1, DatumFactory.createText("Jaime")); + tuple.put(2, DatumFactory.createText("Lannister")); Document md = new Document(); - documentSerializer.serialize(tuple,md); + documentSerializer.serialize(tuple, md); - assertEquals("Kingslayer",md.getString("title")); - assertEquals("Jaime",md.getString("first_name")); - assertEquals("Lannister",md.getString("last_name")); + assertEquals("Kingslayer", md.getString("title")); + assertEquals("Jaime", md.getString("first_name")); + assertEquals("Lannister", md.getString("last_name")); } @@ -68,21 +66,21 @@ public void testSerializeIntFloatBoolean() throws IOException { .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) .build(); - MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem,null); + MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem, null); Tuple tuple = new VTuple(4); - tuple.put(0,DatumFactory.createText("Mr")); - tuple.put(1,DatumFactory.createInt4(24)); - tuple.put(2,DatumFactory.createFloat8(165.98)); - tuple.put(3,DatumFactory.createBool(true)); + tuple.put(0, DatumFactory.createText("Mr")); + tuple.put(1, DatumFactory.createInt4(24)); + tuple.put(2, DatumFactory.createFloat8(165.98)); + tuple.put(3, DatumFactory.createBool(true)); Document md = new Document(); - documentSerializer.serialize(tuple,md); + documentSerializer.serialize(tuple, md); - assertEquals("Mr",md.get("title")); - assertEquals(24,md.get("age")); - assertEquals(165.98,md.get("height")); - assertEquals(true,md.get("single")); + assertEquals("Mr", md.get("title")); + assertEquals(24, md.get("age")); + assertEquals(165.98, md.get("height")); + assertEquals(true, md.get("single")); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 9cc52ade91..4f80f273d3 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -23,7 +23,7 @@ import java.net.URI; -public class TestMongoDBQueryTest extends QueryTestCaseBase{ +public class TestMongoDBQueryTest extends QueryTestCaseBase { static MongoDBTestServer server = MongoDBTestServer.getInstance(); static URI uri = server.getURI(); @@ -33,21 +33,20 @@ public TestMongoDBQueryTest() { } @BeforeClass - public static void setup() throws Exception - { + public static void setup() throws Exception { QueryTestCaseBase.testingCluster.getMaster().refresh(); - // TablespaceManager.addTableSpaceForTest(new ExampleHttpFileTablespace("http_example", uri, configElements)); + // TablespaceManager.addTableSpaceForTest(new ExampleHttpFileTablespace("http_example", uri, configElements)); } @AfterClass - public static void tearDownClass() throws Exception { + public static void tearDownClass() throws Exception { server.stop(); } @Before public void prepareTables() throws TajoException { - if(!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { executeString("create table got (title,first_name,last_name) tablespace test_spacename using mongodb"); executeString("create table github (*) tablespace test_spacename using mongodb"); // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); @@ -56,7 +55,7 @@ public void prepareTables() throws TajoException { @After public void dropTables() throws TajoException { - if(!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { executeString("drop table got"); executeString("drop table github"); } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index c665478118..802c6ff6df 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -31,21 +31,21 @@ import java.io.IOException; import java.net.URI; -import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import static org.junit.Assert.*; -/** - * Created by janaka on 6/2/16. - */ + public class TestMongoDBTableSpace { //mongodb://:@ds017231.mlab.com:17231/tajo_test static MongoDBTestServer server = MongoDBTestServer.getInstance(); static URI uri = server.getURI(); @Test - public void testTablespaceHandler() - { + public void testTablespaceHandler() { assertTrue((TablespaceManager.getByName(server.SPACE_NAME)) instanceof MongoDBTableSpace); assertEquals(server.SPACE_NAME, (TablespaceManager.getByName(server.SPACE_NAME).getName())); @@ -55,7 +55,7 @@ public void testTablespaceHandler() //Test the URI same assertEquals(uri.toASCIIString(), TablespaceManager.get(uri).getUri().toASCIIString()); - } + } @Test(timeout = 1000, expected = TajoRuntimeException.class) public void testCreateTable() throws IOException, TajoException { @@ -77,7 +77,7 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { server.getURI()); //Test create and delete if meta data provided - if(MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + if (MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { space.createTable(tableDesc, false); //Check whether the created table is in the collection @@ -98,7 +98,7 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { //Purg the table space.purgeTable(tableDesc); final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); - assertFalse(found_after.contains( "Table1")); + assertFalse(found_after.contains("Table1")); } } @@ -107,21 +107,21 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { public void testTableVolume() throws IOException, TajoException { Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); Map tableSizes = new HashMap(); - tableSizes.put("github",4); - tableSizes.put("got",5); - for (String tbl:server.collectionNames) { + tableSizes.put("github", 4); + tableSizes.put("got", 5); + for (String tbl : server.collectionNames) { - // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); + // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); - // long a = 1;b + // long a = 1;b TableDesc tbDesc = new TableDesc( IdentifierUtil.buildFQName(server.MAPPEDDBNAME, tbl), SchemaBuilder.builder() .build(), null, - space.getTableUri(null,null, tbl)); + space.getTableUri(null, null, tbl)); - assertEquals((int)tableSizes.get(tbl), space.getTableVolume(tbDesc,Optional.empty())); + assertEquals((int) tableSizes.get(tbl), space.getTableVolume(tbDesc, Optional.empty())); } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json index b3c74e8123..b6434eaeab 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file1.json @@ -1,6 +1,194 @@ [ - {"id":"2937257753","type":"PushEvent","actor":{"id":5266949,"login":"hardrubic","gravatar_id":"","url":"https://api.github.com/users/hardrubic","avatar_url":"https://avatars.githubusercontent.com/u/5266949?"},"repo":{"id":38299397,"name":"hardrubic/rxJavaTest","url":"https://api.github.com/repos/hardrubic/rxJavaTest"},"payload":{"push_id":712081726,"size":1,"distinct_size":1,"ref":"refs/heads/master","head":"ea79d7a424f2693b70b9496726f315a5711b6fe7","before":"613f05557ad353f4bedc6df54128f8091ed1f1e9","commits":[{"sha":"ea79d7a424f2693b70b9496726f315a5711b6fe7","author":{"email":"dgzx106@163.com","name":"hardrubic"},"message":"增加rxJava例子","distinct":true,"url":"https://api.github.com/repos/hardrubic/rxJavaTest/commits/ea79d7a424f2693b70b9496726f315a5711b6fe7"}]},"public":true,"created_at":"2015-07-01T00:00:01Z"}, - {"id":"2937257758","type":"WatchEvent","actor":{"id":11455393,"login":"chrischjh","gravatar_id":"","url":"https://api.github.com/users/chrischjh","avatar_url":"https://avatars.githubusercontent.com/u/11455393?"},"repo":{"id":18218031,"name":"dead-horse/co-and-koa-talk","url":"https://api.github.com/repos/dead-horse/co-and-koa-talk"},"payload":{"action":"started"},"public":true,"created_at":"2015-07-01T00:00:01Z"}, - {"id":"2937257759","type":"CreateEvent","actor":{"id":206379,"login":"gvn","gravatar_id":"","url":"https://api.github.com/users/gvn","avatar_url":"https://avatars.githubusercontent.com/u/206379?"},"repo":{"id":24345476,"name":"gvn/webmaker-android","url":"https://api.github.com/repos/gvn/webmaker-android"},"payload":{"ref":"use-self-building","ref_type":"branch","master_branch":"master","description":"Webmaker for Firefox OS & Android","pusher_type":"user"},"public":true,"created_at":"2015-07-01T00:00:01Z"}, - {"id":"2937257761","type":"ForkEvent","actor":{"id":1088854,"login":"CAOakleyII","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","avatar_url":"https://avatars.githubusercontent.com/u/1088854?"},"repo":{"id":11909954,"name":"skycocker/chromebrew","url":"https://api.github.com/repos/skycocker/chromebrew"},"payload":{"forkee":{"id":38339291,"name":"chromebrew","full_name":"CAOakleyII/chromebrew","owner":{"login":"CAOakleyII","id":1088854,"avatar_url":"https://avatars.githubusercontent.com/u/1088854?v=3","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","html_url":"https://github.com/CAOakleyII","followers_url":"https://api.github.com/users/CAOakleyII/followers","following_url":"https://api.github.com/users/CAOakleyII/following{/other_user}","gists_url":"https://api.github.com/users/CAOakleyII/gists{/gist_id}","starred_url":"https://api.github.com/users/CAOakleyII/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CAOakleyII/subscriptions","organizations_url":"https://api.github.com/users/CAOakleyII/orgs","repos_url":"https://api.github.com/users/CAOakleyII/repos","events_url":"https://api.github.com/users/CAOakleyII/events{/privacy}","received_events_url":"https://api.github.com/users/CAOakleyII/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/CAOakleyII/chromebrew","description":"Package manager for Chrome OS","fork":true,"url":"https://api.github.com/repos/CAOakleyII/chromebrew","forks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/forks","keys_url":"https://api.github.com/repos/CAOakleyII/chromebrew/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CAOakleyII/chromebrew/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CAOakleyII/chromebrew/teams","hooks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/hooks","issue_events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/events{/number}","events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/events","assignees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/assignees{/user}","branches_url":"https://api.github.com/repos/CAOakleyII/chromebrew/branches{/branch}","tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/tags","blobs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/refs{/sha}","trees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CAOakleyII/chromebrew/statuses/{sha}","languages_url":"https://api.github.com/repos/CAOakleyII/chromebrew/languages","stargazers_url":"https://api.github.com/repos/CAOakleyII/chromebrew/stargazers","contributors_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contributors","subscribers_url":"https://api.github.com/repos/CAOakleyII/chromebrew/subscribers","subscription_url":"https://api.github.com/repos/CAOakleyII/chromebrew/subscription","commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/commits{/sha}","git_commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/commits{/sha}","comments_url":"https://api.github.com/repos/CAOakleyII/chromebrew/comments{/number}","issue_comment_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/comments{/number}","contents_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contents/{+path}","compare_url":"https://api.github.com/repos/CAOakleyII/chromebrew/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CAOakleyII/chromebrew/merges","archive_url":"https://api.github.com/repos/CAOakleyII/chromebrew/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CAOakleyII/chromebrew/downloads","issues_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues{/number}","pulls_url":"https://api.github.com/repos/CAOakleyII/chromebrew/pulls{/number}","milestones_url":"https://api.github.com/repos/CAOakleyII/chromebrew/milestones{/number}","notifications_url":"https://api.github.com/repos/CAOakleyII/chromebrew/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CAOakleyII/chromebrew/labels{/name}","releases_url":"https://api.github.com/repos/CAOakleyII/chromebrew/releases{/id}","created_at":"2015-07-01T00:00:00Z","updated_at":"2015-06-28T10:11:09Z","pushed_at":"2015-06-09T07:46:57Z","git_url":"git://github.com/CAOakleyII/chromebrew.git","ssh_url":"git@github.com:CAOakleyII/chromebrew.git","clone_url":"https://github.com/CAOakleyII/chromebrew.git","svn_url":"https://github.com/CAOakleyII/chromebrew","homepage":"http://skycocker.github.io/chromebrew/","size":846,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master","public":true}},"public":true,"created_at":"2015-07-01T00:00:01Z"} + { + "id": "2937257753", + "type": "PushEvent", + "actor": { + "id": 5266949, + "login": "hardrubic", + "gravatar_id": "", + "url": "https://api.github.com/users/hardrubic", + "avatar_url": "https://avatars.githubusercontent.com/u/5266949?" + }, + "repo": { + "id": 38299397, + "name": "hardrubic/rxJavaTest", + "url": "https://api.github.com/repos/hardrubic/rxJavaTest" + }, + "payload": { + "push_id": 712081726, + "size": 1, + "distinct_size": 1, + "ref": "refs/heads/master", + "head": "ea79d7a424f2693b70b9496726f315a5711b6fe7", + "before": "613f05557ad353f4bedc6df54128f8091ed1f1e9", + "commits": [ + { + "sha": "ea79d7a424f2693b70b9496726f315a5711b6fe7", + "author": { + "email": "dgzx106@163.com", + "name": "hardrubic" + }, + "message": "增加rxJava例子", + "distinct": true, + "url": "https://api.github.com/repos/hardrubic/rxJavaTest/commits/ea79d7a424f2693b70b9496726f315a5711b6fe7" + } + ] + }, + "public": true, + "created_at": "2015-07-01T00:00:01Z" + }, + { + "id": "2937257758", + "type": "WatchEvent", + "actor": { + "id": 11455393, + "login": "chrischjh", + "gravatar_id": "", + "url": "https://api.github.com/users/chrischjh", + "avatar_url": "https://avatars.githubusercontent.com/u/11455393?" + }, + "repo": { + "id": 18218031, + "name": "dead-horse/co-and-koa-talk", + "url": "https://api.github.com/repos/dead-horse/co-and-koa-talk" + }, + "payload": { + "action": "started" + }, + "public": true, + "created_at": "2015-07-01T00:00:01Z" + }, + { + "id": "2937257759", + "type": "CreateEvent", + "actor": { + "id": 206379, + "login": "gvn", + "gravatar_id": "", + "url": "https://api.github.com/users/gvn", + "avatar_url": "https://avatars.githubusercontent.com/u/206379?" + }, + "repo": { + "id": 24345476, + "name": "gvn/webmaker-android", + "url": "https://api.github.com/repos/gvn/webmaker-android" + }, + "payload": { + "ref": "use-self-building", + "ref_type": "branch", + "master_branch": "master", + "description": "Webmaker for Firefox OS & Android", + "pusher_type": "user" + }, + "public": true, + "created_at": "2015-07-01T00:00:01Z" + }, + { + "id": "2937257761", + "type": "ForkEvent", + "actor": { + "id": 1088854, + "login": "CAOakleyII", + "gravatar_id": "", + "url": "https://api.github.com/users/CAOakleyII", + "avatar_url": "https://avatars.githubusercontent.com/u/1088854?" + }, + "repo": { + "id": 11909954, + "name": "skycocker/chromebrew", + "url": "https://api.github.com/repos/skycocker/chromebrew" + }, + "payload": { + "forkee": { + "id": 38339291, + "name": "chromebrew", + "full_name": "CAOakleyII/chromebrew", + "owner": { + "login": "CAOakleyII", + "id": 1088854, + "avatar_url": "https://avatars.githubusercontent.com/u/1088854?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/CAOakleyII", + "html_url": "https://github.com/CAOakleyII", + "followers_url": "https://api.github.com/users/CAOakleyII/followers", + "following_url": "https://api.github.com/users/CAOakleyII/following{/other_user}", + "gists_url": "https://api.github.com/users/CAOakleyII/gists{/gist_id}", + "starred_url": "https://api.github.com/users/CAOakleyII/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/CAOakleyII/subscriptions", + "organizations_url": "https://api.github.com/users/CAOakleyII/orgs", + "repos_url": "https://api.github.com/users/CAOakleyII/repos", + "events_url": "https://api.github.com/users/CAOakleyII/events{/privacy}", + "received_events_url": "https://api.github.com/users/CAOakleyII/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/CAOakleyII/chromebrew", + "description": "Package manager for Chrome OS", + "fork": true, + "url": "https://api.github.com/repos/CAOakleyII/chromebrew", + "forks_url": "https://api.github.com/repos/CAOakleyII/chromebrew/forks", + "keys_url": "https://api.github.com/repos/CAOakleyII/chromebrew/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/CAOakleyII/chromebrew/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/CAOakleyII/chromebrew/teams", + "hooks_url": "https://api.github.com/repos/CAOakleyII/chromebrew/hooks", + "issue_events_url": "https://api.github.com/repos/CAOakleyII/chromebrew/issues/events{/number}", + "events_url": "https://api.github.com/repos/CAOakleyII/chromebrew/events", + "assignees_url": "https://api.github.com/repos/CAOakleyII/chromebrew/assignees{/user}", + "branches_url": "https://api.github.com/repos/CAOakleyII/chromebrew/branches{/branch}", + "tags_url": "https://api.github.com/repos/CAOakleyII/chromebrew/tags", + "blobs_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/CAOakleyII/chromebrew/statuses/{sha}", + "languages_url": "https://api.github.com/repos/CAOakleyII/chromebrew/languages", + "stargazers_url": "https://api.github.com/repos/CAOakleyII/chromebrew/stargazers", + "contributors_url": "https://api.github.com/repos/CAOakleyII/chromebrew/contributors", + "subscribers_url": "https://api.github.com/repos/CAOakleyII/chromebrew/subscribers", + "subscription_url": "https://api.github.com/repos/CAOakleyII/chromebrew/subscription", + "commits_url": "https://api.github.com/repos/CAOakleyII/chromebrew/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/CAOakleyII/chromebrew/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/CAOakleyII/chromebrew/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/CAOakleyII/chromebrew/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/CAOakleyII/chromebrew/contents/{+path}", + "compare_url": "https://api.github.com/repos/CAOakleyII/chromebrew/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/CAOakleyII/chromebrew/merges", + "archive_url": "https://api.github.com/repos/CAOakleyII/chromebrew/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/CAOakleyII/chromebrew/downloads", + "issues_url": "https://api.github.com/repos/CAOakleyII/chromebrew/issues{/number}", + "pulls_url": "https://api.github.com/repos/CAOakleyII/chromebrew/pulls{/number}", + "milestones_url": "https://api.github.com/repos/CAOakleyII/chromebrew/milestones{/number}", + "notifications_url": "https://api.github.com/repos/CAOakleyII/chromebrew/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/CAOakleyII/chromebrew/labels{/name}", + "releases_url": "https://api.github.com/repos/CAOakleyII/chromebrew/releases{/id}", + "created_at": "2015-07-01T00:00:00Z", + "updated_at": "2015-06-28T10:11:09Z", + "pushed_at": "2015-06-09T07:46:57Z", + "git_url": "git://github.com/CAOakleyII/chromebrew.git", + "ssh_url": "git@github.com:CAOakleyII/chromebrew.git", + "clone_url": "https://github.com/CAOakleyII/chromebrew.git", + "svn_url": "https://github.com/CAOakleyII/chromebrew", + "homepage": "http://skycocker.github.io/chromebrew/", + "size": 846, + "stargazers_count": 0, + "watchers_count": 0, + "language": null, + "has_issues": false, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "open_issues_count": 0, + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "public": true + } + }, + "public": true, + "created_at": "2015-07-01T00:00:01Z" + } ] \ No newline at end of file diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json index ad2fc3f159..180bd32670 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/dataset/file2.json @@ -1,7 +1,27 @@ [ - { "title" : "Hand of the King", "first_name": "Eddard", "last_name": "Stark"}, - { "title" : "Assassin", "first_name": "Arya", "last_name": "Stark"}, - { "title" : "Lady", "first_name": "Sansa", "last_name": "Stark"}, - { "title" : "Dancing Master", "first_name": "Syrio", "last_name": "Forel"}, - { "title" : "Dancing Master's Father", "first_name": "Nyrio", "last_name": "Forel"} + { + "title": "Hand of the King", + "first_name": "Eddard", + "last_name": "Stark" + }, + { + "title": "Assassin", + "first_name": "Arya", + "last_name": "Stark" + }, + { + "title": "Lady", + "first_name": "Sansa", + "last_name": "Stark" + }, + { + "title": "Dancing Master", + "first_name": "Syrio", + "last_name": "Forel" + }, + { + "title": "Dancing Master's Father", + "first_name": "Nyrio", + "last_name": "Forel" + } ] \ No newline at end of file From 0a7dfd012a7d4fb701348abdf5bc851d27c5e389 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Fri, 19 Aug 2016 11:52:04 +0530 Subject: [PATCH 86/94] Removed commented code --- tajo-storage/tajo-storage-mongodb/pom.xml | 35 ----------------------- 1 file changed, 35 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index fa8d43a3fd..28fb574622 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -38,11 +38,6 @@ - - - - - org.apache.rat apache-rat-plugin @@ -73,18 +68,6 @@ - - - - - - - - - - - - org.apache.maven.plugins @@ -154,20 +137,6 @@ maven-surefire-report-plugin - - - - - - - - - - - - - - @@ -284,10 +253,6 @@ - - - - org.mongodb From 82f6a42128361c194db7e3acfdf99d73176463b1 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Sun, 21 Aug 2016 15:10:05 +0900 Subject: [PATCH 87/94] Add test case for CTAS query --- .../apache/tajo/storage/mongodb/TestMongoDBQueryTest.java | 6 ++++++ .../resources/queries/TestMongoDBQueryTest/testCTAS.sql | 1 + 2 files changed, 7 insertions(+) create mode 100644 tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testCTAS.sql diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 4f80f273d3..0548b4100e 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -93,4 +93,10 @@ public void testJoin() throws Exception { public void testInsert() throws Exception { // runSimpleTests(); } + + @SimpleTest + @Test + public void testCTAS() throws Exception { + runSimpleTests(); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testCTAS.sql b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testCTAS.sql new file mode 100644 index 0000000000..ec1605a886 --- /dev/null +++ b/tajo-storage/tajo-storage-mongodb/src/test/resources/queries/TestMongoDBQueryTest/testCTAS.sql @@ -0,0 +1 @@ +CREATE TABLE got2(first_name text,last_name text) AS SELECT 'a','b'; \ No newline at end of file From f9ab536b8f90e76e8ca452a5292c67f76ad6a052 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Mon, 22 Aug 2016 12:04:47 +0530 Subject: [PATCH 88/94] Removed unnecessary code --- .../apache/tajo/storage/mongodb/TestConnectionInfo.java | 5 ----- .../apache/tajo/storage/mongodb/TestMetadataProvider.java | 7 +++++++ .../apache/tajo/storage/mongodb/TestMongoDBAppender.java | 6 ++++++ .../apache/tajo/storage/mongodb/TestMongoDBQueryTest.java | 2 +- .../apache/tajo/storage/mongodb/TestMongoDBTableSpace.java | 7 +++++++ 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index 57135203be..da3e68df1a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -53,9 +53,4 @@ public final void testGetConnectionInfoType2() { assertEquals(1, connInfo.getParams().size()); assertEquals("GMT+9", connInfo.getParams().get("TZ")); } - - @Test - public final void testNewStructure() { - - } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index 7a01439fa0..211d281b63 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -22,6 +22,7 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; +import org.junit.AfterClass; import org.junit.Test; import java.util.Set; @@ -32,6 +33,12 @@ public class TestMetadataProvider { static MongoDBTestServer server = MongoDBTestServer.getInstance(); + + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + @Test public void testGetTablespaceName() throws Exception { //check for Space Name validity diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java index 61b6b90796..02458a19cf 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java @@ -27,6 +27,7 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.*; import org.bson.Document; +import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@ -65,6 +66,11 @@ public void setup() throws Exception { } + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + @Test public void testAddTuple() throws IOException { diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 4f80f273d3..0e9791d0d6 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -41,7 +41,7 @@ public static void setup() throws Exception { @AfterClass public static void tearDownClass() throws Exception { - server.stop(); + // server.stop(); } @Before diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 802c6ff6df..5ec4c5f3b9 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -27,6 +27,7 @@ import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; +import org.junit.AfterClass; import org.junit.Test; import java.io.IOException; @@ -44,6 +45,12 @@ public class TestMongoDBTableSpace { static MongoDBTestServer server = MongoDBTestServer.getInstance(); static URI uri = server.getURI(); + + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + @Test public void testTablespaceHandler() { assertTrue((TablespaceManager.getByName(server.SPACE_NAME)) instanceof MongoDBTableSpace); From a5c956500b1fe22263109aa0d2e5cacfdb49f1a3 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Mon, 22 Aug 2016 14:56:20 +0530 Subject: [PATCH 89/94] Fixed indentation :) --- .../tajo/storage/mongodb/ConnectionInfo.java | 220 +++++------ .../tajo/storage/mongodb/MongoDBAppender.java | 142 +++---- .../mongodb/MongoDBCollectionReader.java | 62 +-- .../mongodb/MongoDBCollectionWriter.java | 50 +-- .../mongodb/MongoDBDocumentDeserializer.java | 358 +++++++++--------- .../mongodb/MongoDBDocumentSerializer.java | 120 +++--- .../tajo/storage/mongodb/MongoDBFragment.java | 12 +- .../storage/mongodb/MongoDBFragmentSerde.java | 44 +-- .../mongodb/MongoDBMetadataProvider.java | 126 +++--- .../tajo/storage/mongodb/MongoDBScanner.java | 172 ++++----- .../storage/mongodb/MongoDBTableSpace.java | 342 ++++++++--------- .../storage/mongodb/MongoDBTestServer.java | 282 +++++++------- .../storage/mongodb/TestConnectionInfo.java | 60 +-- .../storage/mongodb/TestMetadataProvider.java | 112 +++--- .../storage/mongodb/TestMongoDBAppender.java | 96 ++--- .../TestMongoDBDocumentSerializer.java | 100 ++--- .../storage/mongodb/TestMongoDBQueryTest.java | 122 +++--- .../mongodb/TestMongoDBTableSpace.java | 164 ++++---- 18 files changed, 1292 insertions(+), 1292 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java index 5623d69908..441b006a43 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/ConnectionInfo.java @@ -34,131 +34,131 @@ public class ConnectionInfo { - private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - - private MongoClientURI mongoDBURI; - private String scheme; - private String host; - private String dbName; - private String tableName; - private String user; - private String password; - private int port; - private Map params; - - //To create an instance using provided string of a URI - public static ConnectionInfo fromURI(String originalUri) { - return fromURI(URI.create(originalUri)); + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + + private MongoClientURI mongoDBURI; + private String scheme; + private String host; + private String dbName; + private String tableName; + private String user; + private String password; + private int port; + private Map params; + + //To create an instance using provided string of a URI + public static ConnectionInfo fromURI(String originalUri) { + return fromURI(URI.create(originalUri)); + } + + //creates a instance using provided URI + public static ConnectionInfo fromURI(URI originalUri) { + final String uriStr = originalUri.toASCIIString(); + URI uri = originalUri; + + final ConnectionInfo connInfo = new ConnectionInfo(); + connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); + + connInfo.host = uri.getHost(); + connInfo.port = uri.getPort(); + + //Set the db name + String path = uri.getPath(); + if (path != null && !path.isEmpty()) { + String[] pathElements = path.substring(1).split("/"); + if (pathElements.length != 1) { + throw new TajoInternalError("Invalid JDBC path: " + path); + } + connInfo.dbName = pathElements[0]; } - //creates a instance using provided URI - public static ConnectionInfo fromURI(URI originalUri) { - final String uriStr = originalUri.toASCIIString(); - URI uri = originalUri; - - final ConnectionInfo connInfo = new ConnectionInfo(); - connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); - - connInfo.host = uri.getHost(); - connInfo.port = uri.getPort(); - - //Set the db name - String path = uri.getPath(); - if (path != null && !path.isEmpty()) { - String[] pathElements = path.substring(1).split("/"); - if (pathElements.length != 1) { - throw new TajoInternalError("Invalid JDBC path: " + path); - } - connInfo.dbName = pathElements[0]; - } - - //Convert parms into a Map - Map params = new HashMap<>(); - int paramIndex = uriStr.indexOf("?"); - if (paramIndex > 0) { - String parameterPart = uriStr.substring(paramIndex + 1, uriStr.length()); - - String[] eachParam = parameterPart.split("&"); - - for (String each : eachParam) { - String[] keyValues = each.split("="); - if (keyValues.length != 2) { - throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); - } - params.put(keyValues[0], keyValues[1]); - } - } - - if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_TABLE)) { - connInfo.tableName = params.remove(MongoDBTableSpace.CONFIG_KEY_TABLE); - } - - if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_USERNAME)) { - connInfo.user = params.remove(MongoDBTableSpace.CONFIG_KEY_USERNAME); - } - if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_PASSWORD)) { - connInfo.password = params.remove(MongoDBTableSpace.CONFIG_KEY_PASSWORD); - } - - connInfo.params = params; + //Convert parms into a Map + Map params = new HashMap<>(); + int paramIndex = uriStr.indexOf("?"); + if (paramIndex > 0) { + String parameterPart = uriStr.substring(paramIndex + 1, uriStr.length()); - String mongoDbURIStr = ""; + String[] eachParam = parameterPart.split("&"); - //Generate the MongoURI - mongoDbURIStr += connInfo.getScheme(); - mongoDbURIStr += "://"; - if (connInfo.getUser() != null) { - mongoDbURIStr += connInfo.getUser(); - if (connInfo.getPassword() != null) - mongoDbURIStr += ":" + connInfo.getPassword(); - mongoDbURIStr += "@"; + for (String each : eachParam) { + String[] keyValues = each.split("="); + if (keyValues.length != 2) { + throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); } - mongoDbURIStr += connInfo.getHost(); - mongoDbURIStr += ":"; - mongoDbURIStr += connInfo.getPort(); - mongoDbURIStr += "/"; - mongoDbURIStr += connInfo.getDbName(); - - LOG.info(mongoDbURIStr); - connInfo.mongoDBURI = new MongoClientURI(mongoDbURIStr); - return connInfo; + params.put(keyValues[0], keyValues[1]); + } } - public MongoClientURI getMongoDBURI() { - return mongoDBURI; + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_TABLE)) { + connInfo.tableName = params.remove(MongoDBTableSpace.CONFIG_KEY_TABLE); } - public String getScheme() { - return scheme; + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_USERNAME)) { + connInfo.user = params.remove(MongoDBTableSpace.CONFIG_KEY_USERNAME); } - - public String getHost() { - return host; + if (params.containsKey(MongoDBTableSpace.CONFIG_KEY_PASSWORD)) { + connInfo.password = params.remove(MongoDBTableSpace.CONFIG_KEY_PASSWORD); } - public String getDbName() { - return dbName; - } + connInfo.params = params; - public String getTableName() { - return tableName; - } - - public String getUser() { - return user; - } - - public String getPassword() { - return password; - } - - public int getPort() { - return port; - } + String mongoDbURIStr = ""; - public Map getParams() { - return params; + //Generate the MongoURI + mongoDbURIStr += connInfo.getScheme(); + mongoDbURIStr += "://"; + if (connInfo.getUser() != null) { + mongoDbURIStr += connInfo.getUser(); + if (connInfo.getPassword() != null) + mongoDbURIStr += ":" + connInfo.getPassword(); + mongoDbURIStr += "@"; } + mongoDbURIStr += connInfo.getHost(); + mongoDbURIStr += ":"; + mongoDbURIStr += connInfo.getPort(); + mongoDbURIStr += "/"; + mongoDbURIStr += connInfo.getDbName(); + + LOG.info(mongoDbURIStr); + connInfo.mongoDBURI = new MongoClientURI(mongoDbURIStr); + return connInfo; + } + + public MongoClientURI getMongoDBURI() { + return mongoDBURI; + } + + public String getScheme() { + return scheme; + } + + public String getHost() { + return host; + } + + public String getDbName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public int getPort() { + return port; + } + + public Map getParams() { + return params; + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java index e3504bfd09..65334306e2 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java @@ -34,82 +34,82 @@ public class MongoDBAppender implements Appender { - //Given at constructor - private final Configuration conf; - private final Schema schema; - private final TableMeta meta; - private final Path stagingDir; - private final TaskAttemptId taskAttemptId; - private final URI uri; - - - protected boolean inited = false; - private boolean[] columnStatsEnabled; - private boolean tableStatsEnabled; - - private MongoDBCollectionWriter mongoDBCollectionWriter; - - - public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, - Schema schema, TableMeta meta, Path stagingDir, URI uri) { - this.conf = conf; - this.schema = schema; - this.meta = meta; - this.stagingDir = stagingDir; - this.taskAttemptId = taskAttemptId; - this.uri = stagingDir.toUri(); + //Given at constructor + private final Configuration conf; + private final Schema schema; + private final TableMeta meta; + private final Path stagingDir; + private final TaskAttemptId taskAttemptId; + private final URI uri; + + + protected boolean inited = false; + private boolean[] columnStatsEnabled; + private boolean tableStatsEnabled; + + private MongoDBCollectionWriter mongoDBCollectionWriter; + + + public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path stagingDir, URI uri) { + this.conf = conf; + this.schema = schema; + this.meta = meta; + this.stagingDir = stagingDir; + this.taskAttemptId = taskAttemptId; + this.uri = stagingDir.toUri(); + } + + @Override + public void init() throws IOException { + if (inited) { + throw new IllegalStateException("FileAppender is already initialized."); } - - @Override - public void init() throws IOException { - if (inited) { - throw new IllegalStateException("FileAppender is already initialized."); - } - inited = true; - MongoDBDocumentSerializer md = new MongoDBDocumentSerializer(schema, meta); - mongoDBCollectionWriter = new MongoDBCollectionWriter(ConnectionInfo.fromURI(stagingDir.toString()), md); - mongoDBCollectionWriter.init(); - } - - @Override - public void addTuple(Tuple t) throws IOException { - mongoDBCollectionWriter.addTuple(t); + inited = true; + MongoDBDocumentSerializer md = new MongoDBDocumentSerializer(schema, meta); + mongoDBCollectionWriter = new MongoDBCollectionWriter(ConnectionInfo.fromURI(stagingDir.toString()), md); + mongoDBCollectionWriter.init(); + } + + @Override + public void addTuple(Tuple t) throws IOException { + mongoDBCollectionWriter.addTuple(t); + } + + @Override + public void flush() throws IOException { + mongoDBCollectionWriter.write(); + } + + @Override + public long getEstimatedOutputSize() throws IOException { + throw new IOException(new NotImplementedException()); + } + + @Override + public void close() throws IOException { + mongoDBCollectionWriter.close(); + } + + @Override + public void enableStats() { + if (inited) { + throw new IllegalStateException("Should enable this option before init()"); } - @Override - public void flush() throws IOException { - mongoDBCollectionWriter.write(); - } + this.tableStatsEnabled = true; + this.columnStatsEnabled = new boolean[schema.size()]; - @Override - public long getEstimatedOutputSize() throws IOException { - throw new IOException(new NotImplementedException()); - } - - @Override - public void close() throws IOException { - mongoDBCollectionWriter.close(); - } + } - @Override - public void enableStats() { - if (inited) { - throw new IllegalStateException("Should enable this option before init()"); - } + @Override + public void enableStats(List columnList) { - this.tableStatsEnabled = true; - this.columnStatsEnabled = new boolean[schema.size()]; + } - } - - @Override - public void enableStats(List columnList) { - - } - - @Override - public TableStats getStats() { - TableStats stats = new TableStats(); - return stats; - } + @Override + public TableStats getStats() { + TableStats stats = new TableStats(); + return stats; + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index 4bb39813db..c1a7d45219 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -36,46 +36,46 @@ Used within the MongoScanner to read tuples. */ public class MongoDBCollectionReader { - private final CharsetEncoder encoder = CharsetUtil.getEncoder(CharsetUtil.UTF_8); - List documentList; - private ConnectionInfo connectionInfo; - private MongoDBDocumentDeserializer deserializer; - private int targetLength; - private int currentIndex; + private final CharsetEncoder encoder = CharsetUtil.getEncoder(CharsetUtil.UTF_8); + List documentList; + private ConnectionInfo connectionInfo; + private MongoDBDocumentDeserializer deserializer; + private int targetLength; + private int currentIndex; - public MongoDBCollectionReader(ConnectionInfo connectionInfo, MongoDBDocumentDeserializer deserializer, int targetLength) { - this.connectionInfo = connectionInfo; - this.deserializer = deserializer; - this.targetLength = targetLength; - } + public MongoDBCollectionReader(ConnectionInfo connectionInfo, MongoDBDocumentDeserializer deserializer, int targetLength) { + this.connectionInfo = connectionInfo; + this.deserializer = deserializer; + this.targetLength = targetLength; + } - public void init() throws IOException { - currentIndex = 0; - MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); - MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); + public void init() throws IOException { + currentIndex = 0; + MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); - MongoCollection collection = db.getCollection(connectionInfo.getTableName()); - documentList = (List) collection.find().into( - new ArrayList()); - ; + MongoCollection collection = db.getCollection(connectionInfo.getTableName()); + documentList = (List) collection.find().into( + new ArrayList()); + ; - deserializer.init(); - } + deserializer.init(); + } - public Tuple readTuple() throws IOException, TextLineParsingError { - if (currentIndex >= documentList.size()) return null; + public Tuple readTuple() throws IOException, TextLineParsingError { + if (currentIndex >= documentList.size()) return null; - Tuple outTuple = new VTuple(targetLength); + Tuple outTuple = new VTuple(targetLength); - deserializer.deserialize(documentList.get(currentIndex), outTuple); - currentIndex++; - return outTuple; + deserializer.deserialize(documentList.get(currentIndex), outTuple); + currentIndex++; + return outTuple; - } + } - public float getProgress() { - return ((float) currentIndex) / documentList.size(); - } + public float getProgress() { + return ((float) currentIndex) / documentList.size(); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java index ffb4565517..46e9a8d4b1 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java @@ -31,35 +31,35 @@ * Writes tuples into a mongodb collection */ public class MongoDBCollectionWriter { - MongoClient mongoClient; - MongoCollection collection; - MongoDBDocumentSerializer mongoDBDocumentSerializer; - private ConnectionInfo connectionInfo; - private List documentList = new ArrayList<>(); + MongoClient mongoClient; + MongoCollection collection; + MongoDBDocumentSerializer mongoDBDocumentSerializer; + private ConnectionInfo connectionInfo; + private List documentList = new ArrayList<>(); - public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer serializer) { - this.connectionInfo = connectionInfo; - this.mongoDBDocumentSerializer = serializer; - } + public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer serializer) { + this.connectionInfo = connectionInfo; + this.mongoDBDocumentSerializer = serializer; + } - public void init() { - mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); - MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); - collection = db.getCollection(connectionInfo.getTableName()); - } + public void init() { + mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); + collection = db.getCollection(connectionInfo.getTableName()); + } - public void addTuple(Tuple tuple) throws IOException { - Document dc = new Document(); - mongoDBDocumentSerializer.serialize(tuple, dc); - documentList.add(dc); - } + public void addTuple(Tuple tuple) throws IOException { + Document dc = new Document(); + mongoDBDocumentSerializer.serialize(tuple, dc); + documentList.add(dc); + } - public void write() { - collection.insertMany(documentList); - } + public void write() { + collection.insertMany(documentList); + } - public void close() { - mongoClient.close(); - } + public void close() { + mongoClient.close(); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java index 03772585a0..61a4bf651f 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentDeserializer.java @@ -48,196 +48,196 @@ * */ public class MongoDBDocumentDeserializer { - protected final Schema schema; - protected final TableMeta meta; - // Full Path -> Type - private final Map types; - private final String[] projectedPaths; - private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); - private final TimeZone timezone; - private JSONParser parser; - - public MongoDBDocumentDeserializer(Schema schema, TableMeta meta, Column[] projected) { - this.schema = schema; - this.meta = meta; - - projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true); - types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); - - timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, - StorageUtil.TAJO_CONF.getSystemTimezone().getID())); + protected final Schema schema; + protected final TableMeta meta; + // Full Path -> Type + private final Map types; + private final String[] projectedPaths; + private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); + private final TimeZone timezone; + private JSONParser parser; + + public MongoDBDocumentDeserializer(Schema schema, TableMeta meta, Column[] projected) { + this.schema = schema; + this.meta = meta; + + projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true); + types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); + + timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, + StorageUtil.TAJO_CONF.getSystemTimezone().getID())); + } + + + //Initialize and setup parser + public void init() { + parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR); + } + + /** + * @param object + * @param pathElements + * @param depth + * @param fieldIndex + * @param output + * @throws IOException + */ + private void getValue(JSONObject object, + String fullPath, + String[] pathElements, + int depth, + int fieldIndex, + Tuple output) throws IOException { + String fieldName = pathElements[depth]; + + if (!object.containsKey(fieldName)) { + output.put(fieldIndex, NullDatum.get()); } + switch (types.get(fullPath)) { + case BOOLEAN: + String boolStr = object.getAsString(fieldName); + if (boolStr != null) { + output.put(fieldIndex, DatumFactory.createBool(boolStr.equals("true"))); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case CHAR: + String charStr = object.getAsString(fieldName); + if (charStr != null) { + output.put(fieldIndex, DatumFactory.createChar(charStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT1: + case INT2: + Number int2Num = object.getAsNumber(fieldName); + if (int2Num != null) { + output.put(fieldIndex, DatumFactory.createInt2(int2Num.shortValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT4: + Number int4Num = object.getAsNumber(fieldName); + if (int4Num != null) { + output.put(fieldIndex, DatumFactory.createInt4(int4Num.intValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case INT8: + Number int8Num = object.getAsNumber(fieldName); + if (int8Num != null) { + output.put(fieldIndex, DatumFactory.createInt8(int8Num.longValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case FLOAT4: + Number float4Num = object.getAsNumber(fieldName); + if (float4Num != null) { + output.put(fieldIndex, DatumFactory.createFloat4(float4Num.floatValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case FLOAT8: + Number float8Num = object.getAsNumber(fieldName); + if (float8Num != null) { + output.put(fieldIndex, DatumFactory.createFloat8(float8Num.doubleValue())); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TEXT: + String textStr = object.getAsString(fieldName); + if (textStr != null) { + output.put(fieldIndex, DatumFactory.createText(textStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TIMESTAMP: + String timestampStr = object.getAsString(fieldName); + if (timestampStr != null) { + output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr, timezone)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case TIME: + String timeStr = object.getAsString(fieldName); + if (timeStr != null) { + output.put(fieldIndex, DatumFactory.createTime(timeStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case DATE: + String dateStr = object.getAsString(fieldName); + if (dateStr != null) { + output.put(fieldIndex, DatumFactory.createDate(dateStr)); + } else { + output.put(fieldIndex, NullDatum.get()); + } + break; + case BIT: + case BINARY: + case VARBINARY: + case BLOB: { + Object jsonObject = object.getAsString(fieldName); + + if (jsonObject == null) { + output.put(fieldIndex, NullDatum.get()); + break; + } - //Initialize and setup parser - public void init() { - parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR); - } + output.put(fieldIndex, DatumFactory.createBlob(Base64.decodeBase64((String) jsonObject))); + break; + } - /** - * @param object - * @param pathElements - * @param depth - * @param fieldIndex - * @param output - * @throws IOException - */ - private void getValue(JSONObject object, - String fullPath, - String[] pathElements, - int depth, - int fieldIndex, - Tuple output) throws IOException { - String fieldName = pathElements[depth]; - - if (!object.containsKey(fieldName)) { - output.put(fieldIndex, NullDatum.get()); + case RECORD: + JSONObject nestedObject = (JSONObject) object.get(fieldName); + if (nestedObject != null) { + getValue(nestedObject, fullPath + "/" + pathElements[depth + 1], pathElements, depth + 1, fieldIndex, output); + } else { + output.put(fieldIndex, NullDatum.get()); } + break; - switch (types.get(fullPath)) { - case BOOLEAN: - String boolStr = object.getAsString(fieldName); - if (boolStr != null) { - output.put(fieldIndex, DatumFactory.createBool(boolStr.equals("true"))); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case CHAR: - String charStr = object.getAsString(fieldName); - if (charStr != null) { - output.put(fieldIndex, DatumFactory.createChar(charStr)); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case INT1: - case INT2: - Number int2Num = object.getAsNumber(fieldName); - if (int2Num != null) { - output.put(fieldIndex, DatumFactory.createInt2(int2Num.shortValue())); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case INT4: - Number int4Num = object.getAsNumber(fieldName); - if (int4Num != null) { - output.put(fieldIndex, DatumFactory.createInt4(int4Num.intValue())); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case INT8: - Number int8Num = object.getAsNumber(fieldName); - if (int8Num != null) { - output.put(fieldIndex, DatumFactory.createInt8(int8Num.longValue())); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case FLOAT4: - Number float4Num = object.getAsNumber(fieldName); - if (float4Num != null) { - output.put(fieldIndex, DatumFactory.createFloat4(float4Num.floatValue())); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case FLOAT8: - Number float8Num = object.getAsNumber(fieldName); - if (float8Num != null) { - output.put(fieldIndex, DatumFactory.createFloat8(float8Num.doubleValue())); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case TEXT: - String textStr = object.getAsString(fieldName); - if (textStr != null) { - output.put(fieldIndex, DatumFactory.createText(textStr)); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case TIMESTAMP: - String timestampStr = object.getAsString(fieldName); - if (timestampStr != null) { - output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr, timezone)); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case TIME: - String timeStr = object.getAsString(fieldName); - if (timeStr != null) { - output.put(fieldIndex, DatumFactory.createTime(timeStr)); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case DATE: - String dateStr = object.getAsString(fieldName); - if (dateStr != null) { - output.put(fieldIndex, DatumFactory.createDate(dateStr)); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - case BIT: - case BINARY: - case VARBINARY: - case BLOB: { - Object jsonObject = object.getAsString(fieldName); - - if (jsonObject == null) { - output.put(fieldIndex, NullDatum.get()); - break; - } - - output.put(fieldIndex, DatumFactory.createBlob(Base64.decodeBase64((String) jsonObject))); - break; - } - - case RECORD: - JSONObject nestedObject = (JSONObject) object.get(fieldName); - if (nestedObject != null) { - getValue(nestedObject, fullPath + "/" + pathElements[depth + 1], pathElements, depth + 1, fieldIndex, output); - } else { - output.put(fieldIndex, NullDatum.get()); - } - break; - - case NULL_TYPE: - output.put(fieldIndex, NullDatum.get()); - break; - - default: - throw new TajoRuntimeException( - new NotImplementedException("" + types.get(fullPath).name() + " for json")); - } + case NULL_TYPE: + output.put(fieldIndex, NullDatum.get()); + break; + + default: + throw new TajoRuntimeException( + new NotImplementedException("" + types.get(fullPath).name() + " for json")); } + } - //Map the given documents into the given tuple and fill with calues - public void deserialize(Document doc, Tuple output) throws IOException, TextLineParsingError { + //Map the given documents into the given tuple and fill with calues + public void deserialize(Document doc, Tuple output) throws IOException, TextLineParsingError { - String line = doc.toJson(); - JSONObject object; - try { - object = (JSONObject) parser.parse(line); - } catch (ParseException pe) { - throw new TextLineParsingError(line, pe); - } catch (ArrayIndexOutOfBoundsException ae) { - // truncated value - throw new TextLineParsingError(line, ae); - } + String line = doc.toJson(); + JSONObject object; + try { + object = (JSONObject) parser.parse(line); + } catch (ParseException pe) { + throw new TextLineParsingError(line, pe); + } catch (ArrayIndexOutOfBoundsException ae) { + // truncated value + throw new TextLineParsingError(line, ae); + } - for (int i = 0; i < projectedPaths.length; i++) { - String[] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER); - getValue(object, paths[0], paths, 0, i, output); + for (int i = 0; i < projectedPaths.length; i++) { + String[] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER); + getValue(object, paths[0], paths, 0, i, output); - //output.put(i, DatumFactory.createText( doc.get(projectedPaths[i]).toString() )); - } + //output.put(i, DatumFactory.createText( doc.get(projectedPaths[i]).toString() )); } + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java index 9a123e1ed0..11a2ea8547 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java @@ -34,69 +34,69 @@ */ public class MongoDBDocumentSerializer { - private final Schema schema; - private final TableMeta meta; - private final Map types; - String[] paths; - - public MongoDBDocumentSerializer(Schema schema, TableMeta meta) { - this.schema = schema; - this.meta = meta; - - paths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(schema.getAllColumns()), true); - types = SchemaUtil.buildTypeMap(schema.getAllColumns(), paths); - } - - public void setValue(Tuple inputTuple, String fullPath, String[] paths, int depth, int index, Document outputDoc) { - - - String simpleColumnName = paths[depth]; - switch (types.get(fullPath)) { - case INT1: - case UINT1: - case INT2: - case UINT2: - outputDoc.put(simpleColumnName, inputTuple.getInt2(index)); - break; - - case INT4: - case UINT4: - outputDoc.put(simpleColumnName, inputTuple.getInt4(index)); - break; - - case INT8: - case UINT8: - outputDoc.put(simpleColumnName, inputTuple.getInt8(index)); - break; - - case FLOAT4: - outputDoc.put(simpleColumnName, inputTuple.getFloat4(index)); - break; - - case FLOAT8: - outputDoc.put(simpleColumnName, inputTuple.getFloat8(index)); - break; - - case CHAR: - outputDoc.put(simpleColumnName, inputTuple.getChar(index)); - break; - - case BOOLEAN: - outputDoc.put(simpleColumnName, inputTuple.getBool(index)); - break; - - default: - outputDoc.put(simpleColumnName, inputTuple.getText(index)); - break; - } + private final Schema schema; + private final TableMeta meta; + private final Map types; + String[] paths; + + public MongoDBDocumentSerializer(Schema schema, TableMeta meta) { + this.schema = schema; + this.meta = meta; + + paths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(schema.getAllColumns()), true); + types = SchemaUtil.buildTypeMap(schema.getAllColumns(), paths); + } + + public void setValue(Tuple inputTuple, String fullPath, String[] paths, int depth, int index, Document outputDoc) { + + + String simpleColumnName = paths[depth]; + switch (types.get(fullPath)) { + case INT1: + case UINT1: + case INT2: + case UINT2: + outputDoc.put(simpleColumnName, inputTuple.getInt2(index)); + break; + + case INT4: + case UINT4: + outputDoc.put(simpleColumnName, inputTuple.getInt4(index)); + break; + + case INT8: + case UINT8: + outputDoc.put(simpleColumnName, inputTuple.getInt8(index)); + break; + + case FLOAT4: + outputDoc.put(simpleColumnName, inputTuple.getFloat4(index)); + break; + + case FLOAT8: + outputDoc.put(simpleColumnName, inputTuple.getFloat8(index)); + break; + + case CHAR: + outputDoc.put(simpleColumnName, inputTuple.getChar(index)); + break; + + case BOOLEAN: + outputDoc.put(simpleColumnName, inputTuple.getBool(index)); + break; + + default: + outputDoc.put(simpleColumnName, inputTuple.getText(index)); + break; } + } - public void serialize(Tuple inputTuple, Document outputDoc) throws IOException { - for (int i = 0; i < inputTuple.size(); i++) { - String[] newPaths = paths[i].split(NestedPathUtil.PATH_DELIMITER); - setValue(inputTuple, newPaths[0], newPaths, 0, i, outputDoc); - } + public void serialize(Tuple inputTuple, Document outputDoc) throws IOException { + for (int i = 0; i < inputTuple.size(); i++) { + String[] newPaths = paths[i].split(NestedPathUtil.PATH_DELIMITER); + setValue(inputTuple, newPaths[0], newPaths, 0, i, outputDoc); } + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java index babe7c8af6..3775f32ed7 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragment.java @@ -30,10 +30,10 @@ public class MongoDBFragment extends Fragment { - protected MongoDBFragment(URI uri, - String inputSourceId, - long startKey, - long endKey) { - super("MONGODB", uri, inputSourceId, startKey, endKey, endKey - startKey, null); - } + protected MongoDBFragment(URI uri, + String inputSourceId, + long startKey, + long endKey) { + super("MONGODB", uri, inputSourceId, startKey, endKey, endKey - startKey, null); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java index 9099d8e38c..20a7c3c282 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBFragmentSerde.java @@ -30,28 +30,28 @@ */ public class MongoDBFragmentSerde implements FragmentSerde { - @Override - public Builder newBuilder() { - return MongoDBFragmentProto.newBuilder(); - } + @Override + public Builder newBuilder() { + return MongoDBFragmentProto.newBuilder(); + } - @Override - public MongoDBFragmentProto serialize(MongoDBFragment fragment) { - return MongoDBFragmentProto.newBuilder() - .setUri(fragment.getUri().toASCIIString()) - .setTableName(fragment.getInputSourceId()) - .setStartKey(fragment.getStartKey()) - .setEndKey(fragment.getEndKey()) - .build(); - } + @Override + public MongoDBFragmentProto serialize(MongoDBFragment fragment) { + return MongoDBFragmentProto.newBuilder() + .setUri(fragment.getUri().toASCIIString()) + .setTableName(fragment.getInputSourceId()) + .setStartKey(fragment.getStartKey()) + .setEndKey(fragment.getEndKey()) + .build(); + } - @Override - public MongoDBFragment deserialize(MongoDBFragmentProto proto) { - return new MongoDBFragment( - URI.create(proto.getUri()), - proto.getTableName(), - proto.getStartKey(), - proto.getEndKey() - ); - } + @Override + public MongoDBFragment deserialize(MongoDBFragmentProto proto) { + return new MongoDBFragment( + URI.create(proto.getUri()), + proto.getTableName(), + proto.getStartKey(), + proto.getEndKey() + ); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java index 95eb5ce443..3908e0701f 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBMetadataProvider.java @@ -42,77 +42,77 @@ * * */ public class MongoDBMetadataProvider implements MetadataProvider { - MongoDatabase db; - private MongoDBTableSpace tableSpace; - private String mappedDbName; - private ConnectionInfo connectionInfo; - - public MongoDBMetadataProvider(MongoDBTableSpace tableSpace, String dbName) { - this.tableSpace = tableSpace; - this.mappedDbName = dbName; - - connectionInfo = tableSpace.getConnectionInfo(); - MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); - db = mongoClient.getDatabase(connectionInfo.getDbName()); + MongoDatabase db; + private MongoDBTableSpace tableSpace; + private String mappedDbName; + private ConnectionInfo connectionInfo; + + public MongoDBMetadataProvider(MongoDBTableSpace tableSpace, String dbName) { + this.tableSpace = tableSpace; + this.mappedDbName = dbName; + + connectionInfo = tableSpace.getConnectionInfo(); + MongoClient mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); + db = mongoClient.getDatabase(connectionInfo.getDbName()); + } + + @Override + public String getTablespaceName() { + return tableSpace.getName(); + } + + @Override + public URI getTablespaceUri() { + return tableSpace.getUri(); + } + + @Override + public String getDatabaseName() { + return mappedDbName; + } + + @Override + public Collection getSchemas() { + return Collections.EMPTY_SET; + } + + @Override + public Collection getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { + + //Get a list of table(=collection) names + MongoIterable collectionList = db.listCollectionNames(); + + //Map to a string list and return + Collection list = new ArrayList(); + for (String item : collectionList) { + list.add(item); } + return list; + } - @Override - public String getTablespaceName() { - return tableSpace.getName(); - } - - @Override - public URI getTablespaceUri() { - return tableSpace.getUri(); - } - - @Override - public String getDatabaseName() { - return mappedDbName; - } - - @Override - public Collection getSchemas() { - return Collections.EMPTY_SET; - } + @Override + public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException { - @Override - public Collection getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { - - //Get a list of table(=collection) names - MongoIterable collectionList = db.listCollectionNames(); - - //Map to a string list and return - Collection list = new ArrayList(); - for (String item : collectionList) { - list.add(item); - } - return list; - } - - @Override - public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException { - - //Create table description and meta fro a specific table - TableMeta tbMeta = new TableMeta("mongodb", new KeyValueSet()); - TableDesc tbDesc = new TableDesc( - IdentifierUtil.buildFQName(mappedDbName, tableName), - SchemaBuilder.builder() + //Create table description and meta fro a specific table + TableMeta tbMeta = new TableMeta("mongodb", new KeyValueSet()); + TableDesc tbDesc = new TableDesc( + IdentifierUtil.buildFQName(mappedDbName, tableName), + SchemaBuilder.builder() // .add(new Column("title", TajoDataTypes.Type.TEXT)) // .add(new Column("first_name", TajoDataTypes.Type.TEXT)) // .add(new Column("last_name", TajoDataTypes.Type.TEXT)) - .build(), - tbMeta, - tableSpace.getTableUri(null, null, tableName)); + .build(), + tbMeta, + tableSpace.getTableUri(null, null, tableName)); - final TableStats stats = new TableStats(); - stats.setNumRows(-1); // unknown + final TableStats stats = new TableStats(); + stats.setNumRows(-1); // unknown - //Set the raw count - stats.setNumRows(db.getCollection(tableName).count()); + //Set the raw count + stats.setNumRows(db.getCollection(tableName).count()); - tbDesc.setStats(stats); - return tbDesc; - } + tbDesc.setStats(stats); + return tbDesc; + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index 0f7c8cae1e..faa2ba53eb 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -41,104 +41,104 @@ //Todo Remove FileScanner public class MongoDBScanner implements Scanner { - private final TableMeta meta; - private final Schema schema; - private final MongoDBFragment fragment; - private MongoDBCollectionReader collectionReader; - private Column[] targets; - - private boolean inited; - - public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { - this.schema = schema; - this.meta = meta; - this.fragment = (MongoDBFragment) fragment; + private final TableMeta meta; + private final Schema schema; + private final MongoDBFragment fragment; + private MongoDBCollectionReader collectionReader; + private Column[] targets; + + private boolean inited; + + public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { + this.schema = schema; + this.meta = meta; + this.fragment = (MongoDBFragment) fragment; + } + + @Override + public void init() throws IOException { + if (targets == null) { + targets = schema.toArray(); } - @Override - public void init() throws IOException { - if (targets == null) { - targets = schema.toArray(); - } - - reset(); - } - - @Override - public Tuple next() throws IOException { - try { - Tuple t = collectionReader.readTuple(); - return t; - } catch (TextLineParsingError textLineParsingError) { - textLineParsingError.printStackTrace(); - return null; - } - } - - @Override - public void reset() throws IOException { - MongoDBDocumentDeserializer deserializer = new MongoDBDocumentDeserializer(schema, meta, targets); - collectionReader = new MongoDBCollectionReader(ConnectionInfo.fromURI(fragment.getUri()), deserializer, targets.length); - - collectionReader.init(); - } - - @Override - public void close() throws IOException { - return; - } - - @Override - public void pushOperators(LogicalNode planPart) { - throw new TajoRuntimeException(new UnsupportedException()); + reset(); + } + + @Override + public Tuple next() throws IOException { + try { + Tuple t = collectionReader.readTuple(); + return t; + } catch (TextLineParsingError textLineParsingError) { + textLineParsingError.printStackTrace(); + return null; } - - @Override - public boolean isProjectable() { - return false; + } + + @Override + public void reset() throws IOException { + MongoDBDocumentDeserializer deserializer = new MongoDBDocumentDeserializer(schema, meta, targets); + collectionReader = new MongoDBCollectionReader(ConnectionInfo.fromURI(fragment.getUri()), deserializer, targets.length); + + collectionReader.init(); + } + + @Override + public void close() throws IOException { + return; + } + + @Override + public void pushOperators(LogicalNode planPart) { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public boolean isProjectable() { + return false; + } + + @Override + public void setTarget(Column[] targets) { + if (inited) { + throw new IllegalStateException("Should be called before init()"); } + this.targets = targets; + } - @Override - public void setTarget(Column[] targets) { - if (inited) { - throw new IllegalStateException("Should be called before init()"); - } - this.targets = targets; - } + @Override + public boolean isSelectable() { + return false; + } - @Override - public boolean isSelectable() { - return false; - } + @Override + public void setFilter(EvalNode filter) { - @Override - public void setFilter(EvalNode filter) { - - } + } - @Override - public void setLimit(long num) { + @Override + public void setLimit(long num) { - } + } - @Override - public boolean isSplittable() { - return false; - } + @Override + public boolean isSplittable() { + return false; + } - @Override - public float getProgress() { - return collectionReader.getProgress(); - } + @Override + public float getProgress() { + return collectionReader.getProgress(); + } - @Override - public TableStats getInputStats() { - return null; - } + @Override + public TableStats getInputStats() { + return null; + } - @Override - public Schema getSchema() { - return schema; - } + @Override + public Schema getSchema() { + return schema; + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 2e4dc089ba..0f204968f4 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -54,163 +54,163 @@ public class MongoDBTableSpace extends Tablespace { - //Config Keys - public static final String CONFIG_KEY_MAPPED_DATABASE = "mapped_database"; - public static final String CONFIG_KEY_CONN_PROPERTIES = "connection_properties"; - public static final String CONFIG_KEY_USERNAME = "user"; - public static final String CONFIG_KEY_PASSWORD = "password"; - public static final String CONFIG_KEY_TABLE = "table"; - //Table Space Properties - static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined - false, //not movable - true, //writable at the moment - true, // Absolute path - true); // Meta data will be provided - static final FormatProperty FORMAT_PROPERTY = new FormatProperty( - true, // Insert - true, //direct insert - true);// result staging - private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - protected MongoClient mongoClient; - protected MongoDatabase db; - protected String mappedDBName; - //Mongo Client object - private ConnectionInfo connectionInfo; - - public MongoDBTableSpace(String name, URI uri, JSONObject config) { - - super(name, uri, config); - connectionInfo = ConnectionInfo.fromURI(uri); - - //set Connection Properties - if (config.containsKey(CONFIG_KEY_MAPPED_DATABASE)) { - mappedDBName = this.config.getAsString(CONFIG_KEY_MAPPED_DATABASE); - } else { - mappedDBName = getConnectionInfo().getDbName(); - } - } - - @Override - protected void storageInit() throws IOException { - //Todo Extract User Details from Configuration - try { - connectionInfo = ConnectionInfo.fromURI(uri); - mongoClient = new MongoClient(getConnectionInfo().getMongoDBURI()); - db = mongoClient.getDatabase(getConnectionInfo().getDbName()); - } catch (Exception e) { - throw new TajoInternalError(e); - } - } - - @Override - public long getTableVolume(TableDesc table, Optional filter) { - - long count = 0; - try { - String[] nameSplited = IdentifierUtil.splitFQTableName(table.getName()); - count = db.getCollection(nameSplited[1]).count(); - } catch (Exception e) { - throw new TajoInternalError(e); - } - return count; - } - - - @Override - public List getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { - long tableVolume = getTableVolume(tableDesc, Optional.empty()); - MongoDBFragment mongoDBFragment = new MongoDBFragment(tableDesc.getUri(), inputSourceId, 0, tableVolume); - return Lists.newArrayList(mongoDBFragment); - } - - - @Override - public StorageProperty getProperty() { - return STORAGE_PROPERTY; - } - - @Override - public FormatProperty getFormatProperty(TableMeta meta) { - return FORMAT_PROPERTY; - } - - @Override - public void close() { - - } - - @Override - public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException { - return new TupleRange[0]; - } - - @Override - public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws TajoException { - - } - - @Override - public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { - if (tableDesc == null) - throw new TajoRuntimeException(new NotImplementedException()); - MongoCollection table = db.getCollection(tableDesc.getName()); - - //TODO Handle this here. If empty throw exception or what? - boolean ifExist = (table.count() > 0) ? true : false; - - //If meta data provides. Create a table - if (STORAGE_PROPERTY.isMetadataProvided()) - db.createCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())); - } - - @Override - public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { - if (STORAGE_PROPERTY.isMetadataProvided()) - db.getCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())).drop(); - } - - @Override - public void prepareTable(LogicalNode node) throws IOException, TajoException { - return; - } - - @Override - public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException { - return null; - } - - @Override - public void rollbackTable(LogicalNode node) throws IOException, TajoException { - - } - - @Override - public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { - return null; - } - - @Override - public URI getRootUri() { - return uri; - } - - @Override - public URI getTableUri(TableMeta meta, String databaseName, String tableName) { - //ToDo Find a better way this - String tableURI = ""; - if (this.getUri().toASCIIString().contains("?")) - tableURI = this.getUri().toASCIIString() + "&" + CONFIG_KEY_TABLE + "=" + tableName; - else - tableURI = this.getUri().toASCIIString() + "?" + CONFIG_KEY_TABLE + "=" + tableName; - - return URI.create(tableURI); - } - - //@Override - public URI getTableUri(String databaseName, String tableName) { - //ToDo set the TableURI properly - return URI.create(this.getUri() + "&" + CONFIG_KEY_TABLE + "=" + tableName); - } + //Config Keys + public static final String CONFIG_KEY_MAPPED_DATABASE = "mapped_database"; + public static final String CONFIG_KEY_CONN_PROPERTIES = "connection_properties"; + public static final String CONFIG_KEY_USERNAME = "user"; + public static final String CONFIG_KEY_PASSWORD = "password"; + public static final String CONFIG_KEY_TABLE = "table"; + //Table Space Properties + static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", // type is to be defined + false, //not movable + true, //writable at the moment + true, // Absolute path + true); // Meta data will be provided + static final FormatProperty FORMAT_PROPERTY = new FormatProperty( + true, // Insert + true, //direct insert + true);// result staging + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + protected MongoClient mongoClient; + protected MongoDatabase db; + protected String mappedDBName; + //Mongo Client object + private ConnectionInfo connectionInfo; + + public MongoDBTableSpace(String name, URI uri, JSONObject config) { + + super(name, uri, config); + connectionInfo = ConnectionInfo.fromURI(uri); + + //set Connection Properties + if (config.containsKey(CONFIG_KEY_MAPPED_DATABASE)) { + mappedDBName = this.config.getAsString(CONFIG_KEY_MAPPED_DATABASE); + } else { + mappedDBName = getConnectionInfo().getDbName(); + } + } + + @Override + protected void storageInit() throws IOException { + //Todo Extract User Details from Configuration + try { + connectionInfo = ConnectionInfo.fromURI(uri); + mongoClient = new MongoClient(getConnectionInfo().getMongoDBURI()); + db = mongoClient.getDatabase(getConnectionInfo().getDbName()); + } catch (Exception e) { + throw new TajoInternalError(e); + } + } + + @Override + public long getTableVolume(TableDesc table, Optional filter) { + + long count = 0; + try { + String[] nameSplited = IdentifierUtil.splitFQTableName(table.getName()); + count = db.getCollection(nameSplited[1]).count(); + } catch (Exception e) { + throw new TajoInternalError(e); + } + return count; + } + + + @Override + public List getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { + long tableVolume = getTableVolume(tableDesc, Optional.empty()); + MongoDBFragment mongoDBFragment = new MongoDBFragment(tableDesc.getUri(), inputSourceId, 0, tableVolume); + return Lists.newArrayList(mongoDBFragment); + } + + + @Override + public StorageProperty getProperty() { + return STORAGE_PROPERTY; + } + + @Override + public FormatProperty getFormatProperty(TableMeta meta) { + return FORMAT_PROPERTY; + } + + @Override + public void close() { + + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException { + return new TupleRange[0]; + } + + @Override + public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws TajoException { + + } + + @Override + public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { + if (tableDesc == null) + throw new TajoRuntimeException(new NotImplementedException()); + MongoCollection table = db.getCollection(tableDesc.getName()); + + //TODO Handle this here. If empty throw exception or what? + boolean ifExist = (table.count() > 0) ? true : false; + + //If meta data provides. Create a table + if (STORAGE_PROPERTY.isMetadataProvided()) + db.createCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())); + } + + @Override + public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { + if (STORAGE_PROPERTY.isMetadataProvided()) + db.getCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())).drop(); + } + + @Override + public void prepareTable(LogicalNode node) throws IOException, TajoException { + return; + } + + @Override + public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException { + return null; + } + + @Override + public void rollbackTable(LogicalNode node) throws IOException, TajoException { + + } + + @Override + public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { + return null; + } + + @Override + public URI getRootUri() { + return uri; + } + + @Override + public URI getTableUri(TableMeta meta, String databaseName, String tableName) { + //ToDo Find a better way this + String tableURI = ""; + if (this.getUri().toASCIIString().contains("?")) + tableURI = this.getUri().toASCIIString() + "&" + CONFIG_KEY_TABLE + "=" + tableName; + else + tableURI = this.getUri().toASCIIString() + "?" + CONFIG_KEY_TABLE + "=" + tableName; + + return URI.create(tableURI); + } + + //@Override + public URI getTableUri(String databaseName, String tableName) { + //ToDo set the TableURI properly + return URI.create(this.getUri() + "&" + CONFIG_KEY_TABLE + "=" + tableName); + } // @Override // public URI getTableUri(String databaseName, String tableName) { @@ -219,22 +219,22 @@ public URI getTableUri(String databaseName, String tableName) { // } - // Metadata - public MetadataProvider getMetadataProvider() { - return new MongoDBMetadataProvider(this, mappedDBName); - } + // Metadata + public MetadataProvider getMetadataProvider() { + return new MongoDBMetadataProvider(this, mappedDBName); + } - public ConnectionInfo getConnectionInfo() { - return connectionInfo; - } + public ConnectionInfo getConnectionInfo() { + return connectionInfo; + } - //ToDo Make Sure this is not an issue - @Override - public Appender getAppender(OverridableConf queryContext, - TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) + //ToDo Make Sure this is not an issue + @Override + public Appender getAppender(OverridableConf queryContext, + TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) - { - return new MongoDBAppender(null, taskAttemptId, schema, meta, workDir, workDir.toUri()); - } + { + return new MongoDBAppender(null, taskAttemptId, schema, meta, workDir, workDir.toUri()); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java index e8a4c81f59..e47b3d4411 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/MongoDBTestServer.java @@ -49,160 +49,160 @@ public class MongoDBTestServer { - //mongo server details - public static final int PORT = 12345; - public static final String HOST = "localhost"; - public static final String DBNAME = "test_dbname"; - public static final String MAPPEDDBNAME = "test_mapped_dbname"; - //tajo tableSpace name - public static final String SPACE_NAME = "test_spacename"; - private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); - //Mongo Server componenets - private static final MongodStarter starter = MongodStarter.getDefaultInstance(); - // instance for singleton server - private static MongoDBTestServer instance; - //Collection names (table names) to be created inside mongodb - public String[] collectionNames = {"github", "got"}; - private MongodExecutable _mongodExe; - private MongodProcess _mongod; - private MongoClient _mongo; - //File names to load data - private String[] filenames = {"file1.json", "file2.json"}; - - - // private constructor - private MongoDBTestServer() throws IOException, URISyntaxException { - _mongodExe = starter.prepare(new MongodConfigBuilder() - .version(Version.Main.PRODUCTION) - .net(new Net(PORT, Network.localhostIsIPv6())) - - .cmdOptions(new MongoCmdOptionsBuilder(). - useStorageEngine("mmapv1"). - build()) - .build()); - _mongod = _mongodExe.start(); - _mongo = new MongoClient(HOST, PORT); - registerTablespace(); - - loadData(); + //mongo server details + public static final int PORT = 12345; + public static final String HOST = "localhost"; + public static final String DBNAME = "test_dbname"; + public static final String MAPPEDDBNAME = "test_mapped_dbname"; + //tajo tableSpace name + public static final String SPACE_NAME = "test_spacename"; + private static final Log LOG = LogFactory.getLog(MongoDBTableSpace.class); + //Mongo Server componenets + private static final MongodStarter starter = MongodStarter.getDefaultInstance(); + // instance for singleton server + private static MongoDBTestServer instance; + //Collection names (table names) to be created inside mongodb + public String[] collectionNames = {"github", "got"}; + private MongodExecutable _mongodExe; + private MongodProcess _mongod; + private MongoClient _mongo; + //File names to load data + private String[] filenames = {"file1.json", "file2.json"}; + + + // private constructor + private MongoDBTestServer() throws IOException, URISyntaxException { + _mongodExe = starter.prepare(new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(PORT, Network.localhostIsIPv6())) + + .cmdOptions(new MongoCmdOptionsBuilder(). + useStorageEngine("mmapv1"). + build()) + .build()); + _mongod = _mongodExe.start(); + _mongo = new MongoClient(HOST, PORT); + registerTablespace(); + + loadData(); + } + + //server object can be created using this method + public static MongoDBTestServer getInstance() { + if (instance == null) { + try { + instance = new MongoDBTestServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } } - //server object can be created using this method - public static MongoDBTestServer getInstance() { - if (instance == null) { - try { - instance = new MongoDBTestServer(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - return instance; - } + return instance; + } - //To load files - private static File getRequestedFile(String path) throws FileNotFoundException, URISyntaxException { + //To load files + private static File getRequestedFile(String path) throws FileNotFoundException, URISyntaxException { - URL url = ClassLoader.getSystemResource("dataset/" + path); + URL url = ClassLoader.getSystemResource("dataset/" + path); - if (url == null) { - throw new FileNotFoundException(path); - } - return new File(url.toURI()); + if (url == null) { + throw new FileNotFoundException(path); } - - //To stop the server - public void stop() { - _mongod.stop(); - _mongodExe.stop(); - instance = null; + return new File(url.toURI()); + } + + //To stop the server + public void stop() { + _mongod.stop(); + _mongodExe.stop(); + instance = null; + } + + //Returns the url which can be used to connet to the server instance + public URI getURI() { + try { + return new URI("mongodb://" + HOST + ":" + PORT + "/" + DBNAME); + } catch (Exception e) { + return null; } - - //Returns the url which can be used to connet to the server instance - public URI getURI() { - try { - return new URI("mongodb://" + HOST + ":" + PORT + "/" + DBNAME); - } catch (Exception e) { - return null; + } + + //Start mongo process + private MongosProcess startMongos(int port, int defaultConfigPort, String defaultHost) throws UnknownHostException, + IOException { + IMongosConfig mongosConfig = new MongosConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(port, Network.localhostIsIPv6())) + .configDB(defaultHost + ":" + defaultConfigPort) + .build(); + + MongosExecutable mongosExecutable = MongosStarter.getDefaultInstance().prepare(mongosConfig); + MongosProcess mongos = mongosExecutable.start(); + return mongos; + } + + private MongodProcess startMongod(int defaultConfigPort) throws UnknownHostException, IOException { + IMongodConfig mongoConfigConfig = new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(defaultConfigPort, Network.localhostIsIPv6())) + .configServer(true) + .build(); + + MongodExecutable mongodExecutable = MongodStarter.getDefaultInstance().prepare(mongoConfigConfig); + MongodProcess mongod = mongodExecutable.start(); + return mongod; + } + + //Register the new table space for testing + private void registerTablespace() throws IOException { + JSONObject configElements = new JSONObject(); + configElements.put(MongoDBTableSpace.CONFIG_KEY_MAPPED_DATABASE, MAPPEDDBNAME); + + MongoDBTableSpace tablespace = new MongoDBTableSpace(SPACE_NAME, getURI(), configElements); + tablespace.init(new TajoConf()); + + + TablespaceManager.addTableSpaceForTest(tablespace); + } + + //Create tables and Load data into the mongo server instance + private void loadData() throws IOException, URISyntaxException { + MongoDatabase db = _mongo.getDatabase(DBNAME); + for (int i = 0; i < filenames.length; i++) { + + db.createCollection(collectionNames[i]); + MongoCollection coll = db.getCollection(collectionNames[i]); + + String fileContent = new Scanner(getRequestedFile(filenames[i])).useDelimiter("\\Z").next(); + + //Document list + List documentList = new ArrayList(); + try { + JSONArray jsonarray = new JSONArray(fileContent); + for (int j = 0; j < jsonarray.length(); j++) { + String jsonStr = jsonarray.getJSONObject(j).toString(); + documentList.add(Document.parse(jsonStr)); } - } - - //Start mongo process - private MongosProcess startMongos(int port, int defaultConfigPort, String defaultHost) throws UnknownHostException, - IOException { - IMongosConfig mongosConfig = new MongosConfigBuilder() - .version(Version.Main.PRODUCTION) - .net(new Net(port, Network.localhostIsIPv6())) - .configDB(defaultHost + ":" + defaultConfigPort) - .build(); - - MongosExecutable mongosExecutable = MongosStarter.getDefaultInstance().prepare(mongosConfig); - MongosProcess mongos = mongosExecutable.start(); - return mongos; - } - - private MongodProcess startMongod(int defaultConfigPort) throws UnknownHostException, IOException { - IMongodConfig mongoConfigConfig = new MongodConfigBuilder() - .version(Version.Main.PRODUCTION) - .net(new Net(defaultConfigPort, Network.localhostIsIPv6())) - .configServer(true) - .build(); - - MongodExecutable mongodExecutable = MongodStarter.getDefaultInstance().prepare(mongoConfigConfig); - MongodProcess mongod = mongodExecutable.start(); - return mongod; - } - - //Register the new table space for testing - private void registerTablespace() throws IOException { - JSONObject configElements = new JSONObject(); - configElements.put(MongoDBTableSpace.CONFIG_KEY_MAPPED_DATABASE, MAPPEDDBNAME); - - MongoDBTableSpace tablespace = new MongoDBTableSpace(SPACE_NAME, getURI(), configElements); - tablespace.init(new TajoConf()); - - - TablespaceManager.addTableSpaceForTest(tablespace); - } - //Create tables and Load data into the mongo server instance - private void loadData() throws IOException, URISyntaxException { - MongoDatabase db = _mongo.getDatabase(DBNAME); - for (int i = 0; i < filenames.length; i++) { + } catch (JSONException e) { + e.printStackTrace(); + } - db.createCollection(collectionNames[i]); - MongoCollection coll = db.getCollection(collectionNames[i]); + coll.insertMany(documentList); - String fileContent = new Scanner(getRequestedFile(filenames[i])).useDelimiter("\\Z").next(); + FindIterable docs = coll.find(); - //Document list - List documentList = new ArrayList(); - try { - JSONArray jsonarray = new JSONArray(fileContent); - for (int j = 0; j < jsonarray.length(); j++) { - String jsonStr = jsonarray.getJSONObject(j).toString(); - documentList.add(Document.parse(jsonStr)); - } - - } catch (JSONException e) { - e.printStackTrace(); - } - - coll.insertMany(documentList); - - FindIterable docs = coll.find(); - - docs.forEach(new Block() { - @Override - public void apply(final Document document) { - LOG.info(document.toJson()); - } - }); + docs.forEach(new Block() { + @Override + public void apply(final Document document) { + LOG.info(document.toJson()); } + }); } + } - //Return a mongo client which directly connect to the mongo database. - public MongoClient getMongoClient() { - return _mongo; - } + //Return a mongo client which directly connect to the mongo database. + public MongoClient getMongoClient() { + return _mongo; + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index da3e68df1a..ae39df7b28 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -23,34 +23,34 @@ public class TestConnectionInfo { - @Test - public final void testGetConnectionInfoType1() { - - ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass"); - - assertEquals(connInfo.getMongoDBURI().getURI(), "mongodb://testuser:testpass@localhost:1336/db1"); - assertEquals(connInfo.getScheme(), "mongodb"); - assertEquals(connInfo.getHost(), "localhost"); - assertEquals(connInfo.getPort(), 1336); - assertEquals(connInfo.getDbName(), "db1"); - assertEquals(connInfo.getUser(), "testuser"); - assertEquals(connInfo.getPassword(), "testpass"); - assertEquals(connInfo.getTableName(), "tb1"); - } - - @Test - public final void testGetConnectionInfoType2() { - //Create a connection info object - ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); - - assertEquals(connInfo.getScheme(), "mongodb"); - assertEquals(connInfo.getHost(), "localhost"); - assertEquals(connInfo.getPort(), 1336); - assertEquals(connInfo.getDbName(), "db1"); - assertEquals(connInfo.getUser(), "testuser"); - assertEquals(connInfo.getPassword(), "testpass"); - assertEquals(connInfo.getTableName(), "tb1"); - assertEquals(1, connInfo.getParams().size()); - assertEquals("GMT+9", connInfo.getParams().get("TZ")); - } + @Test + public final void testGetConnectionInfoType1() { + + ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass"); + + assertEquals(connInfo.getMongoDBURI().getURI(), "mongodb://testuser:testpass@localhost:1336/db1"); + assertEquals(connInfo.getScheme(), "mongodb"); + assertEquals(connInfo.getHost(), "localhost"); + assertEquals(connInfo.getPort(), 1336); + assertEquals(connInfo.getDbName(), "db1"); + assertEquals(connInfo.getUser(), "testuser"); + assertEquals(connInfo.getPassword(), "testpass"); + assertEquals(connInfo.getTableName(), "tb1"); + } + + @Test + public final void testGetConnectionInfoType2() { + //Create a connection info object + ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); + + assertEquals(connInfo.getScheme(), "mongodb"); + assertEquals(connInfo.getHost(), "localhost"); + assertEquals(connInfo.getPort(), 1336); + assertEquals(connInfo.getDbName(), "db1"); + assertEquals(connInfo.getUser(), "testuser"); + assertEquals(connInfo.getPassword(), "testpass"); + assertEquals(connInfo.getTableName(), "tb1"); + assertEquals(1, connInfo.getParams().size()); + assertEquals("GMT+9", connInfo.getParams().get("TZ")); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index 211d281b63..f3b5ff88c0 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -31,65 +31,65 @@ import static org.junit.Assert.assertTrue; public class TestMetadataProvider { - static MongoDBTestServer server = MongoDBTestServer.getInstance(); - - - @AfterClass - public static void tearDownClass() throws Exception { - // server.stop(); - } - - @Test - public void testGetTablespaceName() throws Exception { - //check for Space Name validity - Tablespace tablespace = TablespaceManager.get(server.getURI()); - MetadataProvider provider = tablespace.getMetadataProvider(); - assertEquals(server.SPACE_NAME, provider.getTablespaceName()); - } - - @Test - public void testGetDatabaseName() throws Exception { - // - Tablespace tablespace = TablespaceManager.get(server.getURI()); - MetadataProvider provider = tablespace.getMetadataProvider(); - assertEquals(MongoDBTestServer.MAPPEDDBNAME, provider.getDatabaseName()); - } - - - @Test - public void testGetSchemas() throws Exception { - Tablespace tablespace = TablespaceManager.get(server.getURI()); - MetadataProvider provider = tablespace.getMetadataProvider(); - assertTrue(provider.getSchemas().isEmpty()); + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + + + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + + @Test + public void testGetTablespaceName() throws Exception { + //check for Space Name validity + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals(server.SPACE_NAME, provider.getTablespaceName()); + } + + @Test + public void testGetDatabaseName() throws Exception { + // + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals(MongoDBTestServer.MAPPEDDBNAME, provider.getDatabaseName()); + } + + + @Test + public void testGetSchemas() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertTrue(provider.getSchemas().isEmpty()); + } + + @Test + public void testGetTables() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + + final Set expected = Sets.newHashSet(server.collectionNames); + expected.add("system.indexes"); + final Set found = Sets.newHashSet(provider.getTables(null, null)); + + assertEquals(expected, found); + } + + @Test + public void testGetTableDescription() throws Exception { + Tablespace tablespace = TablespaceManager.get(server.getURI()); + MetadataProvider provider = tablespace.getMetadataProvider(); + + for (String tableName : server.collectionNames) { + TableDesc table = provider.getTableDesc(null, tableName); + assertEquals(server.MAPPEDDBNAME + "." + tableName, table.getName()); + assertEquals(server.getURI() + "?table=" + tableName, table.getUri().toASCIIString()); + + //ToDo Check the stats } - @Test - public void testGetTables() throws Exception { - Tablespace tablespace = TablespaceManager.get(server.getURI()); - MetadataProvider provider = tablespace.getMetadataProvider(); - final Set expected = Sets.newHashSet(server.collectionNames); - expected.add("system.indexes"); - final Set found = Sets.newHashSet(provider.getTables(null, null)); - - assertEquals(expected, found); - } - - @Test - public void testGetTableDescription() throws Exception { - Tablespace tablespace = TablespaceManager.get(server.getURI()); - MetadataProvider provider = tablespace.getMetadataProvider(); - - for (String tableName : server.collectionNames) { - TableDesc table = provider.getTableDesc(null, tableName); - assertEquals(server.MAPPEDDBNAME + "." + tableName, table.getName()); - assertEquals(server.getURI() + "?table=" + tableName, table.getUri().toASCIIString()); - - //ToDo Check the stats - } - - - } + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java index 02458a19cf..f0bbff62f0 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java @@ -39,68 +39,68 @@ public class TestMongoDBAppender { - static MongoDBTestServer server = MongoDBTestServer.getInstance(); - static URI uri = server.getURI(); - Appender appender; + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + static URI uri = server.getURI(); + Appender appender; - @Before - public void setup() throws Exception { - Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + @Before + public void setup() throws Exception { + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); - //Create Schema manually - Schema schem = SchemaBuilder.builder(). - add(new Column("title", TajoDataTypes.Type.TEXT)) - .add(new Column("first_name", TajoDataTypes.Type.TEXT)) - .add(new Column("last_name", TajoDataTypes.Type.TEXT)) - .add(new Column("age", TajoDataTypes.Type.INT4)) - .add(new Column("height", TajoDataTypes.Type.FLOAT8)) - .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) - .build(); + //Create Schema manually + Schema schem = SchemaBuilder.builder(). + add(new Column("title", TajoDataTypes.Type.TEXT)) + .add(new Column("first_name", TajoDataTypes.Type.TEXT)) + .add(new Column("last_name", TajoDataTypes.Type.TEXT)) + .add(new Column("age", TajoDataTypes.Type.INT4)) + .add(new Column("height", TajoDataTypes.Type.FLOAT8)) + .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) + .build(); - TableMeta meta = space.getMetadataProvider().getTableDesc(null, "got").getMeta(); - appender = space.getAppender(null, null, meta, schem, new Path(server.getURI() + "?" + MongoDBTableSpace.CONFIG_KEY_TABLE + "=got")); - appender.init(); + TableMeta meta = space.getMetadataProvider().getTableDesc(null, "got").getMeta(); + appender = space.getAppender(null, null, meta, schem, new Path(server.getURI() + "?" + MongoDBTableSpace.CONFIG_KEY_TABLE + "=got")); - } + appender.init(); - @AfterClass - public static void tearDownClass() throws Exception { - // server.stop(); - } + } - @Test - public void testAddTuple() throws IOException { + @Test + public void testAddTuple() throws IOException { - //Create a tuple and add to the table - Tuple tuple = new VTuple(6); - tuple.put(0, DatumFactory.createText("Kingslayer")); - tuple.put(1, DatumFactory.createText("Jaime")); - tuple.put(2, DatumFactory.createText("Lannister")); - tuple.put(3, DatumFactory.createInt4(24)); - tuple.put(4, DatumFactory.createFloat8(165.98)); - tuple.put(5, DatumFactory.createBool(true)); - appender.addTuple(tuple); + //Create a tuple and add to the table + Tuple tuple = new VTuple(6); + tuple.put(0, DatumFactory.createText("Kingslayer")); + tuple.put(1, DatumFactory.createText("Jaime")); + tuple.put(2, DatumFactory.createText("Lannister")); + tuple.put(3, DatumFactory.createInt4(24)); + tuple.put(4, DatumFactory.createFloat8(165.98)); + tuple.put(5, DatumFactory.createBool(true)); + appender.addTuple(tuple); - appender.flush(); + appender.flush(); - //Take data from server - MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title", "Kingslayer")); - Document doc = result.first(); + //Take data from server + MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title", "Kingslayer")); + Document doc = result.first(); - //Validate - assertEquals(doc.get("title"), "Kingslayer"); - assertEquals(doc.get("first_name"), "Jaime"); - assertEquals(doc.get("last_name"), "Lannister"); - assertEquals(doc.get("age"), 24); - assertEquals(doc.get("height"), 165.98); - assertEquals(doc.get("single"), true); + //Validate + assertEquals(doc.get("title"), "Kingslayer"); + assertEquals(doc.get("first_name"), "Jaime"); + assertEquals(doc.get("last_name"), "Lannister"); + assertEquals(doc.get("age"), 24); + assertEquals(doc.get("height"), 165.98); + assertEquals(doc.get("single"), true); - //Remove the inserted doc from database - server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").deleteOne(doc); - } + //Remove the inserted doc from database + server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").deleteOne(doc); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java index 50a42ed3cf..12b8642ec4 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java @@ -33,55 +33,55 @@ public class TestMongoDBDocumentSerializer { - @Test - public void testSeriaalizeTextType() throws IOException { - Schema schem = SchemaBuilder.builder(). - add(new Column("title", TajoDataTypes.Type.TEXT)) - .add(new Column("first_name", TajoDataTypes.Type.TEXT)) - .add(new Column("last_name", TajoDataTypes.Type.TEXT)) - .build(); - - MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem, null); - - Tuple tuple = new VTuple(3); - tuple.put(0, DatumFactory.createText("Kingslayer")); - tuple.put(1, DatumFactory.createText("Jaime")); - tuple.put(2, DatumFactory.createText("Lannister")); - Document md = new Document(); - - documentSerializer.serialize(tuple, md); - - assertEquals("Kingslayer", md.getString("title")); - assertEquals("Jaime", md.getString("first_name")); - assertEquals("Lannister", md.getString("last_name")); - - } - - @Test - public void testSerializeIntFloatBoolean() throws IOException { - Schema schem = SchemaBuilder.builder(). - add(new Column("title", TajoDataTypes.Type.TEXT)) - .add(new Column("age", TajoDataTypes.Type.INT4)) - .add(new Column("height", TajoDataTypes.Type.FLOAT8)) - .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) - .build(); - - MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem, null); - - Tuple tuple = new VTuple(4); - tuple.put(0, DatumFactory.createText("Mr")); - tuple.put(1, DatumFactory.createInt4(24)); - tuple.put(2, DatumFactory.createFloat8(165.98)); - tuple.put(3, DatumFactory.createBool(true)); - Document md = new Document(); - - documentSerializer.serialize(tuple, md); - - assertEquals("Mr", md.get("title")); - assertEquals(24, md.get("age")); - assertEquals(165.98, md.get("height")); - assertEquals(true, md.get("single")); - - } + @Test + public void testSeriaalizeTextType() throws IOException { + Schema schem = SchemaBuilder.builder(). + add(new Column("title", TajoDataTypes.Type.TEXT)) + .add(new Column("first_name", TajoDataTypes.Type.TEXT)) + .add(new Column("last_name", TajoDataTypes.Type.TEXT)) + .build(); + + MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem, null); + + Tuple tuple = new VTuple(3); + tuple.put(0, DatumFactory.createText("Kingslayer")); + tuple.put(1, DatumFactory.createText("Jaime")); + tuple.put(2, DatumFactory.createText("Lannister")); + Document md = new Document(); + + documentSerializer.serialize(tuple, md); + + assertEquals("Kingslayer", md.getString("title")); + assertEquals("Jaime", md.getString("first_name")); + assertEquals("Lannister", md.getString("last_name")); + + } + + @Test + public void testSerializeIntFloatBoolean() throws IOException { + Schema schem = SchemaBuilder.builder(). + add(new Column("title", TajoDataTypes.Type.TEXT)) + .add(new Column("age", TajoDataTypes.Type.INT4)) + .add(new Column("height", TajoDataTypes.Type.FLOAT8)) + .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) + .build(); + + MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem, null); + + Tuple tuple = new VTuple(4); + tuple.put(0, DatumFactory.createText("Mr")); + tuple.put(1, DatumFactory.createInt4(24)); + tuple.put(2, DatumFactory.createFloat8(165.98)); + tuple.put(3, DatumFactory.createBool(true)); + Document md = new Document(); + + documentSerializer.serialize(tuple, md); + + assertEquals("Mr", md.get("title")); + assertEquals(24, md.get("age")); + assertEquals(165.98, md.get("height")); + assertEquals(true, md.get("single")); + + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index f9559cfa0d..41c4e84ab5 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -25,78 +25,78 @@ public class TestMongoDBQueryTest extends QueryTestCaseBase { - static MongoDBTestServer server = MongoDBTestServer.getInstance(); - static URI uri = server.getURI(); - - public TestMongoDBQueryTest() { - super(server.MAPPEDDBNAME); - } - - @BeforeClass - public static void setup() throws Exception { - QueryTestCaseBase.testingCluster.getMaster().refresh(); - // TablespaceManager.addTableSpaceForTest(new ExampleHttpFileTablespace("http_example", uri, configElements)); - - } - - @AfterClass - public static void tearDownClass() throws Exception { - // server.stop(); - } - - @Before - public void prepareTables() throws TajoException { - if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { - executeString("create table got (title,first_name,last_name) tablespace test_spacename using mongodb"); - executeString("create table github (*) tablespace test_spacename using mongodb"); - // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); - } + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + static URI uri = server.getURI(); + + public TestMongoDBQueryTest() { + super(server.MAPPEDDBNAME); + } + + @BeforeClass + public static void setup() throws Exception { + QueryTestCaseBase.testingCluster.getMaster().refresh(); + // TablespaceManager.addTableSpaceForTest(new ExampleHttpFileTablespace("http_example", uri, configElements)); + + } + + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + + @Before + public void prepareTables() throws TajoException { + if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + executeString("create table got (title,first_name,last_name) tablespace test_spacename using mongodb"); + executeString("create table github (*) tablespace test_spacename using mongodb"); + // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); } + } - @After - public void dropTables() throws TajoException { - if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { - executeString("drop table got"); - executeString("drop table github"); - } + @After + public void dropTables() throws TajoException { + if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + executeString("drop table got"); + executeString("drop table github"); } + } - @SimpleTest - @Test - public void testSelect() throws Exception { - runSimpleTests(); - } + @SimpleTest + @Test + public void testSelect() throws Exception { + runSimpleTests(); + } - @SimpleTest - @Test - public void testSort() throws Exception { - runSimpleTests(); - } + @SimpleTest + @Test + public void testSort() throws Exception { + runSimpleTests(); + } - @SimpleTest - @Test - public void testGroupby() throws Exception { - runSimpleTests(); - } + @SimpleTest + @Test + public void testGroupby() throws Exception { + runSimpleTests(); + } - @SimpleTest - @Test - public void testJoin() throws Exception { - runSimpleTests(); - } + @SimpleTest + @Test + public void testJoin() throws Exception { + runSimpleTests(); + } - @SimpleTest - @Test - public void testInsert() throws Exception { + @SimpleTest + @Test + public void testInsert() throws Exception { // runSimpleTests(); - } + } - @SimpleTest - @Test - public void testCTAS() throws Exception { - runSimpleTests(); - } + @SimpleTest + @Test + public void testCTAS() throws Exception { + runSimpleTests(); + } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 5ec4c5f3b9..2a7eb1c900 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -41,96 +41,96 @@ public class TestMongoDBTableSpace { - //mongodb://:@ds017231.mlab.com:17231/tajo_test - static MongoDBTestServer server = MongoDBTestServer.getInstance(); - static URI uri = server.getURI(); - - - @AfterClass - public static void tearDownClass() throws Exception { - // server.stop(); - } - - @Test - public void testTablespaceHandler() { - assertTrue((TablespaceManager.getByName(server.SPACE_NAME)) instanceof MongoDBTableSpace); - assertEquals(server.SPACE_NAME, (TablespaceManager.getByName(server.SPACE_NAME).getName())); - - assertTrue((TablespaceManager.get(uri.toASCIIString() + "&table=tb1")) instanceof MongoDBTableSpace); - assertTrue((TablespaceManager.get(uri)) instanceof MongoDBTableSpace); - - - //Test the URI same - assertEquals(uri.toASCIIString(), TablespaceManager.get(uri).getUri().toASCIIString()); - } - - @Test(timeout = 1000, expected = TajoRuntimeException.class) - public void testCreateTable() throws IOException, TajoException { - Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); - space.createTable(null, false); - } - + //mongodb://:@ds017231.mlab.com:17231/tajo_test + static MongoDBTestServer server = MongoDBTestServer.getInstance(); + static URI uri = server.getURI(); + + + @AfterClass + public static void tearDownClass() throws Exception { + // server.stop(); + } + + @Test + public void testTablespaceHandler() { + assertTrue((TablespaceManager.getByName(server.SPACE_NAME)) instanceof MongoDBTableSpace); + assertEquals(server.SPACE_NAME, (TablespaceManager.getByName(server.SPACE_NAME).getName())); + + assertTrue((TablespaceManager.get(uri.toASCIIString() + "&table=tb1")) instanceof MongoDBTableSpace); + assertTrue((TablespaceManager.get(uri)) instanceof MongoDBTableSpace); + + + //Test the URI same + assertEquals(uri.toASCIIString(), TablespaceManager.get(uri).getUri().toASCIIString()); + } + + @Test(timeout = 1000, expected = TajoRuntimeException.class) + public void testCreateTable() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); + space.createTable(null, false); + } + + + //Todo delete only metadat is provide + @Test(timeout = 1000) + public void testCreateTable_and_Purg() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); + + TableDesc tableDesc = new TableDesc( + IdentifierUtil.buildFQName(server.MAPPEDDBNAME, "Table1"), + SchemaBuilder.builder() + .build(), + null, + server.getURI()); + + //Test create and delete if meta data provided + if (MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { + space.createTable(tableDesc, false); + + //Check whether the created table is in the collection + final Set found = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); + assertTrue(found.contains("Table1")); + + //Check whether the created table is in the mongo database + MongoClient mongoClient = server.getMongoClient(); + Boolean foundInMongoDB = false; + MongoIterable collectionNames = mongoClient.getDatabase(server.DBNAME).listCollectionNames(); + for (final String name : collectionNames) { + if (name.equalsIgnoreCase("Table1")) { + foundInMongoDB = true; + } + } + assertTrue(foundInMongoDB); - //Todo delete only metadat is provide - @Test(timeout = 1000) - public void testCreateTable_and_Purg() throws IOException, TajoException { - Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); - - TableDesc tableDesc = new TableDesc( - IdentifierUtil.buildFQName(server.MAPPEDDBNAME, "Table1"), - SchemaBuilder.builder() - .build(), - null, - server.getURI()); - - //Test create and delete if meta data provided - if (MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { - space.createTable(tableDesc, false); - - //Check whether the created table is in the collection - final Set found = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); - assertTrue(found.contains("Table1")); - - //Check whether the created table is in the mongo database - MongoClient mongoClient = server.getMongoClient(); - Boolean foundInMongoDB = false; - MongoIterable collectionNames = mongoClient.getDatabase(server.DBNAME).listCollectionNames(); - for (final String name : collectionNames) { - if (name.equalsIgnoreCase("Table1")) { - foundInMongoDB = true; - } - } - assertTrue(foundInMongoDB); - - //Purg the table - space.purgeTable(tableDesc); - final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); - assertFalse(found_after.contains("Table1")); + //Purg the table + space.purgeTable(tableDesc); + final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); + assertFalse(found_after.contains("Table1")); - } } + } - @Test - public void testTableVolume() throws IOException, TajoException { - Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); - Map tableSizes = new HashMap(); - tableSizes.put("github", 4); - tableSizes.put("got", 5); - for (String tbl : server.collectionNames) { + @Test + public void testTableVolume() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); + Map tableSizes = new HashMap(); + tableSizes.put("github", 4); + tableSizes.put("got", 5); + for (String tbl : server.collectionNames) { - // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); + // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); - // long a = 1;b - TableDesc tbDesc = new TableDesc( - IdentifierUtil.buildFQName(server.MAPPEDDBNAME, tbl), - SchemaBuilder.builder() - .build(), - null, - space.getTableUri(null, null, tbl)); + // long a = 1;b + TableDesc tbDesc = new TableDesc( + IdentifierUtil.buildFQName(server.MAPPEDDBNAME, tbl), + SchemaBuilder.builder() + .build(), + null, + space.getTableUri(null, null, tbl)); - assertEquals((int) tableSizes.get(tbl), space.getTableVolume(tbDesc, Optional.empty())); + assertEquals((int) tableSizes.get(tbl), space.getTableVolume(tbDesc, Optional.empty())); - } } + } } \ No newline at end of file From fdd871ea6ea5ab2e803bdb875360ac1bedef5738 Mon Sep 17 00:00:00 2001 From: Janaka Date: Fri, 26 Aug 2016 01:48:12 +0530 Subject: [PATCH 90/94] Fixed dependency issue --- tajo-storage/tajo-storage-mongodb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index 28fb574622..c0474d4678 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -256,7 +256,7 @@ org.mongodb - mongodb-driver + mongo-java-driver 3.3.0 From 6b1506a19d1d5aa03f31a61cc1e7b1fbda62a87b Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Mon, 5 Sep 2016 17:00:35 +0530 Subject: [PATCH 91/94] Made the new branch Only scan part --- .../src/main/resources/storage-default.xml | 5 - .../src/test/resources/storage-default.xml | 5 - .../tajo/storage/mongodb/MongoDBAppender.java | 115 ------------------ .../mongodb/MongoDBCollectionWriter.java | 65 ---------- .../mongodb/MongoDBDocumentSerializer.java | 102 ---------------- .../storage/mongodb/MongoDBTableSpace.java | 17 +-- .../storage/mongodb/TestMongoDBAppender.java | 106 ---------------- .../TestMongoDBDocumentSerializer.java | 87 ------------- .../storage/mongodb/TestMongoDBQueryTest.java | 1 - 9 files changed, 1 insertion(+), 502 deletions(-) delete mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java delete mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java delete mode 100644 tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java delete mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java delete mode 100644 tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index fd5d0a6c41..e80931c68f 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -214,11 +214,6 @@ org.apache.tajo.storage.hbase.HFileAppender - - tajo.storage.appender-handler.mongodb.class - org.apache.tajo.storage.mongodb.MongoDBAppender - - tajo.storage.text.io.read-buffer.bytes diff --git a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml index 70f226f1e6..1737e221c7 100644 --- a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml @@ -197,11 +197,6 @@ org.apache.tajo.storage.hbase.HFileAppender - - tajo.storage.appender-handler.mongodb.class - org.apache.tajo.storage.mongodb.MongoDBAppender - - tajo.storage.text.io.read-buffer.bytes diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java deleted file mode 100644 index 65334306e2..0000000000 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBAppender.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tajo.storage.mongodb; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.TaskAttemptId; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.exception.NotImplementedException; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.Tuple; - -import java.io.IOException; -import java.net.URI; -import java.util.List; - -public class MongoDBAppender implements Appender { - - //Given at constructor - private final Configuration conf; - private final Schema schema; - private final TableMeta meta; - private final Path stagingDir; - private final TaskAttemptId taskAttemptId; - private final URI uri; - - - protected boolean inited = false; - private boolean[] columnStatsEnabled; - private boolean tableStatsEnabled; - - private MongoDBCollectionWriter mongoDBCollectionWriter; - - - public MongoDBAppender(Configuration conf, TaskAttemptId taskAttemptId, - Schema schema, TableMeta meta, Path stagingDir, URI uri) { - this.conf = conf; - this.schema = schema; - this.meta = meta; - this.stagingDir = stagingDir; - this.taskAttemptId = taskAttemptId; - this.uri = stagingDir.toUri(); - } - - @Override - public void init() throws IOException { - if (inited) { - throw new IllegalStateException("FileAppender is already initialized."); - } - inited = true; - MongoDBDocumentSerializer md = new MongoDBDocumentSerializer(schema, meta); - mongoDBCollectionWriter = new MongoDBCollectionWriter(ConnectionInfo.fromURI(stagingDir.toString()), md); - mongoDBCollectionWriter.init(); - } - - @Override - public void addTuple(Tuple t) throws IOException { - mongoDBCollectionWriter.addTuple(t); - } - - @Override - public void flush() throws IOException { - mongoDBCollectionWriter.write(); - } - - @Override - public long getEstimatedOutputSize() throws IOException { - throw new IOException(new NotImplementedException()); - } - - @Override - public void close() throws IOException { - mongoDBCollectionWriter.close(); - } - - @Override - public void enableStats() { - if (inited) { - throw new IllegalStateException("Should enable this option before init()"); - } - - this.tableStatsEnabled = true; - this.columnStatsEnabled = new boolean[schema.size()]; - - } - - @Override - public void enableStats(List columnList) { - - } - - @Override - public TableStats getStats() { - TableStats stats = new TableStats(); - return stats; - } -} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java deleted file mode 100644 index 46e9a8d4b1..0000000000 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionWriter.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tajo.storage.mongodb; - -import com.mongodb.MongoClient; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import org.apache.tajo.storage.Tuple; -import org.bson.Document; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Writes tuples into a mongodb collection - */ -public class MongoDBCollectionWriter { - MongoClient mongoClient; - MongoCollection collection; - MongoDBDocumentSerializer mongoDBDocumentSerializer; - private ConnectionInfo connectionInfo; - private List documentList = new ArrayList<>(); - - public MongoDBCollectionWriter(ConnectionInfo connectionInfo, MongoDBDocumentSerializer serializer) { - this.connectionInfo = connectionInfo; - this.mongoDBDocumentSerializer = serializer; - } - - public void init() { - mongoClient = new MongoClient(connectionInfo.getMongoDBURI()); - MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); - collection = db.getCollection(connectionInfo.getTableName()); - } - - public void addTuple(Tuple tuple) throws IOException { - Document dc = new Document(); - mongoDBDocumentSerializer.serialize(tuple, dc); - documentList.add(dc); - } - - public void write() { - collection.insertMany(documentList); - } - - public void close() { - mongoClient.close(); - } - -} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java deleted file mode 100644 index 11a2ea8547..0000000000 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBDocumentSerializer.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tajo.storage.mongodb; - -import com.google.common.collect.Lists; -import org.apache.tajo.catalog.NestedPathUtil; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaUtil; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.storage.Tuple; -import org.bson.Document; - -import java.io.IOException; -import java.util.Map; - -/** - * Serialize tuples into Mongo Documents - */ -public class MongoDBDocumentSerializer { - - private final Schema schema; - private final TableMeta meta; - private final Map types; - String[] paths; - - public MongoDBDocumentSerializer(Schema schema, TableMeta meta) { - this.schema = schema; - this.meta = meta; - - paths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(schema.getAllColumns()), true); - types = SchemaUtil.buildTypeMap(schema.getAllColumns(), paths); - } - - public void setValue(Tuple inputTuple, String fullPath, String[] paths, int depth, int index, Document outputDoc) { - - - String simpleColumnName = paths[depth]; - switch (types.get(fullPath)) { - case INT1: - case UINT1: - case INT2: - case UINT2: - outputDoc.put(simpleColumnName, inputTuple.getInt2(index)); - break; - - case INT4: - case UINT4: - outputDoc.put(simpleColumnName, inputTuple.getInt4(index)); - break; - - case INT8: - case UINT8: - outputDoc.put(simpleColumnName, inputTuple.getInt8(index)); - break; - - case FLOAT4: - outputDoc.put(simpleColumnName, inputTuple.getFloat4(index)); - break; - - case FLOAT8: - outputDoc.put(simpleColumnName, inputTuple.getFloat8(index)); - break; - - case CHAR: - outputDoc.put(simpleColumnName, inputTuple.getChar(index)); - break; - - case BOOLEAN: - outputDoc.put(simpleColumnName, inputTuple.getBool(index)); - break; - - default: - outputDoc.put(simpleColumnName, inputTuple.getText(index)); - break; - } - } - - - public void serialize(Tuple inputTuple, Document outputDoc) throws IOException { - for (int i = 0; i < inputTuple.size(); i++) { - String[] newPaths = paths[i].split(NestedPathUtil.PATH_DELIMITER); - setValue(inputTuple, newPaths[0], newPaths, 0, i, outputDoc); - } - } - -} diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 0f204968f4..51e0f9a0aa 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -212,29 +212,14 @@ public URI getTableUri(String databaseName, String tableName) { return URI.create(this.getUri() + "&" + CONFIG_KEY_TABLE + "=" + tableName); } -// @Override -// public URI getTableUri(String databaseName, String tableName) { -// //ToDo set the TableURI properly -// return URI.create(this.getUri()+"&table="+tableName); -// } - - // Metadata public MetadataProvider getMetadataProvider() { return new MongoDBMetadataProvider(this, mappedDBName); } + //Return Connection info for the tablespace public ConnectionInfo getConnectionInfo() { return connectionInfo; } - - //ToDo Make Sure this is not an issue - @Override - public Appender getAppender(OverridableConf queryContext, - TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) - - { - return new MongoDBAppender(null, taskAttemptId, schema, meta, workDir, workDir.toUri()); - } } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java deleted file mode 100644 index f0bbff62f0..0000000000 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBAppender.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tajo.storage.mongodb; - -import com.mongodb.client.MongoIterable; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaBuilder; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.storage.*; -import org.bson.Document; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; - -import static org.junit.Assert.assertEquals; - -public class TestMongoDBAppender { - - - static MongoDBTestServer server = MongoDBTestServer.getInstance(); - static URI uri = server.getURI(); - Appender appender; - - @AfterClass - public static void tearDownClass() throws Exception { - // server.stop(); - } - - @Before - public void setup() throws Exception { - Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); - - - //Create Schema manually - Schema schem = SchemaBuilder.builder(). - add(new Column("title", TajoDataTypes.Type.TEXT)) - .add(new Column("first_name", TajoDataTypes.Type.TEXT)) - .add(new Column("last_name", TajoDataTypes.Type.TEXT)) - .add(new Column("age", TajoDataTypes.Type.INT4)) - .add(new Column("height", TajoDataTypes.Type.FLOAT8)) - .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) - .build(); - - - TableMeta meta = space.getMetadataProvider().getTableDesc(null, "got").getMeta(); - appender = space.getAppender(null, null, meta, schem, new Path(server.getURI() + "?" + MongoDBTableSpace.CONFIG_KEY_TABLE + "=got")); - - appender.init(); - - } - - @Test - public void testAddTuple() throws IOException { - - //Create a tuple and add to the table - Tuple tuple = new VTuple(6); - tuple.put(0, DatumFactory.createText("Kingslayer")); - tuple.put(1, DatumFactory.createText("Jaime")); - tuple.put(2, DatumFactory.createText("Lannister")); - tuple.put(3, DatumFactory.createInt4(24)); - tuple.put(4, DatumFactory.createFloat8(165.98)); - tuple.put(5, DatumFactory.createBool(true)); - appender.addTuple(tuple); - - appender.flush(); - - //Take data from server - MongoIterable result = server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").find(new Document("title", "Kingslayer")); - Document doc = result.first(); - - //Validate - assertEquals(doc.get("title"), "Kingslayer"); - assertEquals(doc.get("first_name"), "Jaime"); - assertEquals(doc.get("last_name"), "Lannister"); - assertEquals(doc.get("age"), 24); - assertEquals(doc.get("height"), 165.98); - assertEquals(doc.get("single"), true); - - - //Remove the inserted doc from database - server.getMongoClient().getDatabase(server.DBNAME).getCollection("got").deleteOne(doc); - } - -} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java deleted file mode 100644 index 12b8642ec4..0000000000 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBDocumentSerializer.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tajo.storage.mongodb; - -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaBuilder; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; -import org.bson.Document; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; - -public class TestMongoDBDocumentSerializer { - - @Test - public void testSeriaalizeTextType() throws IOException { - Schema schem = SchemaBuilder.builder(). - add(new Column("title", TajoDataTypes.Type.TEXT)) - .add(new Column("first_name", TajoDataTypes.Type.TEXT)) - .add(new Column("last_name", TajoDataTypes.Type.TEXT)) - .build(); - - MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem, null); - - Tuple tuple = new VTuple(3); - tuple.put(0, DatumFactory.createText("Kingslayer")); - tuple.put(1, DatumFactory.createText("Jaime")); - tuple.put(2, DatumFactory.createText("Lannister")); - Document md = new Document(); - - documentSerializer.serialize(tuple, md); - - assertEquals("Kingslayer", md.getString("title")); - assertEquals("Jaime", md.getString("first_name")); - assertEquals("Lannister", md.getString("last_name")); - - } - - @Test - public void testSerializeIntFloatBoolean() throws IOException { - Schema schem = SchemaBuilder.builder(). - add(new Column("title", TajoDataTypes.Type.TEXT)) - .add(new Column("age", TajoDataTypes.Type.INT4)) - .add(new Column("height", TajoDataTypes.Type.FLOAT8)) - .add(new Column("single", TajoDataTypes.Type.BOOLEAN)) - .build(); - - MongoDBDocumentSerializer documentSerializer = new MongoDBDocumentSerializer(schem, null); - - Tuple tuple = new VTuple(4); - tuple.put(0, DatumFactory.createText("Mr")); - tuple.put(1, DatumFactory.createInt4(24)); - tuple.put(2, DatumFactory.createFloat8(165.98)); - tuple.put(3, DatumFactory.createBool(true)); - Document md = new Document(); - - documentSerializer.serialize(tuple, md); - - assertEquals("Mr", md.get("title")); - assertEquals(24, md.get("age")); - assertEquals(165.98, md.get("height")); - assertEquals(true, md.get("single")); - - } - -} diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 41c4e84ab5..80beb73d89 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -49,7 +49,6 @@ public void prepareTables() throws TajoException { if (!MongoDBTableSpace.STORAGE_PROPERTY.isMetadataProvided()) { executeString("create table got (title,first_name,last_name) tablespace test_spacename using mongodb"); executeString("create table github (*) tablespace test_spacename using mongodb"); - // executeString("create table github (*) tablespace test_spacename using ex_http_json with ('path'='github.json')"); } } From 5cea27188a1709a828cd8bc7562b8d5cd82c1855 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Mon, 5 Sep 2016 17:03:40 +0530 Subject: [PATCH 92/94] Reformated code --- .../tajo-catalog-common/src/main/proto/CatalogProtos.proto | 1 - .../org/apache/tajo/storage/mongodb/MongoDBTableSpace.java | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 073e2c27b3..f02bfbf227 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -25,7 +25,6 @@ option java_generate_equals_and_hash = true; import "DataTypes.proto"; import "PrimitiveProtos.proto"; - enum OrderType { ORDER_NONE = 0; ASC = 1; diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 51e0f9a0aa..0f1bcc64b5 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; -import org.apache.tajo.TaskAttemptId; import org.apache.tajo.catalog.*; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoException; @@ -38,7 +37,10 @@ import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.schema.IdentifierUtil; -import org.apache.tajo.storage.*; +import org.apache.tajo.storage.FormatProperty; +import org.apache.tajo.storage.StorageProperty; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.fragment.Fragment; import org.bson.Document; From 36bc03c8730d45d90b8cea5aa5ae4c6c069d913f Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Mon, 5 Sep 2016 17:24:49 +0530 Subject: [PATCH 93/94] Removed comments --- .../apache/tajo/storage/mongodb/MongoDBTableSpace.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index 0f204968f4..91649bc279 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -212,13 +212,6 @@ public URI getTableUri(String databaseName, String tableName) { return URI.create(this.getUri() + "&" + CONFIG_KEY_TABLE + "=" + tableName); } -// @Override -// public URI getTableUri(String databaseName, String tableName) { -// //ToDo set the TableURI properly -// return URI.create(this.getUri()+"&table="+tableName); -// } - - // Metadata public MetadataProvider getMetadataProvider() { return new MongoDBMetadataProvider(this, mappedDBName); @@ -228,8 +221,6 @@ public ConnectionInfo getConnectionInfo() { return connectionInfo; } - - //ToDo Make Sure this is not an issue @Override public Appender getAppender(OverridableConf queryContext, TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) From 7e922999a8791bbe0d5e8bf97cc9174230336785 Mon Sep 17 00:00:00 2001 From: Janaka Chathuranga Date: Tue, 6 Sep 2016 10:45:22 +0530 Subject: [PATCH 94/94] Removed unnecessary comments --- tajo-storage/tajo-storage-mongodb/pom.xml | 3 --- .../tajo/storage/mongodb/MongoDBCollectionReader.java | 5 ++--- .../apache/tajo/storage/mongodb/MongoDBScanner.java | 4 ++-- .../apache/tajo/storage/mongodb/MongoDBTableSpace.java | 4 ---- .../tajo/storage/mongodb/TestConnectionInfo.java | 2 +- .../tajo/storage/mongodb/TestMetadataProvider.java | 5 ----- .../tajo/storage/mongodb/TestMongoDBQueryTest.java | 9 ++++----- .../tajo/storage/mongodb/TestMongoDBTableSpace.java | 10 +++------- 8 files changed, 12 insertions(+), 30 deletions(-) diff --git a/tajo-storage/tajo-storage-mongodb/pom.xml b/tajo-storage/tajo-storage-mongodb/pom.xml index c0474d4678..9d6ae363e1 100644 --- a/tajo-storage/tajo-storage-mongodb/pom.xml +++ b/tajo-storage/tajo-storage-mongodb/pom.xml @@ -252,14 +252,11 @@ - - org.mongodb mongo-java-driver 3.3.0 - de.flapdoodle.embed de.flapdoodle.embed.mongo diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java index c1a7d45219..7eb0f563c3 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBCollectionReader.java @@ -56,9 +56,8 @@ public void init() throws IOException { MongoDatabase db = mongoClient.getDatabase(connectionInfo.getDbName()); MongoCollection collection = db.getCollection(connectionInfo.getTableName()); - documentList = (List) collection.find().into( - new ArrayList()); - ; + documentList = collection.find().into( + new ArrayList<>()); deserializer.init(); } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java index faa2ba53eb..f5264481c2 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBScanner.java @@ -47,7 +47,7 @@ public class MongoDBScanner implements Scanner { private MongoDBCollectionReader collectionReader; private Column[] targets; - private boolean inited; + private boolean inited = false; public MongoDBScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { this.schema = schema; @@ -60,7 +60,7 @@ public void init() throws IOException { if (targets == null) { targets = schema.toArray(); } - + inited = true; reset(); } diff --git a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java index c95fc4039e..3c882e601d 100644 --- a/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/main/java/org/apache/tajo/storage/mongodb/MongoDBTableSpace.java @@ -106,7 +106,6 @@ protected void storageInit() throws IOException { @Override public long getTableVolume(TableDesc table, Optional filter) { - long count = 0; try { String[] nameSplited = IdentifierUtil.splitFQTableName(table.getName()); @@ -157,9 +156,6 @@ public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoExc throw new TajoRuntimeException(new NotImplementedException()); MongoCollection table = db.getCollection(tableDesc.getName()); - //TODO Handle this here. If empty throw exception or what? - boolean ifExist = (table.count() > 0) ? true : false; - //If meta data provides. Create a table if (STORAGE_PROPERTY.isMetadataProvided()) db.createCollection(IdentifierUtil.extractSimpleName(tableDesc.getName())); diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java index ae39df7b28..96922865cf 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestConnectionInfo.java @@ -40,7 +40,7 @@ public final void testGetConnectionInfoType1() { @Test public final void testGetConnectionInfoType2() { - //Create a connection info object + ConnectionInfo connInfo = ConnectionInfo.fromURI("mongodb://localhost:1336/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); assertEquals(connInfo.getScheme(), "mongodb"); diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java index f3b5ff88c0..684f76786a 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMetadataProvider.java @@ -49,7 +49,6 @@ public void testGetTablespaceName() throws Exception { @Test public void testGetDatabaseName() throws Exception { - // Tablespace tablespace = TablespaceManager.get(server.getURI()); MetadataProvider provider = tablespace.getMetadataProvider(); assertEquals(MongoDBTestServer.MAPPEDDBNAME, provider.getDatabaseName()); @@ -84,11 +83,7 @@ public void testGetTableDescription() throws Exception { TableDesc table = provider.getTableDesc(null, tableName); assertEquals(server.MAPPEDDBNAME + "." + tableName, table.getName()); assertEquals(server.getURI() + "?table=" + tableName, table.getUri().toASCIIString()); - - //ToDo Check the stats } - - } diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java index 80beb73d89..0cc670d8f9 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBQueryTest.java @@ -35,13 +35,11 @@ public TestMongoDBQueryTest() { @BeforeClass public static void setup() throws Exception { QueryTestCaseBase.testingCluster.getMaster().refresh(); - // TablespaceManager.addTableSpaceForTest(new ExampleHttpFileTablespace("http_example", uri, configElements)); - - } + } @AfterClass public static void tearDownClass() throws Exception { - // server.stop(); + server.stop(); } @Before @@ -90,7 +88,8 @@ public void testJoin() throws Exception { @SimpleTest @Test public void testInsert() throws Exception { -// runSimpleTests(); + //Todo Enable when insert is supported for Metadata provided tableSpaces. + //runSimpleTests(); } @SimpleTest diff --git a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java index 2a7eb1c900..aba0186215 100644 --- a/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java +++ b/tajo-storage/tajo-storage-mongodb/src/test/java/org/apache/tajo/storage/mongodb/TestMongoDBTableSpace.java @@ -70,8 +70,6 @@ public void testCreateTable() throws IOException, TajoException { space.createTable(null, false); } - - //Todo delete only metadat is provide @Test(timeout = 1000) public void testCreateTable_and_Purg() throws IOException, TajoException { Tablespace space = TablespaceManager.getByName(server.SPACE_NAME); @@ -102,7 +100,7 @@ public void testCreateTable_and_Purg() throws IOException, TajoException { } assertTrue(foundInMongoDB); - //Purg the table + //Purg the table and check whether it exists space.purgeTable(tableDesc); final Set found_after = Sets.newHashSet(space.getMetadataProvider().getTables(null, null)); assertFalse(found_after.contains("Table1")); @@ -116,11 +114,9 @@ public void testTableVolume() throws IOException, TajoException { Map tableSizes = new HashMap(); tableSizes.put("github", 4); tableSizes.put("got", 5); - for (String tbl : server.collectionNames) { - // assertEquals(1,space.getTableVolume(tableDesc, Optional.empty())); - - // long a = 1;b + //Check whether the volumes of tables are correct + for (String tbl : server.collectionNames) { TableDesc tbDesc = new TableDesc( IdentifierUtil.buildFQName(server.MAPPEDDBNAME, tbl), SchemaBuilder.builder()