From 8598e8cf58c1d20914d75e5e65649cd7eb498abb Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 17 Aug 2015 12:24:24 +0900 Subject: [PATCH 1/3] TAJO-1775: HCatalogStore need to be deprecated. --- .../tajo/catalog/store/HCatalogStore.java | 33 ++++++ .../tajo/catalog/store/TestHCatalogStore.java | 103 ++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java create mode 100644 tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java new file mode 100644 index 0000000000..c1a9e30c09 --- /dev/null +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog.store; + +import org.apache.hadoop.conf.Configuration; + +@Deprecated +/** + * HCatalogStore is no longer used since Tajo 0.11.0. This is a dummy HCatalogStore class, which is used + * to be compatible with Tajo 0.10.x applications. + */ +public class HCatalogStore extends HiveCatalogStore { + + public HCatalogStore(final Configuration conf) { + super(conf); + } +} diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java new file mode 100644 index 0000000000..c222801042 --- /dev/null +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog.store; + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.KeyValueSet; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestHCatalogStore { + private static final String DB_NAME = "test_hcatalog"; + private static final String CUSTOMER = "customer"; + + private static HCatalogStore store; + private static Path warehousePath; + + @BeforeClass + public static void setUp() throws Exception { + Path testPath = CommonTestingUtil.getTestDir(); + warehousePath = new Path(testPath, "warehouse"); + + //create local hiveMeta + HiveConf conf = new HiveConf(); + String jdbcUri = "jdbc:derby:;databaseName="+testPath.toUri().getPath()+"metastore_db;create=true"; + conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehousePath.toUri().toString()); + conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, jdbcUri); + conf.set(TajoConf.ConfVars.WAREHOUSE_DIR.varname, warehousePath.toUri().toString()); + + // create local HiveCatalogStore. + TajoConf tajoConf = new TajoConf(conf); + store = new HCatalogStore(tajoConf); + store.createDatabase(DB_NAME, null); + } + + @AfterClass + public static void tearDown() throws IOException { + store.close(); + } + + @Test + public void testTableUsingTextFile() throws Exception { + TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); + schema.addColumn("c_name", TajoDataTypes.Type.TEXT); + schema.addColumn("c_address", TajoDataTypes.Type.TEXT); + schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4); + schema.addColumn("c_phone", TajoDataTypes.Type.TEXT); + schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8); + schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT); + schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, + new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, CUSTOMER)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getUri(), table1.getUri()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + assertEquals(StringEscapeUtils.escapeJava(StorageConstants.DEFAULT_FIELD_DELIMITER), + table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER)); + store.dropTable(DB_NAME, CUSTOMER); + } + +} From e649109c3d0ccf83c2f3ba031819513be5a36b9b Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 18 Aug 2015 08:12:50 +0900 Subject: [PATCH 2/3] Remove unnecessary class --- .../tajo/catalog/store/TestHCatalogStore.java | 103 ------------------ 1 file changed, 103 deletions(-) delete mode 100644 tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java deleted file mode 100644 index c222801042..0000000000 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.catalog.store; - -import org.apache.commons.lang.StringEscapeUtils; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.util.CommonTestingUtil; -import org.apache.tajo.util.KeyValueSet; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TestHCatalogStore { - private static final String DB_NAME = "test_hcatalog"; - private static final String CUSTOMER = "customer"; - - private static HCatalogStore store; - private static Path warehousePath; - - @BeforeClass - public static void setUp() throws Exception { - Path testPath = CommonTestingUtil.getTestDir(); - warehousePath = new Path(testPath, "warehouse"); - - //create local hiveMeta - HiveConf conf = new HiveConf(); - String jdbcUri = "jdbc:derby:;databaseName="+testPath.toUri().getPath()+"metastore_db;create=true"; - conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehousePath.toUri().toString()); - conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, jdbcUri); - conf.set(TajoConf.ConfVars.WAREHOUSE_DIR.varname, warehousePath.toUri().toString()); - - // create local HiveCatalogStore. - TajoConf tajoConf = new TajoConf(conf); - store = new HCatalogStore(tajoConf); - store.createDatabase(DB_NAME, null); - } - - @AfterClass - public static void tearDown() throws IOException { - store.close(); - } - - @Test - public void testTableUsingTextFile() throws Exception { - TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); - - org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); - schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); - schema.addColumn("c_name", TajoDataTypes.Type.TEXT); - schema.addColumn("c_address", TajoDataTypes.Type.TEXT); - schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4); - schema.addColumn("c_phone", TajoDataTypes.Type.TEXT); - schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8); - schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT); - schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); - - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, - new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); - store.createTable(table.getProto()); - assertTrue(store.existTable(DB_NAME, CUSTOMER)); - - TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER)); - assertEquals(table.getName(), table1.getName()); - assertEquals(table.getUri(), table1.getUri()); - assertEquals(table.getSchema().size(), table1.getSchema().size()); - for (int i = 0; i < table.getSchema().size(); i++) { - assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); - } - - assertEquals(StringEscapeUtils.escapeJava(StorageConstants.DEFAULT_FIELD_DELIMITER), - table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER)); - store.dropTable(DB_NAME, CUSTOMER); - } - -} From 80d3bd12585267d55c322534209884f113d5d1b4 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 19 Aug 2015 15:09:01 +0900 Subject: [PATCH 3/3] Trigger for the travis CI build --- .../main/java/org/apache/tajo/catalog/store/HCatalogStore.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index c1a9e30c09..d9180fdd46 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -26,7 +26,6 @@ * to be compatible with Tajo 0.10.x applications. */ public class HCatalogStore extends HiveCatalogStore { - public HCatalogStore(final Configuration conf) { super(conf); }